Skip to content
Snippets Groups Projects
Commit 54e6f95c authored by Jan Zickermann's avatar Jan Zickermann
Browse files

OZG-4097 send-attachment: Upload chunks

parent ef39e25e
Branches
Tags
1 merge request!15Ozg 4097 senden und empfangen von anhängen
Showing
with 193 additions and 45 deletions
package de.ozgcloud.nachrichten.postfach.osiv2.model; package de.ozgcloud.nachrichten.postfach.osiv2.model;
import static de.ozgcloud.nachrichten.postfach.osiv2.model.Osi2FileUpload.*;
import java.io.InputStream;
import org.springframework.core.io.AbstractResource;
import de.ozgcloud.nachrichten.postfach.osiv2.storage.LimitedInputStream;
import lombok.Builder; import lombok.Builder;
@Builder @Builder
public record FileChunkInfo( public record FileChunkInfo(
String guid, Osi2FileUpload upload,
String fileName, long chunkIndex
String contentType,
Integer chunkIndex,
Integer totalChunks,
Long totalFileSize
) { ) {
public AbstractResource createUploadResource(InputStream fileInputStream) {
return new AbstractResource() {
@Override
public String getDescription() {
return FileChunkInfo.this.toString();
}
@Override
public InputStream getInputStream() {
return new LimitedInputStream(fileInputStream, CHUNK_SIZE);
}
};
}
} }
package de.ozgcloud.nachrichten.postfach.osiv2.model; package de.ozgcloud.nachrichten.postfach.osiv2.model;
import java.util.UUID;
import de.ozgcloud.apilib.file.OzgCloudFile; import de.ozgcloud.apilib.file.OzgCloudFile;
import lombok.Builder; import lombok.Builder;
...@@ -8,4 +10,17 @@ public record Osi2FileUpload( ...@@ -8,4 +10,17 @@ public record Osi2FileUpload(
String guid, String guid,
OzgCloudFile file OzgCloudFile file
) { ) {
public static final long CHUNK_SIZE = 100L * (2L << 10);
public static Osi2FileUpload from(OzgCloudFile file) {
return Osi2FileUpload.builder()
.guid(UUID.randomUUID().toString())
.file(file)
.build();
}
public long numberOfChunks() {
return (file.getSize() + CHUNK_SIZE - 1) / CHUNK_SIZE;
}
} }
...@@ -8,8 +8,8 @@ import lombok.RequiredArgsConstructor; ...@@ -8,8 +8,8 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
public class LimitedInputStream extends InputStream { public class LimitedInputStream extends InputStream {
private final InputStream parentStream; private final InputStream parentStream;
private final int limit; private final long limit;
private int readOffset = 0; private long readOffset = 0;
@Override @Override
public int read() throws IOException { public int read() throws IOException {
...@@ -18,12 +18,11 @@ public class LimitedInputStream extends InputStream { ...@@ -18,12 +18,11 @@ public class LimitedInputStream extends InputStream {
return result < 0 ? result : buffer[0]; return result < 0 ? result : buffer[0];
} }
@Override @Override
public int read(byte[] buffer, int offset, int length) throws IOException { public int read(byte[] buffer, int offset, int length) throws IOException {
var remainingLength = limit - readOffset; var remainingLength = limit - readOffset;
if (remainingLength > 0) { if (remainingLength > 0) {
var readLength = parentStream.read(buffer, offset, Math.min(remainingLength, length)); var readLength = parentStream.read(buffer, offset, (int) Math.min(remainingLength, length));
if (readLength > 0) { if (readLength > 0) {
readOffset += readLength; readOffset += readLength;
return readLength; return readLength;
......
...@@ -11,6 +11,7 @@ import java.util.concurrent.Executors; ...@@ -11,6 +11,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import java.util.function.BooleanSupplier; import java.util.function.BooleanSupplier;
import java.util.stream.LongStream;
import java.util.stream.Stream; import java.util.stream.Stream;
import de.ozgcloud.apilib.file.OzgCloudFile; import de.ozgcloud.apilib.file.OzgCloudFile;
...@@ -31,7 +32,6 @@ public class Osi2QuarantineService { ...@@ -31,7 +32,6 @@ public class Osi2QuarantineService {
static final Duration POLLING_INTERVAL = Duration.ofSeconds(1); static final Duration POLLING_INTERVAL = Duration.ofSeconds(1);
static final Duration POLLING_TIMEOUT = Duration.ofMinutes(5); static final Duration POLLING_TIMEOUT = Duration.ofMinutes(5);
static final int CHUNK_SIZE = 100 * (2 << 10);
public List<Osi2FileUpload> uploadAttachments(List<String> attachmentIds) { public List<Osi2FileUpload> uploadAttachments(List<String> attachmentIds) {
var ozgCloudFiles = binaryFileService.getFileMetadataOfIds(attachmentIds); var ozgCloudFiles = binaryFileService.getFileMetadataOfIds(attachmentIds);
...@@ -47,23 +47,36 @@ public class Osi2QuarantineService { ...@@ -47,23 +47,36 @@ public class Osi2QuarantineService {
.toList(); .toList();
} }
Osi2FileUpload uploadFileToQuarantine(OzgCloudFile file) { Osi2FileUpload uploadFileToQuarantine(OzgCloudFile file) {
try (var fileInputStream = binaryFileService.streamFileContent(file.getId().toString())) { try (var fileInputStream = binaryFileService.streamFileContent(file.getId().toString())) {
return uploadInputStreamToQuarantine(file, fileInputStream); return uploadInputStreamToQuarantine(file, fileInputStream);
} catch (IOException e) { } catch (IOException e) {
throw new OsiPostfachException("Failed to upload file to quarantine!", e); throw new OsiPostfachException("Failed to close input stream!", e);
} }
} }
Osi2FileUpload uploadInputStreamToQuarantine(OzgCloudFile file, InputStream fileInputStream) { Osi2FileUpload uploadInputStreamToQuarantine(OzgCloudFile file, InputStream fileInputStream) {
// TODO var fileUpload = Osi2FileUpload.from(file);
return null; streamFileChunkInfos(fileUpload)
.forEachOrdered(chunkInfo ->
postfachApiFacadeService.uploadChunk(
chunkInfo,
chunkInfo.createUploadResource(fileInputStream)
)
);
return fileUpload;
} }
Stream<FileChunkInfo> streamFileChunkInfos(OzgCloudFile file) { Stream<FileChunkInfo> streamFileChunkInfos(Osi2FileUpload upload) {
// TODO return LongStream.range(0, upload.numberOfChunks())
return null; .mapToObj(chunkIndex -> buildFileChunkInfo(upload, chunkIndex));
}
private FileChunkInfo buildFileChunkInfo(Osi2FileUpload upload, long chunkIndex) {
return FileChunkInfo.builder()
.upload(upload)
.chunkIndex(chunkIndex)
.build();
} }
...@@ -81,7 +94,7 @@ public class Osi2QuarantineService { ...@@ -81,7 +94,7 @@ public class Osi2QuarantineService {
} catch (TimeoutException e) { } catch (TimeoutException e) {
throw new OsiPostfachException("Expect the scan to complete after %d seconds!".formatted(POLLING_TIMEOUT.getSeconds()), e); throw new OsiPostfachException("Expect the scan to complete after %d seconds!".formatted(POLLING_TIMEOUT.getSeconds()), e);
} catch (InterruptedException e) { } catch (InterruptedException e) {
LOG.error("[waitForVirusScan] Interrupt"); LOG.debug("[waitForVirusScan] Interrupt");
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
} }
} }
...@@ -99,7 +112,6 @@ public class Osi2QuarantineService { ...@@ -99,7 +112,6 @@ public class Osi2QuarantineService {
}).get(timeout.getSeconds(), TimeUnit.SECONDS); }).get(timeout.getSeconds(), TimeUnit.SECONDS);
} }
public void deleteAttachments(List<Osi2FileUpload> osi2FileMetadata) { public void deleteAttachments(List<Osi2FileUpload> osi2FileMetadata) {
// TODO delete on exception // TODO delete on exception
} }
......
...@@ -77,7 +77,11 @@ public interface Osi2RequestMapper { ...@@ -77,7 +77,11 @@ public interface Osi2RequestMapper {
} }
@Mapping(target = "target", ignore = true) @Mapping(target = "target", ignore = true)
@Mapping(target = "uploadUid", source = "guid") @Mapping(target = "uploadUid", source = "upload.guid")
@Mapping(target = "fileName", source = "upload.file.name")
@Mapping(target = "contentType", source = "upload.file.contentType")
@Mapping(target = "totalChunks", expression = "java( (int) fileChunkInfo.upload().numberOfChunks() )")
@Mapping(target = "totalFileSize", source = "upload.file.size")
DomainChunkMetaData mapDomainChunkMetaData(FileChunkInfo fileChunkInfo); DomainChunkMetaData mapDomainChunkMetaData(FileChunkInfo fileChunkInfo);
default String mapMailboxId(PostfachNachricht nachricht) { default String mapMailboxId(PostfachNachricht nachricht) {
......
package de.ozgcloud.nachrichten.postfach.osiv2.factory; package de.ozgcloud.nachrichten.postfach.osiv2.factory;
import java.util.UUID;
import de.ozgcloud.nachrichten.postfach.osiv2.model.FileChunkInfo; import de.ozgcloud.nachrichten.postfach.osiv2.model.FileChunkInfo;
import de.ozgcloud.nachrichten.postfach.osiv2.transfer.Osi2FileUploadTestFactory;
public class FileChunkInfoTestFactory { public class FileChunkInfoTestFactory {
public static final String FILE_UPLOAD_GUID = UUID.randomUUID().toString();
public static final String FILE_NAME = "test-file-name";
public static final String CONTENT_TYPE = "test-content-type";
public static final Integer CHUNK_INDEX = 1; public static final Integer CHUNK_INDEX = 1;
public static final Integer TOTAL_CHUNKS = 10;
public static final Long TOTAL_FILE_SIZE = 1000L;
public static FileChunkInfo create() { public static FileChunkInfo create() {
return createBuilder().build(); return createBuilder().build();
...@@ -20,11 +13,7 @@ public class FileChunkInfoTestFactory { ...@@ -20,11 +13,7 @@ public class FileChunkInfoTestFactory {
public static FileChunkInfo.FileChunkInfoBuilder createBuilder() { public static FileChunkInfo.FileChunkInfoBuilder createBuilder() {
return FileChunkInfo.builder() return FileChunkInfo.builder()
.guid(FILE_UPLOAD_GUID) .upload(Osi2FileUploadTestFactory.create())
.fileName(FILE_NAME) .chunkIndex(CHUNK_INDEX);
.contentType(CONTENT_TYPE)
.chunkIndex(CHUNK_INDEX)
.totalChunks(TOTAL_CHUNKS)
.totalFileSize(TOTAL_FILE_SIZE);
} }
} }
package de.ozgcloud.nachrichten.postfach.osiv2.model;
import static de.ozgcloud.nachrichten.postfach.osiv2.model.Osi2FileUpload.*;
import static org.assertj.core.api.Assertions.*;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import de.ozgcloud.apilib.file.OzgCloudFileTestFactory;
public class Osi2FileUploadTest {
@DisplayName("calculate number of chunks")
@Nested
class TestCalculateNumberOfChunks {
@DisplayName("should return zero")
@Test
void shouldReturnZero() {
var upload = createWithSize(0L);
var result = upload.numberOfChunks();
assertThat(result).isZero();
}
@DisplayName("should return one")
@ParameterizedTest
@ValueSource(longs = { 1, CHUNK_SIZE - 1, CHUNK_SIZE })
void shouldReturnOne(long fileLength) {
var upload = createWithSize(fileLength);
var result = upload.numberOfChunks();
assertThat(result).isEqualTo(1L);
}
@DisplayName("should return two")
@ParameterizedTest
@ValueSource(longs = { CHUNK_SIZE + 1, CHUNK_SIZE * 2 - 1, CHUNK_SIZE * 2 })
void shouldReturnTwo(long fileLength) {
var upload = createWithSize(fileLength);
var result = upload.numberOfChunks();
assertThat(result).isEqualTo(2L);
}
private Osi2FileUpload createWithSize(long size) {
return Osi2FileUpload.builder()
.file(OzgCloudFileTestFactory.createBuilder()
.size(size)
.build())
.build();
}
}
}
package de.ozgcloud.nachrichten.postfach.osiv2.transfer; package de.ozgcloud.nachrichten.postfach.osiv2.transfer;
import static de.ozgcloud.nachrichten.postfach.osiv2.model.Osi2FileUpload.*;
import java.util.UUID; import java.util.UUID;
import de.ozgcloud.apilib.file.OzgCloudFile; import de.ozgcloud.apilib.file.OzgCloudFile;
...@@ -13,7 +15,8 @@ public class Osi2FileUploadTestFactory { ...@@ -13,7 +15,8 @@ public class Osi2FileUploadTestFactory {
public static final String UPLOAD_GUID2 = UUID.randomUUID().toString(); public static final String UPLOAD_GUID2 = UUID.randomUUID().toString();
public static final String UPLOAD_NAME = "upload.txt"; public static final String UPLOAD_NAME = "upload.txt";
public static final String UPLOAD_CONTENT_TYPE = "text/plain"; public static final String UPLOAD_CONTENT_TYPE = "text/plain";
public static final Long UPLOAD_SIZE = 1001L; public static final long NUMBER_OF_CHUNKS = 5;
public static final long UPLOAD_SIZE = CHUNK_SIZE * NUMBER_OF_CHUNKS;
public static Osi2FileUpload create() { public static Osi2FileUpload create() {
return createBuilder().build(); return createBuilder().build();
......
...@@ -31,8 +31,8 @@ class Osi2PostfachServiceTest { ...@@ -31,8 +31,8 @@ class Osi2PostfachServiceTest {
@DisplayName("send message") @DisplayName("send message")
@Nested @Nested
class TestSendMessage { class TestSendMessage {
private PostfachNachricht nachricht = PostfachNachrichtTestFactory.create(); private final PostfachNachricht nachricht = PostfachNachrichtTestFactory.create();
private List<Osi2FileUpload> uploadFiles = List.of(Osi2FileUploadTestFactory.create()); private final List<Osi2FileUpload> uploadFiles = List.of(Osi2FileUploadTestFactory.create());
@BeforeEach @BeforeEach
void mock() { void mock() {
......
package de.ozgcloud.nachrichten.postfach.osiv2.transfer; package de.ozgcloud.nachrichten.postfach.osiv2.transfer;
import static de.ozgcloud.nachrichten.postfach.osiv2.model.Osi2FileUpload.*;
import static de.ozgcloud.nachrichten.postfach.osiv2.transfer.Osi2FileUploadTestFactory.*; import static de.ozgcloud.nachrichten.postfach.osiv2.transfer.Osi2FileUploadTestFactory.*;
import static org.assertj.core.api.Assertions.*; import static org.assertj.core.api.Assertions.*;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*; import static org.mockito.Mockito.*;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.List; import java.util.List;
...@@ -19,6 +20,8 @@ import org.mockito.Spy; ...@@ -19,6 +20,8 @@ import org.mockito.Spy;
import de.ozgcloud.apilib.file.OzgCloudFile; import de.ozgcloud.apilib.file.OzgCloudFile;
import de.ozgcloud.apilib.file.OzgCloudFileId; import de.ozgcloud.apilib.file.OzgCloudFileId;
import de.ozgcloud.apilib.file.OzgCloudFileTestFactory; import de.ozgcloud.apilib.file.OzgCloudFileTestFactory;
import de.ozgcloud.nachrichten.postfach.osiv2.OsiPostfachException;
import de.ozgcloud.nachrichten.postfach.osiv2.model.FileChunkInfo;
import de.ozgcloud.nachrichten.postfach.osiv2.model.Osi2FileUpload; import de.ozgcloud.nachrichten.postfach.osiv2.model.Osi2FileUpload;
import de.ozgcloud.nachrichten.postfach.osiv2.storage.Osi2BinaryFileRemoteService; import de.ozgcloud.nachrichten.postfach.osiv2.storage.Osi2BinaryFileRemoteService;
import lombok.SneakyThrows; import lombok.SneakyThrows;
...@@ -159,9 +162,56 @@ class Osi2QuarantineServiceTest { ...@@ -159,9 +162,56 @@ class Osi2QuarantineServiceTest {
assertThat(result).isEqualTo(uploadFile); assertThat(result).isEqualTo(uploadFile);
} }
@DisplayName("should close inputStream")
@Test
@SneakyThrows
void shouldCloseInputStream() {
uploadFileToQuarantine();
verify(fileInputStream).close();
}
@DisplayName("should throw OsiPostfachException if close fails with IOException")
@Test
@SneakyThrows
void shouldThrowOsiPostfachException() {
doThrow(new IOException("test")).when(fileInputStream).close();
assertThatThrownBy(this::uploadFileToQuarantine)
.isInstanceOf(OsiPostfachException.class);
}
Osi2FileUpload uploadFileToQuarantine() { Osi2FileUpload uploadFileToQuarantine() {
return service.uploadFileToQuarantine(file); return service.uploadFileToQuarantine(file);
} }
} }
@DisplayName("stream file chunk infos")
@Nested
class TestStreamFileChunkInfos {
private final Osi2FileUpload upload = Osi2FileUploadTestFactory.createBuilder()
.file(OzgCloudFileTestFactory.createBuilder()
.size(CHUNK_SIZE * 2)
.build())
.build();
private final FileChunkInfo chunkInfo1 = FileChunkInfo.builder()
.upload(upload)
.chunkIndex(0)
.build();
private final FileChunkInfo chunkInfo2 = FileChunkInfo.builder()
.upload(upload)
.chunkIndex(1)
.build();
@DisplayName("should stream chunk infos")
@Test
void shouldStreamChunkInfos() {
var result = service.streamFileChunkInfos(upload).toList();
assertThat(result).containsExactly(chunkInfo1, chunkInfo2);
}
}
} }
\ No newline at end of file
...@@ -271,7 +271,7 @@ class Osi2RequestMapperTest { ...@@ -271,7 +271,7 @@ class Osi2RequestMapperTest {
void shouldMapUploadGuid() { void shouldMapUploadGuid() {
var result = doMapping(); var result = doMapping();
assertThat(result.getUploadUid()).isEqualTo(UUID.fromString(FileChunkInfoTestFactory.FILE_UPLOAD_GUID)); assertThat(result.getUploadUid()).isEqualTo(UUID.fromString(UPLOAD_GUID));
} }
@DisplayName("should map file name") @DisplayName("should map file name")
...@@ -279,7 +279,7 @@ class Osi2RequestMapperTest { ...@@ -279,7 +279,7 @@ class Osi2RequestMapperTest {
void shouldMapFileName() { void shouldMapFileName() {
var result = doMapping(); var result = doMapping();
assertThat(result.getFileName()).isEqualTo(FileChunkInfoTestFactory.FILE_NAME); assertThat(result.getFileName()).isEqualTo(UPLOAD_NAME);
} }
@DisplayName("should map content type") @DisplayName("should map content type")
...@@ -287,7 +287,7 @@ class Osi2RequestMapperTest { ...@@ -287,7 +287,7 @@ class Osi2RequestMapperTest {
void shouldMapContentType() { void shouldMapContentType() {
var result = doMapping(); var result = doMapping();
assertThat(result.getContentType()).isEqualTo(FileChunkInfoTestFactory.CONTENT_TYPE); assertThat(result.getContentType()).isEqualTo(UPLOAD_CONTENT_TYPE);
} }
@DisplayName("should map chunk index") @DisplayName("should map chunk index")
...@@ -303,7 +303,7 @@ class Osi2RequestMapperTest { ...@@ -303,7 +303,7 @@ class Osi2RequestMapperTest {
void shouldMapTotalChunks() { void shouldMapTotalChunks() {
var result = doMapping(); var result = doMapping();
assertThat(result.getTotalChunks()).isEqualTo(FileChunkInfoTestFactory.TOTAL_CHUNKS); assertThat(result.getTotalChunks()).isEqualTo(NUMBER_OF_CHUNKS);
} }
@DisplayName("should map total file size") @DisplayName("should map total file size")
...@@ -311,7 +311,7 @@ class Osi2RequestMapperTest { ...@@ -311,7 +311,7 @@ class Osi2RequestMapperTest {
void shouldMapTotalFileSize() { void shouldMapTotalFileSize() {
var result = doMapping(); var result = doMapping();
assertThat(result.getTotalFileSize()).isEqualTo(FileChunkInfoTestFactory.TOTAL_FILE_SIZE); assertThat(result.getTotalFileSize()).isEqualTo(UPLOAD_SIZE);
} }
private DomainChunkMetaData doMapping() { private DomainChunkMetaData doMapping() {
......
...@@ -65,7 +65,7 @@ class PostfachApiFacadeServiceTest { ...@@ -65,7 +65,7 @@ class PostfachApiFacadeServiceTest {
@Mock @Mock
MessageExchangeSendMessageResponse messageExchangeSendMessageResponse; MessageExchangeSendMessageResponse messageExchangeSendMessageResponse;
List<Osi2FileUpload> files = List.of(Osi2FileUploadTestFactory.create()); private final List<Osi2FileUpload> files = List.of(Osi2FileUploadTestFactory.create());
private final PostfachNachricht nachricht = PostfachNachrichtTestFactory.create(); private final PostfachNachricht nachricht = PostfachNachrichtTestFactory.create();
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment