From c70f0f34153c05ee8fb98123add7c923baef037e Mon Sep 17 00:00:00 2001 From: August Kilponen Date: Thu, 16 Jan 2025 18:57:57 +0200 Subject: [PATCH 1/2] =?UTF-8?q?Vaihdettu=20k=C3=A4ytt=C3=B6=C3=B6n=20AWS?= =?UTF-8?q?=20SDK=20V2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../lampi-siirtaja/pom.xml | 7 +- .../java/fi/oph/opintopolku/ovara/App.java | 4 +- .../oph/opintopolku/ovara/config/Config.java | 4 +- .../opintopolku/ovara/db/DatabaseToS3.java | 2 +- .../ovara/io/MultiInputStream.java | 9 +- .../opintopolku/ovara/s3/LampiS3Transfer.java | 116 +++++++++++------- 6 files changed, 85 insertions(+), 57 deletions(-) diff --git a/lampi-siirtaja-container/lampi-siirtaja/pom.xml b/lampi-siirtaja-container/lampi-siirtaja/pom.xml index ef7c295..8aa8e13 100644 --- a/lampi-siirtaja-container/lampi-siirtaja/pom.xml +++ b/lampi-siirtaja-container/lampi-siirtaja/pom.xml @@ -16,10 +16,11 @@ - com.amazonaws - aws-java-sdk-s3 - 1.12.780 + software.amazon.awssdk + s3 + 2.30.0 + com.google.guava guava diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/App.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/App.java index e87fe63..b039684 100644 --- a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/App.java +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/App.java @@ -1,10 +1,10 @@ package fi.oph.opintopolku.ovara; -import com.amazonaws.regions.Regions; import fi.oph.opintopolku.ovara.config.Config; import fi.oph.opintopolku.ovara.service.LampiSiirtajaService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.regions.Region; public class App { @@ -22,7 +22,7 @@ public static void main(String[] args) throws Exception { System.getenv("DB_PASSWORD"), System.getenv("OVARA_LAMPI_SIIRTAJA_BUCKET"), System.getenv("LAMPI_S3_BUCKET"), - Regions.EU_WEST_1.getName(), + Region.EU_WEST_1, "fulldump/ovara/v1/"); LampiSiirtajaService service = new LampiSiirtajaService(config); diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/config/Config.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/config/Config.java index 5dcf057..3d3f276 100644 --- a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/config/Config.java +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/config/Config.java @@ -1,5 +1,7 @@ package fi.oph.opintopolku.ovara.config; +import software.amazon.awssdk.regions.Region; + public record Config( String postgresHost, Integer postgresPort, @@ -7,5 +9,5 @@ public record Config( String postgresPassword, String ovaraS3Bucket, String lampiS3Bucket, - String awsRegion, + Region awsRegion, String lampiKeyPrefix) {} diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/DatabaseToS3.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/DatabaseToS3.java index ef6a546..e7034cd 100644 --- a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/DatabaseToS3.java +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/db/DatabaseToS3.java @@ -94,7 +94,7 @@ private S3ExportResult exportTableToS3(String schemaName, String tableName) thro String.format("select * from %s.%s", schemaName, tableName), config.ovaraS3Bucket(), String.format("%s.csv", tableName), - config.awsRegion()); + config.awsRegion().id()); LOG.info( "Scheman {} taulun {} vienti Ovaran S3-ämpäriin valmistui. Tulokset: {}", diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/io/MultiInputStream.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/io/MultiInputStream.java index 4e972ce..3f56fe9 100644 --- a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/io/MultiInputStream.java +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/io/MultiInputStream.java @@ -1,18 +1,19 @@ package fi.oph.opintopolku.ovara.io; -import com.amazonaws.services.s3.model.S3ObjectInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Enumeration; import java.util.Objects; import java.util.function.Supplier; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; public class MultiInputStream extends InputStream { - private final Enumeration> e; + private final Enumeration>> e; private InputStream in; - public MultiInputStream(Enumeration> e) { + public MultiInputStream(Enumeration>> e) { this.e = e; peekNextStream(); } @@ -26,7 +27,7 @@ final void nextStream() throws IOException { private void peekNextStream() { if (e.hasMoreElements()) { - Supplier s = e.nextElement(); + Supplier> s = e.nextElement(); in = s.get(); if (in == null) throw new NullPointerException(); } else { diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/s3/LampiS3Transfer.java b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/s3/LampiS3Transfer.java index 3297f03..bd8e6c8 100644 --- a/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/s3/LampiS3Transfer.java +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/java/fi/oph/opintopolku/ovara/s3/LampiS3Transfer.java @@ -1,8 +1,5 @@ package fi.oph.opintopolku.ovara.s3; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.*; import com.google.common.collect.Iterators; import com.google.gson.Gson; import fi.oph.opintopolku.ovara.config.Config; @@ -19,6 +16,11 @@ import java.util.zip.GZIPOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; public class LampiS3Transfer { @@ -28,8 +30,8 @@ public class LampiS3Transfer { private static final String MANIFEST_FILENAME = "manifest.json"; private final Config config; - private final AmazonS3 ovaraS3Client; - private final AmazonS3 lampiS3Client; + private final S3Client ovaraS3Client; + private final S3Client lampiS3Client; private final AtomicInteger uploadPartId = new AtomicInteger(0); private final Gson gson; @@ -37,14 +39,26 @@ public class LampiS3Transfer { public LampiS3Transfer(Config config) { this.config = config; - this.ovaraS3Client = AmazonS3ClientBuilder.standard().build(); - this.lampiS3Client = AmazonS3ClientBuilder.standard().build(); this.gson = new Gson(); + this.ovaraS3Client = + S3Client.builder() + .region(config.awsRegion()) + .credentialsProvider(ContainerCredentialsProvider.create()) + .build(); + this.lampiS3Client = + S3Client.builder() + .region(config.awsRegion()) + .credentialsProvider(ContainerCredentialsProvider.create()) + .build(); } - private Supplier constructSupplier(String downloadFilename) { - return () -> - ovaraS3Client.getObject(config.ovaraS3Bucket(), downloadFilename).getObjectContent(); + private Supplier> constructSupplier( + String downloadFilename) { + return () -> { + GetObjectRequest getObjectRequest = + GetObjectRequest.builder().bucket(config.ovaraS3Bucket()).key(downloadFilename).build(); + return ovaraS3Client.getObject(getObjectRequest); + }; } public void startGZIPCompressing(OutputStream out, InputStream in) { @@ -65,21 +79,18 @@ public void startGZIPCompressing(OutputStream out, InputStream in) { } } - private PartETag submitTaskForUploading( + private CompletedPart submitTaskForUploading( String uploadFilename, ByteArrayInputStream inputStream, boolean isFinalPart) { int eachPartId = uploadPartId.incrementAndGet(); UploadPartRequest uploadRequest = - new UploadPartRequest() - .withBucketName(config.lampiS3Bucket()) - .withKey(uploadFilename) - .withUploadId(uploadId) - .withPartNumber(eachPartId) - .withPartSize(inputStream.available()) - .withInputStream(inputStream); - - if (isFinalPart) { - uploadRequest.withLastPart(true); - } + UploadPartRequest.builder() + .bucket(config.lampiS3Bucket()) + .key(uploadFilename) + .uploadId(uploadId) + .partNumber(eachPartId) + .build(); + + RequestBody requestBody = RequestBody.fromInputStream(inputStream, inputStream.available()); LOG.info( "Lähetetään tiedoston {} palanen {} jonka koko on {}", @@ -87,10 +98,14 @@ private PartETag submitTaskForUploading( eachPartId, inputStream.available()); - UploadPartResult uploadResult = lampiS3Client.uploadPart(uploadRequest); + UploadPartResponse uploadPartResponse = lampiS3Client.uploadPart(uploadRequest, requestBody); LOG.info("Lähetetty tiedoston {} palanen {}", uploadFilename, eachPartId); - return uploadResult.getPartETag(); + + CompletedPart completedPart = + CompletedPart.builder().partNumber(eachPartId).eTag(uploadPartResponse.eTag()).build(); + + return completedPart; } public String transferToLampi(String filename, String uploadFilename, int numberOfFiles) @@ -101,17 +116,20 @@ public String transferToLampi(String filename, String uploadFilename, int number filename, numberOfFiles); - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentType("text/csv"); - metadata.setContentEncoding("gzip"); - InitiateMultipartUploadRequest initRequest = - new InitiateMultipartUploadRequest(config.lampiS3Bucket(), uploadFilename) - .withObjectMetadata(metadata); - InitiateMultipartUploadResult initResult = lampiS3Client.initiateMultipartUpload(initRequest); + CreateMultipartUploadRequest createRequest = + CreateMultipartUploadRequest.builder() + .bucket(config.lampiS3Bucket()) + .key(uploadFilename) + .contentType("text/csv") + .contentEncoding("gzip") + .build(); + + CreateMultipartUploadResponse createResponse = + lampiS3Client.createMultipartUpload(createRequest); - uploadId = initResult.getUploadId(); + uploadId = createResponse.uploadId(); - List> streamsFList = + List>> streamsFList = IntStream.rangeClosed(1, numberOfFiles) .mapToObj( fileNumber -> { @@ -121,7 +139,7 @@ public String transferToLampi(String filename, String uploadFilename, int number }) .toList(); - Enumeration> streams = + Enumeration>> streams = Iterators.asEnumeration(streamsFList.iterator()); InputStream multiInputStream = new MultiInputStream(streams); @@ -136,7 +154,7 @@ public String transferToLampi(String filename, String uploadFilename, int number int bytesRead, bytesAdded = 0; byte[] data = new byte[UPLOAD_PART_SIZE]; ByteArrayOutputStream bufferOutputStream = new ByteArrayOutputStream(); - List parts = new ArrayList<>(); + List completedParts = new ArrayList<>(); while ((bytesRead = pipedInputStream.read(data, 0, data.length)) != -1) { bufferOutputStream.write(data, 0, bytesRead); @@ -145,30 +163,35 @@ public String transferToLampi(String filename, String uploadFilename, int number bytesAdded += bytesRead; continue; } - PartETag partETag = + CompletedPart completedPart = submitTaskForUploading( uploadFilename, new ByteArrayInputStream(bufferOutputStream.toByteArray()), false); - parts.add(partETag); + completedParts.add(completedPart); bufferOutputStream.reset(); // flush the bufferOutputStream bytesAdded = 0; // reset the bytes added to 0 } - PartETag partETag = + CompletedPart completedPart = submitTaskForUploading( uploadFilename, new ByteArrayInputStream(bufferOutputStream.toByteArray()), true); - parts.add(partETag); + completedParts.add(completedPart); CompleteMultipartUploadRequest completeRequest = - new CompleteMultipartUploadRequest(config.lampiS3Bucket(), uploadFilename, uploadId, parts); - CompleteMultipartUploadResult completeMultipartUploadResult = + CompleteMultipartUploadRequest.builder() + .bucket(config.lampiS3Bucket()) + .key(uploadFilename) + .uploadId(uploadId) + .multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build()) + .build(); + CompleteMultipartUploadResponse completeMultipartUploadResult = lampiS3Client.completeMultipartUpload(completeRequest); LOG.info("Tiedoston {} lähettäminen Lammen S3-ämpäriin valmistui", filename); - return completeMultipartUploadResult.getVersionId(); + return completeMultipartUploadResult.versionId(); } - public void uploadManifest(List manifestItems) { + public void uploadManifest(List manifestItems) throws Exception { String uploadFilename = config.lampiKeyPrefix() + MANIFEST_FILENAME; String json = gson.toJson(manifestItems); @@ -176,10 +199,11 @@ public void uploadManifest(List manifestItems) { LOG.info("Manifest: {}", json); - ObjectMetadata metadata = new ObjectMetadata(); PutObjectRequest putObjectRequest = - new PutObjectRequest(config.lampiS3Bucket(), uploadFilename, inputStream, metadata); + PutObjectRequest.builder().bucket(config.lampiS3Bucket()).key(uploadFilename).build(); + + RequestBody requestBody = RequestBody.fromInputStream(inputStream, inputStream.available()); - lampiS3Client.putObject(putObjectRequest); + lampiS3Client.putObject(putObjectRequest, requestBody); } } From 0e777eca9126ad25277fe2efa1b856214898abc2 Mon Sep 17 00:00:00 2001 From: August Kilponen Date: Thu, 16 Jan 2025 19:52:07 +0200 Subject: [PATCH 2/2] Logbackin konffi --- .../lampi-siirtaja/src/main/resources/logback.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 lampi-siirtaja-container/lampi-siirtaja/src/main/resources/logback.xml diff --git a/lampi-siirtaja-container/lampi-siirtaja/src/main/resources/logback.xml b/lampi-siirtaja-container/lampi-siirtaja/src/main/resources/logback.xml new file mode 100644 index 0000000..751ab2e --- /dev/null +++ b/lampi-siirtaja-container/lampi-siirtaja/src/main/resources/logback.xml @@ -0,0 +1,14 @@ + + + + + %d{yyyy-MM-dd'T'HH:mm:ss.SSSX, Europe/Helsinki} [%t] %-5level %c: %m%n + + + + + + + + +