-
Notifications
You must be signed in to change notification settings - Fork 500
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #10044 from IQSS/6783-s3-tests
add S3 tests, LocalStack, MinIO
- Loading branch information
Showing
12 changed files
with
767 additions
and
61 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| #!/usr/bin/env bash | ||
| # https://stackoverflow.com/questions/53619901/auto-create-s3-buckets-on-localstack | ||
| awslocal s3 mb s3://mybucket |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| Developers can now test S3 locally by using the Dockerized development environment, which now includes both LocalStack and MinIO. API (end to end) tests are in S3AccessIT. | ||
|
|
||
| In addition, a new integration test class (not an API test, the new Testcontainers-based test launched with `mvn verify`) has been added at S3AccessIOLocalstackIT. It uses Testcontainers to spin up Localstack for S3 testing and does not require Dataverse to be running. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
97 changes: 97 additions & 0 deletions
97
src/test/java/edu/harvard/iq/dataverse/api/S3AccessDirectIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| package edu.harvard.iq.dataverse.api; | ||
|
|
||
| import io.restassured.RestAssured; | ||
| import static io.restassured.RestAssured.given; | ||
| import io.restassured.path.json.JsonPath; | ||
| import io.restassured.response.Response; | ||
| import io.restassured.specification.RequestSpecification; | ||
| import java.io.ByteArrayInputStream; | ||
| import java.io.InputStream; | ||
| import java.io.UnsupportedEncodingException; | ||
| import java.net.URLDecoder; | ||
| import java.nio.charset.StandardCharsets; | ||
| import org.apache.commons.lang3.math.NumberUtils; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| public class S3AccessDirectIT { | ||
|
|
||
| @Test | ||
| public void testS3DirectUpload() { | ||
| // TODO: remove all these constants | ||
| RestAssured.baseURI = "https://demo.dataverse.org"; | ||
| String apiToken = ""; | ||
| String datasetPid = "doi:10.70122/FK2/UBWSJU"; | ||
| String datasetId = "2106131"; | ||
| long size = 1000000000l; | ||
|
|
||
| Response getUploadUrls = getUploadUrls(datasetPid, size, apiToken); | ||
| getUploadUrls.prettyPrint(); | ||
| getUploadUrls.then().assertThat().statusCode(200); | ||
|
|
||
| String url = JsonPath.from(getUploadUrls.asString()).getString("data.url"); | ||
| String partSize = JsonPath.from(getUploadUrls.asString()).getString("data.partSize"); | ||
| String storageIdentifier = JsonPath.from(getUploadUrls.asString()).getString("data.storageIdentifier"); | ||
| System.out.println("url: " + url); | ||
| System.out.println("partSize: " + partSize); | ||
| System.out.println("storageIdentifier: " + storageIdentifier); | ||
|
|
||
| System.out.println("uploading file via direct upload"); | ||
| String decodedUrl = null; | ||
| try { | ||
| decodedUrl = URLDecoder.decode(url, StandardCharsets.UTF_8.name()); | ||
| } catch (UnsupportedEncodingException ex) { | ||
| } | ||
|
|
||
| InputStream inputStream = new ByteArrayInputStream("bumble".getBytes(StandardCharsets.UTF_8)); | ||
| Response uploadFileDirect = uploadFileDirect(decodedUrl, inputStream); | ||
| uploadFileDirect.prettyPrint(); | ||
| uploadFileDirect.then().assertThat().statusCode(200); | ||
|
|
||
| // TODO: Use MD5 or whatever Dataverse is configured for and | ||
| // actually calculate it. | ||
| String jsonData = """ | ||
| { | ||
| "description": "My description.", | ||
| "directoryLabel": "data/subdir1", | ||
| "categories": [ | ||
| "Data" | ||
| ], | ||
| "restrict": "false", | ||
| "storageIdentifier": "%s", | ||
| "fileName": "file1.txt", | ||
| "mimeType": "text/plain", | ||
| "checksum": { | ||
| "@type": "SHA-1", | ||
| "@value": "123456" | ||
| } | ||
| } | ||
| """.formatted(storageIdentifier); | ||
| Response addRemoteFile = UtilIT.addRemoteFile(datasetId, jsonData, apiToken); | ||
| addRemoteFile.prettyPrint(); | ||
| addRemoteFile.then().assertThat() | ||
| .statusCode(200); | ||
| } | ||
|
|
||
| static Response getUploadUrls(String idOrPersistentIdOfDataset, long sizeInBytes, String apiToken) { | ||
| String idInPath = idOrPersistentIdOfDataset; // Assume it's a number. | ||
| String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path. | ||
| if (!NumberUtils.isCreatable(idOrPersistentIdOfDataset)) { | ||
| idInPath = ":persistentId"; | ||
| optionalQueryParam = "&persistentId=" + idOrPersistentIdOfDataset; | ||
| } | ||
| RequestSpecification requestSpecification = given(); | ||
| if (apiToken != null) { | ||
| requestSpecification = given() | ||
| .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); | ||
| } | ||
| return requestSpecification.get("/api/datasets/" + idInPath + "/uploadurls?size=" + sizeInBytes + optionalQueryParam); | ||
| } | ||
|
|
||
| static Response uploadFileDirect(String url, InputStream inputStream) { | ||
| return given() | ||
| .header("x-amz-tagging", "dv-state=temp") | ||
| .body(inputStream) | ||
| .put(url); | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.