From 3ac36f189d4e497cc12aa6e88938d5986c75181c Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger <43503240+paullatzelsperger@users.noreply.github.com> Date: Fri, 15 Sep 2023 09:10:20 +0200 Subject: [PATCH] feat: add azure dataplane (#755) * feat: add Azure Blob data plane packages * formatting/checkstyle * small test runtime simplification * add test step for dataplane tests * DEPENDENCIES * pr remarks - formatting * pr remarks - constants --- .github/workflows/verify.yaml | 13 +- DEPENDENCIES | 285 ++++++++-------- .../edc-dataplane-base/build.gradle.kts | 1 + .../cloud-transfer-tests/build.gradle.kts | 45 +++ .../transfer/test/AzureBlobHelper.java | 95 ++++++ .../transfer/test/AzureToAzureTest.java | 229 +++++++++++++ .../transfer/test/MultiCloudTest.java | 223 ++++++++++++ .../transfer/test/ParticipantRuntime.java | 71 ++++ .../transfer/test/RuntimeConfig.java | 83 +++++ .../dataplane/transfer/test/S3ToS3Test.java | 280 ++++++++++++++++ .../transfer/test/TestConstants.java | 73 ++++ .../transfer/test/TestFunctions.java | 53 +++ .../src/test/resources/testfile.json | 317 ++++++++++++++++++ .../edc-dataplane-proxy-e2e/build.gradle.kts | 0 .../proxy/e2e/DpfProxyEndToEndTest.java | 2 +- .../dataplane/proxy/e2e/EdrCacheSetup.java | 0 .../dataplane/proxy/e2e/KeyStoreSetup.java | 0 edc-tests/runtime/dataplane-cloud/README.md | 4 + .../runtime/dataplane-cloud/build.gradle.kts | 38 +++ gradle/libs.versions.toml | 13 +- settings.gradle.kts | 6 +- 21 files changed, 1682 insertions(+), 149 deletions(-) create mode 100644 edc-tests/edc-dataplane/cloud-transfer-tests/build.gradle.kts create mode 100644 edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureBlobHelper.java create mode 100644 edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureToAzureTest.java create mode 100644 edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/MultiCloudTest.java create mode 100644 edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/ParticipantRuntime.java create mode 100644 edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/RuntimeConfig.java create mode 100644 edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/S3ToS3Test.java create mode 100644 edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestConstants.java create mode 100644 edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestFunctions.java create mode 100644 edc-tests/edc-dataplane/cloud-transfer-tests/src/test/resources/testfile.json rename edc-tests/{ => edc-dataplane}/edc-dataplane-proxy-e2e/build.gradle.kts (100%) rename edc-tests/{ => edc-dataplane}/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/DpfProxyEndToEndTest.java (99%) rename edc-tests/{ => edc-dataplane}/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/EdrCacheSetup.java (100%) rename edc-tests/{ => edc-dataplane}/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/KeyStoreSetup.java (100%) create mode 100644 edc-tests/runtime/dataplane-cloud/README.md create mode 100644 edc-tests/runtime/dataplane-cloud/build.gradle.kts diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index dd3f48cd2..2570fdaa8 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -152,6 +152,17 @@ jobs: - name: Run Postgresql E2E tests run: ./gradlew test -DincludeTags="PostgresqlIntegrationTest" -PverboseTest=true + dataplane-tests: + runs-on: ubuntu-latest + needs: [ verify-formatting, verify-license-headers ] + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-java + + - name: Run Azure/S3 dataplane tests + run: ./gradlew -p edc-tests/edc-dataplane test -DincludeTags="AzureCosmosDbIntegrationTest,AwsS3IntegrationTest" + miw-integration-tests: runs-on: ubuntu-latest needs: [ verify-formatting, verify-license-headers ] @@ -183,7 +194,7 @@ jobs: docker exec docker-environment-postgres-1 /opt/seed.sh - name: Run MIW Integration tests - run: | + run: | ./gradlew -p edc-tests/miw-tests test -DincludeTags="MiwIntegrationTest" -PverboseTest=true - name: Run SSI E2E tests run: | diff --git a/DEPENDENCIES b/DEPENDENCIES index ab716079a..4e1875bd8 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -8,10 +8,11 @@ maven/mavencentral/com.azure/azure-core-http-netty/1.13.6, MIT AND Apache-2.0, a maven/mavencentral/com.azure/azure-core/1.41.0, MIT AND Apache-2.0, approved, #9648 maven/mavencentral/com.azure/azure-core/1.42.0, MIT AND Apache-2.0, approved, #10089 maven/mavencentral/com.azure/azure-identity/1.10.0, MIT AND Apache-2.0, approved, #10086 -maven/mavencentral/com.azure/azure-identity/1.9.2, MIT AND Apache-2.0, approved, #9686 maven/mavencentral/com.azure/azure-json/1.0.1, MIT AND Apache-2.0, approved, #7933 -maven/mavencentral/com.azure/azure-security-keyvault-secrets/4.6.4, MIT, approved, #7940 maven/mavencentral/com.azure/azure-security-keyvault-secrets/4.6.5, MIT, approved, #7940 +maven/mavencentral/com.azure/azure-storage-blob/12.23.0, MIT, approved, #9632 +maven/mavencentral/com.azure/azure-storage-common/12.22.0, MIT, approved, #9645 +maven/mavencentral/com.azure/azure-storage-internal-avro/12.8.0, MIT, approved, #9630 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.10.3, Apache-2.0, approved, CQ21280 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.13.3, Apache-2.0, approved, clearlydefined maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.13.5, Apache-2.0, approved, clearlydefined @@ -28,6 +29,7 @@ maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.14.1, Apache-2. maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.14.2, Apache-2.0, approved, #4105 maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.15.2, Apache-2.0, approved, #7934 maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-toml/2.15.2, Apache-2.0, approved, #9160 +maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.13.5, Apache-2.0, approved, #3768 maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.14.2, Apache-2.0, approved, #4300 maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2, Apache-2.0, approved, #9237 maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.13.3, Apache-2.0, approved, #2566 @@ -46,11 +48,8 @@ maven/mavencentral/com.fasterxml.jackson.module/jackson-module-jakarta-xmlbind-a maven/mavencentral/com.fasterxml.jackson/jackson-bom/2.14.2, Apache-2.0, approved, #7931 maven/mavencentral/com.fasterxml.jackson/jackson-bom/2.15.2, Apache-2.0, approved, #7929 maven/mavencentral/com.fasterxml.woodstox/woodstox-core/6.5.1, Apache-2.0, approved, #7950 -maven/mavencentral/com.github.docker-java/docker-java-api/3.3.0, Apache-2.0, approved, #10346 maven/mavencentral/com.github.docker-java/docker-java-api/3.3.3, Apache-2.0, approved, #10346 -maven/mavencentral/com.github.docker-java/docker-java-transport-zerodep/3.3.0, Apache-2.0 AND (Apache-2.0 AND BSD-3-Clause), approved, #7946 maven/mavencentral/com.github.docker-java/docker-java-transport-zerodep/3.3.3, Apache-2.0 AND (Apache-2.0 AND BSD-3-Clause), approved, #7946 -maven/mavencentral/com.github.docker-java/docker-java-transport/3.3.0, Apache-2.0, approved, #7942 maven/mavencentral/com.github.docker-java/docker-java-transport/3.3.3, Apache-2.0, approved, #7942 maven/mavencentral/com.github.stephenc.jcip/jcip-annotations/1.0-1, Apache-2.0, approved, CQ21949 maven/mavencentral/com.google.code.findbugs/jsr305/3.0.2, Apache-2.0, approved, #20 @@ -64,7 +63,6 @@ maven/mavencentral/com.google.guava/listenablefuture/9999.0-empty-to-avoid-confl maven/mavencentral/com.google.j2objc/j2objc-annotations/1.3, Apache-2.0, approved, CQ21195 maven/mavencentral/com.google.protobuf/protobuf-java/3.19.6, BSD-3-Clause, approved, clearlydefined maven/mavencentral/com.microsoft.azure/msal4j-persistence-extension/1.2.0, MIT, approved, clearlydefined -maven/mavencentral/com.microsoft.azure/msal4j/1.13.8, MIT, approved, clearlydefined maven/mavencentral/com.microsoft.azure/msal4j/1.13.9, MIT, approved, clearlydefined maven/mavencentral/com.microsoft.azure/msal4j/1.4.0, MIT, approved, clearlydefined maven/mavencentral/com.nimbusds/content-type/2.2, Apache-2.0, approved, clearlydefined @@ -150,7 +148,7 @@ maven/mavencentral/io.swagger.core.v3/swagger-integration/2.2.10, Apache-2.0, ap maven/mavencentral/io.swagger.core.v3/swagger-jaxrs2-jakarta/2.2.2, Apache-2.0, approved, clearlydefined maven/mavencentral/io.swagger.core.v3/swagger-jaxrs2/2.2.10, Apache-2.0, approved, #9814 maven/mavencentral/io.swagger.core.v3/swagger-models-jakarta/2.2.2, Apache-2.0, approved, #5919 -maven/mavencentral/io.swagger.core.v3/swagger-models/2.2.10, LicenseRef-scancode-proprietary-license AND Apache-2.0, restricted, #10354 +maven/mavencentral/io.swagger.core.v3/swagger-models/2.2.10, LicenseRef-scancode-proprietary-license AND Apache-2.0, restricted, #10520 maven/mavencentral/jakarta.activation/jakarta.activation-api/1.2.1, EPL-2.0 OR BSD-3-Clause OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.jaf maven/mavencentral/jakarta.activation/jakarta.activation-api/2.1.0, EPL-2.0 OR BSD-3-Clause OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.jaf maven/mavencentral/jakarta.annotation/jakarta.annotation-api/2.1.1, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.ca @@ -203,138 +201,142 @@ maven/mavencentral/org.ccil.cowan.tagsoup/tagsoup/1.2.1, Apache-2.0, approved, c maven/mavencentral/org.checkerframework/checker-qual/3.12.0, MIT, approved, clearlydefined maven/mavencentral/org.checkerframework/checker-qual/3.31.0, MIT, approved, clearlydefined maven/mavencentral/org.codehaus.woodstox/stax2-api/4.2.1, BSD-2-Clause, approved, #2670 -maven/mavencentral/org.eclipse.edc/aggregate-service-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/api-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/api-observability/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/asset-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/asset-index-sql/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/asset-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/auth-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/auth-tokenbased/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/autodoc-processor/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/aws-s3-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/boot/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/callback-event-dispatcher/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/callback-http-dispatcher/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/catalog-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/catalog-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/catalog-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/configuration-filesystem/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/connector-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/contract-agreement-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/contract-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/contract-definition-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/contract-definition-store-sql/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/contract-negotiation-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/contract-negotiation-store-sql/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/contract-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/control-api-configuration/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/control-plane-aggregate-services/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/control-plane-api-client-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/control-plane-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/control-plane-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/core-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-aws-s3/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-client/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-framework/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-http-oauth2-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-http-oauth2/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-http-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-http/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-selector-client/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-selector-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-selector-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/data-plane-util/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-api-configuration/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-catalog-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-catalog-http-dispatcher/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-catalog-transform/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-catalog/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-http-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-http-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-negotiation-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-negotiation-http-dispatcher/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-negotiation-transform/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-negotiation/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-transfer-process-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-transfer-process-http-dispatcher/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-transfer-process-transform/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp-transfer-process/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/dsp/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/http-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/http/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/iam-mock/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/jersey-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/jersey-micrometer/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/jersey-providers/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/jetty-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/jetty-micrometer/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/json-ld-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/json-ld/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/junit/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/jwt-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/jwt-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/management-api-configuration/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/management-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/micrometer-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/oauth2-client/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/oauth2-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/policy-definition-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/policy-definition-store-sql/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/policy-engine-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/policy-engine/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/policy-evaluator/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/policy-model/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/policy-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/runtime-metamodel/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/sql-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/sql-lease/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/sql-pool-apache-commons/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/state-machine/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transaction-datasource-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transaction-local/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transaction-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transfer-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transfer-data-plane-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transfer-data-plane/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transfer-process-api/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transfer-process-store-sql/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transfer-pull-http-dynamic-receiver/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transfer-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transform-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/transform-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/util/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/validator-core/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/validator-spi/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/vault-azure/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/vault-hashicorp/0.2.1, Apache-2.0, approved, technology.edc -maven/mavencentral/org.eclipse.edc/web-spi/0.2.1, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/aggregate-service-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/api-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/api-observability/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/asset-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/asset-index-sql/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/asset-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/auth-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/auth-tokenbased/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/autodoc-processor/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/aws-s3-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/aws-s3-test/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/azure-blob-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/azure-test/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/boot/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/callback-event-dispatcher/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/callback-http-dispatcher/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/catalog-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/catalog-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/catalog-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/configuration-filesystem/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/connector-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/contract-agreement-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/contract-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/contract-definition-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/contract-definition-store-sql/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/contract-negotiation-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/contract-negotiation-store-sql/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/contract-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/control-api-configuration/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/control-plane-aggregate-services/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/control-plane-api-client-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/control-plane-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/control-plane-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/core-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-aws-s3/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-azure-storage/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-client/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-framework/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-http-oauth2-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-http-oauth2/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-http-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-http/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-selector-client/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-selector-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-selector-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/data-plane-util/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-api-configuration/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-catalog-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-catalog-http-dispatcher/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-catalog-transform/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-catalog/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-http-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-http-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-negotiation-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-negotiation-http-dispatcher/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-negotiation-transform/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-negotiation/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-transfer-process-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-transfer-process-http-dispatcher/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-transfer-process-transform/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp-transfer-process/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/dsp/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/http-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/http/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/iam-mock/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/jersey-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/jersey-micrometer/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/jersey-providers/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/jetty-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/jetty-micrometer/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/json-ld-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/json-ld/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/junit/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/jwt-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/jwt-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/management-api-configuration/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/management-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/micrometer-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/oauth2-client/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/oauth2-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/policy-definition-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/policy-definition-store-sql/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/policy-engine-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/policy-engine/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/policy-evaluator/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/policy-model/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/policy-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/runtime-metamodel/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/sql-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/sql-lease/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/sql-pool-apache-commons/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/state-machine/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transaction-datasource-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transaction-local/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transaction-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transfer-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transfer-data-plane-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transfer-data-plane/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transfer-process-api/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transfer-process-store-sql/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transfer-pull-http-dynamic-receiver/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transfer-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transform-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/transform-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/util/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/validator-core/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/validator-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/vault-azure/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/vault-hashicorp/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc +maven/mavencentral/org.eclipse.edc/web-spi/0.2.2-20230905-SNAPSHOT, Apache-2.0, approved, technology.edc maven/mavencentral/org.eclipse.jetty.toolchain/jetty-jakarta-servlet-api/5.0.2, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.eclipse.jetty.toolchain/jetty-jakarta-websocket-api/2.0.0, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty.websocket/websocket-core-client/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty.websocket/websocket-core-common/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty.websocket/websocket-core-server/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty.websocket/websocket-jakarta-client/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty.websocket/websocket-jakarta-common/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty.websocket/websocket-jakarta-server/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty.websocket/websocket-servlet/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-alpn-client/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-annotations/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-client/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-http/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-io/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-jndi/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-plus/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-security/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-server/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-servlet/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-util/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-webapp/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty -maven/mavencentral/org.eclipse.jetty/jetty-xml/11.0.15, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty.websocket/websocket-core-client/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty.websocket/websocket-core-common/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty.websocket/websocket-core-server/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty.websocket/websocket-jakarta-client/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty.websocket/websocket-jakarta-common/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty.websocket/websocket-jakarta-server/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty.websocket/websocket-servlet/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-alpn-client/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-annotations/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-client/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-http/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-io/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-jndi/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-plus/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-security/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-server/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-servlet/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-util/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-webapp/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty +maven/mavencentral/org.eclipse.jetty/jetty-xml/11.0.16, EPL-2.0 OR Apache-2.0, approved, rt.jetty maven/mavencentral/org.flywaydb/flyway-core/9.22.0, Apache-2.0, approved, #10349 maven/mavencentral/org.glassfish.hk2.external/aopalliance-repackaged/3.0.4, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.glassfish maven/mavencentral/org.glassfish.hk2/hk2-api/3.0.4, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.glassfish @@ -376,7 +378,9 @@ maven/mavencentral/org.jetbrains/annotations/17.0.0, Apache-2.0, approved, clear maven/mavencentral/org.jetbrains/annotations/24.0.1, Apache-2.0, approved, #7417 maven/mavencentral/org.junit-pioneer/junit-pioneer/2.0.1, EPL-2.0, approved, clearlydefined maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.0, EPL-2.0, approved, #9714 +maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.9.3, EPL-2.0, approved, #3133 maven/mavencentral/org.junit.jupiter/junit-jupiter-engine/5.10.0, EPL-2.0, approved, #9711 +maven/mavencentral/org.junit.jupiter/junit-jupiter-engine/5.9.3, EPL-2.0, approved, #3125 maven/mavencentral/org.junit.jupiter/junit-jupiter-params/5.10.0, EPL-2.0, approved, #9708 maven/mavencentral/org.junit.platform/junit-platform-commons/1.10.0, EPL-2.0, approved, #9715 maven/mavencentral/org.junit.platform/junit-platform-engine/1.10.0, EPL-2.0, approved, #9709 @@ -410,17 +414,11 @@ maven/mavencentral/org.slf4j/slf4j-api/1.7.35, MIT, approved, CQ13368 maven/mavencentral/org.slf4j/slf4j-api/1.7.36, MIT, approved, CQ13368 maven/mavencentral/org.slf4j/slf4j-api/1.7.7, MIT, approved, CQ9827 maven/mavencentral/org.slf4j/slf4j-api/2.0.5, MIT, approved, #5915 -maven/mavencentral/org.slf4j/slf4j-api/2.0.7, MIT, approved, #5915 maven/mavencentral/org.slf4j/slf4j-api/2.0.9, MIT, approved, #5915 -maven/mavencentral/org.testcontainers/database-commons/1.18.3, MIT, approved, clearlydefined maven/mavencentral/org.testcontainers/database-commons/1.19.0, Apache-2.0, approved, #10345 -maven/mavencentral/org.testcontainers/jdbc/1.18.3, MIT, approved, clearlydefined maven/mavencentral/org.testcontainers/jdbc/1.19.0, Apache-2.0, approved, #10348 -maven/mavencentral/org.testcontainers/junit-jupiter/1.18.3, MIT, approved, #7941 maven/mavencentral/org.testcontainers/junit-jupiter/1.19.0, None, restricted, #10344 -maven/mavencentral/org.testcontainers/postgresql/1.18.3, MIT, approved, #9332 maven/mavencentral/org.testcontainers/postgresql/1.19.0, None, restricted, #10350 -maven/mavencentral/org.testcontainers/testcontainers/1.18.3, MIT, approved, #7938 maven/mavencentral/org.testcontainers/testcontainers/1.19.0, None, restricted, #10347 maven/mavencentral/org.yaml/snakeyaml/1.33, Apache-2.0, approved, clearlydefined maven/mavencentral/org.yaml/snakeyaml/2.0, Apache-2.0 AND (Apache-2.0 OR BSD-3-Clause OR EPL-1.0 OR GPL-2.0-or-later OR LGPL-2.1-or-later), approved, #7275 @@ -458,6 +456,7 @@ maven/mavencentral/software.amazon.awssdk/protocol-core/2.20.123, Apache-2.0, ap maven/mavencentral/software.amazon.awssdk/protocol-core/2.20.144, Apache-2.0, approved, #8635 maven/mavencentral/software.amazon.awssdk/regions/2.20.123, Apache-2.0, approved, #8632 maven/mavencentral/software.amazon.awssdk/regions/2.20.144, Apache-2.0, approved, #8632 +maven/mavencentral/software.amazon.awssdk/s3-transfer-manager/2.20.144, Apache-2.0, approved, #10521 maven/mavencentral/software.amazon.awssdk/s3/2.20.123, Apache-2.0, approved, #8623 maven/mavencentral/software.amazon.awssdk/s3/2.20.144, Apache-2.0, approved, #8623 maven/mavencentral/software.amazon.awssdk/sdk-core/2.20.123, Apache-2.0, approved, #8611 diff --git a/edc-dataplane/edc-dataplane-base/build.gradle.kts b/edc-dataplane/edc-dataplane-base/build.gradle.kts index 03a5ee3db..6c223a758 100644 --- a/edc-dataplane/edc-dataplane-base/build.gradle.kts +++ b/edc-dataplane/edc-dataplane-base/build.gradle.kts @@ -31,6 +31,7 @@ dependencies { runtimeOnly(libs.edc.config.filesystem) runtimeOnly(libs.edc.auth.tokenbased) runtimeOnly(libs.edc.dpf.awss3) + runtimeOnly(libs.edc.dpf.azblob) runtimeOnly(libs.edc.dpf.oauth2) runtimeOnly(libs.edc.dpf.http) diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/build.gradle.kts b/edc-tests/edc-dataplane/cloud-transfer-tests/build.gradle.kts new file mode 100644 index 000000000..aa7bbb5e7 --- /dev/null +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/build.gradle.kts @@ -0,0 +1,45 @@ +/* + * + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +plugins { + `java-library` +} + +dependencies { + + testImplementation(project(":edc-tests:e2e-tests")) + testImplementation(libs.edc.junit) + testImplementation(libs.restAssured) + + // test runtime config + testImplementation(libs.edc.config.filesystem) + testImplementation(libs.edc.dpf.http) + testImplementation(libs.edc.auth.tokenbased) + testImplementation(libs.testcontainers.junit) + testImplementation(testFixtures(libs.edc.azure.test)) + testImplementation(libs.edc.aws.s3.core) + testImplementation(testFixtures(libs.edc.aws.s3.test)) + testImplementation(libs.aws.s3) + testImplementation(libs.aws.s3transfer) +} + + + diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureBlobHelper.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureBlobHelper.java new file mode 100644 index 000000000..eaa9c9341 --- /dev/null +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureBlobHelper.java @@ -0,0 +1,95 @@ +/* + * + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.eclipse.tractusx.edc.dataplane.transfer.test; + +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.sas.BlobContainerSasPermission; +import com.azure.storage.blob.sas.BlobServiceSasSignatureValues; + +import java.io.IOException; +import java.io.InputStream; +import java.time.OffsetDateTime; +import java.util.List; + +import static org.eclipse.edc.azure.testfixtures.TestFunctions.getBlobServiceClient; + +/** + * Helper class that internally uses Azure SDK classes to create containers, upload blobs, generate SAS tokens, etc. + */ +public class AzureBlobHelper { + private final String accountName; + private final String key; + private final String host; + private final int port; + private BlobServiceClient blobServiceClient; + + public AzureBlobHelper(String accountName, String key, String host, int port) { + this.accountName = accountName; + this.key = key; + this.host = host; + this.port = port; + } + + public BlobContainerClient createContainer(String containerName) { + return blobClient().createBlobContainer(containerName); + } + + private BlobServiceClient blobClient() { + if (blobServiceClient == null) { + var endpoint = "http://%s:%s/%s".formatted(host, port, accountName); + blobServiceClient = getBlobServiceClient(accountName, key, endpoint); + } + return blobServiceClient; + } + + public void uploadBlob(BlobContainerClient client, InputStream inputStream, String targetBlobName) { + client.getBlobClient(targetBlobName).upload(inputStream, true); + } + + public void uploadBlob(BlobContainerClient client, String resourceName) { + try (var is = Thread.currentThread().getContextClassLoader().getResourceAsStream(resourceName)) { + client.getBlobClient(resourceName).upload(is, true); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public List listBlobs(String container) { + if (blobClient().listBlobContainers().stream().noneMatch(bci -> bci.getName().equalsIgnoreCase(container))) { + return List.of(); + } + return blobClient() + .getBlobContainerClient(container) + .listBlobs() + .stream().map(BlobItem::getName) + .toList(); + } + + public String generateAccountSas(String containerName) { + var expiry = OffsetDateTime.MAX.minusDays(1); + var permissions = BlobContainerSasPermission.parse("w"); + var vals = new BlobServiceSasSignatureValues(expiry, permissions); + return blobClient().getBlobContainerClient(containerName).generateSas(vals); + } +} diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureToAzureTest.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureToAzureTest.java new file mode 100644 index 000000000..68f8190de --- /dev/null +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/AzureToAzureTest.java @@ -0,0 +1,229 @@ +/* + * + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.eclipse.tractusx.edc.dataplane.transfer.test; + +import io.restassured.http.ContentType; +import org.eclipse.edc.azure.testfixtures.annotations.AzureStorageIntegrationTest; +import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.testcontainers.containers.FixedHostPortGenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.io.FileInputStream; +import java.io.IOException; +import java.time.Duration; + +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.junit.testfixtures.TestUtils.getFreePort; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_ACCOUNT_KEY; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_ACCOUNT_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_CONTAINER_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_KEY_ALIAS; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_PROVIDER_ACCOUNT_KEY; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_PROVIDER_ACCOUNT_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_PROVIDER_CONTAINER_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_PROVIDER_KEY_ALIAS; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZURITE_CONTAINER_PORT; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZURITE_DOCKER_IMAGE; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.COMPLETION_MARKER; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.TESTFILE_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.blobDestinationAddress; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.blobSourceAddress; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestFunctions.createSparseFile; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.verify; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; + +/** + * This test is intended to verify transfers within the same cloud provider, i.e. AzureBlob-to-AzureBlob. + * It spins up a fully-fledged dataplane and issues the DataFlowRequest via the data plane's Control API + */ +@SuppressWarnings("resource") +@Testcontainers +@AzureStorageIntegrationTest +public class AzureToAzureTest { + private static final int PROVIDER_CONTROL_PORT = getFreePort(); + + private static final int AZURITE_HOST_PORT = getFreePort(); + // launches the data plane + @RegisterExtension + protected static final ParticipantRuntime DATAPLANE_RUNTIME = new ParticipantRuntime( + ":edc-tests:runtime:dataplane-cloud", + "AzureBlob-Dataplane", + RuntimeConfig.Azure.createDataplane("/control", PROVIDER_CONTROL_PORT, AZURITE_HOST_PORT) + ); + /** + * Currently we have to use one container to host both consumer and provider accounts, because we cannot handle + * two different endpoint templates for provider and consumer. Endpoint templates are configured globally. + * Also, the host-port must be fixed/deterministic, as the {@code PROVIDER_RUNTIME} needs to know it in advance + */ + @Container + private final FixedHostPortGenericContainer azuriteContainer = new FixedHostPortGenericContainer<>(AZURITE_DOCKER_IMAGE) + .withFixedExposedPort(AZURITE_HOST_PORT, AZURITE_CONTAINER_PORT) + .withEnv("AZURITE_ACCOUNTS", AZBLOB_PROVIDER_ACCOUNT_NAME + ":" + AZBLOB_PROVIDER_ACCOUNT_KEY + ";" + AZBLOB_CONSUMER_ACCOUNT_NAME + ":" + AZBLOB_CONSUMER_ACCOUNT_KEY); + private AzureBlobHelper providerBlobHelper; + private AzureBlobHelper consumerBlobHelper; + + @BeforeEach + void setup() { + providerBlobHelper = new AzureBlobHelper(AZBLOB_PROVIDER_ACCOUNT_NAME, AZBLOB_PROVIDER_ACCOUNT_KEY, azuriteContainer.getHost(), azuriteContainer.getMappedPort(AZURITE_CONTAINER_PORT)); + consumerBlobHelper = new AzureBlobHelper(AZBLOB_CONSUMER_ACCOUNT_NAME, AZBLOB_CONSUMER_ACCOUNT_KEY, azuriteContainer.getHost(), azuriteContainer.getMappedPort(AZURITE_CONTAINER_PORT)); + } + + @Test + void transferFile_success() { + // upload file to provider's blob store + var bcc = providerBlobHelper.createContainer(AZBLOB_PROVIDER_CONTAINER_NAME); + providerBlobHelper.uploadBlob(bcc, TESTFILE_NAME); + + // create container in consumer's blob store + consumerBlobHelper.createContainer(AZBLOB_CONSUMER_CONTAINER_NAME); + + DATAPLANE_RUNTIME.getVault().storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, AZBLOB_PROVIDER_ACCOUNT_KEY); + DATAPLANE_RUNTIME.getVault().storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """ + {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} + """.formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME))); + + var request = createFlowRequest(TESTFILE_NAME); + + var url = "http://localhost:%s/control/transfer".formatted(PROVIDER_CONTROL_PORT); + + given().when() + .baseUri(url) + .contentType(ContentType.JSON) + .body(request) + .post() + .then() + .statusCode(200); + + await().pollInterval(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(60)) + .untilAsserted(() -> assertThat(consumerBlobHelper.listBlobs(AZBLOB_CONSUMER_CONTAINER_NAME)) + .isNotEmpty() + .contains(TESTFILE_NAME) + .contains(COMPLETION_MARKER)); + + } + + /** + * This method is used to transfer a large sparse file from the provider's blob store to the consumer's blob store. It follows the following steps: + *
    + *
  1. Uploads the file to the provider's blob store.
    2. + *
  2. Creates a random binary file of size greater than 1GB.
    3. + *
  3. Uploads the large file to the provider's blob store.
    4. + *
  4. Creates a container in the consumer's blob store.
    5. + *
  5. Creates a flow request with the given blob name.
    6. + *
  6. Waits for the transfer to complete by polling the consumer's blob store for the existence of the transferred blob.
    7. + *
+ */ + @ParameterizedTest(name = "File size bytes: {0}") + // 1mb, 512mb, 1gb + @ValueSource(longs = {1024 * 1024 * 512, 1024L * 1024L * 1024L, /*1024L * 1024L * 1024L * 1024 takes extremely long!*/}) + void transferFile_largeFile(long sizeBytes) throws IOException { + // upload file to provider's blob store + var bcc = providerBlobHelper.createContainer(AZBLOB_PROVIDER_CONTAINER_NAME); + + // create random binary file of >1gb in size + var blobName = "largeblob.bin"; + var f = createSparseFile(sizeBytes); + providerBlobHelper.uploadBlob(bcc, new FileInputStream(f), blobName); + + // create container in consumer's blob store + consumerBlobHelper.createContainer(AZBLOB_CONSUMER_CONTAINER_NAME); + + DATAPLANE_RUNTIME.getVault().storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, AZBLOB_PROVIDER_ACCOUNT_KEY); + DATAPLANE_RUNTIME.getVault().storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """ + {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} + """.formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME))); + + var request = createFlowRequest(blobName); + + var url = "http://localhost:%s/control/transfer".formatted(PROVIDER_CONTROL_PORT); + + given().when() + .baseUri(url) + .contentType(ContentType.JSON) + .body(request) + .post() + .then() + .statusCode(200); + + await().pollInterval(Duration.ofSeconds(10)) + .atMost(Duration.ofSeconds(120)) + .untilAsserted(() -> assertThat(consumerBlobHelper.listBlobs(AZBLOB_CONSUMER_CONTAINER_NAME)) + .isNotEmpty() + .contains(blobName) + .contains(COMPLETION_MARKER)); + + } + + @Test + void transferFile_targetContainerNotExist_shouldFail() { + // upload file to provider's blob store + var bcc = providerBlobHelper.createContainer(AZBLOB_PROVIDER_CONTAINER_NAME); + providerBlobHelper.uploadBlob(bcc, TESTFILE_NAME); + + // do NOT create container in consumer's blob store + + DATAPLANE_RUNTIME.getVault().storeSecret(AZBLOB_PROVIDER_KEY_ALIAS, AZBLOB_PROVIDER_ACCOUNT_KEY); + DATAPLANE_RUNTIME.getVault().storeSecret(AZBLOB_CONSUMER_KEY_ALIAS, """ + {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} + """.formatted(consumerBlobHelper.generateAccountSas(AZBLOB_CONSUMER_CONTAINER_NAME))); + + var request = createFlowRequest(TESTFILE_NAME); + + var url = "http://localhost:%s/control/transfer".formatted(PROVIDER_CONTROL_PORT); + + given().when() + .baseUri(url) + .contentType(ContentType.JSON) + .body(request) + .post() + .then() + .statusCode(200); + + // wait until the data plane logs an exception that it cannot transfer the blob + await().pollInterval(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> verify(DATAPLANE_RUNTIME.getContext().getMonitor()) + .severe(eq("Error creating blob for %s on account %s".formatted(TESTFILE_NAME, AZBLOB_CONSUMER_ACCOUNT_NAME)), + isA(IOException.class))); + } + + private DataFlowRequest createFlowRequest(String blobName) { + return DataFlowRequest.Builder.newInstance() + .id("test-request") + .sourceDataAddress(blobSourceAddress(blobName)) + .destinationDataAddress(blobDestinationAddress(blobName)) + .processId("test-process-id") + .trackable(false) + .build(); + } + +} diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/MultiCloudTest.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/MultiCloudTest.java new file mode 100644 index 000000000..b3b366cb0 --- /dev/null +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/MultiCloudTest.java @@ -0,0 +1,223 @@ +/* + * + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.eclipse.tractusx.edc.dataplane.transfer.test; + +import io.restassured.http.ContentType; +import org.eclipse.edc.aws.s3.AwsClientProviderConfiguration; +import org.eclipse.edc.aws.s3.AwsClientProviderImpl; +import org.eclipse.edc.aws.s3.S3BucketSchema; +import org.eclipse.edc.aws.s3.S3ClientRequest; +import org.eclipse.edc.aws.s3.testfixtures.annotations.AwsS3IntegrationTest; +import org.eclipse.edc.azure.testfixtures.annotations.AzureStorageIntegrationTest; +import org.eclipse.edc.junit.testfixtures.TestUtils; +import org.eclipse.edc.spi.types.domain.DataAddress; +import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.testcontainers.containers.FixedHostPortGenericContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +import java.net.URI; +import java.time.Duration; +import java.util.UUID; + +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.junit.testfixtures.TestUtils.getFreePort; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_ACCOUNT_KEY; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_ACCOUNT_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_CONTAINER_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.AZBLOB_CONSUMER_KEY_ALIAS; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.COMPLETION_MARKER; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.MINIO_CONTAINER_PORT; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.MINIO_DOCKER_IMAGE; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.S3_ACCESS_KEY_ID; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.S3_CONSUMER_BUCKET_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.TESTFILE_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.blobDestinationAddress; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestFunctions.listObjects; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; + +@Testcontainers +@AzureStorageIntegrationTest +@AwsS3IntegrationTest +public class MultiCloudTest { + // S3 test constants + public static final String REGION = Region.US_WEST_2.id(); + public static final String BUCKET_NAME = S3_CONSUMER_BUCKET_NAME; + public static final String BLOB_KEY_ALIAS = AZBLOB_CONSUMER_KEY_ALIAS; + private static final String ACCESS_KEY_ID = S3_ACCESS_KEY_ID; // user name + private static final String SECRET_ACCESS_KEY = UUID.randomUUID().toString(); // password + + // Azure Blob test constants + private static final String BLOB_ACCOUNT_NAME = AZBLOB_CONSUMER_ACCOUNT_NAME; + private static final String BLOB_ACCOUNT_KEY = AZBLOB_CONSUMER_ACCOUNT_KEY; + private static final int AZURITE_HOST_PORT = getFreePort(); + private static final String BLOB_CONTAINER_NAME = AZBLOB_CONSUMER_CONTAINER_NAME; + + // General constants, containers etc. + private static final int PROVIDER_CONTROL_PORT = getFreePort(); // port of the control api + @RegisterExtension + protected static final ParticipantRuntime DATAPLANE_RUNTIME = new ParticipantRuntime( + ":edc-tests:runtime:dataplane-cloud", + "MultiCloud-Dataplane", + RuntimeConfig.Azure.createDataplane("/control", PROVIDER_CONTROL_PORT, AZURITE_HOST_PORT) + ); + + @Container + private final GenericContainer s3Container = new GenericContainer<>(MINIO_DOCKER_IMAGE) + .withEnv("MINIO_ROOT_USER", ACCESS_KEY_ID) + .withEnv("MINIO_ROOT_PASSWORD", SECRET_ACCESS_KEY) + .withExposedPorts(MINIO_CONTAINER_PORT); + + @Container + private final FixedHostPortGenericContainer azuriteContainer = new FixedHostPortGenericContainer<>(TestConstants.AZURITE_DOCKER_IMAGE) + .withFixedExposedPort(AZURITE_HOST_PORT, 10000) + .withEnv("AZURITE_ACCOUNTS", BLOB_ACCOUNT_NAME + ":" + BLOB_ACCOUNT_KEY); + + private AzureBlobHelper blobStoreHelper; + private S3Client s3Client; + private String s3EndpointOverride; + + @BeforeEach + void setup() { + blobStoreHelper = new AzureBlobHelper(BLOB_ACCOUNT_NAME, BLOB_ACCOUNT_KEY, azuriteContainer.getHost(), azuriteContainer.getMappedPort(10000)); + s3EndpointOverride = "http://localhost:%s/".formatted(s3Container.getMappedPort(MINIO_CONTAINER_PORT)); + var providerConfig = AwsClientProviderConfiguration.Builder.newInstance() + .endpointOverride(URI.create(s3EndpointOverride)) + .credentialsProvider(() -> AwsBasicCredentials.create(ACCESS_KEY_ID, SECRET_ACCESS_KEY)) + .build(); + s3Client = new AwsClientProviderImpl(providerConfig).s3Client(S3ClientRequest.from(REGION, s3EndpointOverride)); + } + + @Test + void transferFile_azureToS3() { + // create container in Azure Blob, upload file + var bcc = blobStoreHelper.createContainer(BLOB_CONTAINER_NAME); + blobStoreHelper.uploadBlob(bcc, TESTFILE_NAME); + DATAPLANE_RUNTIME.getVault().storeSecret(BLOB_KEY_ALIAS, BLOB_ACCOUNT_KEY); + + // create target bucket in S3 + var r = s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); + assertThat(r.sdkHttpResponse().isSuccessful()).isTrue(); + + // create data flow request + var dfr = DataFlowRequest.Builder.newInstance() + .id("test-request") + .sourceDataAddress(DataAddress.Builder.newInstance() + .type("AzureStorage") + .property("container", BLOB_CONTAINER_NAME) + .property("account", BLOB_ACCOUNT_NAME) + .property("keyName", BLOB_KEY_ALIAS) + .property("blobname", TESTFILE_NAME) + .build() + ) + .destinationDataAddress(DataAddress.Builder.newInstance() + .type(S3BucketSchema.TYPE) + .keyName(TESTFILE_NAME) + .property(S3BucketSchema.REGION, REGION) + .property(S3BucketSchema.BUCKET_NAME, BUCKET_NAME) + .property(S3BucketSchema.ACCESS_KEY_ID, ACCESS_KEY_ID) + .property(S3BucketSchema.SECRET_ACCESS_KEY, SECRET_ACCESS_KEY) + .property(S3BucketSchema.ENDPOINT_OVERRIDE, s3EndpointOverride) + .build() + ) + .processId("test-process-id") + .trackable(false) + .build(); + + var url = "http://localhost:%s/control/transfer".formatted(PROVIDER_CONTROL_PORT); + given().when() + .baseUri(url) + .contentType(ContentType.JSON) + .body(dfr) + .post() + .then() + .statusCode(200); + + await().pollInterval(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(60)) + .untilAsserted(() -> assertThat(listObjects(s3Client, BUCKET_NAME)) + .isNotEmpty() + .contains(TESTFILE_NAME) + .anyMatch(c -> c.endsWith(COMPLETION_MARKER))); + } + + @Test + void transferFile_s3ToAzure() { + // create source bucket in S3, upload file + var b1 = s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); + assertThat(b1.sdkHttpResponse().isSuccessful()).isTrue(); + var putResponse = s3Client.putObject(PutObjectRequest.builder().bucket(BUCKET_NAME).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); + assertThat(putResponse.sdkHttpResponse().isSuccessful()).isTrue(); + + + // create container in consumer's blob store + blobStoreHelper.createContainer(BLOB_CONTAINER_NAME); + DATAPLANE_RUNTIME.getVault().storeSecret(BLOB_KEY_ALIAS, """ + {"sas": "%s","edctype":"dataspaceconnector:azuretoken"} + """.formatted(blobStoreHelper.generateAccountSas(BLOB_CONTAINER_NAME))); + + + // create data flow request + var dfr = DataFlowRequest.Builder.newInstance() + .id("test-request") + .sourceDataAddress(DataAddress.Builder.newInstance() + .type(S3BucketSchema.TYPE) + .keyName(TESTFILE_NAME) + .property(S3BucketSchema.REGION, REGION) + .property(S3BucketSchema.BUCKET_NAME, BUCKET_NAME) + .property(S3BucketSchema.ACCESS_KEY_ID, ACCESS_KEY_ID) + .property(S3BucketSchema.SECRET_ACCESS_KEY, SECRET_ACCESS_KEY) + .property(S3BucketSchema.ENDPOINT_OVERRIDE, s3EndpointOverride) + .build() + ) + .destinationDataAddress(blobDestinationAddress(TESTFILE_NAME)) + .processId("test-process-id") + .trackable(false) + .build(); + + var url = "http://localhost:%s/control/transfer".formatted(PROVIDER_CONTROL_PORT); + given().when() + .baseUri(url) + .contentType(ContentType.JSON) + .body(dfr) + .post() + .then() + .statusCode(200); + + await().pollInterval(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(60)) + .untilAsserted(() -> assertThat(blobStoreHelper.listBlobs(BLOB_CONTAINER_NAME)) + .isNotEmpty() + .contains(TESTFILE_NAME) + .contains(COMPLETION_MARKER)); + } +} diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/ParticipantRuntime.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/ParticipantRuntime.java new file mode 100644 index 000000000..99f25e42d --- /dev/null +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/ParticipantRuntime.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.tractusx.edc.dataplane.transfer.test; + +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; +import org.eclipse.edc.spi.monitor.ConsoleMonitor; +import org.eclipse.edc.spi.monitor.Monitor; +import org.eclipse.edc.spi.security.Vault; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.system.injection.InjectionContainer; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.spy; + +public class ParticipantRuntime extends EdcRuntimeExtension implements BeforeAllCallback, AfterAllCallback { + + + public ParticipantRuntime(String moduleName, String runtimeName, Map properties) { + super(moduleName, runtimeName, properties); + var monitor = spy(new ConsoleMonitor(runtimeName, ConsoleMonitor.Level.DEBUG)); + registerServiceMock(Monitor.class, monitor); + } + + @Override + public void beforeTestExecution(ExtensionContext extensionContext) { + //do nothing - we only want to start the runtime once + } + + @Override + public void afterTestExecution(ExtensionContext context) { + } + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + //only run this once + super.beforeTestExecution(context); + } + + @Override + public void afterAll(ExtensionContext context) throws Exception { + super.afterTestExecution(context); + } + + public Vault getVault() { + return getContext().getService(Vault.class); + } + + @Override + protected void bootExtensions(ServiceExtensionContext context, List> serviceExtensions) { + super.bootExtensions(context, serviceExtensions); + } + +} diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/RuntimeConfig.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/RuntimeConfig.java new file mode 100644 index 000000000..bf4587265 --- /dev/null +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/RuntimeConfig.java @@ -0,0 +1,83 @@ +/* + * + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.eclipse.tractusx.edc.dataplane.transfer.test; + +import java.util.HashMap; +import java.util.Map; + +import static org.eclipse.edc.junit.testfixtures.TestUtils.getFreePort; + +/** + * Configuration baseline for Data-Plane e2e tests + */ +public class RuntimeConfig { + /** + * Configures the data plane token endpoint, and all relevant HTTP contexts + */ + public static Map baseConfig(String controlPath, int controlPort) { + return new HashMap<>() { + { + put("edc.dataplane.token.validation.endpoint", "http://token-validation.com"); + put("web.http.path", "/api"); + put("web.http.port", String.valueOf(getFreePort())); + put("web.http.public.path", "/public"); + put("web.http.public.port", String.valueOf(getFreePort())); + put("web.http.consumer.api.path", "/api/consumer"); + put("web.http.consumer.api.port", String.valueOf(getFreePort())); + put("tx.dpf.consumer.proxy.port", String.valueOf(getFreePort())); + put("web.http.control.path", controlPath); + put("web.http.control.port", String.valueOf(controlPort)); + } + }; + } + + /** + * Azure specific configuration, e.g. access credentials, blobstore endpoint templates, etc. + */ + public static class Azure { + /** + * Creates a configuration for a Provider runtime, running Azure ingress and egress + * + * @param controlPath the controlPath of the control API + * @param controlPort the port of the control API + * @param mappedAzuritePort the host port for the Blob endpoint template. + */ + public static Map createDataplane(String controlPath, int controlPort, Integer mappedAzuritePort) { + var base = baseConfig(controlPath, controlPort); + + base.putAll(new HashMap<>() { + { + put("edc.blobstore.endpoint.template", "http://127.0.0.1:" + mappedAzuritePort + "/%s"); + } + }); + return base; + } + } + + public static class S3 { + + public static Map createDataplane(String controlPath, int controlPort) { + return baseConfig(controlPath, controlPort); + } + } +} + diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/S3ToS3Test.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/S3ToS3Test.java new file mode 100644 index 000000000..10a690a00 --- /dev/null +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/S3ToS3Test.java @@ -0,0 +1,280 @@ +/* + * + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.eclipse.tractusx.edc.dataplane.transfer.test; + +import io.restassured.http.ContentType; +import org.eclipse.edc.aws.s3.AwsClientProviderConfiguration; +import org.eclipse.edc.aws.s3.AwsClientProviderImpl; +import org.eclipse.edc.aws.s3.S3BucketSchema; +import org.eclipse.edc.aws.s3.S3ClientRequest; +import org.eclipse.edc.aws.s3.testfixtures.annotations.AwsS3IntegrationTest; +import org.eclipse.edc.junit.testfixtures.TestUtils; +import org.eclipse.edc.spi.types.domain.DataAddress; +import org.eclipse.edc.spi.types.domain.transfer.DataFlowRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedUpload; +import software.amazon.awssdk.transfer.s3.model.UploadRequest; + +import java.io.File; +import java.net.URI; +import java.time.Duration; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; + +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.junit.testfixtures.TestUtils.getFreePort; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.COMPLETION_MARKER; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.MINIO_CONTAINER_PORT; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.MINIO_DOCKER_IMAGE; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.S3_ACCESS_KEY_ID; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.S3_CONSUMER_BUCKET_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.S3_PROVIDER_BUCKET_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.S3_REGION; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestConstants.TESTFILE_NAME; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestFunctions.createSparseFile; +import static org.eclipse.tractusx.edc.dataplane.transfer.test.TestFunctions.listObjects; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.ArgumentMatchers.startsWith; +import static org.mockito.Mockito.verify; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; + +/** + * This test is intended to verify transfers within the same cloud provider, i.e. S3-to-S3. + * It spins up a fully-fledged dataplane and issues the DataFlowRequest via the data plane's Control API + */ +@Testcontainers +@AwsS3IntegrationTest +public class S3ToS3Test { + private static final String SECRET_ACCESS_KEY = UUID.randomUUID().toString(); // password + private static final int PROVIDER_CONTROL_PORT = getFreePort(); // port of the control api + @RegisterExtension + protected static final ParticipantRuntime DATAPLANE_RUNTIME = new ParticipantRuntime( + ":edc-tests:runtime:dataplane-cloud", + "AwsS3-Dataplane", + RuntimeConfig.S3.createDataplane("/control", PROVIDER_CONTROL_PORT) + ); + @Container + private final GenericContainer providerContainer = new GenericContainer<>(MINIO_DOCKER_IMAGE) + .withEnv("MINIO_ROOT_USER", S3_ACCESS_KEY_ID) + .withEnv("MINIO_ROOT_PASSWORD", SECRET_ACCESS_KEY) + .withExposedPorts(MINIO_CONTAINER_PORT); + + @Container + private final GenericContainer consumerContainer = new GenericContainer<>(MINIO_DOCKER_IMAGE) + .withEnv("MINIO_ROOT_USER", S3_ACCESS_KEY_ID) + .withEnv("MINIO_ROOT_PASSWORD", SECRET_ACCESS_KEY) + .withExposedPorts(MINIO_CONTAINER_PORT); + private S3Client providerClient; + private S3Client consumerClient; + private String providerEndpointOverride; + private String consumerEndpointOverride; + + @BeforeEach + void setup() { + providerEndpointOverride = "http://localhost:%s/".formatted(providerContainer.getMappedPort(MINIO_CONTAINER_PORT)); + var providerConfig = AwsClientProviderConfiguration.Builder.newInstance() + .endpointOverride(URI.create(providerEndpointOverride)) + .credentialsProvider(() -> AwsBasicCredentials.create(S3_ACCESS_KEY_ID, SECRET_ACCESS_KEY)) + .build(); + providerClient = new AwsClientProviderImpl(providerConfig).s3Client(S3ClientRequest.from(S3_REGION, providerEndpointOverride)); + + consumerEndpointOverride = "http://localhost:%s".formatted(consumerContainer.getMappedPort(MINIO_CONTAINER_PORT)); + var consumerConfig = AwsClientProviderConfiguration.Builder.newInstance() + .endpointOverride(URI.create(consumerEndpointOverride)) + .credentialsProvider(() -> AwsBasicCredentials.create(S3_ACCESS_KEY_ID, SECRET_ACCESS_KEY)) + .build(); + consumerClient = new AwsClientProviderImpl(consumerConfig).s3Client(S3ClientRequest.from(S3_REGION, consumerEndpointOverride)); + } + + @Test + void transferFile_success() { + + // create bucket in provider + var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).build()); + assertThat(b1.sdkHttpResponse().isSuccessful()).isTrue(); + // upload test file in provider + var putResponse = providerClient.putObject(PutObjectRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); + assertThat(putResponse.sdkHttpResponse().isSuccessful()).isTrue(); + + // create bucket in consumer + var b2 = consumerClient.createBucket(CreateBucketRequest.builder().bucket(S3_CONSUMER_BUCKET_NAME).build()); + assertThat(b2.sdkHttpResponse().isSuccessful()).isTrue(); + + // initiate data flow request + var request = createFlowRequest(); + var url = "http://localhost:%s/control/transfer".formatted(PROVIDER_CONTROL_PORT); + + given().when() + .baseUri(url) + .contentType(ContentType.JSON) + .body(request) + .post() + .then() + .statusCode(200); + + + await().pollInterval(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(60)) + .untilAsserted(() -> assertThat(listObjects(consumerClient, S3_CONSUMER_BUCKET_NAME)) + .isNotEmpty() + .contains(TESTFILE_NAME) + .anyMatch(c -> c.endsWith(COMPLETION_MARKER))); + } + + @Test + void transferFile_targetContainerNotExist_shouldFail() { + // create bucket in provider + var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).build()); + assertThat(b1.sdkHttpResponse().isSuccessful()).isTrue(); + // upload test file in provider + var putResponse = providerClient.putObject(PutObjectRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).key(TESTFILE_NAME).build(), TestUtils.getFileFromResourceName(TESTFILE_NAME).toPath()); + assertThat(putResponse.sdkHttpResponse().isSuccessful()).isTrue(); + + // do not create bucket in consumer -> will fail! + + // initiate data flow request + var request = createFlowRequest(); + var url = "http://localhost:%s/control/transfer".formatted(PROVIDER_CONTROL_PORT); + + given().when() + .baseUri(url) + .contentType(ContentType.JSON) + .body(request) + .post() + .then() + .statusCode(200); + + + // wait until the data plane logs an exception that it cannot transfer the file + await().pollInterval(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> verify(DATAPLANE_RUNTIME.getContext().getMonitor()).severe(startsWith("Error writing the %s object on the %s bucket: The specified bucket does not exist".formatted(TESTFILE_NAME, S3_CONSUMER_BUCKET_NAME)), + isA(NoSuchBucketException.class))); + } + + @ParameterizedTest(name = "File size bytes: {0}") + // 1mb, 512mb, 1gb + @ValueSource(longs = {1024 * 1024 * 512, 1024L * 1024L * 1024L, 1024L * 1024L * 1024L * 1024}) + void transferfile_largeFile(long sizeBytes) { + + // create large sparse file + var file = createSparseFile(sizeBytes); + + // create bucket in provider + var b1 = providerClient.createBucket(CreateBucketRequest.builder().bucket(S3_PROVIDER_BUCKET_NAME).build()); + assertThat(b1.sdkHttpResponse().isSuccessful()).isTrue(); + // upload test file in provider + uploadLargeFile(file, S3_PROVIDER_BUCKET_NAME, TESTFILE_NAME).thenAccept(completedUpload -> { + // create bucket in consumer + var b2 = consumerClient.createBucket(CreateBucketRequest.builder().bucket(S3_CONSUMER_BUCKET_NAME).build()); + assertThat(b2.sdkHttpResponse().isSuccessful()).isTrue(); + + // initiate data flow request + var request = createFlowRequest(); + var url = "http://localhost:%s/control/transfer".formatted(PROVIDER_CONTROL_PORT); + + given().when() + .baseUri(url) + .contentType(ContentType.JSON) + .body(request) + .post() + .then() + .statusCode(200); + + + await().pollInterval(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(60)) + .untilAsserted(() -> assertThat(listObjects(consumerClient, S3_CONSUMER_BUCKET_NAME)) + .isNotEmpty() + .contains(TESTFILE_NAME) + .anyMatch(c -> c.endsWith(COMPLETION_MARKER))); + }); + + + } + + private CompletableFuture uploadLargeFile(File file, String bucketName, String fileName) { + + var providerConfig = AwsClientProviderConfiguration.Builder.newInstance() + .endpointOverride(URI.create(providerEndpointOverride)) + .credentialsProvider(() -> AwsBasicCredentials.create(S3_ACCESS_KEY_ID, SECRET_ACCESS_KEY)) + .build(); + var asyncClient = new AwsClientProviderImpl(providerConfig).s3AsyncClient(S3_REGION); + var tm = S3TransferManager.builder() + .s3Client(asyncClient) + .build(); + + // TransferManager processes all transfers asynchronously, + // so this call returns immediately. + var upload = tm.upload(UploadRequest.builder() + .putObjectRequest(PutObjectRequest.builder().key(fileName).bucket(bucketName).build()) + .requestBody(AsyncRequestBody.fromFile(file)) + .build()); + + return upload.completionFuture(); + } + + + private DataFlowRequest createFlowRequest() { + return DataFlowRequest.Builder.newInstance() + .id("test-request") + .sourceDataAddress(DataAddress.Builder.newInstance() + .type(S3BucketSchema.TYPE) + .keyName(TESTFILE_NAME) + .property(S3BucketSchema.REGION, S3_REGION) + .property(S3BucketSchema.BUCKET_NAME, S3_PROVIDER_BUCKET_NAME) + .property(S3BucketSchema.ACCESS_KEY_ID, S3_ACCESS_KEY_ID) + .property(S3BucketSchema.SECRET_ACCESS_KEY, SECRET_ACCESS_KEY) + .property(S3BucketSchema.ENDPOINT_OVERRIDE, providerEndpointOverride) + .build() + ) + .destinationDataAddress(DataAddress.Builder.newInstance() + .type(S3BucketSchema.TYPE) + .keyName(TESTFILE_NAME) + .property(S3BucketSchema.REGION, S3_REGION) + .property(S3BucketSchema.BUCKET_NAME, S3_CONSUMER_BUCKET_NAME) + .property(S3BucketSchema.ACCESS_KEY_ID, S3_ACCESS_KEY_ID) + .property(S3BucketSchema.SECRET_ACCESS_KEY, SECRET_ACCESS_KEY) + .property(S3BucketSchema.ENDPOINT_OVERRIDE, consumerEndpointOverride) + .build() + ) + .processId("test-process-id") + .trackable(false) + .build(); + } + +} diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestConstants.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestConstants.java new file mode 100644 index 000000000..e500270be --- /dev/null +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestConstants.java @@ -0,0 +1,73 @@ +/* + * + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.eclipse.tractusx.edc.dataplane.transfer.test; + +import org.eclipse.edc.spi.types.domain.DataAddress; +import software.amazon.awssdk.regions.Region; + +public class TestConstants { + // AZURE BLOB CONSTANTS + public static final String AZURITE_DOCKER_IMAGE = "mcr.microsoft.com/azure-storage/azurite"; + public static final int AZURITE_CONTAINER_PORT = 10000; + public static final String AZBLOB_PROVIDER_ACCOUNT_NAME = "provider"; + public static final String AZBLOB_PROVIDER_ACCOUNT_KEY = "providerkey"; + public static final String AZBLOB_CONSUMER_ACCOUNT_NAME = "consumer"; + public static final String AZBLOB_CONSUMER_ACCOUNT_KEY = "consumerkey"; + // alias under which the provider key is stored in the vault. must end with -key1 + public static final String AZBLOB_PROVIDER_KEY_ALIAS = "providerkey-key1"; + // alias under which the consumer key is stored in the vault. must end with -key1 + public static final String AZBLOB_CONSUMER_KEY_ALIAS = "consumerkey-key1`"; + public static final String AZBLOB_PROVIDER_CONTAINER_NAME = "src-container"; + public static final String AZBLOB_CONSUMER_CONTAINER_NAME = "dest-container"; + // AMAZON S3 CONSTANTS + public static final String MINIO_DOCKER_IMAGE = "bitnami/minio"; + public static final int MINIO_CONTAINER_PORT = 9000; + public static final String S3_REGION = Region.US_WEST_2.id(); + public static final String S3_PROVIDER_BUCKET_NAME = "provider-bucket"; + public static final String S3_CONSUMER_BUCKET_NAME = "consumer-bucket"; + public static final String S3_ACCESS_KEY_ID = "test-access-key"; // user name + // GENERIC CONSTANTS + public static final String TESTFILE_NAME = "testfile.json"; + public static final String COMPLETION_MARKER = ".complete"; + + public static DataAddress blobSourceAddress(String blobName) { + return DataAddress.Builder.newInstance() + .type("AzureStorage") + .property("container", AZBLOB_PROVIDER_CONTAINER_NAME) + .property("account", AZBLOB_PROVIDER_ACCOUNT_NAME) + .property("keyName", AZBLOB_PROVIDER_KEY_ALIAS) + .property("blobname", blobName) + .build(); + } + + public static DataAddress blobDestinationAddress(String blobName) { + return DataAddress.Builder.newInstance() + .type("AzureStorage") + .property("container", AZBLOB_CONSUMER_CONTAINER_NAME) + .property("account", AZBLOB_CONSUMER_ACCOUNT_NAME) + .property("blobname", blobName) + .property("keyName", AZBLOB_CONSUMER_KEY_ALIAS) + .build(); + } + + +} diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestFunctions.java b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestFunctions.java new file mode 100644 index 000000000..e3ecbc58b --- /dev/null +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/java/org/eclipse/tractusx/edc/dataplane/transfer/test/TestFunctions.java @@ -0,0 +1,53 @@ +/* + * + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.eclipse.tractusx.edc.dataplane.transfer.test; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.file.Files; +import java.util.List; + +public class TestFunctions { + public static File createSparseFile(long sizeBytes) { + try { + var name = Files.createTempFile("", ".bin"); + try (var f = new RandomAccessFile(name.toFile(), "rw")) { + f.setLength(sizeBytes); + f.write("foobar".getBytes()); + } + return name.toFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + public static List listObjects(S3Client consumerClient, String bucketName) { + var response = consumerClient.listObjects(ListObjectsRequest.builder().bucket(bucketName).build()); + return response.contents().stream().map(S3Object::key).toList(); + } +} diff --git a/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/resources/testfile.json b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/resources/testfile.json new file mode 100644 index 000000000..f2c63e6c6 --- /dev/null +++ b/edc-tests/edc-dataplane/cloud-transfer-tests/src/test/resources/testfile.json @@ -0,0 +1,317 @@ +[ + { + "_id": "64e878422f54f7451fc8ac5a", + "index": 0, + "guid": "58cd89a8-bbc4-49dc-a82c-9e95a17bd09a", + "isActive": true, + "balance": "$2,609.85", + "picture": "http://placehold.it/32x32", + "age": 34, + "eyeColor": "brown", + "name": "Bennett Ward", + "gender": "male", + "company": "ANIVET", + "email": "bennettward@anivet.com", + "phone": "+1 (946) 427-3749", + "address": "800 Nichols Avenue, Wheaton, Arkansas, 5955", + "about": "In ipsum incididunt proident laborum ipsum aliqua veniam minim irure magna. Nisi quis laboris amet ullamco quis reprehenderit dolor fugiat adipisicing aliqua dolor irure. Ut esse ipsum ea incididunt ipsum in Lorem sit ex veniam. Enim do sit sunt consequat eu et cillum Lorem consectetur dolor incididunt. Quis laboris ipsum magna ut officia labore elit. Elit proident eiusmod sunt laboris do aute officia laboris deserunt est ut deserunt.\r\n", + "registered": "2017-08-18T03:46:19 -02:00", + "latitude": 40.471184, + "longitude": -83.133691, + "tags": [ + "veniam", + "consequat", + "sint", + "magna", + "adipisicing", + "eiusmod", + "proident" + ], + "friends": [ + { + "id": 0, + "name": "Marquez Robinson" + }, + { + "id": 1, + "name": "Juliana Yang" + }, + { + "id": 2, + "name": "Mccarthy Wilkins" + } + ], + "greeting": "Hello, Bennett Ward! You have 6 unread messages.", + "favoriteFruit": "banana" + }, + { + "_id": "64e8784201c692a3fe69fa9b", + "index": 1, + "guid": "b58c8f04-7a95-4e49-8a15-e28e31cbfb22", + "isActive": false, + "balance": "$1,141.67", + "picture": "http://placehold.it/32x32", + "age": 36, + "eyeColor": "brown", + "name": "Janette Avery", + "gender": "female", + "company": "ASSURITY", + "email": "janetteavery@assurity.com", + "phone": "+1 (897) 585-2136", + "address": "128 Amersfort Place, Kenmar, Minnesota, 9633", + "about": "Minim eiusmod consequat tempor nisi. Do non tempor tempor cupidatat consequat qui. Dolor consequat non fugiat id ea cillum deserunt labore elit cupidatat exercitation voluptate cillum.\r\n", + "registered": "2017-12-02T06:49:19 -01:00", + "latitude": 34.258782, + "longitude": -21.998249, + "tags": [ + "eiusmod", + "ullamco", + "minim", + "aliqua", + "do", + "cillum", + "Lorem" + ], + "friends": [ + { + "id": 0, + "name": "Cherry Skinner" + }, + { + "id": 1, + "name": "Conner Reyes" + }, + { + "id": 2, + "name": "Parker Gonzalez" + } + ], + "greeting": "Hello, Janette Avery! You have 10 unread messages.", + "favoriteFruit": "strawberry" + }, + { + "_id": "64e87842f52337c981862e8f", + "index": 2, + "guid": "9ff7da8a-d2b0-4096-b8f6-9352806a9e83", + "isActive": false, + "balance": "$1,335.78", + "picture": "http://placehold.it/32x32", + "age": 28, + "eyeColor": "green", + "name": "Dominique Hanson", + "gender": "female", + "company": "VANTAGE", + "email": "dominiquehanson@vantage.com", + "phone": "+1 (989) 447-3926", + "address": "615 Meserole Avenue, Austinburg, California, 1522", + "about": "Incididunt laborum consequat est nulla est ea sint cupidatat duis nostrud elit anim. Dolore enim laboris dolore ullamco cupidatat ipsum enim nulla mollit tempor. Deserunt et aliqua mollit ut do dolore tempor voluptate amet aliquip duis consectetur nulla Lorem. Labore adipisicing enim cupidatat ipsum adipisicing eiusmod consequat eu nostrud culpa. Nostrud velit consectetur non deserunt dolore nostrud duis culpa voluptate deserunt sunt veniam. Proident minim eiusmod nulla officia magna eu exercitation anim. Tempor et irure esse laborum.\r\n", + "registered": "2016-07-27T05:23:38 -02:00", + "latitude": -55.068801, + "longitude": -25.708949, + "tags": [ + "eiusmod", + "in", + "proident", + "et", + "et", + "nisi", + "culpa" + ], + "friends": [ + { + "id": 0, + "name": "Nita Noel" + }, + { + "id": 1, + "name": "Catalina Melendez" + }, + { + "id": 2, + "name": "Williamson Sampson" + } + ], + "greeting": "Hello, Dominique Hanson! You have 1 unread messages.", + "favoriteFruit": "banana" + }, + { + "_id": "64e87842338e5431dbd0b1ea", + "index": 3, + "guid": "a96e773d-5c23-4d9a-adc9-d6a018314b58", + "isActive": false, + "balance": "$1,913.89", + "picture": "http://placehold.it/32x32", + "age": 38, + "eyeColor": "green", + "name": "Wagner Dennis", + "gender": "male", + "company": "AQUAZURE", + "email": "wagnerdennis@aquazure.com", + "phone": "+1 (937) 431-2082", + "address": "445 Montauk Court, Verdi, Northern Mariana Islands, 5384", + "about": "Tempor nulla sint proident mollit cupidatat ea tempor ex. Duis aliqua nulla magna in consequat excepteur consequat ad. Do nisi nulla laboris id est sunt. Elit tempor cupidatat exercitation commodo do cupidatat quis veniam dolor aliqua dolor nulla dolor fugiat.\r\n", + "registered": "2021-04-26T09:52:32 -02:00", + "latitude": 18.451519, + "longitude": -156.122664, + "tags": [ + "eu", + "nostrud", + "pariatur", + "culpa", + "ex", + "proident", + "adipisicing" + ], + "friends": [ + { + "id": 0, + "name": "Beatriz Short" + }, + { + "id": 1, + "name": "Benjamin Cohen" + }, + { + "id": 2, + "name": "Owen Byrd" + } + ], + "greeting": "Hello, Wagner Dennis! You have 2 unread messages.", + "favoriteFruit": "banana" + }, + { + "_id": "64e878424a3a21770faa9e26", + "index": 4, + "guid": "4930409b-db05-4937-af3e-2036a482df12", + "isActive": false, + "balance": "$2,529.52", + "picture": "http://placehold.it/32x32", + "age": 28, + "eyeColor": "brown", + "name": "Atkins Moses", + "gender": "male", + "company": "SKYPLEX", + "email": "atkinsmoses@skyplex.com", + "phone": "+1 (995) 553-2028", + "address": "448 Cumberland Street, Dana, Federated States Of Micronesia, 7392", + "about": "Et ut ea velit labore dolore ad reprehenderit exercitation dolor adipisicing et id ea. Sunt cupidatat consectetur nostrud minim eiusmod nisi. Sint laborum duis tempor elit dolor consectetur cillum.\r\n", + "registered": "2021-04-12T11:23:36 -02:00", + "latitude": -2.638186, + "longitude": -154.242354, + "tags": [ + "dolor", + "officia", + "est", + "nulla", + "aute", + "laboris", + "nulla" + ], + "friends": [ + { + "id": 0, + "name": "Riddle Christian" + }, + { + "id": 1, + "name": "Polly Schultz" + }, + { + "id": 2, + "name": "Livingston Manning" + } + ], + "greeting": "Hello, Atkins Moses! You have 9 unread messages.", + "favoriteFruit": "banana" + }, + { + "_id": "64e87842884162d3c897e7ad", + "index": 5, + "guid": "b3cb202a-6e46-4095-9592-6366f6da3bc9", + "isActive": false, + "balance": "$3,158.74", + "picture": "http://placehold.it/32x32", + "age": 32, + "eyeColor": "blue", + "name": "Tasha Russell", + "gender": "female", + "company": "BALUBA", + "email": "tasharussell@baluba.com", + "phone": "+1 (826) 541-3686", + "address": "335 Amber Street, Advance, Pennsylvania, 8192", + "about": "Non pariatur irure ea ex anim deserunt ea exercitation est. Esse quis aliqua irure est nisi enim eiusmod nisi exercitation ipsum sint enim tempor laboris. Et tempor tempor tempor dolore sunt et incididunt tempor. Laboris aliqua voluptate duis id enim elit et occaecat anim ut aliquip incididunt consequat ex. Commodo cupidatat pariatur est irure commodo incididunt excepteur ex. Culpa Lorem enim eu dolor anim.\r\n", + "registered": "2017-02-14T10:10:26 -01:00", + "latitude": 29.687589, + "longitude": -76.699133, + "tags": [ + "esse", + "et", + "enim", + "irure", + "ipsum", + "irure", + "commodo" + ], + "friends": [ + { + "id": 0, + "name": "Phoebe Lester" + }, + { + "id": 1, + "name": "Tisha Owens" + }, + { + "id": 2, + "name": "Tiffany Pearson" + } + ], + "greeting": "Hello, Tasha Russell! You have 2 unread messages.", + "favoriteFruit": "banana" + }, + { + "_id": "64e878428448c3ea48fb1094", + "index": 6, + "guid": "6542e7cd-bccc-4af2-9975-e52ae75d2362", + "isActive": false, + "balance": "$3,203.36", + "picture": "http://placehold.it/32x32", + "age": 23, + "eyeColor": "blue", + "name": "Iris Hawkins", + "gender": "female", + "company": "ANOCHA", + "email": "irishawkins@anocha.com", + "phone": "+1 (879) 591-2788", + "address": "745 Douglass Street, Thatcher, Washington, 7675", + "about": "Dolore ipsum et est aute laborum fugiat esse nisi enim id. Veniam culpa tempor Lorem aliqua culpa quis. Veniam voluptate velit ea excepteur ullamco laboris cillum enim adipisicing magna dolor id. Dolor ex sunt proident occaecat.\r\n", + "registered": "2022-04-25T05:38:28 -02:00", + "latitude": -9.450379, + "longitude": -67.567812, + "tags": [ + "eu", + "velit", + "cupidatat", + "incididunt", + "pariatur", + "velit", + "Lorem" + ], + "friends": [ + { + "id": 0, + "name": "Araceli Rosales" + }, + { + "id": 1, + "name": "Lynette Huff" + }, + { + "id": 2, + "name": "Medina Nolan" + } + ], + "greeting": "Hello, Iris Hawkins! You have 6 unread messages.", + "favoriteFruit": "banana" + } +] \ No newline at end of file diff --git a/edc-tests/edc-dataplane-proxy-e2e/build.gradle.kts b/edc-tests/edc-dataplane/edc-dataplane-proxy-e2e/build.gradle.kts similarity index 100% rename from edc-tests/edc-dataplane-proxy-e2e/build.gradle.kts rename to edc-tests/edc-dataplane/edc-dataplane-proxy-e2e/build.gradle.kts diff --git a/edc-tests/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/DpfProxyEndToEndTest.java b/edc-tests/edc-dataplane/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/DpfProxyEndToEndTest.java similarity index 99% rename from edc-tests/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/DpfProxyEndToEndTest.java rename to edc-tests/edc-dataplane/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/DpfProxyEndToEndTest.java index 588f6d698..c1950ee8f 100644 --- a/edc-tests/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/DpfProxyEndToEndTest.java +++ b/edc-tests/edc-dataplane/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/DpfProxyEndToEndTest.java @@ -59,7 +59,7 @@ public class DpfProxyEndToEndTest { public static final String KEYSTORE_PASS = "test123"; - private static final String LAUNCHER_MODULE = ":edc-tests:edc-dataplane-proxy-e2e"; + private static final String LAUNCHER_MODULE = ":edc-tests:edc-dataplane:edc-dataplane-proxy-e2e"; private static final int CONSUMER_HTTP_PORT = getFreePort(); private static final int CONSUMER_PROXY_PORT = getFreePort(); private static final int PRODUCER_HTTP_PORT = getFreePort(); diff --git a/edc-tests/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/EdrCacheSetup.java b/edc-tests/edc-dataplane/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/EdrCacheSetup.java similarity index 100% rename from edc-tests/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/EdrCacheSetup.java rename to edc-tests/edc-dataplane/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/EdrCacheSetup.java diff --git a/edc-tests/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/KeyStoreSetup.java b/edc-tests/edc-dataplane/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/KeyStoreSetup.java similarity index 100% rename from edc-tests/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/KeyStoreSetup.java rename to edc-tests/edc-dataplane/edc-dataplane-proxy-e2e/src/test/java/org/eclipse/tractusx/edc/dataplane/proxy/e2e/KeyStoreSetup.java diff --git a/edc-tests/runtime/dataplane-cloud/README.md b/edc-tests/runtime/dataplane-cloud/README.md new file mode 100644 index 000000000..d779a8f8b --- /dev/null +++ b/edc-tests/runtime/dataplane-cloud/README.md @@ -0,0 +1,4 @@ +# Dataplane Runtime for Testing Purposes + +This module provides a dataplane runtime using AWS S3 and Azure Blob Storage packages to test +several variants of cloud transfers. Please only use for testing purposes! diff --git a/edc-tests/runtime/dataplane-cloud/build.gradle.kts b/edc-tests/runtime/dataplane-cloud/build.gradle.kts new file mode 100644 index 000000000..90724a443 --- /dev/null +++ b/edc-tests/runtime/dataplane-cloud/build.gradle.kts @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +plugins { + `java-library` + id("application") +} + + +dependencies { + + // use basic (all in-mem) data plane + runtimeOnly(project(":edc-dataplane:edc-dataplane-base")) { + exclude("org.eclipse.edc", "api-observability") + } + +// implementation(project(":edc-tests:runtime:extensions")) +} + +application { + mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") +} + +// do not publish +edcBuild { + publish.set(false) +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a1ac0c9f7..bb508d459 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,7 +2,7 @@ format.version = "1.1" [versions] -edc = "0.2.1" +edc = "0.2.2-20230905-SNAPSHOT" postgres = "42.6.0" awaitility = "4.2.0" nimbus = "9.31" @@ -81,6 +81,7 @@ edc-ext-azure-test = { module = "org.eclipse.edc:azure-test", version.ref = "edc edc-ext-jsonld = { module = "org.eclipse.edc:json-ld", version.ref = "edc" } edc-ext-jersey-providers = { module = "org.eclipse.edc:jersey-providers", version.ref = "edc" } + # implementations edc-sql-assetindex = { module = "org.eclipse.edc:asset-index-sql", version.ref = "edc" } edc-sql-contract-definition = { module = "org.eclipse.edc:contract-definition-store-sql", version.ref = "edc" } @@ -94,6 +95,12 @@ edc-sql-pool = { module = "org.eclipse.edc:sql-pool-apache-commons", version.ref # azure stuff edc-azure-vault = { module = "org.eclipse.edc:vault-azure", version.ref = "edc" } edc-azure-identity = { module = "com.azure:azure-identity", version.ref = "azure-identity" } +edc-azure-test = { module = "org.eclipse.edc:azure-test", version.ref = "edc" } + +# EDC aws s3 stuff +edc-aws-s3-core = { module = "org.eclipse.edc:aws-s3-core", version.ref = "edc" } +edc-aws-s3-test = { module = "org.eclipse.edc:aws-s3-test", version.ref = "edc" } +edc-dpf-awss3 = { module = "org.eclipse.edc:data-plane-aws-s3", version.ref = "edc" } # Control Plane implementations edc-controlplane-callback-dispatcher-event = { module = "org.eclipse.edc:callback-event-dispatcher", version.ref = "edc" } @@ -112,7 +119,8 @@ edc-dpf-transfer = { module = "org.eclipse.edc:transfer-data-plane", version.ref edc-dpf-framework = { module = "org.eclipse.edc:data-plane-framework", version.ref = "edc" } edc-dpf-core = { module = "org.eclipse.edc:data-plane-core", version.ref = "edc" } edc-dpf-util = { module = "org.eclipse.edc:data-plane-util", version.ref = "edc" } -edc-dpf-awss3 = { module = "org.eclipse.edc:data-plane-aws-s3", version.ref = "edc" } + +edc-dpf-azblob = { module = "org.eclipse.edc:data-plane-azure-storage", version.ref = "edc" } edc-dpf-http = { module = "org.eclipse.edc:data-plane-http", version.ref = "edc" } edc-dpf-oauth2 = { module = "org.eclipse.edc:data-plane-http-oauth2", version.ref = "edc" } edc-dpf-api = { module = "org.eclipse.edc:data-plane-api", version.ref = "edc" } @@ -141,6 +149,7 @@ apache-sshd-sftp = { module = "org.apache.sshd:sshd-sftp", version.ref = "apache testcontainers-junit = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } testcontainers-postgres = { module = "org.testcontainers:postgresql", version.ref = "testcontainers" } aws-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" } +aws-s3transfer = { module = "software.amazon.awssdk:s3-transfer-manager", version.ref = "aws" } jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } jakartaJson = { module = "org.glassfish:jakarta.json", version.ref = "jakarta-json" } jacksonJsonP = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jakarta-jsonp", version.ref = "jackson" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 86486df88..707cd22b1 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -54,13 +54,16 @@ include("edc-extensions:ssi:ssi-miw-credential-client") include("edc-extensions:ssi:jws2020-crypto-suite") include(":edc-extensions:ssi:ssi-identity-extractor") - +// test modules include(":edc-tests:e2e-tests") include(":edc-tests:miw-tests") include(":edc-tests:runtime:extensions") include(":edc-tests:runtime:runtime-memory") +include(":edc-tests:runtime:dataplane-cloud") include(":edc-tests:runtime:runtime-memory-ssi") include(":edc-tests:runtime:runtime-postgresql") +include(":edc-tests:edc-dataplane:edc-dataplane-proxy-e2e") +include(":edc-tests:edc-dataplane:cloud-transfer-tests") // modules for controlplane artifacts include(":edc-controlplane") @@ -79,7 +82,6 @@ include(":edc-extensions:dataplane-proxy:edc-dataplane-proxy-consumer-api") include(":edc-extensions:dataplane-proxy:edc-dataplane-proxy-provider-spi") include(":edc-extensions:dataplane-proxy:edc-dataplane-proxy-provider-core") include(":edc-extensions:dataplane-proxy:edc-dataplane-proxy-provider-api") -include(":edc-tests:edc-dataplane-proxy-e2e") include(":samples:multi-tenancy")