Skip to content

Commit

Permalink
Azurite TestContainer for Azure blob testing (#549)
Browse files Browse the repository at this point in the history 
* Azurite TestContainer for Azure blob testing

* Add testconteiners for AWS, fix broken multi-tenant test
  • Loading branch information
@andye2004
andye2004 authored May 17, 2021
1 parent 8d456be commit fadf6b6
Show file tree
Hide file tree
Showing 7 changed files with 181 additions and 40 deletions.
6 changes: 6 additions & 0 deletions spring-content-azure-storage/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,12 @@
<artifactId>hibernate-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<version>1.15.3</version>
<scope>test</scope>
</dependency>
</dependencies>

<repositories>
Expand Down
49 changes: 30 additions & 19 deletions ...e-storage/src/test/java/internal/org/springframework/content/azure/it/AzureStorageIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,24 @@
import javax.persistence.Id;
import javax.sql.DataSource;

import com.azure.storage.blob.BlobContainerClient;
import internal.org.springframework.content.azure.config.BlobIdResolverConverter;
import org.apache.commons.io.IOUtils;
import org.hamcrest.Matchers;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.content.azure.config.EnableAzureStorage;
import org.springframework.content.commons.annotations.ContentId;
import org.springframework.content.commons.annotations.ContentLength;
import org.springframework.content.commons.io.DeletableResource;
import org.springframework.content.commons.repository.ContentStore;
import org.springframework.content.commons.utils.PlacementService;
import org.springframework.content.commons.utils.PlacementServiceImpl;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.Resource;
import org.springframework.core.io.WritableResource;
import org.springframework.data.jpa.repository.JpaRepository;
Expand All @@ -52,7 +56,6 @@
import org.springframework.transaction.PlatformTransactionManager;

import com.azure.core.http.rest.PagedIterable;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.blob.models.BlobItem;
import com.github.paulcwarren.ginkgo4j.Ginkgo4jConfiguration;
Expand All @@ -76,8 +79,7 @@ public class AzureStorageIT {

private TestEntityRepository repo;
private TestEntityStore store;
private BlobServiceClientBuilder storage;
private BlobServiceClient client;
private BlobContainerClient client;

private String resourceLocation;

Expand All @@ -91,8 +93,7 @@ public class AzureStorageIT {

repo = context.getBean(TestEntityRepository.class);
store = context.getBean(TestEntityStore.class);
storage = context.getBean(BlobServiceClientBuilder.class);
client = storage.buildClient();
client = context.getBean(BlobContainerClient.class);

RandomString random = new RandomString(5);
resourceLocation = random.nextString();
Expand All @@ -115,9 +116,9 @@ public class AzureStorageIT {
((DeletableResource)genericResource).delete();
}

PagedIterable<BlobItem> blobs = client.getBlobContainerClient("test").listBlobs();
PagedIterable<BlobItem> blobs = client.listBlobs();
for(BlobItem blob : blobs) {
client.getBlobContainerClient("test").getBlobClient(blob.getName()).delete();
client.getBlobClient(blob.getName()).delete();
}
});

Expand Down Expand Up @@ -247,9 +248,9 @@ public class AzureStorageIT {

AfterEach(() -> {

PagedIterable<BlobItem> blobs = client.getBlobContainerClient("test").listBlobs();
PagedIterable<BlobItem> blobs = client.listBlobs();
for(BlobItem blob : blobs) {
client.getBlobContainerClient("test").getBlobClient(blob.getName()).delete();
client.getBlobClient(blob.getName()).delete();
}
});

Expand Down Expand Up @@ -359,18 +360,28 @@ public void test() {
@EnableAzureStorage(basePackages="internal.org.springframework.content.azure.it")
@Import(InfrastructureConfig.class)
public static class TestConfig {
@Bean
public BlobServiceClientBuilder blobServiceClientBuilder() {
return Azurite.getBlobServiceClientBuilder();
}

@Value("#{environment.AZURE_STORAGE_ENDPOINT}")
private String endpoint;

@Value("#{environment.AZURE_STORAGE_CONNECTION_STRING}")
private String connString;
@Bean
public BlobContainerClient blobContainerClient(BlobServiceClientBuilder builder) {
BlobContainerClient client = builder.buildClient().getBlobContainerClient("test");
// No pre-defined containers at start-up, create it on first bean generation
if (!client.exists()) {
client.create();
}
return client;
}

@Bean
public BlobServiceClientBuilder storage() {
return new BlobServiceClientBuilder()
.endpoint(endpoint)
.connectionString(connString);
@Primary
public PlacementService azureStoragePlacementService() {
// Provide default for tests to cover for missing env.AZURE_STORAGE_BUCKET
PlacementService conversion = new PlacementServiceImpl();
conversion.addConverter(new BlobIdResolverConverter("azure-test-bucket"));
return conversion;
}
}

Expand Down
57 changes: 57 additions & 0 deletions ...nt-azure-storage/src/test/java/internal/org/springframework/content/azure/it/Azurite.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package internal.org.springframework.content.azure.it;

import java.io.Serializable;

import com.azure.storage.blob.BlobServiceClientBuilder;
import org.testcontainers.containers.GenericContainer;

/**
* This class provides a TestContainers implementation of Azure storage via
* an Azurite docker container.
*
* Please refer to the following for details:-
* <a href="https://www.testcontainers.org">http://www.testcontainers.org</a>
* <a href="http://github.com/testcontainers/testcontainers-java">http://github.com/testcontainers/testcontainers-java</a>
* <a href="http://github.com/Azure/Azurite">http://github.com/Azure/Azurite</a>
*/

public class Azurite extends GenericContainer<Azurite> implements Serializable {

// Will default to latest tag on every test run
private static final String DOCKER_IMAGE_NAME = "mcr.microsoft.com/azure-storage/azurite";

// Default config as per Azurite docs
private static final int BLOB_SERVICE_PORT = 10000;

@SuppressWarnings("HttpUrlsUsage") // Testing only
private static final String ENDPOINT = "http://%s:%d/%s";
private static final String DEV_ACC_NAME = "devstoreaccount1";
private static final String PROTOCOL = "DefaultEndpointsProtocol=http";
private static final String ACC_NAME = "AccountName=" + DEV_ACC_NAME;
private static final String ACC_KEY =
"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";

private Azurite() {
super(DOCKER_IMAGE_NAME);
this.start();
}

public static BlobServiceClientBuilder getBlobServiceClientBuilder() {
final String host = Singleton.INSTANCE.getContainerIpAddress();
final Integer mappedPort = Singleton.INSTANCE.getMappedPort(BLOB_SERVICE_PORT);
final String endpoint = String.format(ENDPOINT, host, mappedPort, DEV_ACC_NAME);

return new BlobServiceClientBuilder()
.endpoint(endpoint)
.connectionString(String.join(";", PROTOCOL, ACC_NAME, ACC_KEY, "BlobEndpoint=" + endpoint));
}

@SuppressWarnings("unused") // Serializable safe singleton usage
protected Azurite readResolve() {
return Singleton.INSTANCE;
}

private static class Singleton {
private static final Azurite INSTANCE = new Azurite();
}
}
6 changes: 6 additions & 0 deletions spring-content-s3/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,12 @@
<artifactId>hibernate-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>localstack</artifactId>
<version>1.15.3</version>
<scope>test</scope>
</dependency>
</dependencies>

<repositories>
Expand Down
13 changes: 13 additions & 0 deletions ...t-s3/src/test/java/internal/org/springframework/content/s3/config/EnableS3StoresTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,15 @@
import org.springframework.content.commons.annotations.ContentId;
import org.springframework.content.commons.repository.AssociativeStore;
import org.springframework.content.commons.repository.ContentStore;
import org.springframework.content.commons.utils.PlacementService;
import org.springframework.content.commons.utils.PlacementServiceImpl;
import org.springframework.content.s3.S3ObjectIdResolver;
import org.springframework.content.s3.config.*;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.core.convert.converter.ConverterRegistry;
import org.springframework.core.io.Resource;

Expand Down Expand Up @@ -223,6 +226,16 @@ public AmazonS3 getAmazonS3() {
}
};
}

@Bean
@Primary
public PlacementService s3StorePlacementService() {
PlacementService conversion = new PlacementServiceImpl();
conversion.addConverter(
new S3ObjectIdResolverConverter(
new DefaultAssociativeStoreS3ObjectIdResolver(), "aws-test-bucket"));
return conversion;
}
}

@Configuration
Expand Down
51 changes: 51 additions & 0 deletions spring-content-s3/src/test/java/internal/org/springframework/content/s3/it/LocalStack.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package internal.org.springframework.content.s3.it;

import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;

import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import org.apache.http.client.utils.URIBuilder;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.utility.DockerImageName;

public class LocalStack extends LocalStackContainer implements Serializable {

private static final DockerImageName IMAGE_NAME = DockerImageName.parse("localstack/localstack");

private LocalStack() {
super(IMAGE_NAME);
withServices(Service.S3);
start();
}

private static class Singleton {
private static final LocalStack INSTANCE = new LocalStack();
}

public static AmazonS3 getAmazonS3Client() {
return AmazonS3ClientBuilder
.standard()
.withEndpointConfiguration(Singleton.INSTANCE.getEndpointConfiguration(Service.S3))
.withCredentials(Singleton.INSTANCE.getDefaultCredentialsProvider())
.withPathStyleAccessEnabled(true)
.build();
}

@Override
public URI getEndpointOverride(EnabledService service) {
try {
// super method converts localhost to 127.0.0.1 which fails on macos
// need to revert it back to whatever getContainerIpAddress() returns
return new URIBuilder(super.getEndpointOverride(service)).setHost(getContainerIpAddress()).build();
} catch (URISyntaxException e) {
throw new IllegalStateException("Cannot obtain endpoint URL", e);
}
}

@SuppressWarnings("unused") // Serializable safe singleton usage
protected LocalStack readResolve() {
return Singleton.INSTANCE;
}
}
39 changes: 18 additions & 21 deletions spring-content-s3/src/test/java/internal/org/springframework/content/s3/it/S3StoreIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,24 @@
import javax.persistence.Table;
import javax.sql.DataSource;

import internal.org.springframework.content.s3.config.DefaultAssociativeStoreS3ObjectIdResolver;
import internal.org.springframework.content.s3.config.S3ObjectIdResolverConverter;
import org.apache.commons.io.IOUtils;
import org.hamcrest.Matchers;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.content.commons.annotations.ContentId;
import org.springframework.content.commons.annotations.ContentLength;
import org.springframework.content.commons.io.DeletableResource;
import org.springframework.content.commons.repository.ContentStore;
import org.springframework.content.commons.utils.PlacementService;
import org.springframework.content.commons.utils.PlacementServiceImpl;
import org.springframework.content.s3.config.EnableS3Stores;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.core.env.Environment;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.Resource;
import org.springframework.core.io.WritableResource;
import org.springframework.data.jpa.repository.JpaRepository;
Expand All @@ -53,12 +56,7 @@
import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter;
import org.springframework.transaction.PlatformTransactionManager;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.github.paulcwarren.ginkgo4j.Ginkgo4jConfiguration;
import com.github.paulcwarren.ginkgo4j.Ginkgo4jRunner;

Expand All @@ -80,6 +78,7 @@ public class S3StoreIT {

private TestEntityRepository repo;
private TestEntityStore store;
private AmazonS3 client;

private String resourceLocation;

Expand All @@ -93,6 +92,9 @@ public class S3StoreIT {

repo = context.getBean(TestEntityRepository.class);
store = context.getBean(TestEntityStore.class);
client = context.getBean(AmazonS3.class);

client.createBucket("aws-test-bucket");

RandomString random = new RandomString(5);
resourceLocation = random.nextString();
Expand Down Expand Up @@ -343,24 +345,19 @@ public void test() {
@EnableS3Stores(basePackages="internal.org.springframework.content.s3.it")
@Import(InfrastructureConfig.class)
public static class TestConfig {

@Autowired
private Environment env;

public Region region() {
return Region.getRegion(Regions.fromName(System.getenv("AWS_REGION")));
}

@Bean
public BasicAWSCredentials basicAWSCredentials() {
return new BasicAWSCredentials(env.getProperty("AWS_ACCESS_KEY_ID"), env.getProperty("AWS_SECRET_KEY"));
public AmazonS3 client() {
return LocalStack.getAmazonS3Client();
}

@Bean
public AmazonS3 client(AWSCredentials awsCredentials) {
AmazonS3Client amazonS3Client = new AmazonS3Client(awsCredentials);
amazonS3Client.setRegion(region());
return amazonS3Client;
@Primary
public PlacementService s3StorePlacementService() {
PlacementService conversion = new PlacementServiceImpl();
conversion.addConverter(
new S3ObjectIdResolverConverter(
new DefaultAssociativeStoreS3ObjectIdResolver(), "aws-test-bucket"));
return conversion;
}
}

Expand Down

0 comments on commit fadf6b6

Please sign in to comment.