From 5f756bc0154246f189e0e472444332e5097d96c0 Mon Sep 17 00:00:00 2001 From: HH Date: Tue, 22 Nov 2022 15:06:29 +0800 Subject: [PATCH 01/17] SUBMARINE-1349. Fix the syntax error reported in sonarcloud and add init value for apiversion in XGboostjobList.java --- .../submarine/server/manager/ModelVersionManager.java | 10 +++------- .../server/manager/RegisteredModelManager.java | 10 +++------- .../submitter/k8s/model/xgboostjob/XGBoostJobList.java | 2 +- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java index cbfc18d0fa..5802823aad 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java @@ -50,14 +50,10 @@ public class ModelVersionManager { * * @return object */ - public static ModelVersionManager getInstance() { + public static synchronized ModelVersionManager getInstance() { if (manager == null) { - synchronized (ModelVersionManager.class) { - if (manager == null) { - manager = new ModelVersionManager(new ModelVersionService(), new ModelVersionTagService(), - new Client()); - } - } + manager = new ModelVersionManager(new ModelVersionService(), new ModelVersionTagService(), + new Client()); } return manager; } diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java index f8a94dc85a..af13187572 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java @@ -54,14 +54,10 @@ public class RegisteredModelManager { * * @return object */ - public static RegisteredModelManager getInstance() { + public static synchronized RegisteredModelManager getInstance() { if (manager == null) { - synchronized (RegisteredModelManager.class) { - if (manager == null) { - manager = new RegisteredModelManager(new RegisteredModelService(), new ModelVersionService(), - new RegisteredModelTagService(), new Client()); - } - } + manager = new RegisteredModelManager(new RegisteredModelService(), new ModelVersionService(), + new RegisteredModelTagService(), new Client()); } return manager; } diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/xgboostjob/XGBoostJobList.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/xgboostjob/XGBoostJobList.java index 13e52209a9..43b3a776a9 100644 --- a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/xgboostjob/XGBoostJobList.java +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/xgboostjob/XGBoostJobList.java @@ -27,7 +27,7 @@ public class XGBoostJobList implements KubernetesListObject { @SerializedName("apiVersion") - private String apiVersion; + private String apiVersion = XGBoostJob.CRD_XGBOOST_API_VERSION_V1; @SerializedName("kind") private String kind; From 6b3b311841e9d4f26b412cbf3846c9decb2a3935 Mon Sep 17 00:00:00 2001 From: HH Date: Sat, 10 Dec 2022 17:40:04 +0800 Subject: [PATCH 02/17] SUBMARINE-1349. Fix the syntax error reported in sonarcloud, add init value for kind in XGboostjobList.java and make some class to be singleton. --- .../server/manager/ModelManager.java | 13 ++++++------ .../server/manager/ModelVersionManager.java | 13 ++++++------ .../manager/RegisteredModelManager.java | 15 ++++++------- .../server/rest/ExperimentRestApi.java | 2 +- .../apache/submarine/server/s3/Client.java | 21 +++++++------------ .../server/rest/ModelVersionRestApiTest.java | 4 ++-- .../rest/RegisteredModelRestApiTest.java | 2 +- .../submarine/server/s3/ClientTest.java | 2 +- .../model/service/ModelVersionService.java | 7 +++++++ .../model/service/RegisteredModelService.java | 8 +++++++ .../service/RegisteredModelTagService.java | 7 +++++++ .../database/model/ModelVersionTagTest.java | 4 ++-- .../database/model/ModelVersionTest.java | 4 ++-- .../model/RegisteredModelServiceTest.java | 2 +- .../model/RegisteredModelTagServiceTest.java | 4 ++-- .../k8s/model/xgboostjob/XGBoostJobList.java | 4 ++-- 16 files changed, 65 insertions(+), 47 deletions(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelManager.java index 378dd0ddc8..f5cd9aa2b7 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelManager.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelManager.java @@ -56,13 +56,12 @@ private ModelManager(Submitter submitter, ModelVersionService modelVersionServic * * @return object */ + private static class ModelManagerHolder { + private static ModelManager manager = new ModelManager(SubmitterManager.loadSubmitter(), ModelVersionService.getInstance()); + } + public static ModelManager getInstance() { - if (manager == null) { - synchronized (ModelManager.class) { - manager = new ModelManager(SubmitterManager.loadSubmitter(), new ModelVersionService()); - } - } - return manager; + return ModelManager.ModelManagerHolder.manager; } /** @@ -141,7 +140,7 @@ private void setServeInfo(ServeSpec spec, ModelVersionEntity modelVersion){ } private void transferDescription(ServeSpec spec) { - Client s3Client = new Client(); + Client s3Client = Client.getInstance(); String modelUniquePath = String.format("%s-%d-%s", spec.getModelName(), spec.getModelVersion(), spec.getModelId()); String res = new String(s3Client.downloadArtifact(String.format("registry/%s/%s/%d/description.json", diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java index 5802823aad..79850778d0 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java @@ -50,12 +50,13 @@ public class ModelVersionManager { * * @return object */ - public static synchronized ModelVersionManager getInstance() { - if (manager == null) { - manager = new ModelVersionManager(new ModelVersionService(), new ModelVersionTagService(), - new Client()); - } - return manager; + private static class ModelVersionManagerHolder { + private static ModelVersionManager manager = new ModelVersionManager(ModelVersionService.getInstance(), new ModelVersionTagService(), + Client.getInstance()); + } + + public static ModelVersionManager getInstance() { + return ModelVersionManager.ModelVersionManagerHolder.manager; } @VisibleForTesting diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java index af13187572..c365b7c98d 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java @@ -37,7 +37,6 @@ * Registered model manager. */ public class RegisteredModelManager { - private static RegisteredModelManager manager; /* Registered model service */ private final RegisteredModelService registeredModelService; @@ -54,12 +53,14 @@ public class RegisteredModelManager { * * @return object */ - public static synchronized RegisteredModelManager getInstance() { - if (manager == null) { - manager = new RegisteredModelManager(new RegisteredModelService(), new ModelVersionService(), - new RegisteredModelTagService(), new Client()); - } - return manager; + + private static class RegisteredModelManagerHolder { + private static RegisteredModelManager manager = new RegisteredModelManager(RegisteredModelService.getInstance(), ModelVersionService.getInstance(), + RegisteredModelTagService.getInstance(), Client.getInstance()); + } + + public static RegisteredModelManager getInstance() { + return RegisteredModelManager.RegisteredModelManagerHolder.manager; } @VisibleForTesting diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java index f9a1c45bab..fc65ed4515 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java @@ -58,7 +58,7 @@ @Produces({MediaType.APPLICATION_JSON + "; " + RestConstants.CHARSET_UTF8}) public class ExperimentRestApi { private ExperimentManager experimentManager = ExperimentManager.getInstance(); - private final Client minioClient = new Client(); + private final Client minioClient = Client.getInstance(); @VisibleForTesting public void setExperimentManager(ExperimentManager experimentManager) { diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index fd3971446b..029b31889f 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -26,6 +26,8 @@ import java.util.List; import java.util.Stack; import javax.ws.rs.core.Response; +import java.util.Map; +import java.util.HashMap; import io.minio.CopyObjectArgs; import io.minio.CopySource; @@ -50,25 +52,18 @@ public class Client { /* minio client */ public MinioClient minioClient; - /* submarine config */ - private static final SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); - public Client() { minioClient = MinioClient.builder() - .endpoint(conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)) - .credentials( - conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ACCESS_KEY_ID), - conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_SECRET_ACCESS_KEY) - ).build(); + .endpoint(S3Constants.ENDPOINT) + .credentials(S3Constants.ACCESSKEY, S3Constants.SECRETKEY) + .build(); } - public Client(String endpoint) { + private Client(String endpoint) { minioClient = MinioClient.builder() .endpoint(endpoint) - .credentials( - conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ACCESS_KEY_ID), - conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_SECRET_ACCESS_KEY) - ).build(); + .credentials(S3Constants.ACCESSKEY, S3Constants.SECRETKEY) + .build(); } /** diff --git a/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/ModelVersionRestApiTest.java b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/ModelVersionRestApiTest.java index 2a1bc63f71..4169ba5d8e 100644 --- a/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/ModelVersionRestApiTest.java +++ b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/ModelVersionRestApiTest.java @@ -55,9 +55,9 @@ public class ModelVersionRestApiTest { private final String modelVersionModelType = "experiment_123"; private final String modelVersionTag = "testTag"; - private final RegisteredModelService registeredModelService = new RegisteredModelService(); + private final RegisteredModelService registeredModelService = RegisteredModelService.getInstance(); - private final ModelVersionService modelVersionService = new ModelVersionService(); + private final ModelVersionService modelVersionService = ModelVersionService.getInstance(); private static final GsonBuilder gsonBuilder = new GsonBuilder() .registerTypeAdapter(ExperimentId.class, new ExperimentIdSerializer()) diff --git a/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/RegisteredModelRestApiTest.java b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/RegisteredModelRestApiTest.java index c8535b9e7b..b1f9501966 100644 --- a/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/RegisteredModelRestApiTest.java +++ b/submarine-server/server-core/src/test/java/org/apache/submarine/server/rest/RegisteredModelRestApiTest.java @@ -39,7 +39,7 @@ import org.apache.submarine.server.utils.gson.ExperimentIdSerializer; public class RegisteredModelRestApiTest { - private final RegisteredModelService registeredModelService = new RegisteredModelService(); + private final RegisteredModelService registeredModelService = RegisteredModelService.getInstance(); private final String registeredModelName = "testRegisteredModel"; private final String newRegisteredModelName = "newTestRegisteredModel"; private final String registeredModelDescription = "test registered model description"; diff --git a/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java b/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java index 7f86e3ae78..dd1ff3f604 100644 --- a/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java +++ b/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java @@ -27,7 +27,7 @@ public class ClientTest { - private final Client client = new Client("http://localhost:9000"); + private final Client client = Client.getInstance("http://localhost:9000"); private final String testExperimentId = "experiment-sample"; @After diff --git a/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/ModelVersionService.java b/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/ModelVersionService.java index a0560dcc6c..26fd4982cb 100644 --- a/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/ModelVersionService.java +++ b/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/ModelVersionService.java @@ -30,6 +30,13 @@ import java.util.List; public class ModelVersionService { + private static class ModelVersionServiceHolder { + private static ModelVersionService service = new ModelVersionService(); + } + + public static ModelVersionService getInstance() { + return ModelVersionService.ModelVersionServiceHolder.service; + } private static final Logger LOG = LoggerFactory.getLogger(ModelVersionService.class); diff --git a/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/RegisteredModelService.java b/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/RegisteredModelService.java index 408bd71902..0bfa9605ec 100644 --- a/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/RegisteredModelService.java +++ b/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/RegisteredModelService.java @@ -31,6 +31,14 @@ public class RegisteredModelService { + private static class RegisteredModelServiceHolder { + private static RegisteredModelService service = new RegisteredModelService(); + } + + public static RegisteredModelService getInstance() { + return RegisteredModelService.RegisteredModelServiceHolder.service; + } + private static final Logger LOG = LoggerFactory.getLogger(RegisteredModelService.class); public List selectAll() throws SubmarineRuntimeException { diff --git a/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/RegisteredModelTagService.java b/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/RegisteredModelTagService.java index c36e3f791d..fdbb42c634 100644 --- a/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/RegisteredModelTagService.java +++ b/submarine-server/server-database/src/main/java/org/apache/submarine/server/database/model/service/RegisteredModelTagService.java @@ -28,6 +28,13 @@ import org.slf4j.LoggerFactory; public class RegisteredModelTagService { + private static class RegisteredModelTagServiceHolder { + private static RegisteredModelTagService service = new RegisteredModelTagService(); + } + + public static RegisteredModelTagService getInstance() { + return RegisteredModelTagService.RegisteredModelTagServiceHolder.service; + } private static final Logger LOG = LoggerFactory.getLogger(RegisteredModelTagService.class); diff --git a/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/ModelVersionTagTest.java b/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/ModelVersionTagTest.java index 99e3965a74..c551cacbf8 100644 --- a/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/ModelVersionTagTest.java +++ b/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/ModelVersionTagTest.java @@ -33,8 +33,8 @@ public class ModelVersionTagTest { private static final Logger LOG = LoggerFactory.getLogger(ModelVersionTagTest.class); - RegisteredModelService registeredModelService = new RegisteredModelService(); - ModelVersionService modelVersionService = new ModelVersionService(); + RegisteredModelService registeredModelService = RegisteredModelService.getInstance(); + ModelVersionService modelVersionService = ModelVersionService.getInstance(); ModelVersionTagService modelVersionTagService = new ModelVersionTagService(); diff --git a/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/ModelVersionTest.java b/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/ModelVersionTest.java index 05b5c509c3..df3830ec4c 100644 --- a/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/ModelVersionTest.java +++ b/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/ModelVersionTest.java @@ -31,8 +31,8 @@ import org.apache.submarine.server.database.model.service.RegisteredModelService; public class ModelVersionTest { - RegisteredModelService registeredModelService = new RegisteredModelService(); - ModelVersionService modelVersionService = new ModelVersionService(); + RegisteredModelService registeredModelService = RegisteredModelService.getInstance(); + ModelVersionService modelVersionService = ModelVersionService.getInstance(); @After public void cleanAll() { diff --git a/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/RegisteredModelServiceTest.java b/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/RegisteredModelServiceTest.java index 64266be824..0c173ab4c7 100644 --- a/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/RegisteredModelServiceTest.java +++ b/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/RegisteredModelServiceTest.java @@ -29,7 +29,7 @@ import java.util.List; public class RegisteredModelServiceTest { - RegisteredModelService registeredModelService = new RegisteredModelService(); + RegisteredModelService registeredModelService = RegisteredModelService.getInstance(); @After public void cleanAll() { diff --git a/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/RegisteredModelTagServiceTest.java b/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/RegisteredModelTagServiceTest.java index c122da328b..1deec6468a 100644 --- a/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/RegisteredModelTagServiceTest.java +++ b/submarine-server/server-database/src/test/java/org/apache/submarine/server/database/model/RegisteredModelTagServiceTest.java @@ -31,8 +31,8 @@ public class RegisteredModelTagServiceTest { private static final Logger LOG = LoggerFactory.getLogger(RegisteredModelTagServiceTest.class); - RegisteredModelService registeredModelService = new RegisteredModelService(); - RegisteredModelTagService registeredModelTagService = new RegisteredModelTagService(); + RegisteredModelService registeredModelService = RegisteredModelService.getInstance(); + RegisteredModelTagService registeredModelTagService = RegisteredModelTagService.getInstance(); @After public void cleanAll() { diff --git a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/xgboostjob/XGBoostJobList.java b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/xgboostjob/XGBoostJobList.java index 43b3a776a9..0fa6861d7e 100644 --- a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/xgboostjob/XGBoostJobList.java +++ b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/model/xgboostjob/XGBoostJobList.java @@ -30,7 +30,7 @@ public class XGBoostJobList implements KubernetesListObject { private String apiVersion = XGBoostJob.CRD_XGBOOST_API_VERSION_V1; @SerializedName("kind") - private String kind; + private String kind = XGBoostJob.CRD_XGBOOST_KIND_V1 + "List"; @SerializedName("metadata") private V1ListMeta metadata; @@ -55,6 +55,6 @@ public String getApiVersion() { @Override public String getKind() { - return XGBoostJob.CRD_XGBOOST_KIND_V1 + "List"; + return kind; } } From 8ab7029ed2793d69b96ad3ec7432e4e05b2921e9 Mon Sep 17 00:00:00 2001 From: HH Date: Sun, 11 Dec 2022 11:32:46 +0800 Subject: [PATCH 03/17] SUBMARINE-1349. Fix for the style check. --- .../server/manager/ModelManager.java | 3 +- .../server/manager/ModelVersionManager.java | 5 ++-- .../manager/RegisteredModelManager.java | 6 ++-- .../apache/submarine/server/s3/Client.java | 28 +++++++++++++++---- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelManager.java index f5cd9aa2b7..c0f261d54a 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelManager.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelManager.java @@ -57,7 +57,8 @@ private ModelManager(Submitter submitter, ModelVersionService modelVersionServic * @return object */ private static class ModelManagerHolder { - private static ModelManager manager = new ModelManager(SubmitterManager.loadSubmitter(), ModelVersionService.getInstance()); + private static ModelManager manager = new ModelManager(SubmitterManager.loadSubmitter(), + ModelVersionService.getInstance()); } public static ModelManager getInstance() { diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java index 79850778d0..8b8d496384 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/ModelVersionManager.java @@ -51,8 +51,9 @@ public class ModelVersionManager { * @return object */ private static class ModelVersionManagerHolder { - private static ModelVersionManager manager = new ModelVersionManager(ModelVersionService.getInstance(), new ModelVersionTagService(), - Client.getInstance()); + private static ModelVersionManager manager = new ModelVersionManager(ModelVersionService.getInstance(), + new ModelVersionTagService(), + Client.getInstance()); } public static ModelVersionManager getInstance() { diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java index c365b7c98d..2168cc3e55 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java @@ -55,8 +55,10 @@ public class RegisteredModelManager { */ private static class RegisteredModelManagerHolder { - private static RegisteredModelManager manager = new RegisteredModelManager(RegisteredModelService.getInstance(), ModelVersionService.getInstance(), - RegisteredModelTagService.getInstance(), Client.getInstance()); + private static RegisteredModelManager manager = new RegisteredModelManager(RegisteredModelService.getInstance(), + ModelVersionService.getInstance(), + RegisteredModelTagService.getInstance(), + Client.getInstance()); } public static RegisteredModelManager getInstance() { diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index 029b31889f..acab1be04f 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -51,12 +51,30 @@ public class Client { /* minio client */ public MinioClient minioClient; + public static Map clientFactory = new HashMap(); - public Client() { - minioClient = MinioClient.builder() - .endpoint(S3Constants.ENDPOINT) - .credentials(S3Constants.ACCESSKEY, S3Constants.SECRETKEY) - .build(); + public static Client getClient(String endpoint) { + Client client = clientFactory.get(endpoint); + Map clientLocalFactory = clientFactory; + + if (client == null) { + synchronized(Client.class) { + if (client == null) { + client = new Client(endpoint); + clientLocalFactory.put(endpoint, client); + clientFactory = clientLocalFactory; + } + } + } + return client; + } + + public static Client getInstance() { + return getClient(S3Constants.ENDPOINT); + } + + public static Client getInstance(String endpoint) { + return getClient(endpoint); } private Client(String endpoint) { From eaccfbcc722e1fec93f4c796f63116263fdfcacf Mon Sep 17 00:00:00 2001 From: HH Date: Sun, 11 Dec 2022 11:58:51 +0800 Subject: [PATCH 04/17] SUBMARINE-1349. Fix for the style check. --- .../main/java/org/apache/submarine/server/s3/Client.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index acab1be04f..851cd3b81f 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -51,14 +51,14 @@ public class Client { /* minio client */ public MinioClient minioClient; - public static Map clientFactory = new HashMap(); + public static Map clientFactory = new HashMap(); public static Client getClient(String endpoint) { Client client = clientFactory.get(endpoint); - Map clientLocalFactory = clientFactory; + Map clientLocalFactory = clientFactory; if (client == null) { - synchronized(Client.class) { + synchronized (Client.class) { if (client == null) { client = new Client(endpoint); clientLocalFactory.put(endpoint, client); From 0e95639f58297f9ce96fd2695a658f7c184f2395 Mon Sep 17 00:00:00 2001 From: HH Date: Sun, 11 Dec 2022 20:21:00 +0800 Subject: [PATCH 05/17] SUBMARINE-1349. Fix for the style check. --- .../submarine/server/manager/RegisteredModelManager.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java index 2168cc3e55..807b09e6e6 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/manager/RegisteredModelManager.java @@ -55,7 +55,8 @@ public class RegisteredModelManager { */ private static class RegisteredModelManagerHolder { - private static RegisteredModelManager manager = new RegisteredModelManager(RegisteredModelService.getInstance(), + private static RegisteredModelManager manager = new RegisteredModelManager( + RegisteredModelService.getInstance(), ModelVersionService.getInstance(), RegisteredModelTagService.getInstance(), Client.getInstance()); From 3893d017d30ff63f6606d60d266dfce9d92ad6cf Mon Sep 17 00:00:00 2001 From: HH Date: Sun, 18 Dec 2022 11:25:59 +0800 Subject: [PATCH 06/17] SUBMARINE-1349. Change the Client to be singleton with enum class. --- .../apache/submarine/server/s3/Client.java | 50 +++++++++---------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index 851cd3b81f..3596ce728d 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -44,44 +44,40 @@ import org.apache.submarine.commons.utils.SubmarineConfiguration; import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException; -/** - * S3(Minio) default client - */ -public class Client { - /* minio client */ - public MinioClient minioClient; +public enum Client { + DEFAULT(S3Constants.ENDPOINT), CUSTOMER("http://localhost:9000"); + public static Map clientFactory = new HashMap(); + private final String endpoint; + private final MinioClient minioClient; - public static Client getClient(String endpoint) { - Client client = clientFactory.get(endpoint); - Map clientLocalFactory = clientFactory; - if (client == null) { - synchronized (Client.class) { - if (client == null) { - client = new Client(endpoint); - clientLocalFactory.put(endpoint, client); - clientFactory = clientLocalFactory; - } - } + static { + for (Client clientSingleton : Client.values()) { + clientFactory.put(clientSingleton.endpoint, clientSingleton); } - return client; } - public static Client getInstance() { - return getClient(S3Constants.ENDPOINT); + Client(String endpoint) { + this.endpoint = endpoint; + this.minioClient = MinioClient.builder() + .endpoint(endpoint) + .credentials(S3Constants.ACCESSKEY, S3Constants.SECRETKEY) + .build(); } - public static Client getInstance(String endpoint) { - return getClient(endpoint); + public static Client getInstance() { + return clientFactory.get(S3Constants.ENDPOINT); } - private Client(String endpoint) { - minioClient = MinioClient.builder() - .endpoint(endpoint) - .credentials(S3Constants.ACCESSKEY, S3Constants.SECRETKEY) - .build(); + public static Client getInstance(String endpoint) { + try { + return clientFactory.get(endpoint); + } catch (Exception e) { + throw new SubmarineRuntimeException(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), + e.getMessage()); + } } /** From b1ccac1a6ce7f78a40375bef6f33643c1214b278 Mon Sep 17 00:00:00 2001 From: HH Date: Sun, 18 Dec 2022 12:40:23 +0800 Subject: [PATCH 07/17] SUBMARINE-1349. Fix check error. --- .../apache/submarine/server/s3/Client.java | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index 3596ce728d..7afff5633a 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -44,13 +44,21 @@ import org.apache.submarine.commons.utils.SubmarineConfiguration; import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException; - +/** + * S3(Minio) default client + */ public enum Client { - DEFAULT(S3Constants.ENDPOINT), CUSTOMER("http://localhost:9000"); + DEFAULT(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT), CUSTOMER("http://localhost:9000"); + + /* minio client */ + private final MinioClient minioClient; + + /* submarine config */ + private static final SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); public static Map clientFactory = new HashMap(); private final String endpoint; - private final MinioClient minioClient; + static { @@ -61,14 +69,16 @@ public enum Client { Client(String endpoint) { this.endpoint = endpoint; - this.minioClient = MinioClient.builder() - .endpoint(endpoint) - .credentials(S3Constants.ACCESSKEY, S3Constants.SECRETKEY) - .build(); + this.minioClient = MinioClient.builder() + .endpoint(endpoint) + .credentials( + conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ACCESS_KEY_ID), + conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_SECRET_ACCESS_KEY) + ).build(); } public static Client getInstance() { - return clientFactory.get(S3Constants.ENDPOINT); + return clientFactory.get(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT); } public static Client getInstance(String endpoint) { From c51361350ca235ba5708d7de8975a4c66b0a45fe Mon Sep 17 00:00:00 2001 From: HH Date: Sun, 18 Dec 2022 12:51:48 +0800 Subject: [PATCH 08/17] SUBMARINE-1349. Fix check error. --- .../apache/submarine/server/s3/Client.java | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index 7afff5633a..1e0debfe1c 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -48,18 +48,16 @@ * S3(Minio) default client */ public enum Client { - DEFAULT(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT), CUSTOMER("http://localhost:9000"); + /* submarine config */ + private static final SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); /* minio client */ private final MinioClient minioClient; - /* submarine config */ - private static final SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); - public static Map clientFactory = new HashMap(); private final String endpoint; - + DEFAULT(conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)), CUSTOMER("http://localhost:9000"); static { for (Client clientSingleton : Client.values()) { @@ -67,7 +65,7 @@ public enum Client { } } - Client(String endpoint) { + private Client(String endpoint) { this.endpoint = endpoint; this.minioClient = MinioClient.builder() .endpoint(endpoint) @@ -77,8 +75,18 @@ public enum Client { ).build(); } + private Client() { + this.endpoint = conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT); + this.minioClient = MinioClient.builder() + .endpoint(conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)) + .credentials( + conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ACCESS_KEY_ID), + conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_SECRET_ACCESS_KEY) + ).build(); + } + public static Client getInstance() { - return clientFactory.get(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT); + return clientFactory.get(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT.varValue); } public static Client getInstance(String endpoint) { From cc4365b098241092babaed2480be0e82a8ac252f Mon Sep 17 00:00:00 2001 From: HH Date: Sun, 18 Dec 2022 13:09:33 +0800 Subject: [PATCH 09/17] SUBMARINE-1349. Fix check error. --- .../src/main/java/org/apache/submarine/server/s3/Client.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index 1e0debfe1c..57b6b94e8f 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -48,6 +48,9 @@ * S3(Minio) default client */ public enum Client { + DEFAULT(SubmarineConfiguration.getInstance().getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)), + CUSTOMER("http://localhost:9000"); + /* submarine config */ private static final SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); @@ -57,8 +60,6 @@ public enum Client { public static Map clientFactory = new HashMap(); private final String endpoint; - DEFAULT(conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)), CUSTOMER("http://localhost:9000"); - static { for (Client clientSingleton : Client.values()) { clientFactory.put(clientSingleton.endpoint, clientSingleton); From 2d29e99ec6e1599c5197a8f044e400c92c42d9d9 Mon Sep 17 00:00:00 2001 From: HH Date: Sun, 18 Dec 2022 13:16:07 +0800 Subject: [PATCH 10/17] SUBMARINE-1349. Fix check error. --- .../src/main/java/org/apache/submarine/server/s3/Client.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index 57b6b94e8f..2855a3561b 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -66,7 +66,7 @@ public enum Client { } } - private Client(String endpoint) { + Client(String endpoint) { this.endpoint = endpoint; this.minioClient = MinioClient.builder() .endpoint(endpoint) @@ -76,7 +76,7 @@ private Client(String endpoint) { ).build(); } - private Client() { + Client() { this.endpoint = conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT); this.minioClient = MinioClient.builder() .endpoint(conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)) From fc16e0d5a11b41253d05c5cba15116a023ae410b Mon Sep 17 00:00:00 2001 From: HH Date: Sun, 18 Dec 2022 13:48:45 +0800 Subject: [PATCH 11/17] SUBMARINE-1349. Fix check error by remove the static when declaring SubmarineConfiguration conf. --- .../src/main/java/org/apache/submarine/server/s3/Client.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index 2855a3561b..18457633ca 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -52,7 +52,7 @@ public enum Client { CUSTOMER("http://localhost:9000"); /* submarine config */ - private static final SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); + private final SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); /* minio client */ private final MinioClient minioClient; From bf4918a3972b968617a7383dd0980c0564f0aede Mon Sep 17 00:00:00 2001 From: HH Date: Sun, 18 Dec 2022 14:23:36 +0800 Subject: [PATCH 12/17] SUBMARINE-1349. Fix check error by re-uesing get Instance, since the usage of conf may be used in both staitc and non-static. --- .../src/main/java/org/apache/submarine/server/s3/Client.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index 18457633ca..8a435b9cc5 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -87,7 +87,8 @@ public enum Client { } public static Client getInstance() { - return clientFactory.get(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT.varValue); + return clientFactory.get(SubmarineConfiguration.getInstance() + .getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)); } public static Client getInstance(String endpoint) { From 64f9b4fc1dea2fd58eef68164754a0bf4f26aa39 Mon Sep 17 00:00:00 2001 From: HH Date: Thu, 12 Jan 2023 22:30:42 +0800 Subject: [PATCH 13/17] SUBMARINE-1349. Change the enum class for singleton and add unit-test to cover it. --- .../apache/submarine/server/s3/Client.java | 37 +------------------ .../submarine/server/s3/ClientTest.java | 8 +++- 2 files changed, 9 insertions(+), 36 deletions(-) diff --git a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java index 8a435b9cc5..4edf0eaabd 100644 --- a/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java +++ b/submarine-server/server-core/src/main/java/org/apache/submarine/server/s3/Client.java @@ -26,8 +26,6 @@ import java.util.List; import java.util.Stack; import javax.ws.rs.core.Response; -import java.util.Map; -import java.util.HashMap; import io.minio.CopyObjectArgs; import io.minio.CopySource; @@ -48,8 +46,7 @@ * S3(Minio) default client */ public enum Client { - DEFAULT(SubmarineConfiguration.getInstance().getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)), - CUSTOMER("http://localhost:9000"); + INSTANCE(SubmarineConfiguration.getInstance().getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)); /* submarine config */ private final SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); @@ -57,17 +54,7 @@ public enum Client { /* minio client */ private final MinioClient minioClient; - public static Map clientFactory = new HashMap(); - private final String endpoint; - - static { - for (Client clientSingleton : Client.values()) { - clientFactory.put(clientSingleton.endpoint, clientSingleton); - } - } - Client(String endpoint) { - this.endpoint = endpoint; this.minioClient = MinioClient.builder() .endpoint(endpoint) .credentials( @@ -76,28 +63,8 @@ public enum Client { ).build(); } - Client() { - this.endpoint = conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT); - this.minioClient = MinioClient.builder() - .endpoint(conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)) - .credentials( - conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ACCESS_KEY_ID), - conf.getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_SECRET_ACCESS_KEY) - ).build(); - } - public static Client getInstance() { - return clientFactory.get(SubmarineConfiguration.getInstance() - .getString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT)); - } - - public static Client getInstance(String endpoint) { - try { - return clientFactory.get(endpoint); - } catch (Exception e) { - throw new SubmarineRuntimeException(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), - e.getMessage()); - } + return INSTANCE; } /** diff --git a/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java b/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java index dd1ff3f604..8b8bd65751 100644 --- a/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java +++ b/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java @@ -27,7 +27,7 @@ public class ClientTest { - private final Client client = Client.getInstance("http://localhost:9000"); + private final Client client = Client.getInstance(); private final String testExperimentId = "experiment-sample"; @After @@ -79,4 +79,10 @@ public void testCopyObject() { response = client.downloadArtifact(copyPath); Assert.assertArrayEquals(content, response); } + + @Test + public void testSingleton() { + Client testClient = Client.getInstance(); + Assert.assertEquals(testClient, client); + } } From f0263c8ec1c2e54dd403ddacadf81764e667d3f5 Mon Sep 17 00:00:00 2001 From: HH Date: Mon, 13 Feb 2023 00:20:16 +0800 Subject: [PATCH 14/17] SUBMARINE-1349. Add BeforeClass to change submarine endpoint before testing. --- .../submarine/server/s3/ClientTest.java | 74 +++++++++++-------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java b/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java index 8b8bd65751..20f0452e53 100644 --- a/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java +++ b/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java @@ -19,9 +19,13 @@ package org.apache.submarine.server.s3; +import org.apache.submarine.commons.utils.SubmarineConfVars; +import org.apache.submarine.commons.utils.SubmarineConfiguration; + +import org.junit.BeforeClass; import org.junit.After; -import org.junit.Assert; import org.junit.Test; +import org.junit.Assert; import java.util.List; @@ -35,6 +39,12 @@ public void cleanAll() { client.deleteAllArtifacts(); } + @BeforeClass + public static void init() { + SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); + conf.setString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT, "http://localhost:8000"); + } + @Test public void testLogArtifactAndDownloadArtifact() { String path = "sample_folder/sample_file"; @@ -44,41 +54,41 @@ public void testLogArtifactAndDownloadArtifact() { Assert.assertArrayEquals(content, response); } - @Test - public void testListAndDeleteArtifactByExperimentId() { - byte[] content = "0123456789".getBytes(); + @Test + public void testListAndDeleteArtifactByExperimentId() { + byte[] content = "0123456789".getBytes(); - String[] artifactPaths = { - String.format("experiment/%s/1", testExperimentId), - String.format("experiment/%s/2", testExperimentId) - }; - String[] actualResults = { - String.format("s3://%s/experiment/%s/1", S3Constants.BUCKET, testExperimentId), - String.format("s3://%s/experiment/%s/2", S3Constants.BUCKET, testExperimentId) - }; - client.logArtifact(artifactPaths[0], content); - client.logArtifact(artifactPaths[1], content); - List results = client.listArtifact(String.format("experiment/%s", testExperimentId)); - Assert.assertArrayEquals(actualResults, results.toArray()); + String[] artifactPaths = { + String.format("experiment/%s/1", testExperimentId), + String.format("experiment/%s/2", testExperimentId) + }; + String[] actualResults = { + String.format("s3://%s/experiment/%s/1", S3Constants.BUCKET, testExperimentId), + String.format("s3://%s/experiment/%s/2", S3Constants.BUCKET, testExperimentId) + }; + client.logArtifact(artifactPaths[0], content); + client.logArtifact(artifactPaths[1], content); + List results = client.listArtifact(String.format("experiment/%s", testExperimentId)); + Assert.assertArrayEquals(actualResults, results.toArray()); - client.deleteArtifactsByExperiment(testExperimentId); - results = client.listArtifact(testExperimentId); - Assert.assertArrayEquals(new String[0], results.toArray()); - } + client.deleteArtifactsByExperiment(testExperimentId); + results = client.listArtifact(testExperimentId); + Assert.assertArrayEquals(new String[0], results.toArray()); + } - @Test - public void testCopyObject() { - String path = "sample_folder/sample_file"; - byte[] content = "0123456789".getBytes(); - client.logArtifact(path, content); - byte[] response = client.downloadArtifact(path); - Assert.assertArrayEquals(content, response); + @Test + public void testCopyObject() { + String path = "sample_folder/sample_file"; + byte[] content = "0123456789".getBytes(); + client.logArtifact(path, content); + byte[] response = client.downloadArtifact(path); + Assert.assertArrayEquals(content, response); - String copyPath = "sample_folder_copy/sample_file"; - client.copyArtifact(copyPath, path); - response = client.downloadArtifact(copyPath); - Assert.assertArrayEquals(content, response); - } + String copyPath = "sample_folder_copy/sample_file"; + client.copyArtifact(copyPath, path); + response = client.downloadArtifact(copyPath); + Assert.assertArrayEquals(content, response); + } @Test public void testSingleton() { From 725b978fff1d883df3b10ffad931ce1dc034293f Mon Sep 17 00:00:00 2001 From: HH Date: Mon, 13 Feb 2023 00:25:24 +0800 Subject: [PATCH 15/17] SUBMARINE-1349. Remove indentdent. --- .../submarine/server/s3/ClientTest.java | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java b/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java index 20f0452e53..e1957e1df7 100644 --- a/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java +++ b/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java @@ -54,41 +54,41 @@ public void testLogArtifactAndDownloadArtifact() { Assert.assertArrayEquals(content, response); } - @Test - public void testListAndDeleteArtifactByExperimentId() { - byte[] content = "0123456789".getBytes(); + @Test + public void testListAndDeleteArtifactByExperimentId() { + byte[] content = "0123456789".getBytes(); - String[] artifactPaths = { - String.format("experiment/%s/1", testExperimentId), - String.format("experiment/%s/2", testExperimentId) - }; - String[] actualResults = { - String.format("s3://%s/experiment/%s/1", S3Constants.BUCKET, testExperimentId), - String.format("s3://%s/experiment/%s/2", S3Constants.BUCKET, testExperimentId) - }; - client.logArtifact(artifactPaths[0], content); - client.logArtifact(artifactPaths[1], content); - List results = client.listArtifact(String.format("experiment/%s", testExperimentId)); - Assert.assertArrayEquals(actualResults, results.toArray()); + String[] artifactPaths = { + String.format("experiment/%s/1", testExperimentId), + String.format("experiment/%s/2", testExperimentId) + }; + String[] actualResults = { + String.format("s3://%s/experiment/%s/1", S3Constants.BUCKET, testExperimentId), + String.format("s3://%s/experiment/%s/2", S3Constants.BUCKET, testExperimentId) + }; + client.logArtifact(artifactPaths[0], content); + client.logArtifact(artifactPaths[1], content); + List results = client.listArtifact(String.format("experiment/%s", testExperimentId)); + Assert.assertArrayEquals(actualResults, results.toArray()); - client.deleteArtifactsByExperiment(testExperimentId); - results = client.listArtifact(testExperimentId); - Assert.assertArrayEquals(new String[0], results.toArray()); - } + client.deleteArtifactsByExperiment(testExperimentId); + results = client.listArtifact(testExperimentId); + Assert.assertArrayEquals(new String[0], results.toArray()); + } - @Test - public void testCopyObject() { - String path = "sample_folder/sample_file"; - byte[] content = "0123456789".getBytes(); - client.logArtifact(path, content); - byte[] response = client.downloadArtifact(path); - Assert.assertArrayEquals(content, response); + @Test + public void testCopyObject() { + String path = "sample_folder/sample_file"; + byte[] content = "0123456789".getBytes(); + client.logArtifact(path, content); + byte[] response = client.downloadArtifact(path); + Assert.assertArrayEquals(content, response); - String copyPath = "sample_folder_copy/sample_file"; - client.copyArtifact(copyPath, path); - response = client.downloadArtifact(copyPath); - Assert.assertArrayEquals(content, response); - } + String copyPath = "sample_folder_copy/sample_file"; + client.copyArtifact(copyPath, path); + response = client.downloadArtifact(copyPath); + Assert.assertArrayEquals(content, response); + } @Test public void testSingleton() { From f73915ec2b7a731979f5bcee8fc4c6d0e8f375a1 Mon Sep 17 00:00:00 2001 From: HH Date: Thu, 16 Feb 2023 18:42:33 +0800 Subject: [PATCH 16/17] SUBMARINE-1349. Change the minio client initialization to local host when clientTest. --- .../test/java/org/apache/submarine/server/s3/ClientTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java b/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java index e1957e1df7..d3eab75bfa 100644 --- a/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java +++ b/submarine-server/server-core/src/test/java/org/apache/submarine/server/s3/ClientTest.java @@ -31,7 +31,7 @@ public class ClientTest { - private final Client client = Client.getInstance(); + private static Client client; private final String testExperimentId = "experiment-sample"; @After @@ -42,7 +42,8 @@ public void cleanAll() { @BeforeClass public static void init() { SubmarineConfiguration conf = SubmarineConfiguration.getInstance(); - conf.setString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT, "http://localhost:8000"); + conf.setString(SubmarineConfVars.ConfVars.SUBMARINE_S3_ENDPOINT, "http://localhost:9000"); + client = Client.getInstance(); } @Test From fad6b5a2240a15a12c93e0cab97e542576731226 Mon Sep 17 00:00:00 2001 From: HH Date: Tue, 28 Feb 2023 16:24:36 +0800 Subject: [PATCH 17/17] SUBMARINE-1349. export localhost for CI . --- .github/workflows/master.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 9a07b38a55..edb4dd55e8 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -436,6 +436,7 @@ jobs: TEST_MODULES: "-pl :submarine-server-core" run: | echo ">>> mvn $TEST_FLAG $TEST_MODULES -B" + export SUBMARINE_S3_ENDPOINT=http://localhost:9000 mvn $TEST_FLAG $TEST_MODULES -B - name: Build submarine-serve env: