Support importing model to ModelArts (apache#22)

* Importing model to modelarts * Handled comments
jackylk · Aug 7, 2019 · d3db95d · d3db95d
1 parent 416675b
commit d3db95d
Show file tree

Hide file tree

Showing 21 changed files with 1,506 additions and 333 deletions.
diff --git a/fleet/ai-intf/src/main/java/org/apache/carbondata/ai/ModelTrainingAPI.java b/fleet/ai-intf/src/main/java/org/apache/carbondata/ai/ModelTrainingAPI.java
@@ -28,4 +28,6 @@ long startTrainingJob(Map<String, String> options, String modelName,
   void stopTrainingJob(long jobId) throws IOException;
 
   Map<String, String> getTrainingJobInfo(long jobId) throws Exception;
+
+  String importModel(Map<String, String> options, String jobName) throws Exception;
 }
diff --git a/fleet/cloud/pom.xml b/fleet/cloud/pom.xml
@@ -48,7 +48,25 @@
       <artifactId>okhttp</artifactId>
       <version>3.13.1</version>
     </dependency>
-
+    <dependency>
+      <groupId>com.huaweicloud</groupId>
+      <artifactId>esdk-obs-java</artifactId>
+      <version>3.1.3</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.squareup.okhttp3</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

diff --git a/fleet/cloud/src/main/java/com/huawei/cloud/RestConstants.java b/fleet/cloud/src/main/java/com/huawei/cloud/RestConstants.java
@@ -22,18 +22,28 @@
  */
 public interface RestConstants {
 
-  public static String HUAWEI_CLOUD_AUTH_ENDPOINT =
-      "https://iam.cn-north-1.myhuaweicloud.com/v3/auth/tokens";
+  String HUAWEI_ENDPOINT = "cn-north-1.myhuaweicloud.com";
 
-  public static String AUTH_TOKEN_HEADER = "X-Subject-Token";
+  String HUAWEI_CLOUD_AUTH_ENDPOINT = "https://iam." + HUAWEI_ENDPOINT + "/v3/auth/tokens";
 
-  public static String MODELARTS_CN_NORTH_V1_ENDPOINT =
-      "https://modelarts.cn-north-1.myhuaweicloud.com/v1/";
+  String HUAWEI_CLOUD_SECURITYTOKEN_ENDPOINT =
+      "https://iam." + HUAWEI_ENDPOINT + "/v3.0/OS-CREDENTIAL/securitytokens";
 
-  public static String MODELARTS_TRAINING_REST = "training-jobs";
+  String OBS_ENDPOINT =
+      "https://obs." + HUAWEI_ENDPOINT;
 
-  public static String MODELARTS_TRAINING_VERSIONS = "versions";
+  String AUTH_TOKEN_HEADER = "X-Subject-Token";
 
-  public static String SEPARATOR = "/";
+  String MODELARTS_CN_NORTH_V1_ENDPOINT = "https://modelarts." + HUAWEI_ENDPOINT + "/v1/";
+
+  String OBS_URL_SUFFIX = "obs.myhwclouds.com";
+
+  String MODELARTS_TRAINING_REST = "training-jobs";
+
+  String MODELARTS_MODEL = "models";
+
+  String SEPARATOR = "/";
+
+  String MODELARTS_TRAINING_VERSIONS = "versions";
 
 }
diff --git a/fleet/cloud/src/main/java/com/huawei/cloud/credential/LoginRequestManager.java b/fleet/cloud/src/main/java/com/huawei/cloud/credential/LoginRequestManager.java
@@ -20,7 +20,9 @@
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Map;
+import java.util.Objects;
 
+import com.google.gson.Gson;
 import com.huawei.cloud.RestConstants;
 import com.huawei.cloud.util.RestUtil;
 import okhttp3.Call;
@@ -98,6 +100,34 @@ public static LoginInfo login(String username, String password, OkHttpClient cli
     return loginInfo;
   }
 
+  public static Credential getTemporaryAccessKeys(LoginInfo loginInfo, OkHttpClient client)
+      throws Exception {
+    String accessJson = "{\n" +
+        "    \"auth\": {\n" +
+        "        \"identity\": {\n" +
+        "            \"methods\": [\n" +
+        "                \"token\"\n" +
+        "            ],\n" +
+        "            \"token\": {\n" +
+        "                \"id\": \"" + loginInfo.getToken() + "\",\n" +
+        "                \"duration-seconds\": \"3600\"\n" +
+        "\n" +
+        "            }\n" +
+        "        }\n" +
+        "    }\n" +
+        "}";
+
+    Response response =
+        RestUtil.postSync(RestConstants.HUAWEI_CLOUD_SECURITYTOKEN_ENDPOINT, accessJson, client);
+    Gson gson = new Gson();
+    if (response.isSuccessful()) {
+      AccessInfo accessInfo = gson.fromJson(response.body().string(), AccessInfo.class);
+      return accessInfo.getCredential();
+    } else {
+      throw new Exception(response.body().string());
+    }
+  }
+
   public static class LoginInfo {
     private boolean loggedIn = false;
     private String token;
@@ -135,5 +165,73 @@ public boolean isLoggedIn() {
     public void setLoggedIn(boolean loggedIn) {
       this.loggedIn = loggedIn;
     }
+
+    @Override public boolean equals(Object o) {
+      if (this == o) return true;
+      if (o == null || getClass() != o.getClass()) return false;
+      LoginInfo loginInfo = (LoginInfo) o;
+      return Objects.equals(userName, loginInfo.userName);
+    }
+
+    @Override public int hashCode() {
+
+      return Objects.hash(userName);
+    }
+  }
+
+  public static class AccessInfo implements Serializable {
+
+    private Credential credential;
+
+    public Credential getCredential() {
+      return credential;
+    }
+
+    public void setCredential(Credential credential) {
+      this.credential = credential;
+    }
+  }
+
+  public static class Credential implements Serializable {
+
+    private String access;
+
+    private String secret;
+
+    private String expires_at;
+
+    private String securitytoken;
+
+    public String getAccess() {
+      return access;
+    }
+
+    public void setAccess(String access) {
+      this.access = access;
+    }
+
+    public String getSecret() {
+      return secret;
+    }
+
+    public void setSecret(String secret) {
+      this.secret = secret;
+    }
+
+    public String getExpires_at() {
+      return expires_at;
+    }
+
+    public void setExpires_at(String expires_at) {
+      this.expires_at = expires_at;
+    }
+
+    public String getSecuritytoken() {
+      return securitytoken;
+    }
+
+    public void setSecuritytoken(String securitytoken) {
+      this.securitytoken = securitytoken;
+    }
   }
 }