update default batch & refactor workflow (#27)

vesoft-inc · Dec 23, 2021 · 194a63c · 194a63c
1 parent fe32819
commit 194a63c
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 10 deletions.
diff --git a/.github/workflows/deploy_release.yml b/.github/workflows/deploy_release.yml
@@ -1,7 +1,7 @@
 # This workflow will build a Java project with Maven
 # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
 
-name: Deploy Spark Connector release version to Maven Central Repository
+name: release
 
 on:
   release:

diff --git a/.github/workflows/deploy_snapshot.yml b/.github/workflows/deploy_snapshot.yml
@@ -1,7 +1,7 @@
 # This workflow will build a Java project with Maven
 # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
 
-name: Deploy Spark Connector SNAPSHOT version to Maven SNAPSHOT Repository
+name: snapshot
 
 on:
   push:

diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -1,7 +1,7 @@
 # This workflow will build a Java project with Maven
 # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
 
-name: Java CI with Maven
+name: pull_request
 
 on:
   push:

diff --git a/example/pom.xml b/example/pom.xml
@@ -18,9 +18,33 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-deploy-plugin</artifactId>
-                <configuration>
-                    <skip>true</skip>
-                </configuration>
+                <version>2.8.2</version>
+                <executions>
+                    <execution>
+                        <id>default-deploy</id>
+                        <phase>deploy</phase>
+                        <configuration>
+                            <skip>true</skip>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>default-deploy</id>
+                        <phase>deploy</phase>
+                        <goals>
+                            <goal>deploy</goal>
+                        </goals>
+                        <configuration>
+                            <serverId>ossrh</serverId>
+                            <skipNexusStagingDeployMojo>true</skipNexusStagingDeployMojo>
+                        </configuration>
+                    </execution>
+                </executions>
             </plugin>
             <!-- compiler-plugin -->
             <plugin>

diff --git a/nebula-spark-connector/src/main/scala/com/vesoft/nebula/connector/NebulaConfig.scala b/nebula-spark-connector/src/main/scala/com/vesoft/nebula/connector/NebulaConfig.scala
@@ -256,7 +256,7 @@ object WriteNebulaVertexConfig {
     var tagName: String   = _
     var vidPolicy: String = _
     var vidField: String  = _
-    var batch: Int        = 1000
+    var batch: Int        = 512
     var user: String      = "root"
     var passwd: String    = "nebula"
     var writeMode: String = "insert"
@@ -289,7 +289,7 @@ object WriteNebulaVertexConfig {
     }
 
     /**
-      * set data amount for one batch, default is 1000
+      * set data amount for one batch, default is 512
       */
     def withBatch(batch: Int): WriteVertexConfigBuilder = {
       this.batch = batch
@@ -351,6 +351,9 @@ object WriteNebulaVertexConfig {
         case e: Throwable =>
           assert(false, s"optional write mode: insert or update, your write mode is $writeMode")
       }
+      if (writeMode.equalsIgnoreCase(WriteMode.UPDATE.toString)) {
+        assert(batch <= 512, "the maximum number of statements for Nebula is 512")
+      }
       if (!writeMode.equalsIgnoreCase(WriteMode.DELETE.toString)) {
         assert(tagName != null && !tagName.isEmpty, s"config tagName is empty.")
       } else {
@@ -424,7 +427,7 @@ object WriteNebulaEdgeConfig {
     var dstIdField: String = _
     var dstPolicy: String  = _
     var rankField: String  = _
-    var batch: Int         = 1000
+    var batch: Int         = 512
     var user: String       = "root"
     var passwd: String     = "nebula"
 
@@ -485,7 +488,7 @@ object WriteNebulaEdgeConfig {
     }
 
     /**
-      * set data amount for one batch, default is 1000
+      * set data amount for one batch, default is 512
       */
     def withBatch(batch: Int): WriteEdgeConfigBuilder = {
       this.batch = batch
@@ -575,6 +578,9 @@ object WriteNebulaEdgeConfig {
         case e: Throwable =>
           assert(false, s"optional write mode: insert or update, your write mode is $writeMode")
       }
+      if (writeMode.equalsIgnoreCase(WriteMode.UPDATE.toString)) {
+        assert(batch <= 512, "the maximum number of statements for Nebula is 512")
+      }
       assert(edgeName != null && !edgeName.isEmpty, s"config edgeName is empty.")
       LOG.info(
         s"NebulaWriteEdgeConfig={space=$space,edgeName=$edgeName,srcField=$srcIdField," +

diff --git a/nebula-spark-connector/src/test/scala/com/vesoft/nebula/connector/NebulaConfigSuite.scala b/nebula-spark-connector/src/test/scala/com/vesoft/nebula/connector/NebulaConfigSuite.scala
@@ -101,6 +101,28 @@ class NebulaConfigSuite extends AnyFunSuite with BeforeAndAfterAll {
     assert(writeNebulaConfig.getSpace.equals("test"))
   }
 
+  test("wrong batch size for update") {
+    assertThrows[AssertionError](
+      WriteNebulaVertexConfig
+        .builder()
+        .withSpace("test")
+        .withTag("tag")
+        .withVidField("vId")
+        .withWriteMode(WriteMode.UPDATE)
+        .withBatch(513)
+        .build())
+    assertThrows[AssertionError](
+      WriteNebulaEdgeConfig
+        .builder()
+        .withSpace("test")
+        .withEdge("edge")
+        .withSrcIdField("src")
+        .withDstIdField("dst")
+        .withWriteMode(WriteMode.UPDATE)
+        .withBatch(513)
+        .build())
+  }
+
   test("test wrong policy") {
     assertThrows[AssertionError](
       WriteNebulaVertexConfig