Load Test: reduce dataflow jobs and workder node

ollionorg · Jan 16, 2025 · 29a071f · 29a071f
1 parent 964cc52
commit 29a071f
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 3 deletions.
diff --git a/.github/workflows/java-pr.yml b/.github/workflows/java-pr.yml
@@ -190,7 +190,10 @@ jobs:
         --it-region="asia-south1" \
         --it-project="daring-fiber-439305-v4" \
         --it-artifact-bucket="ollion-teleport-testing" \
-        --it-private-connectivity="datastream-private-connect-asia-south1"
+        --it-private-connectivity="datastream-private-connect-asia-south1" \
+        --max-workers=1 \
+        --num-workers=1 \
+        --autoscaling-algorithm=NONE
     - name: Cleanup Java Environment
       uses: ./.github/actions/cleanup-java-env
       if: always()
diff --git a/cicd/internal/flags/it-flags.go b/cicd/internal/flags/it-flags.go
@@ -88,7 +88,7 @@ func HostIp() string {
 			panic(fmt.Errorf("failed to get gitactions runner host ip: %v", err))
 		} else {
 			// return "-DhostIp=" + string(hostIP)[:len(hostIP)-1]
-			return "-DhostIp=34.93.18.144"
+			return "-DhostIp=34.47.164.248"
 		}
 	}
 	return "-DhostIp=" + dHostIp

diff --git a/...sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlSourceLT.java b/...sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToMySqlSourceLT.java
@@ -76,6 +76,12 @@ public void setup() throws IOException {
                 .name());
 
     createMySQLSchema(jdbcResourceManagers);
+    PipelineLauncher.LaunchConfig.Builder configBuilder = PipelineLauncher.LaunchConfig.builder()
+        .setJobName(testName)
+        .setParameters(getParameters())
+        .addParameter("maxNumWorkers", String.valueOf(maxWorkers))
+        .addParameter("numWorkers", String.valueOf(numWorkers))
+        .addParameter("autoscalingAlgorithm", "NONE");  // Disable autoscaling    
     jobInfo = launchDataflowJob(artifactBucket, numWorkers, maxWorkers);
   }
 
@@ -102,7 +108,7 @@ public void reverseReplication1KTpsLoadTest()
             .setBatchSizeBytes("0")
             .build();
 
-    dataGenerator.execute(Duration.ofMinutes(90));
+    dataGenerator.execute(Duration.ofMinutes(30));
     assertThatPipeline(jobInfo).isRunning();
 
     JDBCRowsCheck check =