Parallelize calls to copy from/to GCS

The -m flag copies from multiple threads and makes a huge performance difference for large trees.
tektoncd · Sep 11, 2020 · 28667d2 · 28667d2
1 parent 1d6b4b1
commit 28667d2
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/task/gcs-download/0.1/gcs-download.yaml b/task/gcs-download/0.1/gcs-download.yaml
@@ -55,7 +55,7 @@ spec:
 
       if [[ "$(params.typeDir)" == "true" ]]; then
         mkdir -p "$DESTINATION"
-        gsutil rsync -d -r "$(params.location)" "$DESTINATION"
+        gsutil -m rsync -d -r "$(params.location)" "$DESTINATION"
       else
         gsutil cp $(params.location) "$DESTINATION"
       fi
diff --git a/task/gcs-upload/0.1/gcs-upload.yaml b/task/gcs-upload/0.1/gcs-upload.yaml
@@ -50,7 +50,7 @@ spec:
       fi
 
       if [[ -d "$SOURCE" ]]; then
-        gsutil rsync -d -r "$SOURCE" "$(params.location)"
+        gsutil -m rsync -d -r "$SOURCE" "$(params.location)"
       else
         gsutil cp "$SOURCE" "$(params.location)"
       fi