Use snapshot files to checkout repositories. (kubeflow#298)

* Add checkout-snapshot.sh

* finish script

* add repo-clone-snapshot to init.sh

* add some args

test-infra/auto-deploy/Dockerfile

@@ -159,3 +159,6 @@ COPY checkout.sh /usr/local/bin
 RUN chmod a+x /usr/local/bin/checkout.sh
 COPY init.sh /usr/local/bin
 RUN chmod a+x /usr/local/bin/init.sh
+COPY repo-clone-snapshot.py /usr/local/bin
+COPY checkout-snapshot.sh /usr/local/bin
+RUN chmod a+x /usr/local/bin/checkout-snapshot.sh

test-infra/auto-deploy/checkout-snapshot.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Simple way to take a snapshot of a github repository at a given commit.
+set -xe
+
+SRC_DIR=$1
+REPO_OWNER=$2
+REPO_NAME=$3
+COMMIT_SHA=$4
+
+ORIGIN_DIR=$PWD
+
+mkdir -p ${SRC_DIR}/${REPO_OWNER}
+REPO_DIR=${SRC_DIR}/${REPO_OWNER}/${REPO_NAME}
+
+echo "Checking out git repo: ${REPO_OWNER}/${REPO_NAME}.git"
+git clone https://github.com/${REPO_OWNER}/${REPO_NAME}.git ${REPO_DIR}
+
+cd ${REPO_DIR}
+
+echo "Taking snapshot at ${COMMIT_SHA}"
+git reset --hard ${COMMIT_SHA}
+
+cd ${ORIGIN_DIR}

test-infra/auto-deploy/init.sh

@@ -8,9 +8,10 @@ PROJECT=$3
 WORKER_CLUSTER=$4
 
 # Check out fresh copy of KF and deployment workflow.
-# TODO(gabrielwen): Need to make a seperate workflow to snapshot repos.
-/usr/local/bin/checkout.sh ${SRC_DIR} ${REPO_OWNER} kubeflow
-/usr/local/bin/checkout.sh ${SRC_DIR} ${REPO_OWNER} testing
+python /usr/local/bin/repo-clone-snapshot.py \
+  --src_dir=${SRC_DIR} \
+  --project=${PROJECT} \
+  --repo_owner=${REPO_OWNER}
 
 PYTHONPATH="${PYTHONPATH}:${SRC_DIR}/${REPO_OWNER}/testing/py"
 export PYTHONPATH

test-infra/auto-deploy/repo-clone-snapshot.py

@@ -0,0 +1,69 @@
+"""Repositories checkout helper script.
+
+Takes as input from latest blob in GCP storage bucket and checkout repositories
+as requested.
+"""
+
+import argparse
+import json
+import logging
+import subprocess
+
+from google.cloud import storage
+
+def main():
+  logging.basicConfig(level=logging.INFO,
+                      format=('%(levelname)s|%(asctime)s'
+                              '|%(pathname)s|%(lineno)d| %(message)s'),
+                      datefmt='%Y-%m-%dT%H:%M:%S',
+                      )
+  logging.getLogger().setLevel(logging.INFO)
+
+  parser = argparse.ArgumentParser()
+
+  parser.add_argument(
+    "--src_dir", default="", type=str,
+    help=("Directory to write repositories to."))
+
+  parser.add_argument(
+    "--project", default="kubeflow-ci", type=str, help=("The project."))
+
+  parser.add_argument(
+    "--zone", default="us-east1-d", type=str, help=("The zone to deploy in."))
+
+  parser.add_argument(
+    "--repo_owner", default="kubeflow", type=str, help=("Repository owner."))
+  parser.add_argument(
+    "--snapshot_bucket",
+    default="kubeflow-ci_deployment-snapshot",
+    type=str, help=("GCP bucket snapshot files written to."))
+
+  args = parser.parse_args()
+  gs_client = storage.Client(project=args.project)
+  bucket = gs_client.get_bucket(args.snapshot_bucket)
+
+  filenames = [b.name for b in bucket.list_blobs()]
+  if not filenames:
+    msg = "Not able to find any snapshot files in " + args.snapshot_bucket
+    logging.error(msg)
+    raise RuntimeError(msg)
+
+  filenames.sort(reverse=True)
+  blob_name = filenames[0] # pylint: disable=unsubscriptable-object
+
+  snapshot = json.loads(bucket.get_blob(blob_name).download_as_string())
+
+  logging.info("Snapshot profile: %s", str(snapshot))
+  for repo in snapshot:
+    logging.info("Checking out: %s at %s", repo, snapshot.get(repo, ""))
+    subprocess.call(("/usr/local/bin/checkout-snapshot.sh"
+                     " {src_dir} {repo_owner} {repo_name} {sha}").format(
+                       src_dir=args.src_dir,
+                       repo_owner=args.repo_owner,
+                       repo_name=repo,
+                       sha=snapshot.get(repo, ""),
+                     ),
+                    shell=True)
+
+if __name__ == '__main__':
+  main()