forked from kubeflow/testing
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Auto deploy job needs to use the new kfctl syntax; also use unique names
Related to kubeflow#471 * Don't set name in the spec because we want to infer it form directory. * Create a new script to deploy with a unique name * Related to: kubeflow#444 * Update cleanup script to clean up new auto-deployed clusters
- Loading branch information
Jeremy Lewi
committed
Oct 18, 2019
1 parent
f8302c8
commit bc78726
Showing
5 changed files
with
307 additions
and
81 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,201 @@ | ||
| """Create a Kubeflow instance. | ||
| The purpose of this script is to automate the creation of Kubeflow Deployments | ||
| corresponding to different versions of Kubeflow. | ||
| This script should replace create_kf_instance. Unlike create_kf_instance | ||
| we no longer need to recycle kf app names because of IAP so we can | ||
| use unique names which greatly simplifies things. | ||
| This greatly simplifieds things. In particular, we don't need to do any | ||
| cleanup in this script because we will rely on cleanup_ci to GC old auto | ||
| deployments. | ||
| """ | ||
| import argparse | ||
| import datetime | ||
| import logging | ||
| import json | ||
| import os | ||
| import re | ||
| import requests | ||
| import shutil | ||
| import subprocess | ||
| import tempfile | ||
| import uuid | ||
| import yaml | ||
|
|
||
| from googleapiclient import discovery | ||
| from google.cloud import storage | ||
| from kubeflow.testing import util | ||
| from retrying import retry | ||
| from oauth2client.client import GoogleCredentials | ||
|
|
||
| @retry(wait_fixed=60000, stop_max_attempt_number=5) | ||
| def run_with_retry(*args, **kwargs): | ||
| util.run(*args, **kwargs) | ||
|
|
||
| def build_kfctl_go(args): | ||
| """Build kfctl go.""" | ||
| build_dir = os.path.join(args.kubeflow_repo, "bootstrap") | ||
| # We need to use retry builds because when building in the test cluster | ||
| # we see intermittent failures pulling dependencies | ||
| util.run(["make", "build-kfctl"], cwd=build_dir) | ||
| kfctl_path = os.path.join(build_dir, "bin", "kfctl") | ||
|
|
||
| return kfctl_path | ||
|
|
||
| def deploy_with_kfctl_go(kfctl_path, args, app_dir, env, labels=None): | ||
| """Deploy Kubeflow using kfctl go binary.""" | ||
| # username and password are passed as env vars and won't appear in the logs | ||
| # | ||
| # We need to edit and rewrite the config file to the app dir because | ||
| # kfctl uses the path of the config file as the app dir.s | ||
| logging.warning("Loading configs %s.", args.kfctl_config) | ||
|
|
||
| if args.kfctl_config.startswith("http"): | ||
| response = requests.get(args.kfctl_config) | ||
| raw_config = response.content | ||
| else: | ||
| with open(args.kfctl_config) as hf: | ||
| raw_config = hf.read() | ||
|
|
||
| config_spec = yaml.load(raw_config) | ||
|
|
||
| # We need to specify a valid email because | ||
| # 1. We need to create appropriate RBAC rules to allow the current user | ||
| # to create the required K8s resources. | ||
| # 2. Setting the IAM policy will fail if the email is invalid. | ||
| email = util.run(["gcloud", "config", "get-value", "account"]) | ||
|
|
||
| if not email: | ||
| raise ValueError("Could not determine GCP account being used.") | ||
|
|
||
| config_spec["spec"]["project"] = args.project | ||
| config_spec["spec"]["email"] = email | ||
| config_spec["spec"]["zone"] = args.zone | ||
|
|
||
| config_spec["spec"] = util.filter_spartakus(config_spec["spec"]) | ||
|
|
||
| # Remove name because we will auto infer from directory. | ||
| if "name" in config_spec["metadata"]: | ||
| logging.info("Deleting name in kfdef spec.") | ||
| del config_spec["metadata"]["name"] | ||
|
|
||
| if not "labels" in config_spec["metadata"]: | ||
| config_spec["metadata"]["labels"] = {} | ||
|
|
||
| if labels: | ||
| config_spec["metadata"]["labels"].update(labels) | ||
|
|
||
| logging.info("KFDefSpec:\n%s", yaml.safe_dump(config_spec)) | ||
|
|
||
| if not os.path.exists(app_dir): | ||
| logging.info("Creating app dir %s", app_dir) | ||
| os.makedirs(app_dir) | ||
|
|
||
| config_file = os.path.join(app_dir, "kf_config.yaml") | ||
| with open(config_file, "w") as hf: | ||
| logging.info("Writing file %s", config_file) | ||
| yaml.dump(config_spec, hf) | ||
|
|
||
| util.run([kfctl_path, "apply", "-V", "-f", config_file], env=env) | ||
|
|
||
| def main(): # pylint: disable=too-many-locals,too-many-statements | ||
| logging.basicConfig(level=logging.INFO, | ||
| format=('%(levelname)s|%(asctime)s' | ||
| '|%(pathname)s|%(lineno)d| %(message)s'), | ||
| datefmt='%Y-%m-%dT%H:%M:%S', | ||
| ) | ||
| logging.getLogger().setLevel(logging.INFO) | ||
|
|
||
| parser = argparse.ArgumentParser() | ||
|
|
||
| parser.add_argument( | ||
| "--project", default="kubeflow-ci-deployment", type=str, | ||
| help=("The project.")) | ||
|
|
||
| parser.add_argument( | ||
| "--zone", default="us-east1-d", type=str, help=("The zone to deploy in.")) | ||
| parser.add_argument( | ||
| "--oauth_file", | ||
| default=("gs://kubeflow-ci-deployment_kf-data/" | ||
| "kf-iap-oauth.kubeflow-ci-deployment.yaml"), | ||
| type=str, help=("The file containing the OAuth client ID & secret" | ||
| "for IAP.")) | ||
|
|
||
| parser.add_argument( | ||
| "--kubeflow_repo", | ||
| default="/src/kubeflow/kubeflow", | ||
| type=str, help=("Path to the Kubeflow repo to use")) | ||
|
|
||
| parser.add_argument( | ||
| "--kfctl_config", | ||
| default=("https://raw.githubusercontent.com/kubeflow/manifests" | ||
| "/master/kfdef/kfctl_gcp_iap.yaml"), | ||
| type=str, help=("Path to the kfctl config to use")) | ||
|
|
||
| parser.add_argument( | ||
| "--apps_dir", | ||
| default=os.getcwd(), | ||
| type=str, help=("Directory to store kubeflow apps.")) | ||
|
|
||
| parser.add_argument( | ||
| "--name", type=str, default="kf-vmaster-{uid}", | ||
| help=("Name for the deployment. This can be a python format string " | ||
| "with the variable uid. Uid will automatically be substituted " | ||
| "for a unique value based on the time.")) | ||
|
|
||
| parser.add_argument( | ||
| "--job_name", | ||
| default="", type=str, help=("Pod name running the job.")) | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| util.maybe_activate_service_account() | ||
|
|
||
| bucket, blob_path = util.split_gcs_uri(args.oauth_file) | ||
|
|
||
| client = storage.Client(project=args.project) | ||
| bucket = client.get_bucket(bucket) | ||
|
|
||
| blob = bucket.get_blob(blob_path) | ||
| contents = blob.download_as_string() | ||
|
|
||
| oauth_info = yaml.load(contents) | ||
|
|
||
| git_describe = util.run(["git", "describe", "--tags", "--always", "--dirty"], | ||
| cwd=args.kubeflow_repo).strip("'") | ||
|
|
||
| kfctl_path = build_kfctl_go(args) | ||
|
|
||
| # We need to keep the name short to avoid hitting limits with certificates. | ||
| uid = datetime.datetime.now().strftime("%m%d") + "-" | ||
| uid = uid + uuid.uuid4().hex[0:3] | ||
|
|
||
| args.name = args.name.format(uid=uid) | ||
| logging.info("Using name %s", args.name) | ||
|
|
||
| app_dir = os.path.join(args.apps_dir, args.name) | ||
|
|
||
| if not os.path.exists(args.apps_dir): | ||
| os.makedirs(args.apps_dir) | ||
|
|
||
| env = {} | ||
| env.update(os.environ) | ||
| env.update(oauth_info) | ||
|
|
||
| labels = { "GIT_LABEL": git_describe, | ||
| "PURPOSE": "kf-test-cluster", | ||
| } | ||
|
|
||
| label_args = [] | ||
| for k, v in labels.items(): | ||
| # labels can only take as input alphanumeric characters, hyphens, and | ||
| # underscores. Replace not valid characters with hyphens. | ||
| val = v.lower().replace("\"", "") | ||
| val = re.sub(r"[^a-z0-9\-_]", "-", val) | ||
| label_args.append("{key}={val}".format(key=k.lower(), val=val)) | ||
|
|
||
| deploy_with_kfctl_go(kfctl_path, args, app_dir, env, labels=labels) | ||
|
|
||
| if __name__ == "__main__": | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.