Added redisbench-admin deploy subtool (#351)

* Add the hability to override GIT_REPO and GIT_ORG

* added redisbench-admin deploy subtool

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.8.7"
+version = "0.8.8"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "README.md"

redisbench_admin/cli.py

@@ -14,6 +14,8 @@
 from redisbench_admin import __version__
 from redisbench_admin.compare.args import create_compare_arguments
 from redisbench_admin.compare.compare import compare_command_logic
+from redisbench_admin.deploy.args import create_deploy_arguments
+from redisbench_admin.deploy.deploy import deploy_command_logic
 from redisbench_admin.export.args import create_export_arguments
 from redisbench_admin.export.export import export_command_logic
 from redisbench_admin.extract.args import create_extract_arguments
@@ -91,7 +93,8 @@ def main():
         parser = create_grafana_api_arguments(parser)
     elif requested_tool == "--version":
         print_version(project_name, project_version)
-        sys.exit(0)
+    elif requested_tool == "deploy":
+        parser = create_deploy_arguments(parser)
     elif requested_tool == "--help":
         print_help(project_name, project_version)
         sys.exit(0)
@@ -100,6 +103,7 @@ def main():
             "compare",
             "run-local",
             "run-remote",
+            "deploy",
             "export",
             "extract",
             "watchdog",
@@ -150,6 +154,8 @@ def main():
         watchdog_command_logic(args, project_name, project_version)
     if requested_tool == "compare":
         compare_command_logic(args, project_name, project_version)
+    if requested_tool == "deploy":
+        deploy_command_logic(args, project_name, project_version)
     if requested_tool == "grafana-api":
         grafana_api_command_logic(args, project_name, project_version)
 

redisbench_admin/deploy/__init__.py

@@ -0,0 +1,5 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#

redisbench_admin/deploy/args.py

@@ -0,0 +1,37 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+import os
+
+DEFAULT_PRIVATE_KEY = "/tmp/benchmarks.redislabs.pem"
+TERRAFORM_BIN_PATH = os.getenv("TERRAFORM_BIN_PATH", "terraform")
+INFRA_TIMEOUT_SECS = int(os.getenv("INFRA_TIMEOUT_SECS", "1200"))
+GIT_ORG = os.getenv("GIT_ORG", None)
+GIT_REPO = os.getenv("GIT_REPO", None)
+
+
+def create_deploy_arguments(parser):
+    parser.add_argument(
+        "--private_key",
+        required=False,
+        default=DEFAULT_PRIVATE_KEY,
+        type=str,
+        help="Use this key for ssh connections.",
+    )
+    parser.add_argument("--terraform_bin_path", type=str, default=TERRAFORM_BIN_PATH)
+    parser.add_argument("--infra_timeout_secs", type=int, default=INFRA_TIMEOUT_SECS)
+    parser.add_argument("--github_actor", type=str, default=None, nargs="?", const="")
+    parser.add_argument("--github_repo", type=str, default=GIT_REPO)
+    parser.add_argument("--github_org", type=str, default=GIT_ORG)
+    parser.add_argument("--github_sha", type=str, default=None, nargs="?", const="")
+    parser.add_argument("--github_branch", type=str, default=None, nargs="?", const="")
+    parser.add_argument("--inventory_git", required=True, type=str)
+    parser.add_argument("--inventory_local_dir", default=None, type=str)
+    parser.add_argument("--set_env_vars_json", default="", type=str)
+    parser.add_argument("--setup_name_sufix", type=str, default="")
+    parser.add_argument(
+        "--destroy", help="destroy the current env", action="store_true"
+    )
+    return parser

redisbench_admin/deploy/deploy.py

@@ -0,0 +1,103 @@
+#  Apache License Version 2.0
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+import json
+import logging
+import os
+
+from redisbench_admin.run.git import git_vars_crosscheck
+from redisbench_admin.run.ssh import ssh_pem_check
+from redisbench_admin.utils.remote import (
+    fetch_remote_setup_git_url,
+    setup_remote_environment,
+)
+from python_terraform import Terraform
+
+from redisbench_admin.utils.utils import EC2_PRIVATE_PEM
+
+
+def deploy_command_logic(args, project_name, project_version):
+    logging.info(
+        "Using: {project_name} {project_version}".format(
+            project_name=project_name, project_version=project_version
+        )
+    )
+    terraform_bin_path = args.terraform_bin_path
+    tf_github_org = args.github_org
+    tf_github_actor = args.github_actor
+    tf_github_repo = args.github_repo
+    tf_github_sha = args.github_sha
+    tf_github_branch = args.github_branch
+    infra_timeout_secs = args.infra_timeout_secs
+
+    (
+        tf_github_actor,
+        tf_github_branch,
+        tf_github_org,
+        tf_github_repo,
+        tf_github_sha,
+    ) = git_vars_crosscheck(
+        tf_github_actor, tf_github_branch, tf_github_org, tf_github_repo, tf_github_sha
+    )
+
+    private_key = args.private_key
+    ssh_pem_check(EC2_PRIVATE_PEM, private_key)
+    inventory_git = args.inventory_git
+    inventory_local_dir = args.inventory_local_dir
+    destroy = args.destroy
+    if inventory_local_dir is not None:
+        if os.path.isdir(inventory_local_dir) is False:
+            os.mkdir(inventory_local_dir)
+    (
+        remote_setup,
+        deployment_type,
+        remote_id,
+    ) = fetch_remote_setup_git_url(inventory_git, inventory_local_dir, destroy)
+    tf = Terraform(
+        working_dir=remote_setup,
+        terraform_bin_path=terraform_bin_path,
+    )
+    tf_setup_name_sufix = "{}-{}".format(args.setup_name_sufix, tf_github_sha)
+    tf_setup_name = "{}{}".format(remote_setup, tf_setup_name_sufix)
+    terraform_backend_key = "benchmarks/infrastructure/{}.tfstate".format(tf_setup_name)
+    tf_triggering_env = "redisbench-admin-deploy"
+    logging.info("Setting an infra timeout of {} secs".format(infra_timeout_secs))
+    if args.destroy is False:
+        (tf_return_code, _, _, _, _, _, _,) = setup_remote_environment(
+            tf,
+            tf_github_sha,
+            tf_github_actor,
+            tf_setup_name,
+            tf_github_org,
+            tf_github_repo,
+            tf_triggering_env,
+            infra_timeout_secs,
+        )
+        logging.info("Deploy terraform return code {}".format(tf_return_code))
+        env_json = tf.output()
+        output_dict = {}
+        for k, v in env_json.items():
+            k = k.upper()
+            if type(v["value"]) == list:
+                output_dict[k] = v["value"][0]
+            else:
+                output_dict[k] = v["value"]
+
+        logging.info("JSON env variables {}".format(output_dict))
+        if args.set_env_vars_json != "":
+            with open(args.set_env_vars_json, "w") as json_fd:
+                json.dump(output_dict, json_fd)
+    else:
+        _, _, _ = tf.init(
+            capture_output=True,
+            backend_config={"key": terraform_backend_key},
+        )
+        logging.info("Refreshing remote state")
+        _, _, _ = tf.refresh()
+        logging.info("Triggering destroy")
+        output = tf.destroy(capture_output=True)
+        logging.info("Finished destroying the remote env. Output:")
+        for line in output[1].split("\n"):
+            print(line)

redisbench_admin/run/args.py

@@ -87,17 +87,17 @@ def common_run_args(parser):
         default="defaults.yml",
         help="specify the defaults file containing spec topologies, common metric extractions,etc...",
     )
-    parser.add_argument("--github_actor", type=str, default=None, nargs="?", const="")
-    parser.add_argument("--github_repo", type=str, default=GIT_REPO)
-    parser.add_argument("--github_org", type=str, default=GIT_ORG)
-    parser.add_argument("--github_sha", type=str, default=None, nargs="?", const="")
     parser.add_argument(
         "--required-module",
         default=None,
         action="append",
         help="path to the module file. "
         "You can use `--required-module` more than once",
     )
+    parser.add_argument("--github_actor", type=str, default=None, nargs="?", const="")
+    parser.add_argument("--github_repo", type=str, default=GIT_REPO)
+    parser.add_argument("--github_org", type=str, default=GIT_ORG)
+    parser.add_argument("--github_sha", type=str, default=None, nargs="?", const="")
     parser.add_argument("--github_branch", type=str, default=None, nargs="?", const="")
     parser.add_argument("--triggering_env", type=str, default=TRIGGERING_ENV)
     parser.add_argument(

redisbench_admin/utils/remote.py

@@ -504,6 +504,39 @@ def fetch_remote_id_from_config(
     return setup
 
 
+def fetch_remote_setup_git_url(git_url, temporary_dir=None, destroy=False):
+    setup_type = None
+    setup = None
+    branch = "master"
+    path = "/"
+
+    tree_pos = git_url.find("tree")
+    if tree_pos > 0:
+        repo = git_url[: tree_pos - 1]
+        remaining = git_url[tree_pos + 5 :].split("/")
+        if len(remaining) > 0:
+            branch = remaining[0]
+        if len(remaining) > 1:
+            path = path + "/".join(remaining[1:])
+
+    terraform_working_dir = common_tf(branch, path, repo, temporary_dir, destroy)
+    return terraform_working_dir, setup_type, setup
+
+
+def common_tf(branch, path, repo, temporary_dir=None, destroy=False):
+    if temporary_dir is None:
+        temporary_dir = tempfile.mkdtemp()
+    if destroy is False:
+        logging.info(
+            "Fetching infrastructure definition from git repo {}/{} (branch={}). Using local dir {} to store state".format(
+                repo, path, branch, temporary_dir
+            )
+        )
+        git.Repo.clone_from(repo, temporary_dir, branch=branch, depth=1)
+    terraform_working_dir = temporary_dir + path
+    return terraform_working_dir
+
+
 def fetch_remote_setup_from_config(
     remote_setup_config,
     repo="https://github.com/RedisLabsModules/testing-infrastructure.git",
@@ -518,14 +551,7 @@ def fetch_remote_setup_from_config(
             setup = remote_setup_property["setup"]
     # fetch terraform folder
     path = "/terraform/{}-{}".format(setup_type, setup)
-    temporary_dir = tempfile.mkdtemp()
-    logging.info(
-        "Fetching infrastructure definition from git repo {}/{} (branch={})".format(
-            repo, path, branch
-        )
-    )
-    git.Repo.clone_from(repo, temporary_dir, branch=branch, depth=1)
-    terraform_working_dir = temporary_dir + path
+    terraform_working_dir = common_tf(branch, path, repo)
     return terraform_working_dir, setup_type, setup