Performance Testing (opensearch-project#314)

* Performance Testing workflow

Signed-off-by: Owais Kazi <owaiskazi19@gmail.com>

bundle-workflow/src/perf_test.py

@@ -0,0 +1,46 @@
+import argparse
+import os
+
+import yaml
+
+from git.git_repository import GitRepository
+from manifests.bundle_manifest import BundleManifest
+from system.working_directory import WorkingDirectory
+from test_workflow.perf_test_cluster import Cluster
+from test_workflow.perf_test_suite import PerfTestSuite
+
+"""
+    Entry point for Performance Test with bundle manifest, config file containing the required arguments for running
+    rally test and the stack name for the cluster. Will call out in test.sh with perf as argument
+"""
+
+parser = argparse.ArgumentParser(description="Test an OpenSearch Bundle")
+parser.add_argument('--bundle-manifest', type=argparse.FileType('r'), help="Bundle Manifest file.")
+parser.add_argument('--stack', dest='stack', help='Stack name for performance test')
+parser.add_argument('--config', type=argparse.FileType('r'), help="Config file.")
+args = parser.parse_args()
+
+manifest = BundleManifest.from_file(args.bundle_manifest)
+
+config = yaml.load(args.config, Loader=yaml.FullLoader)
+
+
+def get_infra_repo_url():
+    if "GITHUB_TOKEN" in os.environ:
+        return "https://${GITHUB_TOKEN}@github.com/opensearch-project/opensearch-infra.git"
+    return "https://github.com/opensearch-project/opensearch-infra.git"
+
+
+current_workspace = os.path.join(os.getcwd(), 'infra')
+cloned_repo = GitRepository(get_infra_repo_url(), 'main', current_workspace)
+security = False
+for component in manifest.components:
+    if component.name == 'security':
+        security = True
+
+with WorkingDirectory(current_workspace) as curdir:
+    with Cluster.create(manifest, config, args.stack, security) as test_cluster_endpoint:
+
+        os.chdir(current_workspace)
+        perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, security, current_workspace)
+        perf_test_suite.execute()

bundle-workflow/src/system/working_directory.py

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import os
+from contextlib import contextmanager
+
+
+@contextmanager
+def WorkingDirectory(path):
+    try:
+        saved_path = os.getcwd()
+        yield os.chdir(path)
+    finally:
+        os.chdir(saved_path)

bundle-workflow/src/test_workflow/perf_test_cluster.py

@@ -0,0 +1,74 @@
+import json
+import os
+import subprocess
+from contextlib import contextmanager
+
+from test_workflow.test_cluster import TestCluster
+
+
+class Cluster:
+    @contextmanager
+    def create(manifest, config, stack_name, security):
+        perf_test_cluster = PerfTestCluster(manifest, config, stack_name, security)
+        try:
+            perf_test_cluster.create()
+            yield perf_test_cluster.endpoint()
+        finally:
+            perf_test_cluster.destroy()
+
+
+class PerfTestCluster(TestCluster):
+    """
+    Represents a performance test cluster. This class deploys the opensearch bundle with CDK and returns the private IP.
+    """
+
+    def __init__(self, bundle_manifest, config, stack_name, security):
+        self.manifest = bundle_manifest
+        self.work_dir = 'tools/cdk/mensor/single-node/'
+        self.stack_name = stack_name
+        self.cluster_endpoint = None
+        self.cluster_port = None
+        self.output_file = 'output.json'
+        self.ip_address = None
+        self.security = 'enable' if security else 'disable'
+        role = config['Constants']['Role']
+        params_dict = {
+            'url': self.manifest.build.location,
+            'security_group_id': config['Constants']['SecurityGroupId'],
+            'vpc_id': config['Constants']['VpcId'],
+            'account_id': config['Constants']['AccountId'],
+            'region': config['Constants']['Region'],
+            'stack_name': self.stack_name,
+            'security': self.security,
+            'architecture': self.manifest.build.architecture,
+        }
+        params_list = []
+        for key, value in params_dict.items():
+            params_list.append(f' -c {key}={value}')
+        role_params = f' --require-approval=never --plugin cdk-assume-role-credential-plugin'\
+                      f' -c assume-role-credentials:writeIamRoleName={role} -c assume-role-credentials:readIamRoleName={role} '
+        self.params = ''.join(params_list) + role_params
+
+    def create(self):
+        os.chdir(self.work_dir)
+        command = f'cdk deploy {self.params} --outputs-file {self.output_file}'
+        print(f'Executing "{command}" in {os.getcwd()}')
+        subprocess.check_call(command, cwd=os.getcwd(), shell=True)
+        with open(self.output_file, 'r') as read_file:
+            load_output = json.load(read_file)
+        self.ip_address = load_output[self.stack_name]['PrivateIp']
+        print('Private IP:', self.ip_address)
+
+    def endpoint(self):
+        self.cluster_endpoint = self.ip_address
+        return self.cluster_endpoint
+
+    def port(self):
+        self.cluster_port = 443 if self.security == 'enable' else 9200
+        return self.cluster_port
+
+    def destroy(self):
+        os.chdir(self.work_dir)
+        command = f'cdk destroy {self.params} --force'
+        print(f'Executing "{command}" in {os.getcwd()}')
+        subprocess.check_call(command, cwd=os.getcwd(), shell=True)

bundle-workflow/src/test_workflow/perf_test_suite.py

@@ -0,0 +1,33 @@
+import os
+import subprocess
+
+from system.working_directory import WorkingDirectory
+
+
+class PerfTestSuite:
+    """
+    Represents a performance test suite. This class runs rally test on the deployed cluster with the provided IP.
+    """
+
+    def __init__(self, bundle_manifest, endpoint, security, current_workspace):
+        self.manifest = bundle_manifest
+        self.work_dir = 'tools/cdk/mensor/mensor_tests'
+        self.endpoint = endpoint
+        self.security = security
+        self.current_workspace = current_workspace
+        self.command = f'pipenv run python test_config.py -i {self.endpoint} -b {self.manifest.build.id}'\
+                       f' -a {self.manifest.build.architecture} '
+
+    def execute(self):
+        try:
+            with WorkingDirectory(self.work_dir):
+                dir = os.getcwd()
+                subprocess.check_call('python3 -m pipenv install', cwd=dir, shell=True)
+                subprocess.check_call('pipenv install', cwd=dir, shell=True)
+
+                if self.security:
+                    subprocess.check_call(f'{self.command} -s', cwd=dir, shell=True)
+                else:
+                    subprocess.check_call(f'{self.command}', cwd=dir, shell=True)
+        finally:
+            os.chdir(self.current_workspace)

bundle-workflow/tests/test_perf_workflow/data/test_manifest.yaml

@@ -0,0 +1,63 @@
+build:
+  architecture: x64
+  id: 41d5ae25183d4e699e92debfbe3f83bd
+  location: https://artifacts.opensearch.org/bundles/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/opensearch-1.0.0-linux-x64.tar.gz
+  name: OpenSearch
+  version: 1.0.0
+components:
+  - commit_id: fb25458f38c30a7ab06de21b0068f1fe3ad56134
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/bundle/opensearch-min-1.0.0-linux-x64.tar.gz
+    name: OpenSearch
+    ref: 1.0
+    repository: https://github.com/saratvemulapalli/OpenSearch.git
+  - commit_id: 7fad9529358259de529763c1c923fd947817a3bd
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-job-scheduler-1.0.0.0.zip
+    name: job-scheduler
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/job-scheduler.git
+  - commit_id: 65bb94fb7d46a88b07b61622585ed701918b19c5
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-sql-1.0.0.0.zip
+    name: sql
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/sql.git
+  - commit_id: a14ccd49389ca41446acc3200e3e870cde15a68e
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-alerting-1.0.0.0.zip
+    name: alerting
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/alerting.git
+  - commit_id: 2e21d59749526baa8e4666168643c4594cdadf79
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-security-1.0.0.0.zip
+    name: security
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/security.git
+  - commit_id: 091fe9f6612cd7e85054918036587bcd3c67eab1
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-index-management-1.0.0.0.zip
+    name: index-management
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/index-management.git
+  - commit_id: 9b29a99b05f2c8cd9d54dc868994cab8460ff0db
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-knn-1.0.0.0.zip
+    name: k-NN
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/k-NN.git
+  - commit_id: 502c96e54fae1cec9fee1fafd77ad92fde9d2459
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-anomaly-detection-1.0.0.0.zip
+    name: anomaly-detection
+    ref: 1.0
+    repository: https://github.com/opensearch-project/anomaly-detection.git
+  - commit_id: bd31e80adf6d52c1b4662d0d2cc9b30d8ae14309
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-asynchronous-search-1.0.0.0.zip
+    name: asynchronous-search
+    ref: main
+    repository: https://github.com/opensearch-project/asynchronous-search.git
+  - commit_id: 72705e2dfcad760c5de7609891700aa11d767884
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-reports-scheduler-1.0.0.0.zip
+    name: dashboards-reports
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/dashboards-reports.git
+  - commit_id: fd745a77c19df4991254b495cf0ec3730c66534d
+    location: https://artifacts.opensearch.org/builds/1.0.0/41d5ae25183d4e699e92debfbe3f83bd/plugins/opensearch-notebooks-1.0.0.0.zip
+    name: dashboards-notebooks
+    ref: 1.0.0.0
+    repository: https://github.com/opensearch-project/dashboards-notebooks.git
+schema-version: '1.0'

bundle-workflow/tests/test_perf_workflow/test_perf_test_cluster.py

@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+from manifests.bundle_manifest import BundleManifest
+from test_workflow.perf_test_cluster import PerfTestCluster
+
+
+class TestPerfCluster(unittest.TestCase):
+    def setUp(self):
+        self.data_path = os.path.realpath(
+            os.path.join(os.path.dirname(__file__), "data")
+        )
+        self.manifest_filename = os.path.join(
+            self.data_path, "test_manifest.yaml"
+        )
+        self.manifest = BundleManifest.from_path(self.manifest_filename)
+        self.stack_name = 'stack'
+        self.security = 'disable'
+        config = {
+            'Constants': {
+                'SecurityGroupId': 'sg-00000000',
+                'VpcId': 'vpc-12345',
+                'AccountId': '12345678',
+                'Region': 'us-west-2',
+                'Role': 'role-arn'
+            }
+        }
+        self.perf_test_cluster = PerfTestCluster(
+            bundle_manifest=self.manifest, config=config, stack_name=self.stack_name, security=self.security
+        )
+
+    def test_create(self):
+        mock_file = MagicMock(side_effect=[{"stack": {"PrivateIp": "10.10.10.10"}}])
+        with patch('test_workflow.perf_test_cluster.os.chdir') as mock_chdir:
+            with patch("subprocess.check_call") as mock_check_call:
+                with patch("builtins.open", MagicMock()):
+                    with patch("json.load", mock_file):
+                        self.perf_test_cluster.create()
+                        mock_chdir.assert_called_once_with('tools/cdk/mensor/single-node/')
+                        self.assertEqual(mock_check_call.call_count, 1)
+
+    def test_endpoint(self):
+        self.assertEqual(self.perf_test_cluster.endpoint(), None)
+
+    def test_port(self):
+        self.assertEqual(self.perf_test_cluster.port(), 443)
+
+    def test_destroy(self):
+        with patch('test_workflow.perf_test_cluster.os.chdir') as mock_chdir:
+            with patch("subprocess.check_call") as mock_check_call:
+                self.perf_test_cluster.destroy()
+                mock_chdir.assert_called_once_with('tools/cdk/mensor/single-node/')
+                self.assertEqual(mock_check_call.call_count, 1)

bundle-workflow/tests/test_perf_workflow/test_perf_test_suite.py

@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import os
+import unittest
+from unittest.mock import patch
+
+from manifests.bundle_manifest import BundleManifest
+from test_workflow.perf_test_suite import PerfTestSuite
+
+
+class TestPerfSuite(unittest.TestCase):
+    def setUp(self):
+        os.chdir(os.path.dirname(__file__))
+        self.manifest = BundleManifest.from_path("data/test_manifest.yaml")
+        self.endpoint = None
+        self.perf_test_suite = PerfTestSuite(
+            bundle_manifest=self.manifest, endpoint=None, security=False, current_workspace='current_workspace'
+        )
+
+    def test_execute(self):
+        with patch('test_workflow.perf_test_suite.os.chdir'):
+            with patch("subprocess.check_call") as mock_check_call:
+                self.perf_test_suite.execute()
+                self.assertEqual(mock_check_call.call_count, 3)