diff --git a/README.md b/README.md index c86d352..7948f90 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,10 @@ The pipeline sets environment variables from GitHub secrets and variables and ru ## Setup +### Running locally + +If you're not using CI tooling and running the script locally you will need to set the below environment variables. There is an option to store these variables in a `.env` file which will get loaded in using `load_dotenv()` at the beginning of the script + ### Configure Repo 1. Set the following git secrets in Settings > Secrets and Variables > Actions: diff --git a/cicd/migration_helpers.py b/cicd/migration_helpers.py index 49905c1..d396d4e 100644 --- a/cicd/migration_helpers.py +++ b/cicd/migration_helpers.py @@ -1,10 +1,13 @@ -import requests import logging import json import os import time -from ib_helpers import upload_chunks, read_file_through_api, package_solution, unzip_files, compile_solution, \ +import requests +from zipfile import ZipFile +from pathlib import Path + +from cicd.ib_helpers import upload_chunks, read_file_through_api, package_solution, unzip_files, compile_solution, \ copy_file_within_ib, read_file_content_from_ib, get_file_metadata, create_folder_if_it_does_not_exists, \ wait_until_job_finishes @@ -98,7 +101,7 @@ def compile_and_package_ib_solution(ib_host, api_token, solution_directory_path, return compile_resp, solution_resp -def download_ibsolution(ib_host, api_token, solution_path, write_to_local=True): +def download_ibsolution(ib_host, api_token, solution_path, write_to_local=False, unzip_solution=False): """ Get the bytes content of an .ibsolution file @@ -110,13 +113,22 @@ def download_ibsolution(ib_host, api_token, solution_path, write_to_local=True): """ # TODO: Check if file exists first - # Read in .ibsolution file resp = read_file_through_api(ib_host, api_token, solution_path) if write_to_local: - with open(solution_path.split('/')[-1], 'wb') as fd: + solution_name = Path(solution_path).name + with open(solution_name, 'wb') as fd: fd.write(resp.content) + if unzip_solution: + zip_path = Path(solution_path).with_suffix(".zip").name + with open(zip_path, 'wb') as fd: + fd.write(resp.content) + with ZipFile(zip_path, "r") as zip_ref: + unzip_dir = Path(Path(zip_path).parent, Path(zip_path).stem) + zip_ref.extractall(unzip_dir) + os.remove(zip_path) + return resp diff --git a/cicd/promote_solution.py b/cicd/promote_solution.py index 0da4926..d860bda 100644 --- a/cicd/promote_solution.py +++ b/cicd/promote_solution.py @@ -166,7 +166,7 @@ def copy_solution_to_working_dir(new_solution_dir): else: ib_solution_path = get_latest_ibsolution_path(SOURCE_IB_API_TOKEN, SOURCE_FILES_API, SOURCE_COMPILED_SOLUTIONS_PATH) - resp = download_ibsolution(SOURCE_IB_HOST, SOURCE_IB_API_TOKEN, ib_solution_path, True) + resp = download_ibsolution(SOURCE_IB_HOST, SOURCE_IB_API_TOKEN, ib_solution_path) target_path = os.path.join(TARGET_IB_PATH, ib_solution_path.split('/')[-1]) upload_file(TARGET_IB_HOST, TARGET_IB_API_TOKEN, target_path, resp.content) @@ -198,7 +198,7 @@ def copy_solution_to_working_dir(new_solution_dir): if args.download_ibsolution or args.local_flow or args.remote_flow: ib_solution_path = get_latest_ibsolution_path(TARGET_IB_API_TOKEN, TARGET_FILES_API, TARGET_IB_PATH) - download_ibsolution(TARGET_IB_HOST, TARGET_IB_API_TOKEN, ib_solution_path) + download_ibsolution(TARGET_IB_HOST, TARGET_IB_API_TOKEN, ib_solution_path, write_to_local=True, unzip_solution=True) if args.set_github_actions_env_var: if args.local: diff --git a/cicd/solution_migration_udf.py b/cicd/solution_migration_udf.py index 7ece625..053f3f8 100644 --- a/cicd/solution_migration_udf.py +++ b/cicd/solution_migration_udf.py @@ -46,7 +46,7 @@ def migrate_solution(source_ib_host, target_ib_host, source_api_token, target_ap file_path = os.path.join(solution_build_dir_path, f'{package["name"]}-{package["version"]}.ibsolution') # Download ibsolution from dev environment - resp = download_ibsolution(source_ib_host, source_api_token, file_path, False) + resp = download_ibsolution(source_ib_host, source_api_token, file_path) # Upload IB Solution to Prod solution_name = f'{package["name"]}-{package["version"]}.ibsolution' diff --git a/requirements.txt b/requirements.txt index f229360..df7458c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ requests +python-dotenv diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures.py b/tests/fixtures.py new file mode 100644 index 0000000..0b19f99 --- /dev/null +++ b/tests/fixtures.py @@ -0,0 +1,26 @@ +import pytest + +_MOCK_IB_HOST_URL = "https://instbase-fake-testing-url.com" +_MOCK_API_TOKEN = "fake-testing-token" +_MOCK_AUTH_HEADERS = {"Authorization": f"Bearer {_MOCK_API_TOKEN}"} + + +@pytest.fixture +def ib_host_url(): + """Fixture for a mock testing IB host URL + + Returns: + str: Mock IB host URL + """ + return _MOCK_IB_HOST_URL + + +@pytest.fixture +def ib_api_token(): + """Fixture for a mock API token + + Returns: + str: Mock API token + """ + + return _MOCK_API_TOKEN diff --git a/cicd/test_ib_helpers.py b/tests/test_ib_helpers.py similarity index 61% rename from cicd/test_ib_helpers.py rename to tests/test_ib_helpers.py index 6548071..08ffff0 100644 --- a/cicd/test_ib_helpers.py +++ b/tests/test_ib_helpers.py @@ -1,33 +1,8 @@ """Collection of unit tests for IB Helpers""" from unittest import mock -import pytest from requests.models import Response -from .ib_helpers import upload_file - -_MOCK_IB_HOST_URL = "https://instbase-fake-testing-url.com" -_MOCK_API_TOKEN = "fake-testing-token" -_MOCK_AUTH_HEADERS = {"Authorization": f"Bearer {_MOCK_API_TOKEN}"} - - -@pytest.fixture -def ib_host_url(): - """Fixture for a mock testing IB host URL - - Returns: - str: Mock IB host URL - """ - return _MOCK_IB_HOST_URL - - -@pytest.fixture -def ib_api_token(): - """Fixture for a mock API token - - Returns: - str: Mock API token - """ - - return _MOCK_API_TOKEN +from cicd.ib_helpers import upload_file +from tests.fixtures import ib_host_url, ib_api_token, _MOCK_IB_HOST_URL, _MOCK_AUTH_HEADERS @mock.patch("cicd.ib_helpers.requests") diff --git a/tests/test_migration_helpers.py b/tests/test_migration_helpers.py new file mode 100644 index 0000000..830ee1b --- /dev/null +++ b/tests/test_migration_helpers.py @@ -0,0 +1,58 @@ +"""Collection of unit tests for IB Helpers""" +from unittest.mock import mock_open, patch, Mock +from requests.models import Response +from cicd.migration_helpers import download_ibsolution +from tests.fixtures import ib_host_url, ib_api_token, _MOCK_IB_HOST_URL, _MOCK_AUTH_HEADERS + + +@patch("cicd.ib_helpers.requests") +def test_download_ibsolution(mock_requests, ib_host_url, ib_api_token): + mocked_response = Mock(spec=Response) + mocked_response.status_code = 200 + mock_requests.get.return_value = mocked_response + solution_path = "Test Space/Test Subspace/fs/Instabase Drive/solution/dummy_solution-0.0.1.ibsolution" + + resp = download_ibsolution( + ib_host_url, ib_api_token, solution_path + ) + + mock_requests.get.assert_called_with( + f"{_MOCK_IB_HOST_URL}/api/v2/files/{solution_path}", + headers=_MOCK_AUTH_HEADERS, + verify=False, + params={'expect-node-type': 'file'} + ) + assert resp.status_code == 200 + + +@patch('cicd.migration_helpers.os.remove') +@patch("cicd.migration_helpers.ZipFile") +@patch('builtins.open', new_callable=mock_open) +@patch("cicd.ib_helpers.requests") +def test_download_ibsolution_and_unzip(mock_requests, mock_open, mock_zip, mock_remove, ib_host_url, ib_api_token): + mocked_response = Mock(spec=Response) + mocked_response.status_code = 200 + mock_requests.get.return_value = mocked_response + solution_path = "Test Space/Test Subspace/fs/Instabase Drive/solution/dummy_solution-0.0.1.ibsolution" + + resp = download_ibsolution( + ib_host_url, ib_api_token, solution_path, True, True + ) + + mock_zip.assert_called_with( + 'dummy_solution-0.0.1.zip', + 'r' + ) + + mock_remove.assert_called_with( + 'dummy_solution-0.0.1.zip' + ) + + mock_requests.get.assert_called_with( + f"{_MOCK_IB_HOST_URL}/api/v2/files/{solution_path}", + headers=_MOCK_AUTH_HEADERS, + verify=False, + params={'expect-node-type': 'file'} + ) + assert resp.status_code == 200 + diff --git a/tox.ini b/tox.ini index b7fc432..fdf13bc 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ description = install pytest in a virtual environment and invoke it on the tests deps = -rrequirements.txt -rrequirements-test.txt -commands = pytest --cov --cov-append cicd {posargs} +commands = pytest --cov --cov-append tests {posargs} [testenv:clean] deps = coverage