From 0575c5331a1ef7e25bb466ac1580a9d77b8e6ca2 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Fri, 13 Oct 2023 16:53:59 -0700 Subject: [PATCH 1/5] Allow airbyte-ci to run from anywhere --- airbyte-ci/connectors/pipelines/README.md | 1 + .../pipelines/commands/airbyte_ci.py | 28 ++++++++++++++++--- .../pipelines/pipelines/dagger_run.py | 3 ++ .../connectors/pipelines/pyproject.toml | 2 +- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 74217e48a040..ecac0517be61 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -381,6 +381,7 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 1.9.0 | [#31369](https://github.com/airbytehq/airbyte/pull/31369) | Run airbyte-ci from any where in airbyte project | | 1.8.0 | [#31369](https://github.com/airbytehq/airbyte/pull/31369) | Remove metadata test commands | | 1.7.2 | [#31343](https://github.com/airbytehq/airbyte/pull/31343) | Bind Pytest integration tests to a dockerhost. | | 1.7.1 | [#31332](https://github.com/airbytehq/airbyte/pull/31332) | Disable Gradle step caching on source-postgres. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py index b14a4d1a6bad..f5161565c2e1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py @@ -5,12 +5,14 @@ """This module is the CLI entrypoint to the airbyte-ci commands.""" import importlib -import os -import subprocess from typing import List - +import git import click +import os +import logging + from github import PullRequest +from pathlib import Path from pipelines import github, main_logger from pipelines.bases import CIContext from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH @@ -63,6 +65,24 @@ def get_latest_version() -> str: return line.split("=")[1].strip().replace('"', "") raise Exception("Could not find version in pyproject.toml. Please ensure you are running from the root of the airbyte repo.") +def get_airbyte_repo() -> git.Repo: + """Get the airbyte repo.""" + return git.Repo(search_parent_directories=True) + + +def get_airbyte_repo_path_with_fallback() -> Path: + """Get the airbyte repo path.""" + try: + return get_airbyte_repo().working_tree_dir + except git.exc.InvalidGitRepositoryError: + logging.warning("Could not find the airbyte repo, falling back to the current working directory.") + path = Path.cwd() + logging.warning(f"Using {path} as the airbyte repo path.") + return path + +def set_working_directory_to_root() -> None: + """Set the working directory to the root of the airbyte repo.""" + os.chdir(get_airbyte_repo_path_with_fallback()) def get_modified_files( git_branch: str, git_revision: str, diffed_branch: str, is_local: bool, ci_context: CIContext, pull_request: PullRequest @@ -90,7 +110,6 @@ def get_modified_files( # COMMANDS - @click.group(help="Airbyte CI top-level command group.") @click.version_option(__installed_version__) @click.option("--is-local/--is-ci", default=True) @@ -180,6 +199,7 @@ def airbyte_ci( airbyte_ci.add_command(connectors) airbyte_ci.add_command(metadata) airbyte_ci.add_command(test) +set_working_directory_to_root() if __name__ == "__main__": airbyte_ci() diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py index df84c79b35be..9ebf2424833a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py @@ -15,6 +15,8 @@ import pkg_resources import requests +from pipelines.commands.airbyte_ci import set_working_directory_to_root + LOGGER = logging.getLogger(__name__) BIN_DIR = Path.home() / "bin" BIN_DIR.mkdir(exist_ok=True) @@ -88,6 +90,7 @@ def check_dagger_cli_install() -> str: def main(): + set_working_directory_to_root() os.environ[DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[0]] = DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[1] exit_code = 0 if len(sys.argv) > 1 and any([arg in ARGS_DISABLING_TUI for arg in sys.argv]): diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 39144ba2ed02..1fdfda0c9189 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "1.8.0" +version = "1.9.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From 28424b7563f8df0aa265b801518255ec00387e37 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Fri, 13 Oct 2023 16:58:23 -0700 Subject: [PATCH 2/5] Update changelog --- airbyte-ci/connectors/pipelines/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index ecac0517be61..3ae6565d0b8f 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -381,7 +381,7 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| 1.9.0 | [#31369](https://github.com/airbytehq/airbyte/pull/31369) | Run airbyte-ci from any where in airbyte project | +| 1.9.0 | [#31412](https://github.com/airbytehq/airbyte/pull/31412) | Run airbyte-ci from any where in airbyte project | | 1.8.0 | [#31369](https://github.com/airbytehq/airbyte/pull/31369) | Remove metadata test commands | | 1.7.2 | [#31343](https://github.com/airbytehq/airbyte/pull/31343) | Bind Pytest integration tests to a dockerhost. | | 1.7.1 | [#31332](https://github.com/airbytehq/airbyte/pull/31332) | Disable Gradle step caching on source-postgres. | From 7dc9d326a96ac36b8136c0676696c8d03c9abfa6 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Mon, 16 Oct 2023 16:30:18 -0700 Subject: [PATCH 3/5] Code review comments --- .../pipelines/commands/airbyte_ci.py | 46 ++++++++++++++++--- .../pipelines/pipelines/dagger_run.py | 1 - 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py index f5161565c2e1..00594f9761d9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py @@ -5,14 +5,14 @@ """This module is the CLI entrypoint to the airbyte-ci commands.""" import importlib -from typing import List -import git -import click -import os import logging +import os +from pathlib import Path +from typing import List +import click +import git from github import PullRequest -from pathlib import Path from pipelines import github, main_logger from pipelines.bases import CIContext from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH @@ -65,9 +65,36 @@ def get_latest_version() -> str: return line.split("=")[1].strip().replace('"', "") raise Exception("Could not find version in pyproject.toml. Please ensure you are running from the root of the airbyte repo.") + +def _validate_airbyte_repo(repo: git.Repo) -> bool: + """Check if any of the remotes are the airbyte repo.""" + expected_repo_name = "airbytehq/airbyte" + for remote in repo.remotes: + if expected_repo_name in remote.url: + return True + + warning_message = f""" + ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ + + It looks like you are not running this command from the airbyte repo ({expected_repo_name}). + + If this command is run from outside the airbyte repo, it will not work properly. + + Please run this command your local airbyte project. + + ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ + """ + + logging.warning(warning_message) + + return False + + def get_airbyte_repo() -> git.Repo: """Get the airbyte repo.""" - return git.Repo(search_parent_directories=True) + repo = git.Repo(search_parent_directories=True) + _validate_airbyte_repo(repo) + return repo def get_airbyte_repo_path_with_fallback() -> Path: @@ -80,9 +107,13 @@ def get_airbyte_repo_path_with_fallback() -> Path: logging.warning(f"Using {path} as the airbyte repo path.") return path + def set_working_directory_to_root() -> None: """Set the working directory to the root of the airbyte repo.""" - os.chdir(get_airbyte_repo_path_with_fallback()) + working_dir = get_airbyte_repo_path_with_fallback() + logging.info(f"Setting working directory to {working_dir}") + os.chdir(working_dir) + def get_modified_files( git_branch: str, git_revision: str, diffed_branch: str, is_local: bool, ci_context: CIContext, pull_request: PullRequest @@ -110,6 +141,7 @@ def get_modified_files( # COMMANDS + @click.group(help="Airbyte CI top-level command group.") @click.version_option(__installed_version__) @click.option("--is-local/--is-ci", default=True) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py index a88e7382b207..f26cb4976ad8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py @@ -14,7 +14,6 @@ import pkg_resources import requests - from pipelines.commands.airbyte_ci import set_working_directory_to_root LOGGER = logging.getLogger(__name__) From 50f85f31bd345c212999edf0ce384563b8b74e3e Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 18 Oct 2023 17:44:24 -0700 Subject: [PATCH 4/5] Fix import --- airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py index f26cb4976ad8..7c3ee1c12ff2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py @@ -14,7 +14,7 @@ import pkg_resources import requests -from pipelines.commands.airbyte_ci import set_working_directory_to_root +from pipelines.cli.airbyte_ci import set_working_directory_to_root LOGGER = logging.getLogger(__name__) BIN_DIR = Path.home() / "bin" From 619eebebb069c45ec3d00e52c2dc9ca5328b3f16 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 18 Oct 2023 18:03:41 -0700 Subject: [PATCH 5/5] Update docstring --- airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 69e247f0b72e..90f59e503020 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -97,7 +97,7 @@ def get_airbyte_repo() -> git.Repo: def get_airbyte_repo_path_with_fallback() -> Path: - """Get the airbyte repo path.""" + """Get the path to the airbyte repo.""" try: return get_airbyte_repo().working_tree_dir except git.exc.InvalidGitRepositoryError: