Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

airbyte-ci: Add upgrade cdk command #33313

Merged
merged 26 commits into from
Dec 19, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
1b984f1
source defined primary key
Dec 1, 2023
7b39d11
update docusaurus
Dec 4, 2023
806e26f
remove unrelated changes
Dec 4, 2023
6624da4
remove unrelated changes
Dec 4, 2023
ee847f0
Merge branch 'master' into flash1293/update-docusaurus
Dec 5, 2023
7d2f58d
Merge remote-tracking branch 'origin/master' into flash1293/update-do…
Dec 7, 2023
5564fc8
cleanup
Dec 7, 2023
b8aec3a
Merge branch 'master' into flash1293/update-docusaurus
Dec 11, 2023
0d42b01
Merge branch 'master' into flash1293/update-docusaurus
Dec 11, 2023
da6d1d2
Merge branch 'master' into flash1293/update-docusaurus
Dec 11, 2023
ee7ef31
add upgrade CDK command
Dec 11, 2023
46b76eb
Merge remote-tracking branch 'origin/master' into flash1293/upgrade-c…
Dec 11, 2023
2a30eb0
Merge remote-tracking branch 'origin/master' into flash1293/upgrade-c…
Dec 11, 2023
867e427
try to run only step
Dec 12, 2023
c77890e
remove
Dec 12, 2023
64324a1
Merge remote-tracking branch 'origin/master' into flash1293/upgrade-c…
Dec 14, 2023
84c0662
review comments and tests
Dec 14, 2023
d499e9f
format
Dec 14, 2023
9528948
update readme
Dec 14, 2023
f75f2a1
Merge branch 'master' into flash1293/upgrade-cdk-command
Dec 15, 2023
0ad032f
Merge remote-tracking branch 'origin/master' into flash1293/upgrade-c…
Dec 18, 2023
006a010
review comments
Dec 18, 2023
6341690
Merge remote-tracking branch 'origin/master' into flash1293/upgrade-c…
Dec 18, 2023
cb7adf3
review comments
Dec 18, 2023
f3a3d89
Merge remote-tracking branch 'origin/master' into flash1293/upgrade-c…
Dec 19, 2023
6dba66f
bump airbyte-ci version
Dec 19, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
review comments and tests
  • Loading branch information
Joe Reuter committed Dec 14, 2023
commit 84c066224bb1dfddb76cdf622fbd1385f90e5b2b
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@


@click.command(cls=DaggerPipelineCommand, short_help="Upgrade CDK version")
@click.argument("target-version", type=str)
@click.argument("target-cdk-version", type=str)
@click.pass_context
async def bump_version(
ctx: click.Context,
target_version: str,
target_cdk_version: str,
) -> bool:
"""Upgrade CDK version"""

Expand Down Expand Up @@ -49,7 +49,7 @@ async def bump_version(
ctx.obj["concurrency"],
ctx.obj["dagger_logs_path"],
ctx.obj["execute_timeout"],
target_version,
target_cdk_version,
)

return True
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
#

import re
import os

from dagger import Container
from pipelines.airbyte_ci.connectors.context import ConnectorContext
from pipelines.airbyte_ci.connectors.reports import ConnectorReport
from pipelines.helpers import git
Expand All @@ -17,39 +17,41 @@ class SetCDKVersion(Step):
def __init__(
self,
context: ConnectorContext,
repo_dir: Container,
new_version: str,
):
super().__init__(context)
self.repo_dir = repo_dir
self.new_version = new_version

async def _run(self) -> StepResult:
context: ConnectorContext = self.context
setup_py_path = context.connector.code_directory / "setup.py"
if not setup_py_path.exists():
connector_dir = await context.get_connector_dir(include=["setup.py"])
if not (await connector_dir.entries()):
return StepResult(
self,
StepStatus.SKIPPED,
StepStatus.FAILURE,
stdout="Connector does not have a setup.py file.",
output_artifact=self.repo_dir,
)
setup_py = connector_dir.file("setup.py")
setup_py_content = await setup_py.contents()
try:
updated_setup_py = self.update_cdk_version(setup_py_path.read_text())
updated_setup_py = self.update_cdk_version(setup_py_content)
full_og_connector_dir = await context.get_connector_dir()
updated_connector_dir = full_og_connector_dir.with_new_file("setup.py", updated_setup_py)
diff = full_og_connector_dir.diff(updated_connector_dir)
await diff.export(os.path.join(git.get_git_repo_path(), context.connector.code_directory))
return StepResult(
self,
StepStatus.SUCCESS,
stdout=f"Updated CDK version to {self.new_version}",
output_artifact=diff
)
except Exception as e:
return StepResult(
self,
StepStatus.FAILURE,
stdout=f"Could not add changelog entry: {e}",
output_artifact=self.container_with_airbyte_repo,
stderr=f"Could not set CDK version: {e}",
exc_info=e,
)
updated_repo_dir = self.repo_dir.with_new_file(str(setup_py_path), updated_setup_py)
return StepResult(
self,
StepStatus.SUCCESS,
stdout=f"Updated CDK version to {self.new_version}",
output_artifact=updated_repo_dir,
)

def update_cdk_version(self, og_setup_py_content: str) -> str:
airbyte_cdk_dependency = re.search(
Expand All @@ -73,21 +75,16 @@ async def run_connector_cdk_upgrade_pipeline(
context (ConnectorContext): The initialized connector context.

Returns:
ConnectorReport: The reports holding the base image version upgrade results.
ConnectorReport: The reports holding the CDK version set results.
"""
async with semaphore:
steps_results = []
async with context:
og_repo_dir = await context.get_repo_dir()

set_cdk_version = SetCDKVersion(
context,
og_repo_dir,
target_version,
)
set_cdk_version_result = await set_cdk_version.run()
steps_results.append(set_cdk_version_result)
final_repo_dir = set_cdk_version_result.output_artifact
await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path()))
context.report = ConnectorReport(context, steps_results, name="CONNECTOR VERSION CDK UPGRADE RESULTS")
return context.report
88 changes: 75 additions & 13 deletions airbyte-ci/connectors/pipelines/tests/test_upgrade_cdk.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@
import json
import random
from typing import List
from unittest.mock import MagicMock, AsyncMock

import anyio
import pytest
from dagger import Directory
from connector_ops.utils import Connector, ConnectorLanguage
from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline
from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline
Expand All @@ -19,14 +21,31 @@
]

@pytest.fixture
def connector_with_poetry():
return Connector("destination-duckdb")
def sample_connector():
return Connector("source-pokeapi")

def get_sample_setup_py(airbyte_cdk_dependency: str):
return f"""from setuptools import find_packages, setup

MAIN_REQUIREMENTS = [
"{airbyte_cdk_dependency}",
]

setup(
name="source_pokeapi",
description="Source implementation for Pokeapi.",
author="Airbyte",
author_email="contact@airbyte.io",
packages=find_packages(),
install_requires=MAIN_REQUIREMENTS,
)
"""

@pytest.fixture
def context_for_connector_with_poetry(connector_with_poetry, dagger_client, current_platform):
def connector_context(sample_connector, dagger_client, current_platform):
context = ConnectorContext(
pipeline_name="test unit tests",
connector=connector_with_poetry,
pipeline_name="test",
connector=sample_connector,
git_branch="test",
git_revision="test",
report_output_prefix="test",
Expand All @@ -35,15 +54,58 @@ def context_for_connector_with_poetry(connector_with_poetry, dagger_client, curr
targeted_platforms=[current_platform],
)
context.dagger_client = dagger_client
context.connector_secrets = {}
return context

async def test_run_connector_cdk_upgrade_pipeline(context_for_connector_with_poetry):
# og_repo_dir = await context.get_repo_dir()
# report = await upgrade_cdk_pipeline.run_connector_cdk_upgrade_pipeline(context_for_connector_with_poetry, semaphore, "6.6.6")
og_repo_dir = await context_for_connector_with_poetry.get_repo_dir()
step = upgrade_cdk_pipeline.SetCDKVersion(context_for_connector_with_poetry, og_repo_dir, "6.6.6")
@pytest.mark.parametrize(
"setup_py_content, expected_setup_py_content",
[
(get_sample_setup_py("airbyte-cdk"), get_sample_setup_py("airbyte-cdk>=6.6.6")),
(get_sample_setup_py("airbyte-cdk[file-based]"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")),
(get_sample_setup_py("airbyte-cdk==1.2.3"), get_sample_setup_py("airbyte-cdk>=6.6.6")),
(get_sample_setup_py("airbyte-cdk>=1.2.3"), get_sample_setup_py("airbyte-cdk>=6.6.6")),
(get_sample_setup_py("airbyte-cdk[file-based]>=1.2.3"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")),
],
)
async def test_run_connector_cdk_upgrade_pipeline(connector_context: ConnectorContext, setup_py_content: str, expected_setup_py_content: str):
full_og_connector_dir = await connector_context.get_connector_dir()
updated_connector_dir = full_og_connector_dir.with_new_file("setup.py", setup_py_content)

# For this test, replace the actual connector dir with an updated version that sets the setup.py contents
connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir)

# Mock the diff method to record the resulting directory and return a mock to not actually export the diff to the repo
updated_connector_dir.diff = MagicMock(return_value=AsyncMock())
step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6")
step_result = await step.run()
assert step_result.status == StepStatus.SUCCESS

# Check that the resulting directory that got passed to the mocked diff method looks as expected
resulting_directory: Directory = await full_og_connector_dir.diff(updated_connector_dir.diff.call_args[0][0])
files = await resulting_directory.entries()
assert files == ["setup.py"]
setup_py = resulting_directory.file("setup.py")
actual_setup_py_content = await setup_py.contents()
assert expected_setup_py_content == actual_setup_py_content

# Assert that the diff was exported to the repo
assert updated_connector_dir.diff.return_value.export.call_count == 1

# print(report.steps_results[-1])
print(step_result)
async def test_fail_connector_cdk_upgrade_pipeline_on_missing_setup_py(connector_context: ConnectorContext):
full_og_connector_dir = await connector_context.get_connector_dir()
updated_connector_dir = full_og_connector_dir.without_file("setup.py")

connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir)

step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6")
step_result = await step.run()
assert step_result.status == StepStatus.FAILURE

async def test_fail_connector_cdk_upgrade_pipeline_on_missing_airbyte_cdk(connector_context: ConnectorContext):
full_og_connector_dir = await connector_context.get_connector_dir()
updated_connector_dir = full_og_connector_dir.with_new_file("setup.py", get_sample_setup_py("another-lib==1.2.3"))

connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir)

step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6")
step_result = await step.run()
assert step_result.status == StepStatus.FAILURE