[airbyte-ci] Improved PR generation flow and a bugfix (#38582)

Co-authored-by: Brian Leonard <brian.leonard@airbyte.io>

airbyte-ci/connectors/pipelines/README.md

@@ -488,6 +488,7 @@ Get source-openweather up to date. If there are changes, bump the version and ad
 - `airbyte-ci connectors --name=source-openweather up_to_date --dev`: forces update if there are only dev changes
 - `airbyte-ci connectors --name=source-openweather up_to_date --dep pytest@^8.10 --dep airbyte-cdk@0.80.0`: allows update to toml files as well
 - `airbyte-ci connectors --name=source-openweather up_to_date --pull`: make a pull request for it
+- `airbyte-ci connectors --name=source-openweather up_to_date --no-bump`: don't change the version or changelog
 
 ### Other things it could do
 
@@ -538,6 +539,7 @@ Modify the selected connector metadata to use the latest base image version.
 
 Upgrade the base image for source-openweather:
 `airbyte-ci connectors --name=source-openweather upgrade_base_image`
+`airbyte-ci connectors --name=source-openweather upgrade_base_image --changelog --bump patch --pull-request-number 123`
 
 ### Options
 
@@ -569,6 +571,7 @@ Migrate connectors the poetry package manager.
 
 Migrate source-openweather to use the base image:
 `airbyte-ci connectors --name=source-openweather migrate-to-poetry`
+`airbyte-ci connectors --name=source-openweather migrate-to-poetry --changelog --bump patch`
 
 ### <a id="connectors-migrate_to_inline_schemas"></a>`connectors migrate_to_inline_schemas` command
 
@@ -616,7 +619,7 @@ Options:
 #### Examples
 
 Make a PR for all changes, bump the version and make a changelog in those PRs. They will be on the branch ci_update/round2/<connector-name>:
-`airbyte-ci connectors pull_request -m "upgrading connectors" -b ci_update/round2 --bump patch --changelog`
+`airbyte-ci connectors --modified pull_request -m "upgrading connectors" -b ci_update/round2 --bump patch --changelog`
 
 Do it just for a few connectors:
 `airbyte-ci connectors --name source-aha --name source-quickbooks pull_request -m "upgrading connectors" -b ci_update/round2 --bump patch --changelog`
@@ -744,7 +747,8 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 ## Changelog
 
 | Version | PR                                                         | Description                                                                                                                  |
-|---------|------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|
+| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.14.1  | [#38582](https://github.com/airbytehq/airbyte/pull/38582)  | Fixed bugs in `up_to_date` flags, `pull_request` version change logic.                                                       |
 | 4.14.0  | [#38281](https://github.com/airbytehq/airbyte/pull/38281)  | Conditionally run test suites according to `connectorTestSuitesOptions` in metadata files.                                   |
 | 4.13.3  | [#38221](https://github.com/airbytehq/airbyte/pull/38221)  | Add dagster cloud dev deployment pipeline opitions                                                                           |
 | 4.13.2  | [#38246](https://github.com/airbytehq/airbyte/pull/38246)  | Remove invalid connector test step options.                                                                                  |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py

@@ -45,12 +45,23 @@ def __init__(self, context: ConnectorContext) -> None:
         connector = context.connector
         if connector.metadata_file_path.is_file():
             self.metadata_content = connector.metadata_file_path.read_text()
+        else:
+            self.metadata_content = None
+
         if connector.dockerfile_file_path.is_file():
             self.dockerfile_content = connector.dockerfile_file_path.read_text()
+        else:
+            self.dockerfile_content = None
+
         if connector.pyproject_file_path.is_file():
             self.poetry_content = connector.pyproject_file_path.read_text()
+        else:
+            self.poetry_content = None
+
         if connector.documentation_file_path and connector.documentation_file_path.is_file():
             self.documentation_content = connector.documentation_file_path.read_text()
+        else:
+            self.documentation_content = None
 
     async def _run(self) -> StepResult:
         connector = self.context.connector
@@ -142,14 +153,16 @@ def __init__(
 
     async def get_repo_dir(self) -> Directory:
         if not self.repo_dir:
-            self.repo_dir = await self.context.get_connector_dir()
+            self.repo_dir = await self.context.get_repo_dir()
         return self.repo_dir
 
     async def _run(self) -> StepResult:
         result = await self.update_metadata()
         if result.status is not StepStatus.SUCCESS:
             return result
 
+        # Update the version of the connector in the Dockerfile.
+        # TODO: This can be removed once we ditch all Dockerfiles from connectors.
         if self.context.connector.dockerfile_file_path.is_file():
             result = await self.update_dockerfile()
             if result.status is not StepStatus.SUCCESS:
@@ -241,7 +254,7 @@ async def update_package_version(self) -> StepResult:
             )
 
         content = await dagger_read_file(repo_dir, file_path)
-        new_content = re.sub(r"(?<=\bversion = \")(.*)(?=\")", self.new_version, content)
+        new_content = re.sub(r"(?<=^version = \")(.*)(?=\")", self.new_version, content)
         self.repo_dir = await dagger_write_file(repo_dir, file_path, new_content)
 
         if self.export:

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py

@@ -16,10 +16,27 @@
     short_help="Make the selected connectors use our base image: remove dockerfile, update metadata.yaml and update documentation.",
 )
 @click.option("--pull-request-number", type=str, required=False, default=None)
+@click.option(
+    "--changelog",
+    help="Add message to the changelog.",
+    type=bool,
+    is_flag=True,
+    required=False,
+    default=False,
+)
+@click.option(
+    "--bump",
+    help="Bump the metadata.yaml version. Can be `major`, `minor`, or `patch`.",
+    type=click.Choice(["patch", "minor", "major"]),
+    required=False,
+    default=None,
+)
 @click.pass_context
 async def migrate_to_base_image(
     ctx: click.Context,
     pull_request_number: str | None,
+    changelog: bool,
+    bump: str | None,
 ) -> bool:
     """
     Bump a connector version: update metadata.yaml, changelog and delete legacy files.
@@ -64,6 +81,8 @@ async def migrate_to_base_image(
         ctx.obj["dagger_logs_path"],
         ctx.obj["execute_timeout"],
         pull_request_number,
+        changelog,
+        bump,
     )
 
     return True

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py

@@ -15,6 +15,7 @@
 from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext
 from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report
 from pipelines.helpers import git
+from pipelines.helpers.connectors.yaml import read_dagger_yaml, write_dagger_yaml
 from pipelines.models.steps import Step, StepResult, StepStatus
 
 if TYPE_CHECKING:
@@ -68,7 +69,7 @@ async def _run(self) -> StepResult:
             )
 
         metadata_path = self.context.connector.metadata_file_path
-        current_metadata = yaml.safe_load(await self.repo_dir.file(str(metadata_path)).contents())
+        current_metadata = await read_dagger_yaml(self.repo_dir, metadata_path)
         current_base_image_address = current_metadata.get("data", {}).get("connectorBuildOptions", {}).get("baseImage")
 
         if current_base_image_address is None and not self.set_if_not_exists:
@@ -87,7 +88,7 @@ async def _run(self) -> StepResult:
                 output=self.repo_dir,
             )
         updated_metadata = self.update_base_image_in_metadata(current_metadata, latest_base_image_address)
-        updated_repo_dir = self.repo_dir.with_new_file(str(metadata_path), contents=yaml.safe_dump(updated_metadata))
+        updated_repo_dir = write_dagger_yaml(self.repo_dir, updated_metadata, metadata_path)
 
         return StepResult(
             step=self,
@@ -284,7 +285,7 @@ async def run_connector_base_image_upgrade_pipeline(context: ConnectorContext, s
 
 
 async def run_connector_migration_to_base_image_pipeline(
-    context: ConnectorContext, semaphore: "Semaphore", pull_request_number: str | None
+    context: ConnectorContext, semaphore: "Semaphore", pull_request_number: str | None, changelog: bool, bump: str | None
 ) -> Report:
     async with semaphore:
         steps_results = []
@@ -324,14 +325,19 @@ async def run_connector_migration_to_base_image_pipeline(
 
             latest_repo_dir_state = update_base_image_in_metadata_result.output
             # BUMP CONNECTOR VERSION IN METADATA
-            new_version = get_bumped_version(context.connector.version, "patch")
-            bump_version_in_metadata = SetConnectorVersion(context, new_version, latest_repo_dir_state, False)
-            bump_version_in_metadata_result = await bump_version_in_metadata.run()
-            steps_results.append(bump_version_in_metadata_result)
+            if bump:
+                new_version = get_bumped_version(context.connector.version, bump)
+            else:
+                new_version = None
+
+            if new_version:
+                bump_version_in_metadata = SetConnectorVersion(context, new_version, latest_repo_dir_state, False)
+                bump_version_in_metadata_result = await bump_version_in_metadata.run()
+                steps_results.append(bump_version_in_metadata_result)
+                latest_repo_dir_state = bump_version_in_metadata_result.output
 
-            latest_repo_dir_state = bump_version_in_metadata_result.output
             # ADD CHANGELOG ENTRY only if the PR number is provided.
-            if pull_request_number is not None:
+            if new_version and changelog and pull_request_number is not None:
                 add_changelog_entry = AddChangelogEntry(
                     context,
                     new_version,

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_poetry/commands.py

@@ -10,14 +10,27 @@
 from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
 
 
+@click.option(
+    "--changelog",
+    help="Add message to the changelog.",
+    type=bool,
+    is_flag=True,
+    required=False,
+    default=False,
+)
+@click.option(
+    "--bump",
+    help="Bump the metadata.yaml version. Can be `major`, `minor`, or `patch`.",
+    type=click.Choice(["patch", "minor", "major"]),
+    required=False,
+    default=None,
+)
 @click.command(
     cls=DaggerPipelineCommand,
     short_help="Migrate the selected connectors to poetry.",
 )
 @click.pass_context
-async def migrate_to_poetry(
-    ctx: click.Context,
-) -> bool:
+async def migrate_to_poetry(ctx: click.Context, changelog: bool, bump: str | None) -> bool:
 
     connectors_contexts = [
         ConnectorContext(
@@ -38,7 +51,7 @@ async def migrate_to_poetry(
             ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
             ci_git_user=ctx.obj["ci_git_user"],
             ci_github_access_token=ctx.obj["ci_github_access_token"],
-            enable_report_auto_open=True,
+            enable_report_auto_open=False,
             docker_hub_username=ctx.obj.get("docker_hub_username"),
             docker_hub_password=ctx.obj.get("docker_hub_password"),
             s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),
@@ -54,6 +67,8 @@ async def migrate_to_poetry(
         ctx.obj["concurrency"],
         ctx.obj["dagger_logs_path"],
         ctx.obj["execute_timeout"],
+        changelog,
+        bump,
     )
 
     return True