From e7fe22bd7d77d17b27ce5de11f72e830dbac569e Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 11:51:52 +0100 Subject: [PATCH 01/18] add auto-bump logic to CDK command --- .../publish-cdk-command-manually.yml | 594 ++++++++++-------- tools/bin/bump_cdk.py | 115 ++++ 2 files changed, 434 insertions(+), 275 deletions(-) create mode 100644 tools/bin/bump_cdk.py diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index a5b18a4327ec5..b08e9778e1e4e 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -22,6 +22,9 @@ on: changelog-message: description: "Changelog message to be added to CHANGELOG.md" required: false + connector-selector: + description: "Connectors to bump (comma separated list of connector names or colon separated key value pairs of metadata fields)" + required: false concurrency: group: publish-airbyte-cdk @@ -57,54 +60,54 @@ jobs: github-token: ${{ env.PAT }} label: ${{ github.run_id }}-publisher - build-cdk: - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Build CDK Package - run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build - - name: Post failure to Slack channel dev-connectors-extensibility - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": "Error during `build-cdk` while publishing Python CDK!", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Error while publishing Python CDK!" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # build-cdk: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/setup-python@v4 + # with: + # python-version: "3.10" + # - uses: actions/setup-java@v3 + # with: + # distribution: "zulu" + # java-version: "17" + # - name: Checkout Airbyte + # uses: actions/checkout@v3 + # with: + # repository: ${{ github.event.inputs.repo }} + # ref: ${{ github.event.inputs.gitref }} + # - name: Build CDK Package + # run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build + # - name: Post failure to Slack channel dev-connectors-extensibility + # if: ${{ failure() }} + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": "Error during `build-cdk` while publishing Python CDK!", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "Error while publishing Python CDK!" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} bump-version: - needs: build-cdk + # needs: build-cdk if: github.event.inputs.release-type != 'none' runs-on: ubuntu-latest outputs: @@ -128,178 +131,272 @@ jobs: new_version="$(grep -i 'current_version = ' .bumpversion.cfg | sed -e 's/.* = //')" awk -v NEW_VERSION="$new_version" -v CHANGELOG_MESSAGE="${{ github.event.inputs.changelog-message }}" 'NR==3{print "## " NEW_VERSION "\n" CHANGELOG_MESSAGE "\n"}1' CHANGELOG.md > tmp && mv tmp CHANGELOG.md echo NEW_VERSION=$new_version >> $GITHUB_OUTPUT - - name: Commit and Push Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - file_pattern: airbyte-cdk/python/setup.py airbyte-cdk/python/.bumpversion.cfg airbyte-cdk/python/CHANGELOG.md airbyte-cdk/python/Dockerfile - commit_message: 🤖 Bump ${{ github.event.inputs.release-type }} version of Python CDK - commit_user_name: Octavia Squidington III - commit_user_email: octavia-squidington-iii@users.noreply.github.com - - name: Post failure to Slack channel dev-connectors-extensibility - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": "Error during `bump-version` while publishing Python CDK!", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Error while publishing Python CDK!" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # - name: Commit and Push Changes + # uses: stefanzweifel/git-auto-commit-action@v4 + # with: + # file_pattern: airbyte-cdk/python/setup.py airbyte-cdk/python/.bumpversion.cfg airbyte-cdk/python/CHANGELOG.md airbyte-cdk/python/Dockerfile + # commit_message: 🤖 Bump ${{ github.event.inputs.release-type }} version of Python CDK + # commit_user_name: Octavia Squidington III + # commit_user_email: octavia-squidington-iii@users.noreply.github.com + # - name: Post failure to Slack channel dev-connectors-extensibility + # if: ${{ failure() }} + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": "Error during `bump-version` while publishing Python CDK!", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "Error while publishing Python CDK!" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - publish-cdk: - needs: bump-version - runs-on: ubuntu-latest - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Publish Python Package - uses: mariamrf/py-package-publish-action@v1.1.0 - with: - # specify the same version as in ~/.python-version - python_version: "3.10" - pip_version: "23.2" - subdir: "airbyte-cdk/python/" - env: - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - - name: Post failure to Slack channel dev-connectors-extensibility - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": "Error during `publish-cdk` while publishing Python CDK!", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Error while publishing Python CDK!" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # publish-cdk: + # needs: bump-version + # runs-on: ubuntu-latest + # steps: + # - name: Checkout Airbyte + # uses: actions/checkout@v3 + # with: + # repository: ${{ github.event.inputs.repo }} + # ref: ${{ github.event.inputs.gitref }} + # - name: Publish Python Package + # uses: mariamrf/py-package-publish-action@v1.1.0 + # with: + # # specify the same version as in ~/.python-version + # python_version: "3.10" + # pip_version: "23.2" + # subdir: "airbyte-cdk/python/" + # env: + # TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + # TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + # - name: Post failure to Slack channel dev-connectors-extensibility + # if: ${{ failure() }} + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": "Error during `publish-cdk` while publishing Python CDK!", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "Error while publishing Python CDK!" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - publish-docker-image: - timeout-minutes: 240 - needs: - - start-publish-docker-image-runner-0 - - publish-cdk - runs-on: runner-pool-${{ github.run_id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: airbytehq/airbyte - ref: master - - name: Install Java - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Publish source-declarative-manifest - id: publish - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - TZ: UTC - uses: Wandalen/wretry.action@v1.0.42 - with: - command: | - docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-cdk/python false - attempt_limit: 3 - attempt_delay: 5000 in # ms - - name: Post failure to Slack channel dev-connectors-extensibility - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": "Error during `publish-docker-image` while publishing Python CDK!", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Error while publishing Docker image following Python CDK release!" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # publish-docker-image: + # timeout-minutes: 240 + # needs: + # - start-publish-docker-image-runner-0 + # - publish-cdk + # runs-on: runner-pool-${{ github.run_id }} + # steps: + # - name: Checkout Airbyte + # uses: actions/checkout@v3 + # with: + # repository: airbytehq/airbyte + # ref: master + # - name: Install Java + # uses: actions/setup-java@v3 + # with: + # distribution: "zulu" + # java-version: "17" + # - name: Install Python + # uses: actions/setup-python@v4 + # with: + # python-version: "3.10" + # - name: Publish source-declarative-manifest + # id: publish + # env: + # DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + # DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + # # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + # TZ: UTC + # uses: Wandalen/wretry.action@v1.0.42 + # with: + # command: | + # docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} + # ./tools/integrations/manage.sh publish airbyte-cdk/python false + # attempt_limit: 3 + # attempt_delay: 5000 in # ms + # - name: Post failure to Slack channel dev-connectors-extensibility + # if: ${{ failure() }} + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": "Error during `publish-docker-image` while publishing Python CDK!", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "Error while publishing Docker image following Python CDK release!" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - update-connector-builder: + # update-connector-builder: + # needs: + # - bump-version + # - publish-docker-image + # runs-on: ubuntu-latest + # steps: + # - uses: actions/setup-python@v4 + # with: + # python-version: "3.9" + # - name: Checkout Airbyte Platform Internal + # uses: actions/checkout@v3 + # with: + # repository: airbytehq/airbyte-platform-internal + # token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + # - name: Update CDK version + # run: | + # PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) + # sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/Dockerfile + # sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" airbyte-connector-builder-server-wrapped/Dockerfile + # sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/requirements.in + # echo ${{needs.bump-version.outputs.new_cdk_version}} > oss/airbyte-connector-builder-resources/CDK_VERSION + # cd oss/airbyte-connector-builder-server + # pip install pip-tools + # pip-compile --upgrade + # - name: Create Pull Request + # id: create-pull-request + # uses: peter-evans/create-pull-request@v4 + # with: + # token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + # commit-message: Updating CDK version following release + # title: Updating CDK version following release + # body: This is an automatically generated PR triggered by a CDK release + # branch: automatic-cdk-release + # base: master + # delete-branch: true + # - name: Post success to Slack channel dev-connectors-extensibility + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": "A new version of Python CDK has been released!", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # - name: Post failure to Slack channel dev-connectors-extensibility + # if: ${{ failure() }} + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": ":warning: Could not automatically create a PR for Connector Builder>\n" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + + bump-connectors: needs: - bump-version - - publish-docker-image runs-on: ubuntu-latest steps: - uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.10" - name: Checkout Airbyte Platform Internal uses: actions/checkout@v3 with: - repository: airbytehq/airbyte-platform-internal + repository: airbytehq/airbyte token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - - name: Update CDK version - run: | - PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) - sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/Dockerfile - sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" airbyte-connector-builder-server-wrapped/Dockerfile - sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/requirements.in - echo ${{needs.bump-version.outputs.new_cdk_version}} > oss/airbyte-connector-builder-resources/CDK_VERSION - cd oss/airbyte-connector-builder-server - pip install pip-tools - pip-compile --upgrade - name: Create Pull Request id: create-pull-request uses: peter-evans/create-pull-request@v4 @@ -308,78 +405,25 @@ jobs: commit-message: Updating CDK version following release title: Updating CDK version following release body: This is an automatically generated PR triggered by a CDK release - branch: automatic-cdk-release - base: master + branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} + base: flash1293/auto-bump delete-branch: true - - name: Post success to Slack channel dev-connectors-extensibility - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": "A new version of Python CDK has been released!", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - - name: Post failure to Slack channel dev-connectors-extensibility - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true + - name: Update CDK version + run: | + pipx install --editable --force airbyte-ci/connectors/pipelines/ + python ./tools/bin/bump_cdk.py --pr ${{ steps.cpr.outputs.pull-request-number }} --selector=${{github.event.inputs.connector-selector}} --changelog="${{github.event.inputs.changelog-message}}"" --version="${{needs.bump-version.outputs.new_cdk_version}}" + # Run create-pull-request again to update the PR with the changes (need to do like this because we need the PR number to create this) + - name: Update Pull Request + id: create-pull-request + uses: peter-evans/create-pull-request@v4 with: - channel-id: C04J1M66D8B - payload: | - { - "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":warning: Could not automatically create a PR for Connector Builder>\n" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + commit-message: Updating CDK version following release + title: Updating CDK version following release + body: This is an automatically generated PR triggered by a CDK release + branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} + base: flash1293/auto-bump + delete-branch: true # In case of self-hosted EC2 errors, remove this block. stop-publish-docker-image-runner-0: diff --git a/tools/bin/bump_cdk.py b/tools/bin/bump_cdk.py new file mode 100644 index 0000000000000..997446ac0f4dd --- /dev/null +++ b/tools/bin/bump_cdk.py @@ -0,0 +1,115 @@ +# This script takes an arg with a connector selector, then bumps the CDK version of all connectors that match that selector +# Example: python tools/bin/bump_cdk.py --selector=source-s3 <- this targets the s3 source connector +# Example: python tools/bin/bump_cdk.py --selector=connectorSubtype:vectorstore <- this targets all connectors with subtype vectorstore +# Example: python tools/bin/bump_cdk.py --selector=connectorSubtype:vectorstore,source-s3 <- this targets all connectors with subtype vectorstore and the s3 source connector + +# Step 1: Get the connector selector from the command line + +import argparse +import os +import re +import subprocess +import sys + +parser = argparse.ArgumentParser(description='Bump CDK version for connectors') +parser.add_argument('--selector', required=True, help='Connector selector') +parser.add_argument('--pr', required=True, help='PR Number') +parser.add_argument('--version', required=True, help='New CDK version number') +parser.add_argument('--changelog', required=True, help='Changelog entry') +args = parser.parse_args() + +# store the selector in a variable +selector = args.selector + +# Split the selector into an array by comma, then turn into key value pairs (if there is a colon, split on that, otherwise the value is the value and the key is connector) +# Example: source-s3 -> {name: source-s3} +# Example: connectorSubtype:vectorstore -> {connectorSubtype: vectorstore} +# Example: connectorSubtype:vectorstore,source-s3 -> {connectorSubtype: vectorstore, name: source-s3} +# Example: connectorSubtype:vectorstore,source-s3,connectorSubtype:vectorstore -> {connectorSubtype: vectorstore, name: source-s3, connectorSubtype: vectorstore} + +# Split the selector into an array by comma +selectorArray = selector.split(',') +# Create a new array to store the key value pairs +selectorArrayKeyValue = [] +# Loop through the selector array +for selector in selectorArray: + # If the selector has a colon, split on that + if ':' in selector: + selectorKeyValue = selector.split(':') + selectorArrayKeyValue.append({selectorKeyValue[0]: selectorKeyValue[1]}) + # Otherwise, the key is name and the value is the selector + else: + selectorArrayKeyValue.append({'connector': selector}) + +# Get all folders in airbyte-integrations/connectors +connectorFolders = os.listdir('./airbyte-integrations/connectors') + +connectors_to_bump = [] + +# Loop through the connector folders and read the metadata.yaml file as object +for connectorFolder in connectorFolders: + # Get the manifest file path + manifestFilePath = f'./airbyte-integrations/connectors/{connectorFolder}/metadata.yaml' + # Check if the manifest file exists + if os.path.exists(manifestFilePath): + # Read the manifest file as object + manifestFile = open(manifestFilePath, 'r') + manifestFileObject = manifestFile.read() + manifestFile.close() + # Loop through the selector array key value pairs + for selector in selectorArrayKeyValue: + # Loop through the key value pairs + for key, value in selector.items(): + # If the key is connector, check if the connector name matches the value + if key == 'connector': + if connectorFolder == value: + connectors_to_bump.append(connectorFolder) + # If the key is connectorSubtype, check if the connectorSubtype matches the value + # TODO use actual yaml parsing for this + elif re.search(f'{key}: {value}', manifestFileObject): + connectors_to_bump.append(connectorFolder) + +print("Found connectors to bump:") +print(connectors_to_bump) + +current_cdk_version = args.version + +print(f"New CDK version: {current_cdk_version}") + +# Step 2: Bump the CDK version for each connector +# To bump the CDK version, do the following: +# * Check if there is a setup.py file in the connector folder +# * If there is, read the setup.py file as object +# * Find a string that describes the airbyte-cdk dependency. It might have an extra definition after it, like [file-based], or a version specification like >=0.53.5 or both +# * If there is no version, add the version >=0.53.5 +# * If there is a version, bump it to 0.53.5 (leave >= or == or whatever in place) + +# Loop through the connectors to bump +for connector in connectors_to_bump: + # Get the setup file path + setupFilePath = f'./airbyte-integrations/connectors/{connector}/setup.py' + # Check if the setup file exists + if os.path.exists(setupFilePath): + # Read the setup file as object + setupFile = open(setupFilePath, 'r') + setupFileObject = setupFile.read() + setupFile.close() + # Find the airbyte-cdk dependency + airbyte_cdk_dependency = re.search(r'airbyte-cdk(?P<extra>\[[a-zA-Z0-9-]*\])?(?P<version>[<>=!~]+[0-9]*\.[0-9]*\.[0-9]*)?', setupFileObject) + # If there is no airbyte-cdk dependency, add the version + if airbyte_cdk_dependency is not None: + + new_version = f"airbyte_cdk{airbyte_cdk_dependency.group('extra') or ''}=={current_cdk_version}" + setupFileObject = setupFileObject.replace(airbyte_cdk_dependency.group(), new_version) + print(f"Updated {connector}") + else: + print(f"No airbyte-cdk dependency found, skipping {connector}") + # Write the setup file + setupFile = open(setupFilePath, 'w') + setupFile.write(setupFileObject) + setupFile.close() + + # Bump version via airbyte-ci = airbyte-ci connectors --name={connector} bump_version patch {pr number from args} "Update CDK version: {changelog from args}" + subprocess.run(f"airbyte-ci connectors --name={connector} bump_version patch {args.pr} \"Update CDK version: {args.changelog}\"", shell=True) + else: + print(f"No setup.py found, skipping {connector}") \ No newline at end of file From bf3c56e68b3130a0b45e37d9703254fcfc840f08 Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 11:55:02 +0100 Subject: [PATCH 02/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index b08e9778e1e4e..ffef34bc5d37a 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -411,10 +411,10 @@ jobs: - name: Update CDK version run: | pipx install --editable --force airbyte-ci/connectors/pipelines/ - python ./tools/bin/bump_cdk.py --pr ${{ steps.cpr.outputs.pull-request-number }} --selector=${{github.event.inputs.connector-selector}} --changelog="${{github.event.inputs.changelog-message}}"" --version="${{needs.bump-version.outputs.new_cdk_version}}" + python ./tools/bin/bump_cdk.py --pr ${{ steps.create-pull-request.outputs.pull-request-number }} --selector=${{github.event.inputs.connector-selector}} --changelog="${{github.event.inputs.changelog-message}}"" --version="${{needs.bump-version.outputs.new_cdk_version}}" # Run create-pull-request again to update the PR with the changes (need to do like this because we need the PR number to create this) - name: Update Pull Request - id: create-pull-request + id: update-pull-request uses: peter-evans/create-pull-request@v4 with: token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} From f93a3462139cb803e7774194d058e946f9bbb9d4 Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 11:57:18 +0100 Subject: [PATCH 03/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index ffef34bc5d37a..120aff0497103 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -431,7 +431,7 @@ jobs: name: Stop Build EC2 Runner needs: - start-publish-docker-image-runner-0 # required to get output from the start-runner job - - publish-docker-image # required to wait when the main job is done + - bump-connectors # required to wait when the main job is done runs-on: ubuntu-latest steps: - name: Configure AWS credentials From 6af2db22d6ae16b6ac49d0b87bc061c04c6207ab Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 12:00:27 +0100 Subject: [PATCH 04/18] fix --- .../publish-cdk-command-manually.yml | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index 120aff0497103..d373bfdc8819f 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -34,31 +34,31 @@ jobs: # We are using these runners because they are the same as the one for `publish-command.yml` # One problem we had using `ubuntu-latest` for example is that the user is not root and some commands would fail in # `manage.sh` (specifically `apt-get`) - start-publish-docker-image-runner-0: - name: Start Build EC2 Runner 0 - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: airbytehq/airbyte - ref: master - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ env.PAT }} - label: ${{ github.run_id }}-publisher + # start-publish-docker-image-runner-0: + # name: Start Build EC2 Runner 0 + # runs-on: ubuntu-latest + # outputs: + # label: ${{ steps.start-ec2-runner.outputs.label }} + # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + # steps: + # - name: Checkout Airbyte + # uses: actions/checkout@v3 + # with: + # repository: airbytehq/airbyte + # ref: master + # - name: Check PAT rate limits + # run: | + # ./tools/bin/find_non_rate_limited_PAT \ + # ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ + # ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} + # - name: Start AWS Runner + # id: start-ec2-runner + # uses: ./.github/actions/start-aws-runner + # with: + # aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + # github-token: ${{ env.PAT }} + # label: ${{ github.run_id }}-publisher # build-cdk: # runs-on: ubuntu-latest @@ -392,7 +392,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.10" - - name: Checkout Airbyte Platform Internal + - name: Checkout Airbyte uses: actions/checkout@v3 with: repository: airbytehq/airbyte @@ -411,7 +411,7 @@ jobs: - name: Update CDK version run: | pipx install --editable --force airbyte-ci/connectors/pipelines/ - python ./tools/bin/bump_cdk.py --pr ${{ steps.create-pull-request.outputs.pull-request-number }} --selector=${{github.event.inputs.connector-selector}} --changelog="${{github.event.inputs.changelog-message}}"" --version="${{needs.bump-version.outputs.new_cdk_version}}" + python ./tools/bin/bump_cdk.py --pr ${{ steps.create-pull-request.outputs.pull-request-number }} --selector=${{github.event.inputs.connector-selector}} --changelog="${{github.event.inputs.changelog-message}}" --version="${{needs.bump-version.outputs.new_cdk_version}}" # Run create-pull-request again to update the PR with the changes (need to do like this because we need the PR number to create this) - name: Update Pull Request id: update-pull-request @@ -426,31 +426,31 @@ jobs: delete-branch: true # In case of self-hosted EC2 errors, remove this block. - stop-publish-docker-image-runner-0: - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - name: Stop Build EC2 Runner - needs: - - start-publish-docker-image-runner-0 # required to get output from the start-runner job - - bump-connectors # required to wait when the main job is done - runs-on: ubuntu-latest - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Stop EC2 runner - uses: airbytehq/ec2-github-runner@base64v1.1.0 - with: - mode: stop - github-token: ${{ env.PAT }} - label: ${{ needs.start-publish-docker-image-runner-0.outputs.label }} - ec2-instance-id: ${{ needs.start-publish-docker-image-runner-0.outputs.ec2-instance-id }} + # stop-publish-docker-image-runner-0: + # if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + # name: Stop Build EC2 Runner + # needs: + # - start-publish-docker-image-runner-0 # required to get output from the start-runner job + # - bump-connectors # required to wait when the main job is done + # runs-on: ubuntu-latest + # steps: + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + # aws-region: us-east-2 + # - name: Checkout Airbyte + # uses: actions/checkout@v3 + # - name: Check PAT rate limits + # run: | + # ./tools/bin/find_non_rate_limited_PAT \ + # ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ + # ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} + # - name: Stop EC2 runner + # uses: airbytehq/ec2-github-runner@base64v1.1.0 + # with: + # mode: stop + # github-token: ${{ env.PAT }} + # label: ${{ needs.start-publish-docker-image-runner-0.outputs.label }} + # ec2-instance-id: ${{ needs.start-publish-docker-image-runner-0.outputs.ec2-instance-id }} From 0f65e96c590f57bb9a4c77e99965f8b43be4ac11 Mon Sep 17 00:00:00 2001 From: flash1293 <flash1293@users.noreply.github.com> Date: Wed, 22 Nov 2023 11:04:12 +0000 Subject: [PATCH 05/18] Automated Commit - Formatting Changes --- tools/bin/bump_cdk.py | 46 ++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/tools/bin/bump_cdk.py b/tools/bin/bump_cdk.py index 997446ac0f4dd..b37e10f082312 100644 --- a/tools/bin/bump_cdk.py +++ b/tools/bin/bump_cdk.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + # This script takes an arg with a connector selector, then bumps the CDK version of all connectors that match that selector # Example: python tools/bin/bump_cdk.py --selector=source-s3 <- this targets the s3 source connector # Example: python tools/bin/bump_cdk.py --selector=connectorSubtype:vectorstore <- this targets all connectors with subtype vectorstore @@ -11,11 +13,11 @@ import subprocess import sys -parser = argparse.ArgumentParser(description='Bump CDK version for connectors') -parser.add_argument('--selector', required=True, help='Connector selector') -parser.add_argument('--pr', required=True, help='PR Number') -parser.add_argument('--version', required=True, help='New CDK version number') -parser.add_argument('--changelog', required=True, help='Changelog entry') +parser = argparse.ArgumentParser(description="Bump CDK version for connectors") +parser.add_argument("--selector", required=True, help="Connector selector") +parser.add_argument("--pr", required=True, help="PR Number") +parser.add_argument("--version", required=True, help="New CDK version number") +parser.add_argument("--changelog", required=True, help="Changelog entry") args = parser.parse_args() # store the selector in a variable @@ -28,32 +30,32 @@ # Example: connectorSubtype:vectorstore,source-s3,connectorSubtype:vectorstore -> {connectorSubtype: vectorstore, name: source-s3, connectorSubtype: vectorstore} # Split the selector into an array by comma -selectorArray = selector.split(',') +selectorArray = selector.split(",") # Create a new array to store the key value pairs selectorArrayKeyValue = [] # Loop through the selector array for selector in selectorArray: # If the selector has a colon, split on that - if ':' in selector: - selectorKeyValue = selector.split(':') + if ":" in selector: + selectorKeyValue = selector.split(":") selectorArrayKeyValue.append({selectorKeyValue[0]: selectorKeyValue[1]}) # Otherwise, the key is name and the value is the selector else: - selectorArrayKeyValue.append({'connector': selector}) + selectorArrayKeyValue.append({"connector": selector}) # Get all folders in airbyte-integrations/connectors -connectorFolders = os.listdir('./airbyte-integrations/connectors') +connectorFolders = os.listdir("./airbyte-integrations/connectors") connectors_to_bump = [] # Loop through the connector folders and read the metadata.yaml file as object for connectorFolder in connectorFolders: # Get the manifest file path - manifestFilePath = f'./airbyte-integrations/connectors/{connectorFolder}/metadata.yaml' + manifestFilePath = f"./airbyte-integrations/connectors/{connectorFolder}/metadata.yaml" # Check if the manifest file exists if os.path.exists(manifestFilePath): # Read the manifest file as object - manifestFile = open(manifestFilePath, 'r') + manifestFile = open(manifestFilePath, "r") manifestFileObject = manifestFile.read() manifestFile.close() # Loop through the selector array key value pairs @@ -61,12 +63,12 @@ # Loop through the key value pairs for key, value in selector.items(): # If the key is connector, check if the connector name matches the value - if key == 'connector': + if key == "connector": if connectorFolder == value: connectors_to_bump.append(connectorFolder) # If the key is connectorSubtype, check if the connectorSubtype matches the value # TODO use actual yaml parsing for this - elif re.search(f'{key}: {value}', manifestFileObject): + elif re.search(f"{key}: {value}", manifestFileObject): connectors_to_bump.append(connectorFolder) print("Found connectors to bump:") @@ -87,15 +89,17 @@ # Loop through the connectors to bump for connector in connectors_to_bump: # Get the setup file path - setupFilePath = f'./airbyte-integrations/connectors/{connector}/setup.py' + setupFilePath = f"./airbyte-integrations/connectors/{connector}/setup.py" # Check if the setup file exists if os.path.exists(setupFilePath): # Read the setup file as object - setupFile = open(setupFilePath, 'r') + setupFile = open(setupFilePath, "r") setupFileObject = setupFile.read() setupFile.close() # Find the airbyte-cdk dependency - airbyte_cdk_dependency = re.search(r'airbyte-cdk(?P<extra>\[[a-zA-Z0-9-]*\])?(?P<version>[<>=!~]+[0-9]*\.[0-9]*\.[0-9]*)?', setupFileObject) + airbyte_cdk_dependency = re.search( + r"airbyte-cdk(?P<extra>\[[a-zA-Z0-9-]*\])?(?P<version>[<>=!~]+[0-9]*\.[0-9]*\.[0-9]*)?", setupFileObject + ) # If there is no airbyte-cdk dependency, add the version if airbyte_cdk_dependency is not None: @@ -105,11 +109,13 @@ else: print(f"No airbyte-cdk dependency found, skipping {connector}") # Write the setup file - setupFile = open(setupFilePath, 'w') + setupFile = open(setupFilePath, "w") setupFile.write(setupFileObject) setupFile.close() # Bump version via airbyte-ci = airbyte-ci connectors --name={connector} bump_version patch {pr number from args} "Update CDK version: {changelog from args}" - subprocess.run(f"airbyte-ci connectors --name={connector} bump_version patch {args.pr} \"Update CDK version: {args.changelog}\"", shell=True) + subprocess.run( + f'airbyte-ci connectors --name={connector} bump_version patch {args.pr} "Update CDK version: {args.changelog}"', shell=True + ) else: - print(f"No setup.py found, skipping {connector}") \ No newline at end of file + print(f"No setup.py found, skipping {connector}") From 9a3ac8901dead92a9a701fcee9e4694ea4d51483 Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 12:06:40 +0100 Subject: [PATCH 06/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 6 +++++- tools/bin/bump_cdk.py | 9 +++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index d373bfdc8819f..23fee6894e847 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -397,6 +397,10 @@ jobs: with: repository: airbytehq/airbyte token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + - name: Update CDK version + run: | + pipx install --editable --force airbyte-ci/connectors/pipelines/ + python ./tools/bin/bump_cdk.py --selector=${{github.event.inputs.connector-selector}} --version="${{needs.bump-version.outputs.new_cdk_version}}" - name: Create Pull Request id: create-pull-request uses: peter-evans/create-pull-request@v4 @@ -408,7 +412,7 @@ jobs: branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} base: flash1293/auto-bump delete-branch: true - - name: Update CDK version + - name: Bump connector versions and changelog run: | pipx install --editable --force airbyte-ci/connectors/pipelines/ python ./tools/bin/bump_cdk.py --pr ${{ steps.create-pull-request.outputs.pull-request-number }} --selector=${{github.event.inputs.connector-selector}} --changelog="${{github.event.inputs.changelog-message}}" --version="${{needs.bump-version.outputs.new_cdk_version}}" diff --git a/tools/bin/bump_cdk.py b/tools/bin/bump_cdk.py index 997446ac0f4dd..bf91983849bb0 100644 --- a/tools/bin/bump_cdk.py +++ b/tools/bin/bump_cdk.py @@ -13,9 +13,9 @@ parser = argparse.ArgumentParser(description='Bump CDK version for connectors') parser.add_argument('--selector', required=True, help='Connector selector') -parser.add_argument('--pr', required=True, help='PR Number') +parser.add_argument('--pr', required=False, help='PR Number') parser.add_argument('--version', required=True, help='New CDK version number') -parser.add_argument('--changelog', required=True, help='Changelog entry') +parser.add_argument('--changelog', required=False, help='Changelog entry') args = parser.parse_args() # store the selector in a variable @@ -109,7 +109,8 @@ setupFile.write(setupFileObject) setupFile.close() - # Bump version via airbyte-ci = airbyte-ci connectors --name={connector} bump_version patch {pr number from args} "Update CDK version: {changelog from args}" - subprocess.run(f"airbyte-ci connectors --name={connector} bump_version patch {args.pr} \"Update CDK version: {args.changelog}\"", shell=True) + if args.pr and args.changelog: + # Bump connector version via airbyte-ci = airbyte-ci connectors --name={connector} bump_version patch {pr number from args} "Update CDK version: {changelog from args}" + subprocess.run(f"airbyte-ci connectors --name={connector} bump_version patch {args.pr} \"Update CDK version: {args.changelog}\"", shell=True) else: print(f"No setup.py found, skipping {connector}") \ No newline at end of file From 5cc88311bb431c32edc648e99c25de5f09056edd Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 12:16:03 +0100 Subject: [PATCH 07/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index 23fee6894e847..704604af0f1a0 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -399,7 +399,6 @@ jobs: token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - name: Update CDK version run: | - pipx install --editable --force airbyte-ci/connectors/pipelines/ python ./tools/bin/bump_cdk.py --selector=${{github.event.inputs.connector-selector}} --version="${{needs.bump-version.outputs.new_cdk_version}}" - name: Create Pull Request id: create-pull-request @@ -412,6 +411,12 @@ jobs: branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} base: flash1293/auto-bump delete-branch: true + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: airbytehq/airbyte + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} - name: Bump connector versions and changelog run: | pipx install --editable --force airbyte-ci/connectors/pipelines/ From 40a417a3677cd8ebe704e1ff26576eb0eedfc348 Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 12:17:25 +0100 Subject: [PATCH 08/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index 704604af0f1a0..2a32b0979fab8 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -416,7 +416,7 @@ jobs: with: repository: airbytehq/airbyte token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} + ref: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} - name: Bump connector versions and changelog run: | pipx install --editable --force airbyte-ci/connectors/pipelines/ From ade561038dcc20027e5a9161fcc08852677b623c Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 12:26:08 +0100 Subject: [PATCH 09/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index 2a32b0979fab8..bbb333d0bb5b6 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -417,6 +417,11 @@ jobs: repository: airbytehq/airbyte token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} ref: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} + # This is required because airbyte-ci is using origin/master to check for changed files + - name: Set Origin + run: | + git remote add origin https://github.com/airbytehq/airbyte.git + git fetch origin master --depth=1 - name: Bump connector versions and changelog run: | pipx install --editable --force airbyte-ci/connectors/pipelines/ From 531050e4e138727fe39f223e7fafd21e2a84ae33 Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 12:27:22 +0100 Subject: [PATCH 10/18] fix --- tools/bin/bump_cdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/bump_cdk.py b/tools/bin/bump_cdk.py index 972adb097d14a..93cf521ac107c 100644 --- a/tools/bin/bump_cdk.py +++ b/tools/bin/bump_cdk.py @@ -103,7 +103,7 @@ # If there is no airbyte-cdk dependency, add the version if airbyte_cdk_dependency is not None: - new_version = f"airbyte_cdk{airbyte_cdk_dependency.group('extra') or ''}=={current_cdk_version}" + new_version = f"airbyte-cdk{airbyte_cdk_dependency.group('extra') or ''}=={current_cdk_version}" setupFileObject = setupFileObject.replace(airbyte_cdk_dependency.group(), new_version) print(f"Updated {connector}") else: From 998f2852960bae30c46decd0ef3b6d9a7dc6a484 Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 14:23:03 +0100 Subject: [PATCH 11/18] fix --- tools/bin/bump_cdk.py | 76 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/tools/bin/bump_cdk.py b/tools/bin/bump_cdk.py index 93cf521ac107c..74b4cb586c3a4 100644 --- a/tools/bin/bump_cdk.py +++ b/tools/bin/bump_cdk.py @@ -12,6 +12,7 @@ import re import subprocess import sys +import datetime parser = argparse.ArgumentParser(description="Bump CDK version for connectors") parser.add_argument("--selector", required=True, help="Connector selector") @@ -108,15 +109,82 @@ print(f"Updated {connector}") else: print(f"No airbyte-cdk dependency found, skipping {connector}") + continue # Write the setup file setupFile = open(setupFilePath, "w") setupFile.write(setupFileObject) setupFile.close() if args.pr and args.changelog: - # Bump version via airbyte-ci = airbyte-ci connectors --name={connector} bump_version patch {pr number from args} "Update CDK version: {changelog from args}" - subprocess.run( - f'airbyte-ci connectors --name={connector} bump_version patch {args.pr} "Update CDK version: {args.changelog}"', shell=True - ) + # Bump version in metadata.yaml: + # * Read metadata.yaml + # * Get current version (field is dockerImageTag: ) + # * Bump the patch version + # * Write metadata.yaml back + + # Get the manifest file path + manifestFilePath = f"./airbyte-integrations/connectors/{connector}/metadata.yaml" + manifestFile = open(manifestFilePath, "r") + manifestFileObject = manifestFile.read() + manifestFile.close() + + # Get the current version + current_version = re.search(r"dockerImageTag: (?P<version>[0-9]*\.[0-9]*\.[0-9]*)", manifestFileObject) + if current_version is not None: + # Split by ., then bump the last number + version_numbers = current_version.group("version").split(".") + version_numbers[-1] = str(int(version_numbers[-1]) + 1) + new_version = ".".join(version_numbers) + manifestFileObject = manifestFileObject.replace(current_version.group(), f"dockerImageTag: {new_version}") + # write back + manifestFile = open(manifestFilePath, "w") + manifestFile.write(manifestFileObject) + manifestFile.close() + + # if there is a Dockerfile, bump the version there too (line starts with LABEL io.airbyte.version=) + dockerFilePath = f"./airbyte-integrations/connectors/{connector}/Dockerfile" + if os.path.exists(dockerFilePath): + dockerFile = open(dockerFilePath, "r") + dockerFileObject = dockerFile.read() + dockerFile.close() + dockerFileObject = re.sub(r"LABEL io.airbyte.version=.*", f"LABEL io.airbyte.version={new_version}", dockerFileObject) + # write back + dockerFile = open(dockerFilePath, "w") + dockerFile.write(dockerFileObject) + dockerFile.close() + else: + print(f"No dockerImageTag found, skipping bumping tag for {connector}") + + # Add changelog entry in documentation file: + # * Read documentation file (docs/source|destination/{connector-name}.md) + # * Find the changelog table (line with "| Version | Date") + # * Add a new line with the new version, date, and changelog entry + # * Write documentation file back + + # Get the documentation file path + just_the_name = connector.replace("source-", "").replace("destination-", "") + documentationFilePath = f"./docs/integrations/sources/{just_the_name}.md" if os.path.exists( + f"./docs/integrations/sources/{just_the_name}.md" + ) else f"./docs/integrations/destinations/{just_the_name}.md" + + documentationFile = open(documentationFilePath, "r") + documentationFileObject = documentationFile.read() + documentationFile.close() + + # Find the changelog table + changelog_table = re.search(r"\| Version \| Date.*\n.*\n", documentationFileObject) + if changelog_table is not None: + # Add a new line with the new version, date, and changelog entry + new_changelog_entry = f"| {new_version} | {datetime.datetime.now().strftime('%Y-%m-%d')} | [{args.pr}](https://github.com/airbytehq/airbyte/pull/{args.pr}) | {args.changelog} |\n" + documentationFileObject = documentationFileObject.replace( + changelog_table.group(), f"{changelog_table.group()}\n{new_changelog_entry}" + ) + else: + print(f"No changelog table found, skipping adding changelog entry for {connector}") + + # write back + documentationFile = open(documentationFilePath, "w") + documentationFile.write(documentationFileObject) + documentationFile.close() else: print(f"No setup.py found, skipping {connector}") From 2b9f6ca2b0bb353e6d1fdb35dd2d2ece3c54ef4a Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 14:23:50 +0100 Subject: [PATCH 12/18] fix --- tools/bin/bump_cdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/bump_cdk.py b/tools/bin/bump_cdk.py index 74b4cb586c3a4..17243974b8d30 100644 --- a/tools/bin/bump_cdk.py +++ b/tools/bin/bump_cdk.py @@ -177,7 +177,7 @@ # Add a new line with the new version, date, and changelog entry new_changelog_entry = f"| {new_version} | {datetime.datetime.now().strftime('%Y-%m-%d')} | [{args.pr}](https://github.com/airbytehq/airbyte/pull/{args.pr}) | {args.changelog} |\n" documentationFileObject = documentationFileObject.replace( - changelog_table.group(), f"{changelog_table.group()}\n{new_changelog_entry}" + changelog_table.group(), f"{changelog_table.group()}{new_changelog_entry}" ) else: print(f"No changelog table found, skipping adding changelog entry for {connector}") From 7edca83b163f38fe6f62c558948691f2a3ee88b7 Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 14:25:28 +0100 Subject: [PATCH 13/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index bbb333d0bb5b6..a96d016a18c7b 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -417,14 +417,8 @@ jobs: repository: airbytehq/airbyte token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} ref: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} - # This is required because airbyte-ci is using origin/master to check for changed files - - name: Set Origin - run: | - git remote add origin https://github.com/airbytehq/airbyte.git - git fetch origin master --depth=1 - name: Bump connector versions and changelog run: | - pipx install --editable --force airbyte-ci/connectors/pipelines/ python ./tools/bin/bump_cdk.py --pr ${{ steps.create-pull-request.outputs.pull-request-number }} --selector=${{github.event.inputs.connector-selector}} --changelog="${{github.event.inputs.changelog-message}}" --version="${{needs.bump-version.outputs.new_cdk_version}}" # Run create-pull-request again to update the PR with the changes (need to do like this because we need the PR number to create this) - name: Update Pull Request From bdc247f89334eea656de671fa0f0d1341a0ef3d0 Mon Sep 17 00:00:00 2001 From: flash1293 <flash1293@users.noreply.github.com> Date: Wed, 22 Nov 2023 13:30:00 +0000 Subject: [PATCH 14/18] Automated Commit - Formatting Changes --- tools/bin/bump_cdk.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/bin/bump_cdk.py b/tools/bin/bump_cdk.py index 17243974b8d30..32ab24ba39c19 100644 --- a/tools/bin/bump_cdk.py +++ b/tools/bin/bump_cdk.py @@ -8,11 +8,11 @@ # Step 1: Get the connector selector from the command line import argparse +import datetime import os import re import subprocess import sys -import datetime parser = argparse.ArgumentParser(description="Bump CDK version for connectors") parser.add_argument("--selector", required=True, help="Connector selector") @@ -139,7 +139,7 @@ # write back manifestFile = open(manifestFilePath, "w") manifestFile.write(manifestFileObject) - manifestFile.close() + manifestFile.close() # if there is a Dockerfile, bump the version there too (line starts with LABEL io.airbyte.version=) dockerFilePath = f"./airbyte-integrations/connectors/{connector}/Dockerfile" @@ -151,7 +151,7 @@ # write back dockerFile = open(dockerFilePath, "w") dockerFile.write(dockerFileObject) - dockerFile.close() + dockerFile.close() else: print(f"No dockerImageTag found, skipping bumping tag for {connector}") @@ -163,9 +163,11 @@ # Get the documentation file path just_the_name = connector.replace("source-", "").replace("destination-", "") - documentationFilePath = f"./docs/integrations/sources/{just_the_name}.md" if os.path.exists( + documentationFilePath = ( f"./docs/integrations/sources/{just_the_name}.md" - ) else f"./docs/integrations/destinations/{just_the_name}.md" + if os.path.exists(f"./docs/integrations/sources/{just_the_name}.md") + else f"./docs/integrations/destinations/{just_the_name}.md" + ) documentationFile = open(documentationFilePath, "r") documentationFileObject = documentationFile.read() @@ -181,7 +183,7 @@ ) else: print(f"No changelog table found, skipping adding changelog entry for {connector}") - + # write back documentationFile = open(documentationFilePath, "w") documentationFile.write(documentationFileObject) From 91c9b4824838a6db0fedcd3dec477d9a62fcaefc Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 14:31:17 +0100 Subject: [PATCH 15/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index a96d016a18c7b..8cbd8f5638529 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -408,7 +408,7 @@ jobs: commit-message: Updating CDK version following release title: Updating CDK version following release body: This is an automatically generated PR triggered by a CDK release - branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} + branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}}-${{github.event.inputs.connector-selector}} base: flash1293/auto-bump delete-branch: true - name: Checkout Airbyte @@ -416,7 +416,7 @@ jobs: with: repository: airbytehq/airbyte token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - ref: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} + ref: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}}-${{github.event.inputs.connector-selector}} - name: Bump connector versions and changelog run: | python ./tools/bin/bump_cdk.py --pr ${{ steps.create-pull-request.outputs.pull-request-number }} --selector=${{github.event.inputs.connector-selector}} --changelog="${{github.event.inputs.changelog-message}}" --version="${{needs.bump-version.outputs.new_cdk_version}}" @@ -429,7 +429,7 @@ jobs: commit-message: Updating CDK version following release title: Updating CDK version following release body: This is an automatically generated PR triggered by a CDK release - branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} + branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}}-${{github.event.inputs.connector-selector}} base: flash1293/auto-bump delete-branch: true From a84e164631c566cd1245096ee0acef4faf45f26e Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 14:36:18 +0100 Subject: [PATCH 16/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index 8cbd8f5638529..3e4a1454a1311 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -399,7 +399,7 @@ jobs: token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - name: Update CDK version run: | - python ./tools/bin/bump_cdk.py --selector=${{github.event.inputs.connector-selector}} --version="${{needs.bump-version.outputs.new_cdk_version}}" + python ./tools/bin/bump_cdk.py --selector=${{github.event.inputs.connector-selector}} --version="x.y.z" - name: Create Pull Request id: create-pull-request uses: peter-evans/create-pull-request@v4 From da54fa3ccc5c7990b54b67819b2b05b4e7a5cd88 Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 14:38:33 +0100 Subject: [PATCH 17/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index 3e4a1454a1311..51f6357393978 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -408,7 +408,7 @@ jobs: commit-message: Updating CDK version following release title: Updating CDK version following release body: This is an automatically generated PR triggered by a CDK release - branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}}-${{github.event.inputs.connector-selector}} + branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} base: flash1293/auto-bump delete-branch: true - name: Checkout Airbyte @@ -416,7 +416,7 @@ jobs: with: repository: airbytehq/airbyte token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - ref: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}}-${{github.event.inputs.connector-selector}} + ref: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} - name: Bump connector versions and changelog run: | python ./tools/bin/bump_cdk.py --pr ${{ steps.create-pull-request.outputs.pull-request-number }} --selector=${{github.event.inputs.connector-selector}} --changelog="${{github.event.inputs.changelog-message}}" --version="${{needs.bump-version.outputs.new_cdk_version}}" @@ -429,7 +429,7 @@ jobs: commit-message: Updating CDK version following release title: Updating CDK version following release body: This is an automatically generated PR triggered by a CDK release - branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}}-${{github.event.inputs.connector-selector}} + branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} base: flash1293/auto-bump delete-branch: true From ec617f8074eef6c0191483781190dfdd427f846f Mon Sep 17 00:00:00 2001 From: Joe Reuter <joe@airbyte.io> Date: Wed, 22 Nov 2023 14:42:24 +0100 Subject: [PATCH 18/18] fix --- .github/workflows/publish-cdk-command-manually.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index 51f6357393978..e1a7a51a4bafa 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -399,7 +399,7 @@ jobs: token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - name: Update CDK version run: | - python ./tools/bin/bump_cdk.py --selector=${{github.event.inputs.connector-selector}} --version="x.y.z" + python ./tools/bin/bump_cdk.py --selector=${{github.event.inputs.connector-selector}} --version="0.0.0" - name: Create Pull Request id: create-pull-request uses: peter-evans/create-pull-request@v4