diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index a5b18a4327ec..e1a7a51a4baf 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -22,6 +22,9 @@ on: changelog-message: description: "Changelog message to be added to CHANGELOG.md" required: false + connector-selector: + description: "Connectors to bump (comma separated list of connector names or colon separated key value pairs of metadata fields)" + required: false concurrency: group: publish-airbyte-cdk @@ -31,80 +34,80 @@ jobs: # We are using these runners because they are the same as the one for `publish-command.yml` # One problem we had using `ubuntu-latest` for example is that the user is not root and some commands would fail in # `manage.sh` (specifically `apt-get`) - start-publish-docker-image-runner-0: - name: Start Build EC2 Runner 0 - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: airbytehq/airbyte - ref: master - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ env.PAT }} - label: ${{ github.run_id }}-publisher + # start-publish-docker-image-runner-0: + # name: Start Build EC2 Runner 0 + # runs-on: ubuntu-latest + # outputs: + # label: ${{ steps.start-ec2-runner.outputs.label }} + # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + # steps: + # - name: Checkout Airbyte + # uses: actions/checkout@v3 + # with: + # repository: airbytehq/airbyte + # ref: master + # - name: Check PAT rate limits + # run: | + # ./tools/bin/find_non_rate_limited_PAT \ + # ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ + # ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} + # - name: Start AWS Runner + # id: start-ec2-runner + # uses: ./.github/actions/start-aws-runner + # with: + # aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + # github-token: ${{ env.PAT }} + # label: ${{ github.run_id }}-publisher - build-cdk: - runs-on: ubuntu-latest - steps: - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Build CDK Package - run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build - - name: Post failure to Slack channel dev-connectors-extensibility - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": "Error during `build-cdk` while publishing Python CDK!", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Error while publishing Python CDK!" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on \n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # build-cdk: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/setup-python@v4 + # with: + # python-version: "3.10" + # - uses: actions/setup-java@v3 + # with: + # distribution: "zulu" + # java-version: "17" + # - name: Checkout Airbyte + # uses: actions/checkout@v3 + # with: + # repository: ${{ github.event.inputs.repo }} + # ref: ${{ github.event.inputs.gitref }} + # - name: Build CDK Package + # run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build + # - name: Post failure to Slack channel dev-connectors-extensibility + # if: ${{ failure() }} + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": "Error during `build-cdk` while publishing Python CDK!", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "Error while publishing Python CDK!" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on \n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} bump-version: - needs: build-cdk + # needs: build-cdk if: github.event.inputs.release-type != 'none' runs-on: ubuntu-latest outputs: @@ -128,178 +131,275 @@ jobs: new_version="$(grep -i 'current_version = ' .bumpversion.cfg | sed -e 's/.* = //')" awk -v NEW_VERSION="$new_version" -v CHANGELOG_MESSAGE="${{ github.event.inputs.changelog-message }}" 'NR==3{print "## " NEW_VERSION "\n" CHANGELOG_MESSAGE "\n"}1' CHANGELOG.md > tmp && mv tmp CHANGELOG.md echo NEW_VERSION=$new_version >> $GITHUB_OUTPUT - - name: Commit and Push Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - file_pattern: airbyte-cdk/python/setup.py airbyte-cdk/python/.bumpversion.cfg airbyte-cdk/python/CHANGELOG.md airbyte-cdk/python/Dockerfile - commit_message: 🤖 Bump ${{ github.event.inputs.release-type }} version of Python CDK - commit_user_name: Octavia Squidington III - commit_user_email: octavia-squidington-iii@users.noreply.github.com - - name: Post failure to Slack channel dev-connectors-extensibility - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": "Error during `bump-version` while publishing Python CDK!", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Error while publishing Python CDK!" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on \n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # - name: Commit and Push Changes + # uses: stefanzweifel/git-auto-commit-action@v4 + # with: + # file_pattern: airbyte-cdk/python/setup.py airbyte-cdk/python/.bumpversion.cfg airbyte-cdk/python/CHANGELOG.md airbyte-cdk/python/Dockerfile + # commit_message: 🤖 Bump ${{ github.event.inputs.release-type }} version of Python CDK + # commit_user_name: Octavia Squidington III + # commit_user_email: octavia-squidington-iii@users.noreply.github.com + # - name: Post failure to Slack channel dev-connectors-extensibility + # if: ${{ failure() }} + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": "Error during `bump-version` while publishing Python CDK!", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "Error while publishing Python CDK!" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on \n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - publish-cdk: - needs: bump-version - runs-on: ubuntu-latest - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Publish Python Package - uses: mariamrf/py-package-publish-action@v1.1.0 - with: - # specify the same version as in ~/.python-version - python_version: "3.10" - pip_version: "23.2" - subdir: "airbyte-cdk/python/" - env: - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - - name: Post failure to Slack channel dev-connectors-extensibility - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": "Error during `publish-cdk` while publishing Python CDK!", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Error while publishing Python CDK!" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on \n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # publish-cdk: + # needs: bump-version + # runs-on: ubuntu-latest + # steps: + # - name: Checkout Airbyte + # uses: actions/checkout@v3 + # with: + # repository: ${{ github.event.inputs.repo }} + # ref: ${{ github.event.inputs.gitref }} + # - name: Publish Python Package + # uses: mariamrf/py-package-publish-action@v1.1.0 + # with: + # # specify the same version as in ~/.python-version + # python_version: "3.10" + # pip_version: "23.2" + # subdir: "airbyte-cdk/python/" + # env: + # TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + # TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + # - name: Post failure to Slack channel dev-connectors-extensibility + # if: ${{ failure() }} + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": "Error during `publish-cdk` while publishing Python CDK!", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "Error while publishing Python CDK!" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on \n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - publish-docker-image: - timeout-minutes: 240 - needs: - - start-publish-docker-image-runner-0 - - publish-cdk - runs-on: runner-pool-${{ github.run_id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: airbytehq/airbyte - ref: master - - name: Install Java - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Publish source-declarative-manifest - id: publish - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - TZ: UTC - uses: Wandalen/wretry.action@v1.0.42 - with: - command: | - docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-cdk/python false - attempt_limit: 3 - attempt_delay: 5000 in # ms - - name: Post failure to Slack channel dev-connectors-extensibility - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": "Error during `publish-docker-image` while publishing Python CDK!", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Error while publishing Docker image following Python CDK release!" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on \n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # publish-docker-image: + # timeout-minutes: 240 + # needs: + # - start-publish-docker-image-runner-0 + # - publish-cdk + # runs-on: runner-pool-${{ github.run_id }} + # steps: + # - name: Checkout Airbyte + # uses: actions/checkout@v3 + # with: + # repository: airbytehq/airbyte + # ref: master + # - name: Install Java + # uses: actions/setup-java@v3 + # with: + # distribution: "zulu" + # java-version: "17" + # - name: Install Python + # uses: actions/setup-python@v4 + # with: + # python-version: "3.10" + # - name: Publish source-declarative-manifest + # id: publish + # env: + # DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + # DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + # # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + # TZ: UTC + # uses: Wandalen/wretry.action@v1.0.42 + # with: + # command: | + # docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} + # ./tools/integrations/manage.sh publish airbyte-cdk/python false + # attempt_limit: 3 + # attempt_delay: 5000 in # ms + # - name: Post failure to Slack channel dev-connectors-extensibility + # if: ${{ failure() }} + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": "Error during `publish-docker-image` while publishing Python CDK!", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "Error while publishing Docker image following Python CDK release!" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on \n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + + # update-connector-builder: + # needs: + # - bump-version + # - publish-docker-image + # runs-on: ubuntu-latest + # steps: + # - uses: actions/setup-python@v4 + # with: + # python-version: "3.9" + # - name: Checkout Airbyte Platform Internal + # uses: actions/checkout@v3 + # with: + # repository: airbytehq/airbyte-platform-internal + # token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + # - name: Update CDK version + # run: | + # PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) + # sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/Dockerfile + # sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" airbyte-connector-builder-server-wrapped/Dockerfile + # sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/requirements.in + # echo ${{needs.bump-version.outputs.new_cdk_version}} > oss/airbyte-connector-builder-resources/CDK_VERSION + # cd oss/airbyte-connector-builder-server + # pip install pip-tools + # pip-compile --upgrade + # - name: Create Pull Request + # id: create-pull-request + # uses: peter-evans/create-pull-request@v4 + # with: + # token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + # commit-message: Updating CDK version following release + # title: Updating CDK version following release + # body: This is an automatically generated PR triggered by a CDK release + # branch: automatic-cdk-release + # base: master + # delete-branch: true + # - name: Post success to Slack channel dev-connectors-extensibility + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": "A new version of Python CDK has been released!", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on \n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + # - name: Post failure to Slack channel dev-connectors-extensibility + # if: ${{ failure() }} + # uses: slackapi/slack-github-action@v1.23.0 + # continue-on-error: true + # with: + # channel-id: C04J1M66D8B + # payload: | + # { + # "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated", + # "blocks": [ + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": ":warning: Could not automatically create a PR for Connector Builder>\n" + # } + # }, + # { + # "type": "section", + # "text": { + # "type": "mrkdwn", + # "text": "See details on \n" + # } + # } + # ] + # } + # env: + # SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - update-connector-builder: + bump-connectors: needs: - bump-version - - publish-docker-image runs-on: ubuntu-latest steps: - uses: actions/setup-python@v4 with: - python-version: "3.9" - - name: Checkout Airbyte Platform Internal + python-version: "3.10" + - name: Checkout Airbyte uses: actions/checkout@v3 with: - repository: airbytehq/airbyte-platform-internal + repository: airbytehq/airbyte token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - name: Update CDK version run: | - PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) - sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/Dockerfile - sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" airbyte-connector-builder-server-wrapped/Dockerfile - sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/requirements.in - echo ${{needs.bump-version.outputs.new_cdk_version}} > oss/airbyte-connector-builder-resources/CDK_VERSION - cd oss/airbyte-connector-builder-server - pip install pip-tools - pip-compile --upgrade + python ./tools/bin/bump_cdk.py --selector=${{github.event.inputs.connector-selector}} --version="0.0.0" - name: Create Pull Request id: create-pull-request uses: peter-evans/create-pull-request@v4 @@ -308,105 +408,57 @@ jobs: commit-message: Updating CDK version following release title: Updating CDK version following release body: This is an automatically generated PR triggered by a CDK release - branch: automatic-cdk-release - base: master + branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} + base: flash1293/auto-bump delete-branch: true - - name: Post success to Slack channel dev-connectors-extensibility - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": "A new version of Python CDK has been released!", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on \n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - - name: Post failure to Slack channel dev-connectors-extensibility - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.23.0 - continue-on-error: true - with: - channel-id: C04J1M66D8B - payload: | - { - "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":warning: Could not automatically create a PR for Connector Builder>\n" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "See details on \n" - } - } - ] - } - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - - # In case of self-hosted EC2 errors, remove this block. - stop-publish-docker-image-runner-0: - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - name: Stop Build EC2 Runner - needs: - - start-publish-docker-image-runner-0 # required to get output from the start-runner job - - publish-docker-image # required to wait when the main job is done - runs-on: ubuntu-latest - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - name: Checkout Airbyte uses: actions/checkout@v3 - - name: Check PAT rate limits + with: + repository: airbytehq/airbyte + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + ref: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} + - name: Bump connector versions and changelog run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Stop EC2 runner - uses: airbytehq/ec2-github-runner@base64v1.1.0 + python ./tools/bin/bump_cdk.py --pr ${{ steps.create-pull-request.outputs.pull-request-number }} --selector=${{github.event.inputs.connector-selector}} --changelog="${{github.event.inputs.changelog-message}}" --version="${{needs.bump-version.outputs.new_cdk_version}}" + # Run create-pull-request again to update the PR with the changes (need to do like this because we need the PR number to create this) + - name: Update Pull Request + id: update-pull-request + uses: peter-evans/create-pull-request@v4 with: - mode: stop - github-token: ${{ env.PAT }} - label: ${{ needs.start-publish-docker-image-runner-0.outputs.label }} - ec2-instance-id: ${{ needs.start-publish-docker-image-runner-0.outputs.ec2-instance-id }} + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + commit-message: Updating CDK version following release + title: Updating CDK version following release + body: This is an automatically generated PR triggered by a CDK release + branch: flash1293/auto-bump-${{needs.bump-version.outputs.new_cdk_version}} + base: flash1293/auto-bump + delete-branch: true + + # In case of self-hosted EC2 errors, remove this block. + # stop-publish-docker-image-runner-0: + # if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + # name: Stop Build EC2 Runner + # needs: + # - start-publish-docker-image-runner-0 # required to get output from the start-runner job + # - bump-connectors # required to wait when the main job is done + # runs-on: ubuntu-latest + # steps: + # - name: Configure AWS credentials + # uses: aws-actions/configure-aws-credentials@v1 + # with: + # aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + # aws-region: us-east-2 + # - name: Checkout Airbyte + # uses: actions/checkout@v3 + # - name: Check PAT rate limits + # run: | + # ./tools/bin/find_non_rate_limited_PAT \ + # ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ + # ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} + # - name: Stop EC2 runner + # uses: airbytehq/ec2-github-runner@base64v1.1.0 + # with: + # mode: stop + # github-token: ${{ env.PAT }} + # label: ${{ needs.start-publish-docker-image-runner-0.outputs.label }} + # ec2-instance-id: ${{ needs.start-publish-docker-image-runner-0.outputs.ec2-instance-id }} diff --git a/tools/bin/bump_cdk.py b/tools/bin/bump_cdk.py new file mode 100644 index 000000000000..32ab24ba39c1 --- /dev/null +++ b/tools/bin/bump_cdk.py @@ -0,0 +1,192 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +# This script takes an arg with a connector selector, then bumps the CDK version of all connectors that match that selector +# Example: python tools/bin/bump_cdk.py --selector=source-s3 <- this targets the s3 source connector +# Example: python tools/bin/bump_cdk.py --selector=connectorSubtype:vectorstore <- this targets all connectors with subtype vectorstore +# Example: python tools/bin/bump_cdk.py --selector=connectorSubtype:vectorstore,source-s3 <- this targets all connectors with subtype vectorstore and the s3 source connector + +# Step 1: Get the connector selector from the command line + +import argparse +import datetime +import os +import re +import subprocess +import sys + +parser = argparse.ArgumentParser(description="Bump CDK version for connectors") +parser.add_argument("--selector", required=True, help="Connector selector") +parser.add_argument("--pr", required=False, help="PR Number") +parser.add_argument("--version", required=True, help="New CDK version number") +parser.add_argument("--changelog", required=False, help="Changelog entry") +args = parser.parse_args() + +# store the selector in a variable +selector = args.selector + +# Split the selector into an array by comma, then turn into key value pairs (if there is a colon, split on that, otherwise the value is the value and the key is connector) +# Example: source-s3 -> {name: source-s3} +# Example: connectorSubtype:vectorstore -> {connectorSubtype: vectorstore} +# Example: connectorSubtype:vectorstore,source-s3 -> {connectorSubtype: vectorstore, name: source-s3} +# Example: connectorSubtype:vectorstore,source-s3,connectorSubtype:vectorstore -> {connectorSubtype: vectorstore, name: source-s3, connectorSubtype: vectorstore} + +# Split the selector into an array by comma +selectorArray = selector.split(",") +# Create a new array to store the key value pairs +selectorArrayKeyValue = [] +# Loop through the selector array +for selector in selectorArray: + # If the selector has a colon, split on that + if ":" in selector: + selectorKeyValue = selector.split(":") + selectorArrayKeyValue.append({selectorKeyValue[0]: selectorKeyValue[1]}) + # Otherwise, the key is name and the value is the selector + else: + selectorArrayKeyValue.append({"connector": selector}) + +# Get all folders in airbyte-integrations/connectors +connectorFolders = os.listdir("./airbyte-integrations/connectors") + +connectors_to_bump = [] + +# Loop through the connector folders and read the metadata.yaml file as object +for connectorFolder in connectorFolders: + # Get the manifest file path + manifestFilePath = f"./airbyte-integrations/connectors/{connectorFolder}/metadata.yaml" + # Check if the manifest file exists + if os.path.exists(manifestFilePath): + # Read the manifest file as object + manifestFile = open(manifestFilePath, "r") + manifestFileObject = manifestFile.read() + manifestFile.close() + # Loop through the selector array key value pairs + for selector in selectorArrayKeyValue: + # Loop through the key value pairs + for key, value in selector.items(): + # If the key is connector, check if the connector name matches the value + if key == "connector": + if connectorFolder == value: + connectors_to_bump.append(connectorFolder) + # If the key is connectorSubtype, check if the connectorSubtype matches the value + # TODO use actual yaml parsing for this + elif re.search(f"{key}: {value}", manifestFileObject): + connectors_to_bump.append(connectorFolder) + +print("Found connectors to bump:") +print(connectors_to_bump) + +current_cdk_version = args.version + +print(f"New CDK version: {current_cdk_version}") + +# Step 2: Bump the CDK version for each connector +# To bump the CDK version, do the following: +# * Check if there is a setup.py file in the connector folder +# * If there is, read the setup.py file as object +# * Find a string that describes the airbyte-cdk dependency. It might have an extra definition after it, like [file-based], or a version specification like >=0.53.5 or both +# * If there is no version, add the version >=0.53.5 +# * If there is a version, bump it to 0.53.5 (leave >= or == or whatever in place) + +# Loop through the connectors to bump +for connector in connectors_to_bump: + # Get the setup file path + setupFilePath = f"./airbyte-integrations/connectors/{connector}/setup.py" + # Check if the setup file exists + if os.path.exists(setupFilePath): + # Read the setup file as object + setupFile = open(setupFilePath, "r") + setupFileObject = setupFile.read() + setupFile.close() + # Find the airbyte-cdk dependency + airbyte_cdk_dependency = re.search( + r"airbyte-cdk(?P\[[a-zA-Z0-9-]*\])?(?P[<>=!~]+[0-9]*\.[0-9]*\.[0-9]*)?", setupFileObject + ) + # If there is no airbyte-cdk dependency, add the version + if airbyte_cdk_dependency is not None: + + new_version = f"airbyte-cdk{airbyte_cdk_dependency.group('extra') or ''}=={current_cdk_version}" + setupFileObject = setupFileObject.replace(airbyte_cdk_dependency.group(), new_version) + print(f"Updated {connector}") + else: + print(f"No airbyte-cdk dependency found, skipping {connector}") + continue + # Write the setup file + setupFile = open(setupFilePath, "w") + setupFile.write(setupFileObject) + setupFile.close() + + if args.pr and args.changelog: + # Bump version in metadata.yaml: + # * Read metadata.yaml + # * Get current version (field is dockerImageTag: ) + # * Bump the patch version + # * Write metadata.yaml back + + # Get the manifest file path + manifestFilePath = f"./airbyte-integrations/connectors/{connector}/metadata.yaml" + manifestFile = open(manifestFilePath, "r") + manifestFileObject = manifestFile.read() + manifestFile.close() + + # Get the current version + current_version = re.search(r"dockerImageTag: (?P[0-9]*\.[0-9]*\.[0-9]*)", manifestFileObject) + if current_version is not None: + # Split by ., then bump the last number + version_numbers = current_version.group("version").split(".") + version_numbers[-1] = str(int(version_numbers[-1]) + 1) + new_version = ".".join(version_numbers) + manifestFileObject = manifestFileObject.replace(current_version.group(), f"dockerImageTag: {new_version}") + # write back + manifestFile = open(manifestFilePath, "w") + manifestFile.write(manifestFileObject) + manifestFile.close() + + # if there is a Dockerfile, bump the version there too (line starts with LABEL io.airbyte.version=) + dockerFilePath = f"./airbyte-integrations/connectors/{connector}/Dockerfile" + if os.path.exists(dockerFilePath): + dockerFile = open(dockerFilePath, "r") + dockerFileObject = dockerFile.read() + dockerFile.close() + dockerFileObject = re.sub(r"LABEL io.airbyte.version=.*", f"LABEL io.airbyte.version={new_version}", dockerFileObject) + # write back + dockerFile = open(dockerFilePath, "w") + dockerFile.write(dockerFileObject) + dockerFile.close() + else: + print(f"No dockerImageTag found, skipping bumping tag for {connector}") + + # Add changelog entry in documentation file: + # * Read documentation file (docs/source|destination/{connector-name}.md) + # * Find the changelog table (line with "| Version | Date") + # * Add a new line with the new version, date, and changelog entry + # * Write documentation file back + + # Get the documentation file path + just_the_name = connector.replace("source-", "").replace("destination-", "") + documentationFilePath = ( + f"./docs/integrations/sources/{just_the_name}.md" + if os.path.exists(f"./docs/integrations/sources/{just_the_name}.md") + else f"./docs/integrations/destinations/{just_the_name}.md" + ) + + documentationFile = open(documentationFilePath, "r") + documentationFileObject = documentationFile.read() + documentationFile.close() + + # Find the changelog table + changelog_table = re.search(r"\| Version \| Date.*\n.*\n", documentationFileObject) + if changelog_table is not None: + # Add a new line with the new version, date, and changelog entry + new_changelog_entry = f"| {new_version} | {datetime.datetime.now().strftime('%Y-%m-%d')} | [{args.pr}](https://github.com/airbytehq/airbyte/pull/{args.pr}) | {args.changelog} |\n" + documentationFileObject = documentationFileObject.replace( + changelog_table.group(), f"{changelog_table.group()}{new_changelog_entry}" + ) + else: + print(f"No changelog table found, skipping adding changelog entry for {connector}") + + # write back + documentationFile = open(documentationFilePath, "w") + documentationFile.write(documentationFileObject) + documentationFile.close() + else: + print(f"No setup.py found, skipping {connector}")