forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
airbyte-ci: run on github hosted runners (airbytehq#34316)
- Loading branch information
1 parent
da5171c
commit 506ddeb
Showing
21 changed files
with
370 additions
and
480 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| name: "Get Dagger Engine Image" | ||
| description: "Pulls the Dagger Engine Image or load from cache" | ||
|
|
||
| inputs: | ||
| dagger_engine_image: | ||
| description: "Image name of the Dagger Engine" | ||
| required: true | ||
| path_to_dagger_engine_image_cache: | ||
| description: "Path to the Dagger Engine image cache" | ||
| required: false | ||
| default: "/home/runner/dagger-engine-image-cache" | ||
|
|
||
| runs: | ||
| using: "composite" | ||
| steps: | ||
| - name: Create local image cache directory | ||
| id: create-dagger-engine-image-cache-dir | ||
| shell: bash | ||
| run: mkdir -p ${{ inputs.path_to_dagger_engine_image_cache }} | ||
|
|
||
| - name: Restore dagger engine image cache | ||
| id: dagger-engine-image-cache-restore | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| path: ${{ inputs.path_to_dagger_engine_image_cache }} | ||
| key: ${{ inputs.dagger_engine_image }} | ||
|
|
||
| # If no GitHub Action cache hit, pull the image and save it locally as tar to the cache directory | ||
| - name: Pull dagger engine image | ||
| id: pull-dagger-engine-image | ||
| if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true' | ||
| shell: bash | ||
| run: | | ||
| set -x | ||
| docker pull ${{ inputs.dagger_engine_image }} | ||
| docker save -o ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar ${{ inputs.dagger_engine_image }} | ||
| # If no GitHub Action cache hit, save the path to the image cache directory to the Github Action cache | ||
| - name: Save dagger engine image cache | ||
| id: dagger-engine-image-cache-save | ||
| if: steps.dagger-engine-image-cache-restore.outputs.cache-hit != 'true' | ||
| uses: actions/cache/save@v4 | ||
| with: | ||
| path: ${{ inputs.path_to_dagger_engine_image_cache }} | ||
| key: ${{ inputs.dagger_engine_image }} | ||
|
|
||
| # If GitHub Action cache hit, load the image tar restored from the cache | ||
| - name: Load dagger engine image from cache | ||
| if: steps.dagger-engine-image-cache-restore.outputs.cache-hit == 'true' | ||
| shell: bash | ||
| run: | | ||
| set -x | ||
| docker load -i ${{ inputs.path_to_dagger_engine_image_cache }}/image.tar |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| name: "Install Airbyte CI" | ||
| description: "Install Airbyte CI from source or from a binary according to changed files. Pulls the Dagger Engine image according to the dagger version used in airbyte-ci." | ||
|
|
||
| inputs: | ||
| airbyte_ci_binary_url: | ||
| description: "URL to airbyte-ci binary" | ||
| required: false | ||
| default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci | ||
| path_to_airbyte_ci_source: | ||
| description: "Path to airbyte-ci source" | ||
| required: false | ||
| default: airbyte-ci/connectors/pipelines | ||
| runs: | ||
| using: "composite" | ||
| steps: | ||
| - name: Get changed files | ||
| uses: tj-actions/changed-files@v39 | ||
| id: changes | ||
| with: | ||
| files_yaml: | | ||
| pipelines: | ||
| - '${{ inputs.path_to_airbyte_ci_source }}/**' | ||
| - name: Determine how Airbyte CI should be installed | ||
| shell: bash | ||
| id: determine-install-mode | ||
| run: | | ||
| if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then | ||
| echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source." | ||
| echo "install-mode=source" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "install-mode=binary" >> $GITHUB_OUTPUT | ||
| fi | ||
| - name: Install Airbyte CI from binary | ||
| id: install-airbyte-ci-binary | ||
| if: steps.determine-install-mode.outputs.install-mode == 'binary' | ||
| shell: bash | ||
| run: | | ||
| curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin | ||
| sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci | ||
| sudo chmod +x /usr/local/bin/airbyte-ci | ||
| - name: Install Python 3.10 | ||
| id: install-python-3-10 | ||
| uses: actions/setup-python@v4 | ||
| if: steps.determine-install-mode.outputs.install-mode == 'source' | ||
| with: | ||
| python-version: "3.10" | ||
| token: ${{ inputs.github_token }} | ||
|
|
||
| - name: Install Airbyte CI from source | ||
| id: install-airbyte-ci-source | ||
| if: steps.determine-install-mode.outputs.install-mode == 'source' | ||
| shell: bash | ||
| run: | | ||
| pip install --upgrade pip | ||
| pip install pipx | ||
| pipx ensurepath | ||
| pipx install ${{ inputs.path_to_airbyte_ci_source }} | ||
| - name: Get dagger engine image name | ||
| id: get-dagger-engine-image-name | ||
| shell: bash | ||
| run: | | ||
| dagger_engine_image=$(airbyte-ci --ci-requirements | tail -n 1 | jq -r '.dagger_engine_image') | ||
| echo "dagger_engine_image=${dagger_engine_image}" >> "$GITHUB_OUTPUT" | ||
| - name: Get dagger engine image | ||
| id: get-dagger-engine-image | ||
| uses: ./.github/actions/get-dagger-engine-image | ||
| with: | ||
| dagger_engine_image: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }} | ||
|
|
||
| outputs: | ||
| install_mode: | ||
| description: "Whether Airbyte CI was installed from source or from a binary" | ||
| value: ${{ steps.determine-install-mode.outputs.install-mode }} | ||
| dagger_engine_image_name: | ||
| description: "Dagger engine image name" | ||
| value: ${{ steps.get-dagger-engine-image-name.outputs.dagger_engine_image }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.