diff --git a/.github/workflows/test-dbt-installation-docker.yml b/.github/workflows/test-dbt-installation-docker.yml index 37ab4560..3ac12d3a 100644 --- a/.github/workflows/test-dbt-installation-docker.yml +++ b/.github/workflows/test-dbt-installation-docker.yml @@ -61,42 +61,24 @@ jobs: runs-on: ubuntu-latest outputs: - latest-tags: ${{ steps.get-latest-tags.outputs.result }} + latest-tags: ${{ steps.get-latest-tags.outputs.container-tags }} steps: - name: "Fetch ${{ inputs.package_name }} Tags From ${{ env.GITHUB_PACKAGES_LINK }}" - # Description: Fetch tags for specific container and filter out `latest` tags - # GH API doc: https://docs.github.com/en/rest/packages?apiVersion=2022-11-28#list-package-versions-for-a-package-owned-by-an-organization - uses: actions/github-script@v6 + uses: dbt-labs/actions/fetch-container-tags@v1.1.1 id: get-latest-tags with: + package_name: ${{ inputs.package_name }} + organization: "dbt-labs" + pat: ${{ secrets.GITHUB_TOKEN }} + regex: "latest$" + perform_match_method: "search" retries: 3 - script: | - try { - const response = await github.rest.packages.getAllPackageVersionsForPackageOwnedByOrg({ - package_type: 'container', - package_name: '${{ inputs.package_name }}', - org: 'dbt-labs', - }) - let tags = []; - if (response.data) { - const package_data = response.data; - for (const [key, value] of Object.entries(package_data)) { - if (value.metadata.container.tags) { - tags = tags.concat(value.metadata.container.tags) - } - } - tags = tags.filter(t => t.endsWith('latest')) - } - return tags - } catch (error) { - core.setFailed(error.message); - } - name: "[ANNOTATION] ${{ inputs.package_name }} - tags to test" run: | title="${{ inputs.package_name }} - tags to test" - message="The workflow will run tests for the following tags of the ${{ inputs.package_name }} image: ${{ steps.get-latest-tags.outputs.result }}" + message="The workflow will run tests for the following tags of the ${{ inputs.package_name }} image: ${{ steps.get-latest-tags.outputs.container-tags }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" docker-installation-test: diff --git a/.github/workflows/test-dbt-installation-notify-job-statuses.yml b/.github/workflows/test-dbt-installation-notify-job-statuses.yml index 438715b5..0e222aaa 100644 --- a/.github/workflows/test-dbt-installation-notify-job-statuses.yml +++ b/.github/workflows/test-dbt-installation-notify-job-statuses.yml @@ -77,7 +77,7 @@ jobs: core.debug(files_list); files_list.map(file => { - if (file.includes(${{ inputs.package_name }})) { + if (file.includes("${{ inputs.package_name }}")) { const buffer = fs.readFileSync(`${artifact_folder}/${file}`); const jobs_status = buffer.toString().trim().replace(/['"]+/g, ''); diff --git a/.github/workflows/test-dbt-installation-source.yml b/.github/workflows/test-dbt-installation-source.yml index e32b5b77..c90e0ef3 100644 --- a/.github/workflows/test-dbt-installation-source.yml +++ b/.github/workflows/test-dbt-installation-source.yml @@ -58,41 +58,25 @@ jobs: runs-on: ubuntu-latest outputs: - latest-branches: ${{ steps.get-latest-branches.outputs.result }} + latest-branches: ${{ steps.get-latest-branches.outputs.repo-branches }} steps: - # Description: Fetch protected branches metadata for specific repo and filter out `latest` branches - # GH API doc: https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#list-branches - - name: "Fetch ${{ inputs.package_name }} Protected Branches Metadata" - uses: actions/github-script@v6 + - name: "Fetch dbt-core Latest Branches" + uses: dbt-labs/actions/fetch-repo-branches@v1.1.1 id: get-latest-branches with: + repo_name: ${{ inputs.package_name }} + organization: "dbt-labs" + pat: ${{ secrets.GITHUB_TOKEN }} + fetch_protected_branches_only: true + regex: "^1.[0-9]+.latest$" + perform_match_method: "match" retries: 3 - script: | - try { - const response = await github.rest.repos.listBranches({ - repo: '${{ inputs.package_name }}', - owner: 'dbt-labs', - protected: true, - }) - let branches = [] - if (response.data) { - const branches_data = response.data - for (const [key, value] of Object.entries(branches_data)) { - branches = branches.concat(value.name) - } - const re_latest_branch = new RegExp('^1.*.latest') - branches = branches.filter(b => re_latest_branch.test(b)) - } - return branches - } catch (error) { - core.setFailed(error.message) - } - name: "[ANNOTATION] ${{ inputs.package_name }} - branches to test" run: | title="${{ inputs.package_name }} - branches to test" - message="The workflow will run tests for the following branches of the ${{ inputs.package_name }} repo: ${{ steps.get-latest-branches.outputs.result }}" + message="The workflow will run tests for the following branches of the ${{ inputs.package_name }} repo: ${{ steps.get-latest-branches.outputs.repo-branches }}" echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" source-installation-test: diff --git a/fetch-container-tags/Dockerfile b/fetch-container-tags/Dockerfile new file mode 100644 index 00000000..85532e05 --- /dev/null +++ b/fetch-container-tags/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3-slim AS builder +ADD . /app +WORKDIR /app + +# We are installing a dependency here directly into our app source dir +RUN pip install --target=/app setuptools requests + +# A distroless container image with Python and some basics like SSL certificates +# https://github.com/GoogleContainerTools/distroless +FROM gcr.io/distroless/python3-debian10 +COPY --from=builder /app /app +WORKDIR /app +ENV PYTHONPATH /app +CMD ["/app/main.py"] diff --git a/fetch-container-tags/README.md b/fetch-container-tags/README.md new file mode 100644 index 00000000..cd7fc7ec --- /dev/null +++ b/fetch-container-tags/README.md @@ -0,0 +1,65 @@ +# Fetch Container Tags + +A [GitHub Action](https://github.com/features/actions) for fetching container tags from [GitHub packages](https://ghcr.io). + +Example usage: + +```yaml +name: Example Workflow for Fetch Container Tags +on: push +jobs: + fetch-latest-tags: + runs-on: ubuntu-latest + + outputs: + latest-tags: ${{ steps.fetch-latest-tags.outputs.container-tags }} + + steps: + - uses: actions/checkout@v2 + + - name: "Fetch dbt-postgres Container Tags" + id: get-latest-tags + uses: dbt-labs/actions/fetch-container-tags + with: + package_name: "dbt-postgres" + organization: "dbt-labs" + pat: ${{ secrets.GITHUB_TOKEN }} + regex: "latest$" + perform_match_method: "search" + retries: 3 + + - name: "Display Container Tags" + run: | + echo container latest tags: ${{ steps.fetch-latest-tags.outputs.container-tags }} + + dynamic-matrix: + runs-on: ubuntu-latest + needs: fetch-latest-tags + + strategy: + fail-fast: false + matrix: + tag: ${{ fromJSON(needs.fetch-latest-tags.outputs.latest-tags) }} + + steps: + - name: "Display Tag Name" + run: | + echo container tag: ${{ matrix.tag }} +``` + +### Inputs + +| Property | Required | Default | Description | +| -------------------- | -------- | -------------- | ------------------------------------------------------------- | +| package_name | yes | - | Container name | +| organization | yes | - | Organization that owns the package | +| pat | yes | - | PAT for fetch request | +| regex | no | `empty string` | Filter container tags | +| perform_match_method | no | `match` | Select which method use to filter tags (search/match/findall) | +| retries | no | `3` | Retries for fetch request | + +### Outputs + +| Property | Example | Description | +| -------------- | -------------------------------------------------------------------- | ---------------------- | +| container-tags | `['1.2.latest', 'latest', '1.3.latest', '1.1.latest', '1.0.latest']` | List of container tags | diff --git a/fetch-container-tags/action.yml b/fetch-container-tags/action.yml new file mode 100644 index 00000000..2ebc8b6c --- /dev/null +++ b/fetch-container-tags/action.yml @@ -0,0 +1,29 @@ +name: "Fetch Container Tags" +description: "Gets Container Tags From GitHub Packages" +inputs: + package_name: + description: "Package name" + required: true + organization: + description: "GitHub organization where package is stored" + required: true + pat: + description: "Personal access token" + required: true + regex: + description: "Regexp will be applied to fetch request result" + required: true + perform_match_method: + description: "Set which match method will be used with regex. Supported methods: match, search, findall. Default: match" + required: false + default: "match" + retries: + description: "How many retries before failure" + required: false + default: "3" +outputs: + container-tags: + description: "List of containers tag" +runs: + using: "docker" + image: "Dockerfile" diff --git a/fetch-container-tags/main.py b/fetch-container-tags/main.py new file mode 100644 index 00000000..930389e5 --- /dev/null +++ b/fetch-container-tags/main.py @@ -0,0 +1,154 @@ +import requests +import re +import os +import time +from enum import Enum +from dataclasses import dataclass + + +class ProvidedMatchMethodNotSupportedOrIncorrect(Exception): + """The specified match method is not supported or incorrect""" + pass + + +@dataclass +class FetchRequestData: + package_type: str + package_name: str + organization: str + pat: str + attempts_limit: int + + def get_request_url(self) -> str: + """ + Description: Fetch tags for specific container + GH API doc: https://docs.github.com/en/rest/packages?apiVersion=2022-11-28#list-package-versions-for-a-package-owned-by-an-organization + """ + url = f"https://api.github.com/orgs/{self.organization}/packages/{self.package_type}/{self.package_name}/versions" + return url + + def get_request_headers(self) -> dict: + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.pat}", + "X-GitHub-Api-Version": "2022-11-28", + } + return headers + + +class SupportedMatchMethod(Enum): + MATCH = 'match' + SEARCH = 'search' + FINDALL = 'findall' + + +def set_output(name, value): + os.system(f"""echo "{name}={value}" >> $GITHUB_OUTPUT""") + + +def get_exponential_backoff_in_seconds(attempt_number: int) -> int: + """ + Returns exponential back-off - depending on number of attempt. + Considers that `attempt_number` starts from 0. + Initial back-off - 4 second. + """ + return pow(attempt_number + 2, 2) + + +def fetch_package_metadata(request_data: FetchRequestData): + url: str = request_data.get_request_url() + headers: dict = request_data.get_request_headers() + + print(f"::debug::Start fetching package metadata") + + for attempt in range(request_data.attempts_limit): + print( + f"::debug::Fetching package metadata - attempt {attempt + 1} / {request_data.attempts_limit}") + try: + response = requests.get(url=url, headers=headers) + response.raise_for_status() + except requests.exceptions.RequestException as e: + if attempt == request_data.attempts_limit - 1: + raise RuntimeError(f"{e}") + + if attempt < request_data.attempts_limit - 1: + print( + f"Exception occurred: {type(e).__name__} - {e}. Retrying.") + back_off = get_exponential_backoff_in_seconds(attempt) + print( + f"::debug::Sleep for {back_off} seconds before next attempt") + time.sleep(back_off) + continue + break + + print(f"::debug::Finish fetching metadata") + + return response.json() + + +def get_tags_list(package_metadata) -> list: + tags = [] + for key in package_metadata: + tags += key.get('metadata', {}).get('container', {}).get('tags') + return tags + + +def apply_regex_to_tags(regex: str, tags: list, perform_method: SupportedMatchMethod): + regex = re.compile(regex) + method: function = regex.match + if (perform_method == SupportedMatchMethod.FINDALL): + method = regex.match + if (perform_method == SupportedMatchMethod.SEARCH): + method = regex.search + print(f"::debug::Applying regex {regex} via {perform_method}") + return list(filter(method, tags)) + + +def main(): + package_name = os.environ["INPUT_PACKAGE_NAME"] + package_type = "container" + organization = os.environ["INPUT_ORGANIZATION"] + pat = os.environ["INPUT_PAT"] + attempts_limit = int(os.environ["INPUT_RETRIES"]) + 1 + regex = "" + perform_match_method_input = "" + perform_match_method = -1 + + if os.environ.get('INPUT_REGEX') is not None: + regex = os.environ["INPUT_REGEX"] + + try: + perform_match_method_input = os.environ["INPUT_PERFORM_MATCH_METHOD"].upper( + ) + if hasattr(SupportedMatchMethod, perform_match_method_input): + perform_match_method = SupportedMatchMethod[perform_match_method_input] + else: + raise ProvidedMatchMethodNotSupportedOrIncorrect( + f"Match method {perform_match_method_input} is not supported or incorrect") + except Exception as e: + raise RuntimeError(f"{e}") + + request_data = FetchRequestData( + package_type=package_type, + package_name=package_name, + organization=organization, + pat=pat, + attempts_limit=attempts_limit + ) + + request_response = fetch_package_metadata(request_data) + container_tags = get_tags_list(request_response) + + if (regex != ""): + container_tags = apply_regex_to_tags( + regex, container_tags, perform_match_method) + + print("::group::Parse Semver Outputs") + print(f"container-tags={container_tags}") + print("::endgroup::") + + set_output("container-tags", container_tags) + + +if __name__ == "__main__": + main() diff --git a/fetch-repo-branches/Dockerfile b/fetch-repo-branches/Dockerfile new file mode 100644 index 00000000..85532e05 --- /dev/null +++ b/fetch-repo-branches/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3-slim AS builder +ADD . /app +WORKDIR /app + +# We are installing a dependency here directly into our app source dir +RUN pip install --target=/app setuptools requests + +# A distroless container image with Python and some basics like SSL certificates +# https://github.com/GoogleContainerTools/distroless +FROM gcr.io/distroless/python3-debian10 +COPY --from=builder /app /app +WORKDIR /app +ENV PYTHONPATH /app +CMD ["/app/main.py"] diff --git a/fetch-repo-branches/README.md b/fetch-repo-branches/README.md new file mode 100644 index 00000000..21c03771 --- /dev/null +++ b/fetch-repo-branches/README.md @@ -0,0 +1,67 @@ +# Fetch Repo Branches + +A [GitHub Action](https://github.com/features/actions) for fetching branches via GitHub API. + +Example usage: + +```yaml +name: Example Workflow for Fetch Repo Branches +on: push +jobs: + fetch-latest-branches: + runs-on: ubuntu-latest + + outputs: + latest-branches: ${{ steps.get-latest-branches.outputs.repo-branches }} + + steps: + - uses: actions/checkout@v2 + + - name: "Fetch ${{ inputs.package_name }} Protected Branches Metadata" + uses: dbt-labs/actions/fetch-repo-branches + id: get-latest-branches + with: + repo_name: "dbt-core" + organization: "dbt-labs" + pat: ${{ secrets.GITHUB_TOKEN }} + fetch_protected_branches_only: true + regex: "^1.[0-9]+.latest$" + perform_match_method: "match" + retries: 3 + + - name: "Display Latest Branches" + run: | + echo repo latest branches: ${{ steps.get-latest-branches.outputs.repo-branches }} + + dynamic-matrix: + runs-on: ubuntu-latest + needs: fetch-latest-branches + + strategy: + fail-fast: false + matrix: + branch: ${{ fromJSON(needs.fetch-latest-branches.outputs.latest-branches) }} + + steps: + - name: "Display Branch Name" + run: | + echo branch name: ${{ matrix.branch }} +``` + +### Inputs + +| Property | Required | Default | Description | +| ----------------------------- | -------- | -------------- | ------------------------------------------------------------- | +| repo_name | yes | - | Repo name | +| organization | yes | - | Organization that owns repo | +| pat | yes | - | PAT for fetch request | +| fetch_protected_branches_only | no | `false` | Adjust request to fetch only protected branches | +| regex | no | `empty string` | Filter container tags | +| perform_match_method | no | `match` | Select which method use to filter tags (search/match/findall) | +| retries | no | `3` | Retries for fetch request | + +### Outputs + +| Property | Example | Description | +| ------------- | ------------------------------------------------------------------------ | --------------------------------- | +| repo-branches | `['1.0.latest', '1.1.latest', '1.2.latest', '1.3.latest', '1.4.latest']` | List of branches matching request | diff --git a/fetch-repo-branches/action.yml b/fetch-repo-branches/action.yml new file mode 100644 index 00000000..1f4b3bd8 --- /dev/null +++ b/fetch-repo-branches/action.yml @@ -0,0 +1,34 @@ +name: "Fetch Repo Branches" +description: "Gets Branches List Of Specified Repo" +inputs: + repo_name: + description: "Repo name" + required: true + organization: + description: "GitHub organization where package is stored" + required: true + pat: + description: "Personal access token" + required: true + regex: + description: "Regexp will be applied to fetch request result" + required: false + default: "" + fetch_protected_branches_only: + description: "Adjust request to fetch only protected branches" + required: false + default: "false" + perform_match_method: + description: "Set which match method will be used with regex. Supported methods: match, search, findall. Default: match" + required: false + default: "match" + retries: + description: "How many retries before failure" + required: false + default: "3" +outputs: + repo-branches: + description: "List of available branches" +runs: + using: "docker" + image: "Dockerfile" diff --git a/fetch-repo-branches/main.py b/fetch-repo-branches/main.py new file mode 100644 index 00000000..3dfd0c06 --- /dev/null +++ b/fetch-repo-branches/main.py @@ -0,0 +1,164 @@ +import requests +import re +import os +import json +import time +from enum import Enum +from dataclasses import dataclass + + +class ProvidedMatchMethodNotSupportedOrIncorrect(Exception): + """The specified match method is not supported or incorrect""" + pass + + +@dataclass +class FetchRequestData: + repo_name: str + organization: str + protected_branches_only: bool + pat: str + attempts_limit: int + + def get_request_url(self) -> str: + """ + Description: Fetch branches metadata for specific repo + GH API doc: https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#list-branches + """ + url = f"https://api.github.com/repos/{self.organization}/{self.repo_name}/branches" + return url + + def get_request_parameters(self) -> dict: + parameters = { + "protected": self.protected_branches_only + } + return parameters + + def get_request_headers(self) -> dict: + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.pat}", + "X-GitHub-Api-Version": "2022-11-28", + } + return headers + + +class SupportedMatchMethod(Enum): + MATCH = 'match' + SEARCH = 'search' + FINDALL = 'findall' + + +def set_output(name, value): + os.system(f"""echo "{name}={value}" >> $GITHUB_OUTPUT""") + + +def get_exponential_backoff_in_seconds(attempt_number: int) -> int: + """ + Returns exponential back-off - depending on number of attempt. + Considers that `attempt_number` starts from 0. + Initial back-off - 4 second. + """ + return pow(attempt_number + 2, 2) + + +def fetch_repo_branches(request_data: FetchRequestData): + url: str = request_data.get_request_url() + headers: dict = request_data.get_request_headers() + parameters: dict = request_data.get_request_parameters() + + print(f"::debug::Start fetching package metadata") + + print(parameters) + + for attempt in range(request_data.attempts_limit): + print( + f"::debug::Fetching package metadata - attempt {attempt + 1} / {request_data.attempts_limit}") + try: + response = requests.get( + url=url, params=parameters, headers=headers) + response.raise_for_status() + except requests.exceptions.RequestException as e: + if attempt == request_data.attempts_limit - 1: + raise RuntimeError(f"{e}") + + if attempt < request_data.attempts_limit - 1: + print( + f"Exception occurred: {type(e).__name__} - {e}. Retrying.") + back_off = get_exponential_backoff_in_seconds(attempt) + print( + f"::debug::Sleep for {back_off} seconds before next attempt") + time.sleep(back_off) + continue + break + + print(f"::debug::Finish fetching metadata") + + return response.json() + + +def get_branches_list(package_metadata) -> list: + tags = [] + for key in package_metadata: + tags.append(key["name"]) + return tags + + +def apply_regex_to_list(regex: str, branches: list, perform_method: SupportedMatchMethod): + regex = re.compile(regex) + method: function = regex.match + if (perform_method == SupportedMatchMethod.FINDALL): + method = regex.match + if (perform_method == SupportedMatchMethod.SEARCH): + method = regex.search + print(f"::debug::Applying regex {regex} via {perform_method}") + return list(filter(method, branches)) + + +def main(): + repo_name = os.environ["INPUT_REPO_NAME"] + organization = os.environ["INPUT_ORGANIZATION"] + pat = os.environ["INPUT_PAT"] + protected_branches_only = os.environ["INPUT_FETCH_PROTECTED_BRANCHES_ONLY"] == "true" + attempts_limit = int(os.environ["INPUT_RETRIES"]) + 1 + regex = "" + perform_match_method_input = "" + perform_match_method = -1 + + if os.environ.get('INPUT_REGEX') is not None: + regex = os.environ["INPUT_REGEX"] + + try: + perform_match_method_input = os.environ["INPUT_PERFORM_MATCH_METHOD"].upper( + ) + if hasattr(SupportedMatchMethod, perform_match_method_input): + perform_match_method = SupportedMatchMethod[perform_match_method_input] + else: + raise ProvidedMatchMethodNotSupportedOrIncorrect( + f"Match method {perform_match_method_input} is not supported or incorrect") + except Exception as e: + raise RuntimeError(f"{e}") + + request_data = FetchRequestData( + repo_name=repo_name, + organization=organization, + pat=pat, + protected_branches_only=protected_branches_only, + attempts_limit=attempts_limit + ) + + request_response = fetch_repo_branches(request_data) + branches = get_branches_list(request_response) + + if (regex != ""): + branches = apply_regex_to_list(regex, branches, perform_match_method) + + print("::group::Parse Semver Outputs") + print(f"repo-branches={branches}") + print("::endgroup::") + + set_output("repo-branches", branches) + + +if __name__ == "__main__": + main()