-
Notifications
You must be signed in to change notification settings - Fork 433
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
GHA Docker build flow for PR's (#1883)
* GHA based PR docker build flow
- Loading branch information
1 parent
1a29fe7
commit fa83d50
Showing
7 changed files
with
256 additions
and
64 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| # Copyright 2022 MosaicML Composer authors | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| """Reads in the Docker build matrix and generates a GHA job matrix.""" | ||
|
|
||
| import json | ||
| from argparse import ArgumentParser, FileType, Namespace | ||
| from uuid import uuid4 | ||
|
|
||
| import yaml | ||
|
|
||
|
|
||
| def _parse_args() -> Namespace: | ||
| """Parse command-line arguments. | ||
| Returns: | ||
| Namespace: Command-line arguments. | ||
| """ | ||
| args = ArgumentParser(description='Process a Docker matrix YAML file.') | ||
| args.add_argument('yaml_file', type=FileType('r'), help='The YAML file to be processed.') | ||
| args.add_argument('-b', | ||
| '--build_args', | ||
| action='append', | ||
| required=False, | ||
| help='List of build args to override globally') | ||
|
|
||
| return args.parse_args() | ||
|
|
||
|
|
||
| def main(args: Namespace): | ||
| """Reads in the Docker build matrix and generates a GHA job matrix.""" | ||
| image_configs = yaml.safe_load(args.yaml_file) | ||
|
|
||
| for image_config in image_configs: | ||
|
|
||
| # Convert tags list to a CSV string | ||
| image_config['TAGS'] = ','.join(image_config['TAGS']) | ||
|
|
||
| # Generate a random UUID for staging | ||
| image_config['UUID'] = str(uuid4()) | ||
|
|
||
| # Apply build args override | ||
| if args.build_args is not None: | ||
| for build_arg in args.build_args: | ||
| arg, val = build_arg.split('=') | ||
| if arg in image_config.keys(): | ||
| image_config[arg] = val | ||
|
|
||
| json_string = json.dumps(image_configs) | ||
| print(f"""matrix={{"include": {json_string}}}""") | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| main(_parse_args()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| name: Docker Build | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| build-matrix: | ||
| required: true | ||
| type: string | ||
| context: | ||
| required: true | ||
| type: string | ||
| push: | ||
| required: true | ||
| type: boolean | ||
| staging: | ||
| required: true | ||
| type: boolean | ||
| staging-repo: | ||
| required: false | ||
| type: string | ||
| secrets: | ||
| username: | ||
| required: true | ||
| password: | ||
| required: true | ||
| jobs: | ||
| build-images: | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| fail-fast: true | ||
| matrix: ${{ fromJSON(inputs.build-matrix) }} | ||
| name: "${{ matrix.IMAGE_NAME }}" | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v3 | ||
|
|
||
| - name: Setup QEMU | ||
| uses: docker/setup-qemu-action@v2 | ||
|
|
||
| - name: Setup Docker Buildx | ||
| uses: docker/setup-buildx-action@v2 | ||
|
|
||
| - name: Login to DockerHub | ||
| uses: docker/login-action@v2 | ||
| with: | ||
| username: ${{ secrets.username }} | ||
| password: ${{ secrets.password }} | ||
|
|
||
| - name: Calculate Docker Image Variables | ||
| run: | | ||
| set -euxo pipefail | ||
| ################### | ||
| # Image config | ||
| ################### | ||
| echo "IMAGE_NAME: ${{ matrix.IMAGE_NAME }}" | ||
| echo "BASE_IMAGE: ${{ matrix.BASE_IMAGE }}" | ||
| echo "TARGET: ${{ matrix.TARGET }}" | ||
| echo "PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }}" | ||
| echo "PYTORCH_VERSION: ${{ matrix.PYTORCH_VERSION }}" | ||
| echo "TORCHTEXT_VERSION: ${{ matrix.TORCHTEXT_VERSION }}" | ||
| echo "TORCHVISION_VERSION: ${{ matrix.TORCHVISION_VERSION }}" | ||
| echo "CUDA_VERSION: ${{ matrix.CUDA_VERSION }}" | ||
| echo "AWS_OFI_NCCL_VERSION: ${{ matrix.AWS_OFI_NCCL_VERSION }}" | ||
| echo "COMPOSER_INSTALL_COMMAND: ${{ matrix.COMPOSER_INSTALL_COMMAND }}" | ||
| echo "TAGS: ${{ matrix.TAGS }}" | ||
| echo "UUID: ${{ matrix.UUID }}" | ||
| ################### | ||
| # Calculate the tag | ||
| ################### | ||
| if [[ ${{ inputs.staging }} ]]; then | ||
| STAGING_REPO=${{ inputs.staging-repo }} | ||
| IMAGE_TAG=${STAGING_REPO}:${{ matrix.UUID }} | ||
| IMAGE_CACHE="${STAGING_REPO}:${{ matrix.IMAGE_NAME }}-buildcache" | ||
| else | ||
| IMAGE_TAG=${{ matrix.TAGS }} | ||
| IMAGE_CACHE="${IMAGE_TAG}-buildcache" | ||
| fi | ||
| echo "IMAGE_TAG=${IMAGE_TAG}" >> ${GITHUB_ENV} | ||
| echo "IMAGE_CACHE=${IMAGE_CACHE}" >> ${GITHUB_ENV} | ||
| - name: Build and Push the Docker Image | ||
| uses: docker/build-push-action@v3 | ||
| with: | ||
| context: ${{ inputs.context }} | ||
| tags: ${{ env.IMAGE_TAG }} | ||
| target: ${{ matrix.TARGET }} | ||
| push: ${{ inputs.push }} | ||
| cache-from: type=registry,ref=${{ env.IMAGE_CACHE }} | ||
| cache-to: type=registry,ref=${{ env.IMAGE_CACHE }},mode=max | ||
| build-args: | | ||
| BASE_IMAGE=${{ matrix.BASE_IMAGE }} | ||
| CUDA_VERSION=${{ matrix.CUDA_VERSION }} | ||
| AWS_OFI_NCCL_VERSION=${{ matrix.AWS_OFI_NCCL_VERSION }} | ||
| PYTHON_VERSION=${{ matrix.PYTHON_VERSION }} | ||
| PYTORCH_VERSION=${{ matrix.PYTORCH_VERSION }} | ||
| TORCHTEXT_VERSION=${{ matrix.TORCHTEXT_VERSION }} | ||
| TORCHVISION_VERSION=${{ matrix.TORCHVISION_VERSION }} | ||
| COMPOSER_INSTALL_COMMAND=${{ matrix.COMPOSER_INSTALL_COMMAND }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| name: PR Docker | ||
| on: | ||
| pull_request: | ||
| branches: | ||
| - dev | ||
| - main | ||
| - release/** | ||
| paths: | ||
| - .github/bin/gen_docker_matrix.py | ||
| - .github/workflows/docker** | ||
| - docker/** | ||
| defaults: | ||
| run: | ||
| working-directory: . | ||
| jobs: | ||
| build-image-matrix: | ||
| if: github.repository_owner == 'mosaicml' | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 2 | ||
| outputs: | ||
| matrix: ${{ steps.set-matrix.outputs.matrix }} | ||
| steps: | ||
| - uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: 3.9 | ||
| - uses: actions/checkout@v3 | ||
| - id: set-matrix | ||
| run: | | ||
| # Install yaml dependency | ||
| pip install pyyaml | ||
| # Override package install command for Composer image | ||
| COMPOSER_INSTALL_COMMAND="mosaicml[all]@git+https://github.com/mosaicml/composer.git@${{ github.sha }}" | ||
| # Generate build matrix | ||
| BUILD_MATRIX=$(python .github/bin/gen_docker_matrix.py docker/build_matrix.yaml -b COMPOSER_INSTALL_COMMAND=$COMPOSER_INSTALL_COMMAND) | ||
| echo $BUILD_MATRIX >> $GITHUB_OUTPUT | ||
| docker-build: | ||
| needs: build-image-matrix | ||
| uses: ./.github/workflows/docker-build.yaml | ||
| with: | ||
| build-matrix: ${{ needs.build-image-matrix.outputs.matrix }} | ||
| context: ./docker | ||
| push: true | ||
| staging: true | ||
| staging-repo: mosaicml/ci-staging | ||
| secrets: | ||
| username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| password: ${{ secrets.DOCKER_HUB_PASSWORD }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.