From cf25b83a55394d07912ebff8aa1b71a1924d3a88 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 12 Aug 2020 13:11:27 -0700 Subject: [PATCH 1/3] Move code for tagging/pushing docker images into tox.ini --- .github/workflows/tox.yml | 24 ++++++++++++------------ tox.ini | 14 +++++++++----- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index d6f46bd7b16..94acab36392 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -59,6 +59,16 @@ jobs: run: | sudo DEBIAN_FRONTEND=noninteractive apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox + - name: Try to login to docker.pkg.github.com + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + run: | + TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" + if [ -z "$TOKEN" ]; then + TOKEN="${{ secrets.GITHUB_TOKEN }}" + fi + if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then + echo "::set-env name=DOCKER_PUSH_REPOSITORY::docker.pkg.github.com/${{ github.repository }}/" + fi - run: | set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Copy logs from the docker image or build container @@ -78,20 +88,10 @@ jobs: run: | .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME" if: always() - - name: Push docker images + - name: List docker images run: | if [ -f .tox/$TOX_ENV/Dockertags ]; then - TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" - if [ -z "$TOKEN" ]; then - TOKEN="${{ secrets.GITHUB_TOKEN }}" - fi - echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin - for a in $(cat .tox/$TOX_ENV/Dockertags); do - FULL_TAG=docker.pkg.github.com/${{ github.repository }}/$a - docker tag $a $FULL_TAG - echo Pushing $FULL_TAG - docker push $FULL_TAG - done || echo "(Ignoring errors)" + cat .tox/$TOX_ENV/Dockertags fi if: always() diff --git a/tox.ini b/tox.ini index f0fbeea7134..8de5a8a914d 100644 --- a/tox.ini +++ b/tox.ini @@ -103,6 +103,8 @@ passenv = # Set for example to "with-system-packages configured with-targets-pre with-targets" # to tag intermediate images. docker: DOCKER_TARGETS + # If set, we use this prefix and push to it + docker: DOCKER_PUSH_REPOSITORY local: MAKE local-direct: * @@ -409,8 +411,8 @@ commands = # configure binfmt-support on the Docker host (works locally or remotely, i.e: using boot2docker) docker-{arm64,armhf}: docker run --rm --privileged multiarch/qemu-user-static:register --reset docker: bash -c 'for docker_target in {env:DOCKER_TARGETS:with-targets}; do \ - docker: BUILD_TAG=sage-{envname}-$docker_target:$(git describe --dirty --always); \ - docker: if docker build . -f {envdir}/Dockerfile \ + docker: BUILD_TAG={env:DOCKER_PUSH_REPOSITORY:}sage-{envname}-$docker_target:$(git describe --dirty --always); \ + docker: docker build . -f {envdir}/Dockerfile \ docker: --target $docker_target \ docker: --tag $BUILD_TAG \ docker: --build-arg EXTRA_CONFIGURE_ARGS="{env:CONFIGURE_ARGS}" \ @@ -420,9 +422,11 @@ commands = docker: --build-arg TARGETS_PRE="$(if test -n "$TARGETS_PRE"; then echo $TARGETS_PRE; else echo {posargs:sagelib-build-deps}; fi)" \ docker: --build-arg TARGETS="{posargs:build}" \ docker: --build-arg TARGETS_OPTIONAL="{env:TARGETS_OPTIONAL:ptest}" \ - docker: {env:EXTRA_DOCKER_BUILD_ARGS:}; \ - docker: then echo $BUILD_TAG >> {envdir}/Dockertags; \ - docker: else docker commit $(docker ps -l -q) $BUILD_TAG-failed && echo $BUILD_TAG-failed >> {envdir}/Dockertags; exit 1; fi; \ + docker: {env:EXTRA_DOCKER_BUILD_ARGS:}; status=$?; \ + docker: if [ $status != 0 ]; then BUILD_TAG="$BUILD_TAG-failed"; docker commit $(docker ps -l -q) $BUILD_TAG; fi; \ + docker: echo $BUILD_TAG >> {envdir}/Dockertags; \ + docker: if [ x"{env:DOCKER_PUSH_REPOSITORY:}" != x ]; then echo Pushing $BUILD_TAG; docker push $BUILD_TAG || echo (ignoring errors); fi; \ + docker: if [ $status != 0 ]; then exit $status; fi; \ docker: done' # pathpy checksuite needs tox. #28728: gap fails its test suite. # linbox/cysignals testsuites fail. ppl takes very long. From 51c6cde1c97ec8760f4a030f0f5fac3472100f28 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 12 Aug 2020 14:33:02 -0700 Subject: [PATCH 2/3] fixup --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8de5a8a914d..16b611eaac4 100644 --- a/tox.ini +++ b/tox.ini @@ -425,7 +425,7 @@ commands = docker: {env:EXTRA_DOCKER_BUILD_ARGS:}; status=$?; \ docker: if [ $status != 0 ]; then BUILD_TAG="$BUILD_TAG-failed"; docker commit $(docker ps -l -q) $BUILD_TAG; fi; \ docker: echo $BUILD_TAG >> {envdir}/Dockertags; \ - docker: if [ x"{env:DOCKER_PUSH_REPOSITORY:}" != x ]; then echo Pushing $BUILD_TAG; docker push $BUILD_TAG || echo (ignoring errors); fi; \ + docker: if [ x"{env:DOCKER_PUSH_REPOSITORY:}" != x ]; then echo Pushing $BUILD_TAG; docker push $BUILD_TAG || echo "(ignoring errors)"; fi; \ docker: if [ $status != 0 ]; then exit $status; fi; \ docker: done' # pathpy checksuite needs tox. #28728: gap fails its test suite. From 231b81298cc8d0cbf10dceef08a825126f171959 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 12 Aug 2020 15:36:25 -0700 Subject: [PATCH 3/3] .github-workflows/tox-*.yml: Update similarly --- .github/workflows/tox-experimental.yml | 24 ++++++++++++------------ .github/workflows/tox-gcc_spkg.yml | 24 ++++++++++++------------ .github/workflows/tox-optional.yml | 24 ++++++++++++------------ 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/tox-experimental.yml b/.github/workflows/tox-experimental.yml index b9f09e7c03c..3d016dcf0f3 100644 --- a/.github/workflows/tox-experimental.yml +++ b/.github/workflows/tox-experimental.yml @@ -61,6 +61,16 @@ jobs: run: | sudo DEBIAN_FRONTEND=noninteractive apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox + - name: Try to login to docker.pkg.github.com + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + run: | + TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" + if [ -z "$TOKEN" ]; then + TOKEN="${{ secrets.GITHUB_TOKEN }}" + fi + if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then + echo "::set-env name=DOCKER_PUSH_REPOSITORY::docker.pkg.github.com/${{ github.repository }}/" + fi - run: | set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Copy logs from the docker image or build container @@ -80,20 +90,10 @@ jobs: run: | .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME" if: always() - - name: Push docker images + - name: List docker images run: | if [ -f .tox/$TOX_ENV/Dockertags ]; then - TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" - if [ -z "$TOKEN" ]; then - TOKEN="${{ secrets.GITHUB_TOKEN }}" - fi - echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin - for a in $(cat .tox/$TOX_ENV/Dockertags); do - FULL_TAG=docker.pkg.github.com/${{ github.repository }}/$a - docker tag $a $FULL_TAG - echo Pushing $FULL_TAG - docker push $FULL_TAG - done || echo "(Ignoring errors)" + cat .tox/$TOX_ENV/Dockertags fi if: always() diff --git a/.github/workflows/tox-gcc_spkg.yml b/.github/workflows/tox-gcc_spkg.yml index 53efb6a7a6f..b8562b1dc0d 100644 --- a/.github/workflows/tox-gcc_spkg.yml +++ b/.github/workflows/tox-gcc_spkg.yml @@ -59,6 +59,16 @@ jobs: run: | sudo DEBIAN_FRONTEND=noninteractive apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox + - name: Try to login to docker.pkg.github.com + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + run: | + TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" + if [ -z "$TOKEN" ]; then + TOKEN="${{ secrets.GITHUB_TOKEN }}" + fi + if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then + echo "::set-env name=DOCKER_PUSH_REPOSITORY::docker.pkg.github.com/${{ github.repository }}/" + fi - run: | set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Copy logs from the docker image or build container @@ -78,20 +88,10 @@ jobs: run: | .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME" if: always() - - name: Push docker images + - name: List docker images run: | if [ -f .tox/$TOX_ENV/Dockertags ]; then - TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" - if [ -z "$TOKEN" ]; then - TOKEN="${{ secrets.GITHUB_TOKEN }}" - fi - echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin - for a in $(cat .tox/$TOX_ENV/Dockertags); do - FULL_TAG=docker.pkg.github.com/${{ github.repository }}/$a - docker tag $a $FULL_TAG - echo Pushing $FULL_TAG - docker push $FULL_TAG - done || echo "(Ignoring errors)" + cat .tox/$TOX_ENV/Dockertags fi if: always() diff --git a/.github/workflows/tox-optional.yml b/.github/workflows/tox-optional.yml index 625c6e17a11..a59f4503780 100644 --- a/.github/workflows/tox-optional.yml +++ b/.github/workflows/tox-optional.yml @@ -63,6 +63,16 @@ jobs: run: | sudo DEBIAN_FRONTEND=noninteractive apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox + - name: Try to login to docker.pkg.github.com + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + run: | + TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" + if [ -z "$TOKEN" ]; then + TOKEN="${{ secrets.GITHUB_TOKEN }}" + fi + if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then + echo "::set-env name=DOCKER_PUSH_REPOSITORY::docker.pkg.github.com/${{ github.repository }}/" + fi - run: | set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;" - name: Copy logs from the docker image or build container @@ -82,20 +92,10 @@ jobs: run: | .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME" if: always() - - name: Push docker images + - name: List docker images run: | if [ -f .tox/$TOX_ENV/Dockertags ]; then - TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}" - if [ -z "$TOKEN" ]; then - TOKEN="${{ secrets.GITHUB_TOKEN }}" - fi - echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin - for a in $(cat .tox/$TOX_ENV/Dockertags); do - FULL_TAG=docker.pkg.github.com/${{ github.repository }}/$a - docker tag $a $FULL_TAG - echo Pushing $FULL_TAG - docker push $FULL_TAG - done || echo "(Ignoring errors)" + cat .tox/$TOX_ENV/Dockertags fi if: always()