MCR migration

- Migrate images to Microsoft Container Registry
- Fix Shellcheck issues
- Related to #278

.vsts/pipeline.yml

@@ -3,7 +3,7 @@ name: $(SourceBranch)$(Rev:.r)
 jobs:
   - job: ComponentGovernance
     pool:
-      vmImage: vs2017-win2016
+      vmImage: windows-2019
     condition: ne(variables['Build.Reason'], 'PullRequest')
     variables:
       EnableDetectorPip: true
@@ -143,9 +143,9 @@ jobs:
         condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], ''))
       - powershell: |
           docker version
-          docker login -u="$(docker.username)" -p="$(docker.password)"
-          $cargoImage = "alfpark/batch-shipyard:" + $env:GIT_TAG + "-cargo-windows"
-          $cliImage = "alfpark/batch-shipyard:" + $env:GIT_TAG + "-cli-windows"
+          docker login "$(docker.servername)" -u="$(docker.username)" -p="$(docker.password)"
+          $cargoImage = "$(docker.servername)/public/azure-batch/shipyard:" + $env:GIT_TAG + "-cargo-windows"
+          $cliImage = "$(docker.servername)/public/azure-batch/shipyard:" + $env:GIT_TAG + "-cli-windows"
           pushd docker\\windows\\cargo
           docker build --build-arg GIT_BRANCH=$env:BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$env:BUILD_SOURCEVERSION -t $cargoImage .
           docker push $cargoImage
@@ -241,6 +241,25 @@ jobs:
         condition: >
           and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual'),
           startsWith(variables['Build.SourceBranch'], 'refs/tags/'), eq(variables['python.version'], '3.7'))
+      - script: |
+          set -e
+          set -o pipefail
+          docker version
+          docker login "$(docker.servername)" -u="$(docker.username)" -p="$(docker.password)"
+          singularity_version=$(grep -m1 _SINGULARITY_VERSION convoy/misc.py | cut -d "'" -f 2)
+          echo "Replicating Singularity verison $singularity_version images to MCR"
+          dhImage="alfpark/singularity:${singularity_version}-mnt"
+          docker pull "$dhImage"
+          mcrImage="$(docker.servername)/public/azure-batch/shipyard:${singularity_version}-singularity-mnt"
+          docker tag "$dhImage" "$mcrImage"
+          docker push "$mcrImage"
+          dhImage="alfpark/singularity:${singularity_version}-mnt-resource"
+          docker pull "$dhImage"
+          mcrImage="$(docker.servername)/public/azure-batch/shipyard:${singularity_version}-singularity-mnt-resource"
+          docker tag "$dhImage" "$mcrImage"
+          docker push "$mcrImage"
+        displayName: Replicate Singularity Container Images
+        condition: and(succeeded(), ne(variables['ARTIFACT_CLI'], ''))
       - template: ./pyenv.yml
       - script: |
           set -e
@@ -277,35 +296,38 @@ jobs:
           set -e
           set -o pipefail
           docker version
-          docker login -u="$(docker.username)" -p="$(docker.password)"
+          docker login "$(docker.servername)" -u="$(docker.username)" -p="$(docker.password)"
           if [ -n "$GIT_TAG" ]; then
-            cargoImage="alfpark/batch-shipyard:${DOCKER_TAG}-cargo"
             pushd cargo
+            cargoImage="$(docker.servername)/public/azure-batch/shipyard:${DOCKER_TAG}-cargo"
             docker build --build-arg GIT_BRANCH=$BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$BUILD_SOURCEVERSION -t $cargoImage .
             docker push $cargoImage
             popd
-            cascadeImage="alfpark/batch-shipyard:${DOCKER_TAG}-cascade"
             pushd cascade
-            docker build --build-arg GIT_BRANCH=$BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$BUILD_SOURCEVERSION -t $cascadeImage .
+            cascadeImage="$(docker.servername)/public/azure-batch/shipyard:${DOCKER_TAG}-cascade-docker"
+            docker build --build-arg GIT_BRANCH=$BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$BUILD_SOURCEVERSION -t $cascadeImage -f Dockerfile.docker .
+            docker push $cascadeImage
+            cascadeImage="$(docker.servername)/public/azure-batch/shipyard:${DOCKER_TAG}-cascade-singularity"
+            docker build --build-arg GIT_BRANCH=$BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$BUILD_SOURCEVERSION -t $cascadeImage -f Dockerfile.singularity .
             docker push $cascadeImage
             popd
-            fedImage="alfpark/batch-shipyard:${DOCKER_TAG}-federation"
             pushd federation
+            fedImage="$(docker.servername)/public/azure-batch/shipyard:${DOCKER_TAG}-federation"
             docker build --build-arg GIT_BRANCH=$BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$BUILD_SOURCEVERSION -t $fedImage .
             docker push $fedImage
             popd
-            heimdallImage="alfpark/batch-shipyard:${DOCKER_TAG}-heimdall"
             pushd heimdall
+            heimdallImage="$(docker.servername)/public/azure-batch/shipyard:${DOCKER_TAG}-heimdall"
             docker build --build-arg GIT_BRANCH=$BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$BUILD_SOURCEVERSION -t $heimdallImage .
             docker push $heimdallImage
             popd
-            slurmImage="alfpark/batch-shipyard:${DOCKER_TAG}-slurm"
             pushd slurm
+            slurmImage="$(docker.servername)/public/azure-batch/shipyard:${DOCKER_TAG}-slurm"
             docker build --build-arg GIT_BRANCH=$BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$BUILD_SOURCEVERSION -t $slurmImage .
             docker push $slurmImage
             popd
           fi
-          cliImage="alfpark/batch-shipyard:${DOCKER_TAG}-cli"
+          cliImage="$(docker.servername)/public/azure-batch/shipyard:${DOCKER_TAG}-cli"
           pushd docker/linux/cli
           docker build --build-arg GIT_BRANCH=$BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$BUILD_SOURCEVERSION -t $cliImage .
           docker push $cliImage

cascade/Dockerfile.singularity

@@ -1,7 +1,7 @@
 # Dockerfile for Azure/batch-shipyard (Cascade/Singularity)
 
 # base image containing singularity
-FROM alfpark/singularity:3.3.0-default
+FROM alfpark/singularity:3.3.0
 
 FROM ubuntu:18.04
 MAINTAINER Fred Park <https://github.com/Azure/batch-shipyard>

convoy/batch.py

@@ -5204,7 +5204,8 @@ def add_jobs(
         addlcmds = data.process_input_data(config, bxfile, jobspec)
         if addlcmds is not None:
             if util.is_not_empty(federation_id):
-                tfm = 'alfpark/batch-shipyard:{}-cargo'.format(__version__)
+                tfm = 'mcr.microsoft.com/azure-batch/shipyard:{}-cargo'.format(
+                    __version__)
                 if tfm in addlcmds:
                     raise RuntimeError(
                         'input_data:azure_batch is not supported at the '
@@ -5368,7 +5369,10 @@ def add_jobs(
                             config, 'continue adding job schedule {}'.format(
                                 job_id)):
                         continue
-            jmimgname = 'alfpark/batch-shipyard:{}-cargo'.format(__version__)
+            jmimgname = (
+                'mcr.microsoft.com/azure-batch/shipyard:{}-cargo'.format(
+                    __version__)
+            )
             if is_windows:
                 jmimgname = '{}-windows'.format(jmimgname)
                 jscmdline = (

convoy/data.py

@@ -226,7 +226,8 @@ def process_input_data(config, bxfile, spec, on_task=False):
     :rtype: str
     :return: additonal command
     """
-    tfmimage = 'alfpark/batch-shipyard:{}-cargo'.format(__version__)
+    tfmimage = 'mcr.microsoft.com/azure-batch/shipyard:{}-cargo'.format(
+        __version__)
     is_windows = settings.is_windows_pool(config)
     if is_windows:
         bxcmd = ('powershell -ExecutionPolicy Unrestricted -command '

convoy/fleet.py

@@ -2238,10 +2238,10 @@ def _update_container_images(
                 d='-d' if (settings.batch_shipyard_settings(config)
                            .use_shipyard_docker_image) else '',
                 e='-e {}'.format(envfile),
-                i='-i alfpark/batch-shipyard:{}-cascade-docker'.format(
-                    __version__),
-                j=('-j alfpark/batch-shipyard:{}-cascade-singularity'
-                   .format(__version__)),
+                i=('-i mcr.microsoft.com/azure-batch/shipyard:'
+                   '{}-cascade-docker'.format(__version__)),
+                j=('-j mcr.microsoft.com/azure-batch/shipyard:'
+                   '{}-cascade-singularity'.format(__version__)),
                 ld='-l $log_directory',
                 p='-p {}'.format(
                     settings.batch_shipyard_settings(config)
@@ -2391,8 +2391,9 @@ def _list_docker_images(batch_client, config):
         for out in spout:
             if util.is_not_empty(out):
                 dec = out.split()
-                if (not dec[1].startswith('alfpark/batch-shipyard') and
-                        not dec[1].startswith('alfpark/blobxfer')):
+                if (not dec[1].startswith(
+                        'mcr.microsoft.com/azure-batch/shipyard') and
+                        not dec[1].startswith('mcr.microsoft.com/blobxfer')):
                     node_images[key].add(dec[0])
                     if dec[0] not in all_images:
                         all_images[dec[0]] = dec[1]

docker/singularity/Dockerfile

@@ -3,10 +3,10 @@
 FROM ubuntu:18.04
 MAINTAINER Fred Park <https://github.com/Azure/batch-shipyard>
 
+ARG SINGULARITY_VERSION
 ARG LOCAL_STATE_DIR
 
-ENV SINGULARITY_VERSION=3.3.0 \
-    GO_VERSION=1.12 \
+ENV GO_VERSION=1.12 \
     GOOS=linux \
     GOARCH=amd64
 

docker/singularity/build-images.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -e
+set -o pipefail
+
+SINGULARITY_VERSION=3.3.0
+REPO=alfpark/singularity
+
+LSD=(/var/lib /mnt /mnt/resource)
+TAGS=(default mnt mnt-resource)
+
+i=0
+for lsd in "${LSD[@]}"; do
+    tag="-${TAGS[$i]}"
+    if [ "$tag" == "-default" ]; then
+        tag=""
+    fi
+    di="${REPO}:${SINGULARITY_VERSION}${tag}"
+    docker build --pull -t "$di" --build-arg SINGULARITY_VERSION="$SINGULARITY_VERSION" --build-arg LOCAL_STATE_DIR="$lsd" -f Dockerfile .
+    i=$((i + 1))
+done

docs/01-batch-shipyard-installation.md

@@ -307,31 +307,33 @@ Please see the Upgrading section below for information on upgrading to a new
 release of Batch Shipyard.
 
 ## <a name="docker-install"></a>Batch Shipyard CLI Installation via Docker
-If using the [alfpark/batch-shipyard:latest-cli](https://hub.docker.com/r/alfpark/batch-shipyard)
+If using the
+[mcr.microsoft.com/azure-batch/shipyard:latest-cli](https://hub.docker.com/_/microsoft-azure-batch-shipyard)
 Docker image, then all of the required software is bundled in the image
 itself, however, you will need an installation of the Docker engine on
 your machine. More information on installing Docker to your local machine
 can be found [here](https://www.docker.com/products/overview).
 
 To install:
 ```shell
-docker pull alfpark/batch-shipyard:latest-cli
+docker pull mcr.microsoft.com/azure-batch/shipyard:latest-cli
 ```
 This will pull the CLI Docker image of Batch Shipyard to your local machine.
 You are now ready to execute it with `docker run`. Please see the
 [Batch Shipyard Usage](20-batch-shipyard-usage.md) guide for more information
 on how to execute the Batch Shipyard CLI Docker image.
 
 ## <a name="singularity-install"></a>Batch Shipyard CLI Installation via Singularity
-If using the [alfpark/batch-shipyard-singularity:cli](https://www.singularity-hub.org/collections/204)
+If using the
+[library://alfpark/batch/shipyard:latest-cli](https://cloud.sylabs.io/library/alfpark/batch/shipyard),
 Singularity image, then all of the required software is bundled in the image
 itself, however, you will need an installation of Singularity on your
 machine. More information on installing Singularity to your local machine
 can be found [here](https://www.sylabs.io/singularity/).
 
 To install:
 ```shell
-singularity pull shub://alfpark/batch-shipyard-singularity:cli
+singularity pull library://alfpark/batch/shipyard:cli
 ```
 This will pull the CLI Singularity image of Batch Shipyard to your local
 machine (to the current working directory). You are now ready to execute

docs/20-batch-shipyard-usage.md

@@ -1088,18 +1088,19 @@ task `task1` from a live compute node. Because all portions of the
 `--filespec` option are specified, the tool will not prompt for any input.
 
 ## <a name="container-cli"></a>Batch Shipyard Container Image CLI Invocation
-If using either the Docker image [alfpark/batch-shipyard:latest-cli](https://hub.docker.com/r/alfpark/batch-shipyard)
+If using either the Docker image
+[mcr.microsoft.com/azure-batch/shipyard:latest-cli](https://hub.docker.com/_/microsoft-azure-batch-shipyard)
 or the Singularity image
-[shub://alfpark/batch-shipyard-singularity:cli](https://www.singularity-hub.org/collections/204),
+[library://alfpark/batch/shipyard:latest-cli](https://cloud.sylabs.io/library/alfpark/batch/shipyard),
 then you would invoke Batch Shipyard as:
 
 ```shell
 # if using Docker
-docker run --rm -it alfpark/batch-shipyard:latest-cli \
+docker run --rm -it mcr.microsoft.com/azure-batch/shipyard:latest-cli \
     <command> <subcommand> <options...>
 
 # if using Singularity
-singularity run shub://alfpark/batch-shipyard-singularity:cli \
+singularity run library://alfpark/batch/shipyard:latest-cli \
     <command> <subcommand> <options...>
 ```
 
@@ -1119,14 +1120,14 @@ Shipyard configs are stored in the host path
 docker run --rm -it \
     -v /home/user/batch-shipyard-configs:/configs \
     -w /configs \
-    alfpark/batch-shipyard:latest-cli \
+    mcr.microsoft.com/azure-batch/shipyard:latest-cli \
     <command> <subcommand> <options...>
 
 # if using Singularity
 singularity run \
     -B /home/user/batch-shipyard-configs:/configs \
     --pwd /configs \
-    shub://alfpark/batch-shipyard-singularity:cli \
+    library://alfpark/batch/shipyard:latest-cli \
     <command> <subcommand> <options...>
 ```
 

docs/94-batch-shipyard-major-version-migration.md

@@ -78,7 +78,10 @@ The Docker image name for the CLI has changed. Batch Shipyard Docker images
 now follow the `version-component` naming convetion for tags. Thus, the
 `latest` CLI version will now be `alfpark/batch-shipyard:latest-cli`. This
 will also apply to versioned CLI images. For example, version `3.0.0` will
-be named as `alfpark/batch-shipyard:3.0.0-cli`.
+be named as `alfpark/batch-shipyard:3.0.0-cli`. As of version `3.8.0`,
+Docker images are published to the
+[mcr.microsoft.com/azure-batch/shipyard](https://hub.docker.com/_/microsoft-azure-batch-shipyard)
+on the Microsoft Container Registry instead of Docker Hub.
 
 #### Command Renaming
 Some commands have been placed under sub-commands for better hierarchy

federation/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.5"
 
 services:
   federation:
-    image: alfpark/batch-shipyard:{BATCH_SHIPYARD_VERSION}-federation
+    image: mcr.microsoft.com/azure-batch/shipyard:{BATCH_SHIPYARD_VERSION}-federation
     container_name: federation
     restart: unless-stopped
     privileged: true

heimdall/docker-compose-nonginx.yml

@@ -10,7 +10,7 @@ networks:
 
 services:
   heimdall:
-    image: alfpark/batch-shipyard:{BATCH_SHIPYARD_VERSION}-heimdall
+    image: mcr.microsoft.com/azure-batch/shipyard:{BATCH_SHIPYARD_VERSION}-heimdall
     container_name: heimdall
     restart: unless-stopped
     networks:

heimdall/docker-compose.yml

@@ -10,7 +10,7 @@ networks:
 
 services:
   heimdall:
-    image: alfpark/batch-shipyard:{BATCH_SHIPYARD_VERSION}-heimdall
+    image: mcr.microsoft.com/azure-batch/shipyard:{BATCH_SHIPYARD_VERSION}-heimdall
     container_name: heimdall
     restart: unless-stopped
     networks:

scripts/replicate_batch_shipyard_images.sh

@@ -95,11 +95,11 @@ log INFO "Logging into target Docker registry"
 docker login --username "$username" --password "$password" "$registry"
 
 log INFO "Mirroring Docker images"
-mirror_docker_image "alfpark/blobxfer:$bxver"
-mirror_docker_image "alfpark/batch-shipyard:${bsver}-cascade"
-mirror_docker_image "alfpark/batch-shipyard:${bsver}-cargo"
-mirror_docker_image "alfpark/singularity:${sinver}-ubuntu-16.04"
-#mirror_docker_image "alfpark/singularity:${sinver}-ubuntu-18.04"
-mirror_docker_image "alfpark/singularity:${sinver}-centos-7"
+mirror_docker_image "mcr.microsoft.com/blobxfer:$bxver"
+mirror_docker_image "mcr.microsoft.com/azure-batch/shipyard:${bsver}-cascade-docker"
+mirror_docker_image "mcr.microsoft.com/azure-batch/shipyard:${bsver}-cascade-singularity"
+mirror_docker_image "mcr.microsoft.com/azure-batch/shipyard:${bsver}-cargo"
+mirror_docker_image "mcr.microsoft.com/azure-batch/shipyard:${sinver}-singularity-mnt"
+mirror_docker_image "mcr.microsoft.com/azure-batch/shipyard:${sinver}-singularity-mnt-resource"
 
 log INFO "Docker image mirroring complete"