(PXP-11210) add job for gen3sdk dependencies versions

kube/services/jobs/compare-gen3sdk-dep-versions-job.yaml

@@ -0,0 +1,126 @@
+#
+# run with:
+# gen3 job run compare-gen3sdk-dep-versions [VERBOSE true] [ALL_DEPS true]
+#
+# VERBOSE true
+#   Log standard output from 'pip install' and 'poetry show' commands.
+#   Default is to log only the different/all dependency versions in csv format.
+#
+# ALL_DEPS true
+#  Log versions of all installed dependencies.
+#  Default is to log only the different versions.
+#
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: compare-gen3sdk-dep-versions
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: gen3job
+    spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: karpenter.sh/capacity-type
+                operator: In
+                values:
+                - on-demand
+          - weight: 99
+            preference:
+              matchExpressions:
+              - key: eks.amazonaws.com/capacityType
+                operator: In
+                values:
+                - ONDEMAND
+      serviceAccountName: useryaml-job
+      volumes:
+        - name: shared-data
+          emptyDir: {}
+      containers:
+      - name: awshelper
+        GEN3_AWSHELPER_IMAGE|-image: quay.io/cdis/awshelper:master-|
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 80
+        env:
+        - name: VERBOSE
+          GEN3_VERBOSE|-value: ""-|
+        - name: ALL_DEPS
+          GEN3_ALL_DEPS|-value: ""-|
+        volumeMounts:
+        - name: shared-data
+          mountPath: /mnt/shared
+        resources:
+          limits:
+            cpu: 1
+            memory: 1Gi
+
+        command: ["/bin/bash" ]
+        args:
+          - "-c"
+          - |
+            if [ "$VERBOSE" = true ]; then
+                pip --version
+                pip install gen3
+            else
+                pip install gen3 > /dev/null 2>&1
+            fi
+            pip freeze | sed 's/==/\t/' | sort > pip_freeze.tsv
+            if [ "$VERBOSE" = true ]; then
+                cat pip_freeze.tsv
+            fi
+
+            # poetry - show versions from lock file
+            GITHUB_REPO="raw.githubusercontent.com/uc-cdis/gen3sdk-python"
+            curl -OJLs https://${GITHUB_REPO}/master/poetry.lock
+            curl -OJLs https://${GITHUB_REPO}/master/pyproject.toml
+            if [[ ! -f poetry.lock || ! -f pyproject.toml ]]; then
+                echo -e "ERROR: could not get poetry.lock or pyproject.toml file from ${GITHUB_REPO}"
+                exit 0
+            fi
+            # install the version of poetry that matches the lock file
+            VERSION_STRING=`grep "generated by Poetry" poetry.lock`
+            rx='Poetry ([0-9]+\.[0-9]+\.[0-9]+)'
+            if [[ "$VERSION_STRING" =~ $rx ]]; then
+                POETRY_VERSION="${BASH_REMATCH[1]}"
+            else
+                echo "ERROR: Could not get poetry version from lock file"
+                exit 0
+            fi
+            if [ "$VERBOSE" = true ]; then
+                echo "Installing poetry version $POETRY_VERSION"
+                pip install poetry=="$POETRY_VERSION"
+                PATH=$PATH:~/.local/bin
+                poetry --version
+                poetry env info
+            else
+                pip install poetry=="$POETRY_VERSION" > /dev/null 2>&1
+                PATH=$PATH:~/.local/bin
+            fi
+            poetry show  | sed 's/(\!)//' | awk '{print $1"\t"$2}' | sort > poetry_show.tsv
+            if [ "$VERBOSE" = true ]; then
+                echo "Poetry-show file"
+                cat poetry_show.tsv
+            fi            
+
+            # compare
+            join -j 1 -o 1.1,1.2,2.2 poetry_show.tsv pip_freeze.tsv > join.txt
+            echo "package,poetry-version,pip-version" > gen3_deps_versions.csv
+            cat join.txt | awk '{print $1","$2","$3}' >> gen3_deps_versions.csv
+            # filter out the different versions
+            cat gen3_deps_versions.csv | awk -F ',' '$2 != $3' > mismatched_versions.csv
+
+            if [ "$ALL_VERSIONS" = true ]; then
+                cat gen3_deps_versions.csv
+            else
+                cat mismatched_versions.csv
+            fi
+
+      restartPolicy: Never