Fixes for compare-refdata pipeline (#1996)

* Fix scp path in compare-refdata pipeline * Do not fetch pull requests if refdata source is Azure * Use old fetch method
tardis-sn · May 6, 2022 · 16ce307 · 16ce307
1 parent 9976a20
commit 16ce307
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 10 deletions.
diff --git a/.azure-pipelines/compare-refdata.yml b/.azure-pipelines/compare-refdata.yml
@@ -19,10 +19,9 @@ pr:
     - '*'
 
 variables:
-  system.debug: false
-  results.url: 'http://opensupernova.org/~azuredevops/files/refdata-results'
   pr.number: '$(System.PullRequest.PullRequestNumber)'
   commit.sha: '$(Build.SourceVersion)'
+  results.url: 'http://opensupernova.org/~azuredevops/files/refdata-results'
   #ref1.hash: ''
   #ref2.hash: ''
 
@@ -35,7 +34,7 @@ jobs:
       - template: templates/default.yml
         parameters:
           fetchRefdata: true
-          refdataRepo: 'azure'
+          refdataRepo: 'azure'  # use 'github' when comparing between custom ref hashes
           useMamba: false
 
       - bash: |
@@ -47,9 +46,14 @@ jobs:
           cd $(refdata.dir)
           git remote add upstream $(git remote get-url origin)
           git fetch upstream
-          git fetch upstream "+refs/pull/*/head:refs/remotes/upstream/pr/*"
         displayName: 'Set upstream remote'
 
+      - ${{ if or(startsWith(variables['ref1.hash'], 'upstream/pr'), startsWith(variables['ref2.hash'], 'upstream/pr')) }}:
+        - bash: |
+            cd $(refdata.dir)
+            git fetch upstream "+refs/pull/*/head:refs/remotes/upstream/pr/*"
+          displayName: 'Fetch pull requests'
+
       - bash: |
           cd $(tardis.dir)
           source activate tardis
@@ -97,11 +101,10 @@ jobs:
 
       - bash: |
           ssh azuredevops@opensupernova.org "mkdir -p /home/azuredevops/public_html/files/refdata-results/$(subfolder)"
-          scp $(refdata.dir)/notebooks/ref_data_compare.html azuredevops@opensupernova.org:/home/azuredevops/public_html/files/refdata-results/$(pr.number)/$(commit.sha).html
+          scp $(refdata.dir)/notebooks/ref_data_compare.html azuredevops@opensupernova.org:/home/azuredevops/public_html/files/refdata-results/$(subfolder)/$(commit.sha).html
         displayName: 'Copy files to server'
         condition: succeededOrFailed()
 
-      # Run if the pipeline is triggered by a pull request.
       - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
         - task: GitHubComment@0
           inputs:

diff --git a/.azure-pipelines/templates/default.yml b/.azure-pipelines/templates/default.yml
@@ -38,7 +38,7 @@ steps:
   - ${{ if eq(parameters.useMamba, false) }}:
     - bash: |
         echo "##vso[task.setvariable variable=package.manager]conda"
-      displayName: 'Set environment variables'
+      displayName: 'Set package manager'
 
   - ${{ if eq(parameters.useMamba, true) }}:
     - bash: |
@@ -50,9 +50,14 @@ steps:
     fetchDepth: ${{ parameters.fetchDepth }}
 
   - ${{ if and(eq(parameters.fetchRefdata, true), eq(parameters.refdataRepo, 'azure')) }}:
-    - checkout: git://TARDIS/tardis-refdata
-      lfs: true
-      displayName: 'Fetch reference data (Azure)'
+      # Azure Repos requires token auth for public repositories containing LFS objects (bug).
+      # Fetch reference data from Azure with a PAT until a fix arrives.
+      - bash: |
+          MY_PAT=$(refdata_token)
+          B64_PAT=$(printf ":$MY_PAT" | base64)
+          git -c http.extraHeader="Authorization: Basic ${B64_PAT}" clone https://tardis-sn@dev.azure.com/tardis-sn/TARDIS/_git/tardis-refdata $(refdata.dir)
+          cd $(refdata.dir); git -c http.extraHeader="Authorization: Basic ${B64_PAT}" lfs fetch --all
+        displayName: 'Fetch reference data repository'
 
   - ${{ if and(eq(parameters.fetchRefdata, true), eq(parameters.refdataRepo, 'github')) }}:
     - bash: |