From cea4cd5aed351f19c0d6129e1e508c1d7179c751 Mon Sep 17 00:00:00 2001
From: Matthias Koeppe <mkoeppe@math.ucdavis.edu>
Date: Sat, 3 Oct 2020 18:16:09 -0700
Subject: [PATCH] ci-cygwin-standard.yml: More stages, continue-on-error: true

---
 .github/workflows/ci-cygwin-standard.yml | 254 +++++++++++++++++++++--
 1 file changed, 234 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/ci-cygwin-standard.yml b/.github/workflows/ci-cygwin-standard.yml
index c133d863a58..b1729444867 100644
--- a/.github/workflows/ci-cygwin-standard.yml
+++ b/.github/workflows/ci-cygwin-standard.yml
@@ -146,7 +146,7 @@ jobs:
     env:
       STAGE: ii-a
       PREVIOUS_STAGES: i-*
-      TARGETS: cvxopt rpy2 networkx
+      TARGETS: cvxopt rpy2
       LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
 
@@ -481,11 +481,91 @@ jobs:
 
 ############################################## stage-iii ##########################################
 
-  cygwin-stage-iii:
+  cygwin-stage-iii-a:
     env:
-      STAGE: iii
+      STAGE: iii-a
       PREVIOUS_STAGES: ii-*
-      TARGETS: build
+      TARGETS: sagelib
+      LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
+
+    needs: [cygwin-stage-ii-a, cygwin-stage-ii-b, cygwin-stage-ii-c, cygwin-stage-ii-d, cygwin-stage-ii-e]
+
+    runs-on: windows-latest
+
+    continue-on-error: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        pkgs: [standard]
+    steps:
+    - run: |
+        git config --global core.autocrlf false
+        git config --global core.symlinks true
+    - uses: actions/checkout@v1
+    - name: install cygwin and minimal prerequisites with choco
+      shell: bash {0}
+      run: |
+        choco --version
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        choco install $PACKAGES --source cygwin
+    - name: bootstrap
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && env && ./bootstrap'
+    - name: install additional cygwin packages with choco
+      if: contains(matrix.pkgs, 'standard')
+      shell: bash {0}
+      run: |
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/*/distros/cygwin.txt)
+        choco install $PACKAGES --source cygwin
+    - uses: actions/download-artifact@v2
+      with:
+        name: ${{ env.LOCAL_ARTIFACT_NAME }}
+        path: C:\\tools\\cygwin\\tmp
+    - name: Extract sage-local artifact
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && .github/workflows/extract-sage-local.sh /tmp/sage-local-${{ env.PREVIOUS_STAGES }}.tar'
+    - name: configure
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && ./configure $CONFIGURE_ARGS'
+    - name: make
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make -k -w V=0 $TARGETS'
+    - name: Prepare logs artifact
+      shell: bash
+      run: |
+        mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; for a in local/var/tmp/sage/build/*; do if [ -d $a ]; then tar -c --remove-files -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a).tar" $a; fi; done; cp -r logs/*  "artifacts/$LOGS_ARTIFACT_NAME"
+      if: always()
+    - uses: actions/upload-artifact@v2
+      with:
+        path: artifacts
+        name: ${{ env.LOGS_ARTIFACT_NAME }}
+      if: always()
+    - name: Print out logs for immediate inspection
+      # The markup in the output is a GitHub Actions logging command
+      # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions
+      shell: bash
+      run: |
+        find "artifacts/$LOGS_ARTIFACT_NAME" -type f -name "*.log" -exec sh -c 'if tail -20 "{}" 2>/dev/null | grep "^Error" >/dev/null; then echo :":"error file={}:":" ==== LOG FILE {} CONTAINS AN ERROR ====; cat {} ; fi' \;
+      if: always()
+    - name: Prepare sage-local artifact
+      # We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
+      # We remove the local/lib64 link, which will be recreated by the next stage.
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && rm -f local/lib64; tar -cf /tmp/sage-local-${{ env.STAGE }}.tar --remove-files local'
+      if: always()
+    - uses: actions/upload-artifact@v2
+      with:
+        path: C:\\tools\\cygwin\\tmp\\sage-local-${{ env.STAGE }}.tar
+        name: ${{ env.LOCAL_ARTIFACT_NAME }}
+      if: always()
+
+  cygwin-stage-iii-b:
+    env:
+      STAGE: iii-b
+      PREVIOUS_STAGES: ii-*
+      TARGETS: networkx
       LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
 
@@ -493,6 +573,8 @@ jobs:
 
     runs-on: windows-latest
 
+    continue-on-error: true
+
     strategy:
       fail-fast: false
       matrix:
@@ -550,15 +632,97 @@ jobs:
 
 ############################################## stage-iv ##########################################
 
-  cygwin-stage-iv-a:
+  cygwin-stage-iv:
     env:
-      STAGE: iv-a
-      PREVIOUS_STAGES: iii
+      STAGE: iv
+      PREVIOUS_STAGES: iii-*
+      TARGETS: build
+      LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
+
+    needs: [cygwin-stage-iii-a, cygwin-stage-iii-b]
+
+    runs-on: windows-latest
+
+    continue-on-error: true
+
+    strategy:
+      fail-fast: false
+      matrix:
+        pkgs: [standard]
+    steps:
+    - run: |
+        git config --global core.autocrlf false
+        git config --global core.symlinks true
+    - uses: actions/checkout@v1
+    - name: install cygwin and minimal prerequisites with choco
+      shell: bash {0}
+      run: |
+        choco --version
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        choco install $PACKAGES --source cygwin
+    - name: bootstrap
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && env && ./bootstrap'
+    - name: install additional cygwin packages with choco
+      if: contains(matrix.pkgs, 'standard')
+      shell: bash {0}
+      run: |
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/*/distros/cygwin.txt)
+        choco install $PACKAGES --source cygwin
+    - uses: actions/download-artifact@v2
+      with:
+        name: ${{ env.LOCAL_ARTIFACT_NAME }}
+        path: C:\\tools\\cygwin\\tmp
+    - name: Extract sage-local artifact
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && .github/workflows/extract-sage-local.sh /tmp/sage-local-${{ env.PREVIOUS_STAGES }}.tar'
+    - name: configure
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && ./configure $CONFIGURE_ARGS'
+    - name: make
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make -k -w V=0 $TARGETS'
+    - name: Prepare logs artifact
+      shell: bash
+      run: |
+        mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; for a in local/var/tmp/sage/build/*; do if [ -d $a ]; then tar -c --remove-files -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a).tar" $a; fi; done; cp -r logs/*  "artifacts/$LOGS_ARTIFACT_NAME"
+      if: always()
+    - uses: actions/upload-artifact@v2
+      with:
+        path: artifacts
+        name: ${{ env.LOGS_ARTIFACT_NAME }}
+      if: always()
+    - name: Print out logs for immediate inspection
+      # The markup in the output is a GitHub Actions logging command
+      # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions
+      shell: bash
+      run: |
+        find "artifacts/$LOGS_ARTIFACT_NAME" -type f -name "*.log" -exec sh -c 'if tail -20 "{}" 2>/dev/null | grep "^Error" >/dev/null; then echo :":"error file={}:":" ==== LOG FILE {} CONTAINS AN ERROR ====; cat {} ; fi' \;
+      if: always()
+    - name: Prepare sage-local artifact
+      # We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
+      # We remove the local/lib64 link, which will be recreated by the next stage.
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && rm -f local/lib64; tar -cf /tmp/sage-local-${{ env.STAGE }}.tar --remove-files local'
+      if: always()
+    - uses: actions/upload-artifact@v2
+      with:
+        path: C:\\tools\\cygwin\\tmp\\sage-local-${{ env.STAGE }}.tar
+        name: ${{ env.LOCAL_ARTIFACT_NAME }}
+      if: always()
+
+############################################## stage-v ###########################################
+
+  cygwin-stage-v-a:
+    env:
+      STAGE: v-a
+      PREVIOUS_STAGES: iv
       TARGETS: ptest
       LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
 
-    needs: [cygwin-stage-iii]
+    needs: [cygwin-stage-iv]
 
     runs-on: windows-latest
 
@@ -617,15 +781,15 @@ jobs:
         name: ${{ env.LOCAL_ARTIFACT_NAME }}
       if: always()
 
-  cygwin-stage-iv-b:
+  cygwin-stage-v-b:
     env:
-      STAGE: iv-b
-      PREVIOUS_STAGES: iii
+      STAGE: v-b
+      PREVIOUS_STAGES: iv
       TARGETS: 4ti2 pynormaliz topcom lrslib latte_int cryptominisat
       LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
 
-    needs: [cygwin-stage-iii]
+    needs: [cygwin-stage-iv]
 
     runs-on: windows-latest
 
@@ -684,15 +848,15 @@ jobs:
         name: ${{ env.LOCAL_ARTIFACT_NAME }}
       if: always()
 
-  cygwin-stage-iv-c:
+  cygwin-stage-v-c:
     env:
-      STAGE: iv-c
-      PREVIOUS_STAGES: iii
+      STAGE: v-c
+      PREVIOUS_STAGES: iv
       TARGETS: sage_numerical_backends_coin
       LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
 
-    needs: [cygwin-stage-iii]
+    needs: [cygwin-stage-iv]
 
     runs-on: windows-latest
 
@@ -751,15 +915,15 @@ jobs:
         name: ${{ env.LOCAL_ARTIFACT_NAME }}
       if: always()
 
-  cygwin-stage-iv-d:
+  cygwin-stage-v-d:
     env:
-      STAGE: iv-d
-      PREVIOUS_STAGES: iii
+      STAGE: v-d
+      PREVIOUS_STAGES: iv
       TARGETS: qepcad  barvinok isl qhull primecount plantri kenzo libsemigroups mcqd meataxe mpfrcx openssl p_group_cohomology rst2ipynb sirocco tdlib tides
       LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
 
-    needs: [cygwin-stage-iii]
+    needs: [cygwin-stage-iv]
 
     runs-on: windows-latest
 
@@ -817,3 +981,53 @@ jobs:
         path: C:\\tools\\cygwin\\tmp\\sage-local-${{ env.STAGE }}.tar
         name: ${{ env.LOCAL_ARTIFACT_NAME }}
       if: always()
+
+  cygwin-stage-v-e:
+    env:
+      STAGE: v-e
+      PREVIOUS_STAGES: iv
+      TARGETS:
+      LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
+      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
+
+    needs: [cygwin-stage-iv]
+
+    runs-on: windows-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        pkgs: [standard]
+    steps:
+    - run: |
+        git config --global core.autocrlf false
+        git config --global core.symlinks true
+    - uses: actions/checkout@v1
+    - name: install cygwin and minimal prerequisites with choco
+      shell: bash {0}
+      run: |
+        choco --version
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
+        choco install $PACKAGES --source cygwin
+    - name: bootstrap
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && env && ./bootstrap'
+    - name: install additional cygwin packages with choco
+      if: contains(matrix.pkgs, 'standard')
+      shell: bash {0}
+      run: |
+        PACKAGES=$(sed 's/#.*//;' ./build/pkgs/*/distros/cygwin.txt)
+        choco install $PACKAGES --source cygwin
+    - uses: actions/download-artifact@v2
+      with:
+        name: ${{ env.LOCAL_ARTIFACT_NAME }}
+        path: C:\\tools\\cygwin\\tmp
+    - name: Extract sage-local artifact
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && .github/workflows/extract-sage-local.sh'
+    - name: configure
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && ./configure $CONFIGURE_ARGS'
+    - name: make
+      run: |
+        C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make -k -w V=0 $TARGETS'