Merge pull request #1608 from poldracklab/ci/workspace_to_cache

CI: Use caches to pass data between jobs
nipreps · May 5, 2019 · ac63888 · ac63888
2 parents ef7542e + c6cac05
commit ac63888
Showing 1 changed file with 47 additions and 50 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -15,7 +15,7 @@ jobs:
             apk add --no-cache pigz python3
       - restore_cache:
           keys:
-            - docker-v3-{{ .Branch }}-{{ epoch }}
+            - docker-v3-{{ .Branch }}-{{ .Revision }}
             - docker-v3-{{ .Branch }}-
             - docker-v3-master-
             - docker-v3-
@@ -66,13 +66,16 @@ jobs:
           command: |
             mkdir -p /tmp/cache
             docker save ubuntu:xenial-20161213 poldracklab/fmriprep:latest \
-            | pigz -8 -p 3 > /tmp/cache/docker.tar.gz
+            | pigz -3 > /tmp/cache/docker.tar.gz
 
       - persist_to_workspace:
           root: /tmp
           paths:
-            - cache/docker.tar.gz
             - src/fmriprep
+      - save_cache:
+         key: docker-v3-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
+         paths:
+            - /tmp/cache/docker.tar.gz
 
   get_data:
     machine:
@@ -82,7 +85,7 @@ jobs:
     steps:
       - restore_cache:
           keys:
-            - data-v6-{{ epoch }}
+            - data-v6-{{ .Revision }}
             - data-v6-
       - run:
           name: Get test data from ds000005
@@ -144,14 +147,12 @@ jobs:
       - persist_to_workspace:
           root: /tmp
           paths:
-            - data
             - fslicense
             - ds005/nipype.cfg
-            - ds005/derivatives
             - ds054/nipype.cfg
             - ds210/nipype.cfg
       - save_cache:
-         key: data-v6-{{ epoch }}
+         key: data-v6-{{ .Revision }}-{{ epoch }}
          paths:
             - /tmp/data
             - /tmp/ds005/derivatives/freesurfer
@@ -164,7 +165,7 @@ jobs:
     steps:
       - restore_cache:
           keys:
-            - regression-v3-{{ epoch }}
+            - regression-v3-{{ .Revision }}
             - regression-v3-
       - run:
           name: Get truncated BOLD series
@@ -187,28 +188,11 @@ jobs:
             else
               echo "Pre-computed masks were cached"
             fi
-      - persist_to_workspace:
-          root: /tmp
-          paths:
-            - data
       - save_cache:
-         key: regression-v3-{{ epoch }}
+         key: regression-v3-{{ .Revision }}-{{ epoch }}
          paths:
             - /tmp/data
 
-  update_cache:
-    machine:
-      # Ubuntu 14.04 with Docker 17.10.0-ce
-      image: circleci/classic:201711-01
-    working_directory: /tmp/src/fmriprep
-    steps:
-      - attach_workspace:
-          at: /tmp
-      - save_cache:
-         key: docker-v3-{{ .Branch }}-{{ epoch }}
-         paths:
-            - /tmp/cache/docker.tar.gz
-
   test_pytest:
     machine:
       image: circleci/classic:201711-01
@@ -227,6 +211,12 @@ jobs:
 
       - attach_workspace:
           at: /tmp
+      - restore_cache:
+          keys:
+            - docker-v3-{{ .Branch }}-{{ .Revision }}
+      - restore_cache:
+          keys:
+            - regression-v3-{{ .Revision }}
       - run:
           name: Load Docker image layer cache
           no_output_timeout: 30m
@@ -340,7 +330,13 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - ds005-anat-v14-{{ .Branch }}-{{ epoch }}
+            - docker-v3-{{ .Branch }}-{{ .Revision }}
+      - restore_cache:
+          keys:
+            - data-v6-{{ .Revision }}
+      - restore_cache:
+          keys:
+            - ds005-anat-v14-{{ .Branch }}-{{ .Revision }}
             - ds005-anat-v14-{{ .Branch }}
             - ds005-anat-v14-master
             - ds005-anat-v14-
@@ -384,7 +380,7 @@ jobs:
             rm -rf /tmp/ds005/work/reportlets
             rm -rf /tmp/ds005/derivatives/fmriprep
       - save_cache:
-         key: ds005-anat-v14-{{ .Branch }}-{{ epoch }}
+         key: ds005-anat-v14-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
          paths:
             - /tmp/ds005/work
 
@@ -484,7 +480,13 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - ds054-anat-v12-{{ .Branch }}-{{ epoch }}
+            - docker-v3-{{ .Branch }}-{{ .Revision }}
+      - restore_cache:
+          keys:
+            - data-v6-{{ .Revision }}
+      - restore_cache:
+          keys:
+            - ds054-anat-v12-{{ .Branch }}-{{ .Revision }}
             - ds054-anat-v12-{{ .Branch }}
             - ds054-anat-v12-master
             - ds054-anat-v12-
@@ -527,7 +529,7 @@ jobs:
             rm -rf /tmp/ds054/work/reportlets
             rm -rf /tmp/ds054/derivatives/fmriprep
       - save_cache:
-         key: ds054-anat-v12-{{ .Branch }}-{{ epoch }}
+         key: ds054-anat-v12-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
          paths:
             - /tmp/ds054/work
 
@@ -613,8 +615,14 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - ds210-anat-v10-{{ .Branch }}-{{ epoch }}
-            - ds210-anat-v10-{{ .Branch }}
+            - docker-v3-{{ .Branch }}-{{ .Revision }}
+      - restore_cache:
+          keys:
+            - data-v6-{{ .Revision }}
+      - restore_cache:
+          keys:
+            - ds210-anat-v10-{{ .Branch }}-{{ .Revision }}
+            - ds210-anat-v10-{{ .Branch }}-
             - ds210-anat-v10-master
             - ds210-anat-v10-
       - run:
@@ -656,7 +664,7 @@ jobs:
             rm -rf /tmp/ds210/work/reportlets
             rm -rf /tmp/ds210/derivatives/fmriprep
       - save_cache:
-         key: ds210-anat-v10-{{ .Branch }}-{{ epoch }}
+         key: ds210-anat-v10-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
          paths:
             - /tmp/ds210/work
 
@@ -700,8 +708,9 @@ jobs:
     working_directory: /tmp/src/fmriprep
     steps:
 
-      - attach_workspace:
-          at: /tmp
+      - restore_cache:
+          keys:
+            - docker-v3-{{ .Branch }}-{{ .Revision }}
       - run:
           name: Load Docker image layer cache
           no_output_timeout: 30m
@@ -728,9 +737,9 @@ jobs:
       image: circleci/classic:201711-01
     working_directory: /tmp/src/fmriprep
     steps:
-
-      - attach_workspace:
-          at: /tmp
+      - restore_cache:
+          keys:
+            - docker-v3-{{ .Branch }}-{{ .Revision }}
       - run:
           name: Load Docker image layer cache
           no_output_timeout: 30m
@@ -937,18 +946,6 @@ workflows:
             tags:
               only: /.*/
 
-      - update_cache:
-          requires:
-            - build
-          filters:
-            branches:
-              ignore:
-                - /docs?\/.*/
-                - /tests?\/.*/
-                - /docker\/.*/
-            tags:
-              only: /.*/
-
       - test_pytest:
           requires:
             - build