diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8a5984c1683..cc5836e1809 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,8 @@ stages:
   - finalize-status
 
 include:
-  - local: '.gitlab-condition.yml'
+  - local: '.gitlab/condition.yml'
+  - local: '.gitlab/image.yml'
 
 # Templates with reasonable defaults for builds and tests
 .variables_template: &default_variables
@@ -40,7 +41,6 @@ include:
 .before_script_template: &default_before_script
   - export NUM_CORES=${CI_PARALLELISM}
   - export OMP_NUM_THREADS=${NUM_CORES}
-  - export CUDA_VISIBLE_DEVICES=$((RANDOM % 2))
   - export CCACHE_DIR=${CCACHE_DIR}
   - export CCACHE_MAXSIZE=${CCACHE_MAXSIZE}
 
@@ -62,6 +62,7 @@ include:
     - if [ -n "${CUDA_ARCH}" ]; then
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
+      export CUDA_VISIBLE_DEVICES=$((RANDOM % 2));
       fi
     - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then export SYCL_DEVICE_TYPE; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
@@ -95,6 +96,7 @@ include:
     - if [ -n "${CUDA_ARCH}" ]; then
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
+      export CUDA_VISIBLE_DEVICES=$((RANDOM % 2));
       fi
     - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then export SYCL_DEVICE_TYPE; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
@@ -149,56 +151,54 @@ include:
 
 status_pending:
   stage: init-status
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .pr_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     STATUS_CONTEXT: "quick"
   script: |
     curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
     https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
     -d "{\"state\":\"pending\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
-  extends:
-    - .pr_condition
-  dependencies: []
-  tags:
-    - private_ci
-    - cpu
 
 status_success:
   stage: finalize-status
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .pr_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     STATUS_CONTEXT: "quick"
+  # we always exit with the code 3 such that it will process when retrying
   script: |
     curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
     https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
     -d "{\"state\":\"success\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
-  extends:
-    - .pr_condition
-  dependencies: []
-  tags:
-    - private_ci
-    - cpu
+    exit 3
+  allow_failure:
+    exit_codes: 3
 
 status_failure:
   stage: finalize-status
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .pr_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     STATUS_CONTEXT: "quick"
+  # we always exit with the code 3 such that it will process when retrying
   script: |
     curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
     https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
     -d "{\"state\":\"failure\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
+    exit 3
   when: on_failure
-  extends:
-    - .pr_condition
-  dependencies: []
-  tags:
-    - private_ci
-    - cpu
+  allow_failure:
+    exit_codes: 3
+
 
 sync:
   stage: sync
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     GIT_STRATEGY: none
     PRIVATE_REPO: git@gitlab.com:ginkgo-project/ginkgo.git
@@ -215,13 +215,12 @@ sync:
     - develop
   except:
     - schedules
-  tags:
-    - private_ci
-    - cpu
 
 trigger_pipeline:
   stage: trigger_pipeline
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .pr_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     STATUS_CONTEXT: "quick"
   script:
@@ -231,7 +230,7 @@ trigger_pipeline:
       if [[ "${PR_ID}" != "null" ]]; then
         echo "Find the corresponding Pull Request - ${PR_ID}"
         echo "Check whether the PR contains ST:ready-to-merge or ST:run-full-test label"
-        ENABLE_FULL_PIPELINE=$(curl -X GET -s -H "Accept: application/vnd.github.v3+json" \
+        ENABLE_FULL_PIPELINE=$(curl -X GET -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
           "https://api.github.com/repos/ginkgo-project/ginkgo/issues/${PR_ID}" | jq -r \
           'any( [.labels | .[] | .name ] | .[] ; . == "ST:ready-to-merge" or . == "ST:run-full-test")')
         if [[ "${ENABLE_FULL_PIPELINE}" == "true" ]]; then
@@ -244,21 +243,18 @@ trigger_pipeline:
       else
         echo "Can not find the corresponding Pull Request"
       fi
-  extends:
-    - .pr_condition
+  # Override variables condition
   only:
     variables:
       - $RUN_CI_TAG && $STATUS_CONTEXT == "quick"
-  dependencies: []
-  tags:
-    - private_ci
-    - cpu
 
 # Build jobs
 # Job with example runs.
 build/cuda90/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda90-gnu5-llvm39
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -267,16 +263,12 @@ build/cuda90/gcc/all/debug/shared:
     FAST_TESTS: "ON"
     RUN_EXAMPLES: "ON"
     CUDA_ARCH: 35
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda90/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda90-gnu5-llvm39
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -286,17 +278,13 @@ build/cuda90/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # cuda 9.1 and friends
 build/cuda91/gcc/all/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda91-gnu6-llvm40
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -305,16 +293,12 @@ build/cuda91/gcc/all/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda91/clang/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda91-gnu6-llvm40
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -323,18 +307,14 @@ build/cuda91/clang/all/release/shared:
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 
 # cuda 9.2 and friends
-build/cuda92/gcc/all/release/debug:
+build/cuda92/gcc/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -342,18 +322,14 @@ build/cuda92/gcc/all/release/debug:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Make sure that our jobs run when HWLOC is
 # forcibly switched off
 build/cuda92/intel/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -364,12 +340,6 @@ build/cuda92/intel/cuda/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Build CUDA NVIDIA without omp
 build/cuda92/intel/cuda_wo_omp/release/shared:
@@ -396,7 +366,9 @@ build/cuda92/intel/cuda_wo_omp/release/shared:
 # third-party HWLOC.
 build/cuda100/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -405,16 +377,12 @@ build/cuda100/gcc/all/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     CUDA_ARCH: 35
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda100/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -425,16 +393,12 @@ build/cuda100/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda100/intel/cuda/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -443,17 +407,13 @@ build/cuda100/intel/cuda/release/shared:
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # cuda 10.1 and friends
 build/cuda101/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -462,16 +422,12 @@ build/cuda101/gcc/all/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda101/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -482,17 +438,13 @@ build/cuda101/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # clang-cuda with cuda 10.1 and friends
 build/clang-cuda101/gcc/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     CUDA_COMPILER: "clang++"
@@ -501,16 +453,12 @@ build/clang-cuda101/gcc/all/release/shared:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/clang-cuda101/clang/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -522,19 +470,15 @@ build/clang-cuda101/clang/cuda/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # cuda 10.2 and friends
 
 # works when there is no hwloc and tpl hwloc is also switched off.
 build/cuda102/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -544,17 +488,13 @@ build/cuda102/gcc/all/debug/shared:
     FAST_TESTS: "ON"
     BUILD_HWLOC: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Use TPL hwloc when no system hwloc is available
 build/cuda102/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -565,16 +505,12 @@ build/cuda102/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda102/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -585,17 +521,13 @@ build/cuda102/intel/cuda/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # cuda 11.0 and friends
 build/cuda110/gcc/cuda/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -603,16 +535,12 @@ build/cuda110/gcc/cuda/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda110/clang/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -622,16 +550,12 @@ build/cuda110/clang/cuda/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda110/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -642,17 +566,13 @@ build/cuda110/intel/cuda/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # HIP AMD
 build/amd/gcc/hip/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-amd-gnu8-llvm7
+  extends:
+    - .quick_test_condition
+    - .use_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -660,16 +580,12 @@ build/amd/gcc/hip/debug/shared:
     RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - amd
-    - gpu
 
 build/amd/clang/hip/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-amd-gnu8-llvm7
+  extends:
+    - .quick_test_condition
+    - .use_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -678,12 +594,6 @@ build/amd/clang/hip/release/static:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - amd
-    - gpu
 
 # Build HIP AMD without omp
 build/amd/clang/hip_wo_omp/release/shared:
@@ -706,7 +616,9 @@ build/amd/clang/hip_wo_omp/release/shared:
 # no cuda but latest gcc and clang
 build/nocuda/gcc/core/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_REFERENCE: "OFF"
@@ -714,30 +626,24 @@ build/nocuda/gcc/core/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     BUILD_HWLOC: "OFF"
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cpu
 
 build/nocuda/clang/core/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
     CXX_COMPILER: "clang++"
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Release"
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cpu
 
 build/nocuda/intel/core/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8-intel
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -745,29 +651,22 @@ build/nocuda/intel/core/debug/shared:
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - cpu
 
 build/nocuda/gcc/omp/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_TYPE: "Release"
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cpu
 
 build/nocuda/clang/omp/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .full_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -776,15 +675,12 @@ build/nocuda/clang/omp/debug/static:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cpu
 
 build/nocuda/intel/omp/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8-intel
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -792,16 +688,12 @@ build/nocuda/intel/omp/release/static:
     BUILD_OMP: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - cpu
 
 build/dpcpp/cpu/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-oneapi
+  extends:
+    - .quick_test_condition
+    - .use_gko-oneapi
   variables:
     <<: *default_variables
     C_COMPILER: "gcc"
@@ -810,131 +702,99 @@ build/dpcpp/cpu/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     SYCL_DEVICE_TYPE: "CPU"
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - cpu
 
 # Job with important warnings as error
 warnings:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     CXX_FLAGS: "-Werror=pedantic -pedantic-errors"
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: yes
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Ensure kernel modules do not depend on core
 no-circular-deps:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     EXTRA_CMAKE_FLAGS: '-DGINKGO_CHECK_CIRCULAR_DEPS=on'
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: no
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Ensure Ginkgo builds from a subdirectory
 subdir-build:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .full_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     CI_PROJECT_PATH_SUFFIX: "/test_subdir"
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: no
-  tags:
-    - private_ci
-    - cpu
 
 # Ensure Ginkgo can be used when exporting the build directory
 export-build:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .full_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     EXPORT_BUILD_DIR: "ON"
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: no
-  tags:
-    - private_ci
-    - cpu
 
 # Run clang-tidy and iwyu
 clang-tidy:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     EXTRA_CMAKE_FLAGS: '-DGINKGO_WITH_CLANG_TIDY=ON'
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: yes
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 iwyu:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_CUDA: "HIP"
     EXTRA_CMAKE_FLAGS: '-DGINKGO_WITH_IWYU=ON'
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: yes
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Code analysis, coverage and reporting tool
 # For short living branches or PRs, try to detect an open PR
 sonarqube_cov_:
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - PR_ID=$(curl "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
@@ -957,24 +817,19 @@ sonarqube_cov_:
       -Dsonar.cfamily.gcov.reportsPath=build/Testing/CoverageInfo
       ${sonar_branching}
     - bash <(curl -s https://codecov.io/bash) -f "\!*examples*" -f "\!*third_party*" -f "\!*c\\+\\+*" -f "\!*benchmark*"
-  dependencies: []
   except:
     refs:
       - develop
       - master
       - tags
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # For long living branches, do not detect the PR. A PR would always be detected
 # (the one that was merged).
 sonarqube_cov:
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=COVERAGE
@@ -984,19 +839,14 @@ sonarqube_cov:
       -Dsonar.cfamily.gcov.reportsPath=build/Testing/CoverageInfo
       -Dsonar.branch.name=${CI_COMMIT_REF_NAME}
     - bash <(curl -s https://codecov.io/bash) -f "\!*examples*" -f "\!*third_party*" -f "\!*c\\+\\+*" -f "\!*benchmark*"
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 
 # Deploy documentation to github-pages
 gh-pages:
   stage: deploy
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .deploy_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     PUBLIC_REPO: git@github.com:ginkgo-project/ginkgo.git
@@ -1026,17 +876,13 @@ gh-pages:
     - git add -A
     - git diff --quiet HEAD ||
       (git commit -m "Update documentation from ${CURRENT_SHA}" && git push)
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cpu
 
 
 threadsanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - LD_PRELOAD=/usr/local/lib/libomp.so
@@ -1045,75 +891,48 @@ threadsanitizer:
         -DCTEST_MEMORYCHECK_TYPE=ThreadSanitizer
         -DCTEST_MEMORYCHECK_SANITIZER_OPTIONS=ignore_noninstrumented_modules=1
         --timeout 6000
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 leaksanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=LSAN
       -DCTEST_MEMORYCHECK_TYPE=LeakSanitizer
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 addresssanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=ASAN
       -DCTEST_MEMORYCHECK_TYPE=AddressSanitizer
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 undefinedsanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     # the Gold linker is required because of a linker flag issues given by UBsan
     # in the Ubuntu setup we are using.
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=UBSAN
       -DCTEST_MEMORYCHECK_TYPE=UndefinedBehaviorSanitizer
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 cudamemcheck:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=RelWithDebInfo
       -DCTEST_MEMORYCHECK_TYPE=CudaMemcheck
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Benchmark build
 .benchmark_before_script_template: &default_benchmark_before_script
@@ -1136,7 +955,8 @@ cudamemcheck:
 
 fineci-benchmark-build:
   stage: benchmark-build
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .use_gko-nocuda-gnu9-llvm8-intel
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1175,10 +995,6 @@ fineci-benchmark-build:
     - schedules
 #    - develop
 #    - master
-  tags:
-    - private_ci
-    - cpu
-    - cuda
 
 
 # Benchmark runs
@@ -1217,7 +1033,8 @@ fineci-benchmark-build:
 
 fineci-benchmark-cuda:
   stage: benchmark-cuda
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .use_gko-nocuda-gnu9-llvm8-intel
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1226,10 +1043,6 @@ fineci-benchmark-cuda:
     BENCHMARK_REPO: git@github.com:ginkgo-project/ginkgo-data.git
     SYSTEM_NAME: K20Xm
   <<: *default_benchmark
-  tags:
-    - private_ci
-    - cpu
-    - cuda
 
 # fineci-benchmark-omp:
 #   stage: benchmark-omp
@@ -1255,7 +1068,8 @@ fineci-benchmark-cuda:
 
 new-issue-on-failure:
   stage: on-failure
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .use_gko-nocuda-gnu9-llvm8
   script: curl --request POST "https://gitlab.com/api/v4/projects/${PROJECT_ID}/issues?private_token=${BOT_ACCESS_TOKEN}&title=Error%20in%20${CI_PROJECT_NAME}%20with%20pipeline%20${CI_PIPELINE_ID}%20for%20commit%20${CI_COMMIT_SHA}&labels&description=${CI_PIPELINE_URL}"
   when: on_failure
   only:
@@ -1263,6 +1077,3 @@ new-issue-on-failure:
       - develop
       - master
   dependencies: []
-  tags:
-    - private_ci
-    - cpu
diff --git a/.gitlab-condition.yml b/.gitlab/condition.yml
similarity index 88%
rename from .gitlab-condition.yml
rename to .gitlab/condition.yml
index b9162544bca..fffb88738a5 100644
--- a/.gitlab-condition.yml
+++ b/.gitlab/condition.yml
@@ -7,6 +7,7 @@
       - develop
       - master
       - tags
+  dependencies: []
 
 .full_test_condition:
   only:
@@ -14,11 +15,13 @@
         - $RUN_CI_TAG && $STATUS_CONTEXT == "full"
         - $RUN_CI_TAG && ($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop")
         - $RUN_CI_TAG && $CI_COMMIT_TAG
+  dependencies: []
 
 .quick_test_condition:
   only:
     variables:
       - $RUN_CI_TAG && $STATUS_CONTEXT == null
+  dependencies: []
 
 .deploy_condition:
   only:
@@ -30,3 +33,4 @@
       - $RUN_CI_TAG
   except:
       - schedules
+  dependencies: []
diff --git a/.gitlab/image.yml b/.gitlab/image.yml
new file mode 100644
index 00000000000..3fca4782061
--- /dev/null
+++ b/.gitlab/image.yml
@@ -0,0 +1,82 @@
+.use_gko-nocuda-gnu9-llvm8:
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  tags:
+    - private_ci
+    - cpu
+
+.use_gko-nocuda-gnu9-llvm8-intel:
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  tags:
+    - private_ci
+    - cpu
+    - cuda
+
+.use_gko-cuda90-gnu5-llvm39:
+  image: localhost:5000/gko-cuda90-gnu5-llvm39
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda91-gnu6-llvm40:
+  image: localhost:5000/gko-cuda91-gnu6-llvm40
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda92-gnu7-llvm50-intel2017:
+  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda100-gnu7-llvm60-intel2018:
+  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda101-gnu8-llvm7-intel2019:
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda101-gnu8-llvm10-intel2019:
+  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda102-gnu8-llvm8-intel2019:
+  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda110-gnu9-llvm9-intel2020:
+  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-amd-gnu8-llvm7:
+  image: localhost:5000/gko-amd-gnu8-llvm7
+  tags:
+    - private_ci
+    - amd
+    - gpu
+
+.use_gko-oneapi:
+  image: localhost:5000/gko-oneapi
+  tags:
+    - private_ci
+    - cuda
+    - cpu