diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d4b98db13b77b..8519d0f88fc7c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -136,7 +136,7 @@ default: # exclude cargo-check-benches from such runs .test-refs-check-benches: rules: - - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_IMAGE =~ /staging$/ + - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_IMAGE =~ /staging$/ when: never - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" @@ -329,10 +329,20 @@ cancel-pipeline-test-linux-stable-int: needs: - job: test-linux-stable-int -cancel-pipeline-cargo-check-subkey: +cancel-pipeline-cargo-check-each-crate-1: extends: .cancel-pipeline-template needs: - - job: cargo-check-subkey + - job: "cargo-check-each-crate 1/2" + +cancel-pipeline-cargo-check-each-crate-2: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 2/2" + +cancel-pipeline-cargo-check-each-crate-macos: + extends: .cancel-pipeline-template + needs: + - job: cargo-check-each-crate-macos cancel-pipeline-check-tracing: extends: .cancel-pipeline-template diff --git a/scripts/ci/gitlab/check-each-crate.sh b/scripts/ci/gitlab/check-each-crate.sh new file mode 100755 index 0000000000000..27e2cf947787a --- /dev/null +++ b/scripts/ci/gitlab/check-each-crate.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +## A script that checks each workspace crate individually. +## It's relevant to check workspace crates individually because otherwise their compilation problems +## due to feature misconfigurations won't be caught, as exemplified by +## https://github.com/paritytech/substrate/issues/12705 + +set -Eeu -o pipefail +shopt -s inherit_errexit + +set -x + +target_group="$1" +groups_total="$2" + +readarray -t workspace_crates < <(\ + cargo tree --workspace --depth 0 | \ + awk '{ if (length($1) == 0 || substr($1, 1, 1) == "[") { skip } else { print $1 } }' +) + +crates_total=${#workspace_crates[*]} + +if [ "$crates_total" -lt "$groups_total" ]; then + # `crates_total / groups_total` would result in 0, so round it up to 1 + crates_per_group=1 +else + # We add `crates_total % groups_total > 0` (which evaluates to 1 in case there's a remainder for + # `crates_total % groups_total`) to round UP `crates_total / groups_total` 's + # potentially-fractional result to the nearest integer. Doing that ensures that we'll not miss any + # crate in case `crates_total / groups_total` would normally result in a fractional number, since + # in those cases Bash would round DOWN the result to the nearest integer. For example, if + # `crates_total = 5` and `groups_total = 2`, then `crates_total / groups_total` would round down + # to 2; since the loop below would then step by 2, we'd miss the 5th crate. + crates_per_group=$(( (crates_total / groups_total) + (crates_total % groups_total > 0) )) +fi + +group=1 +for ((i=0; i < crates_total; i += crates_per_group)); do + if [ $group -eq "$target_group" ]; then + for crate in "${workspace_crates[@]:$i:$crates_per_group}"; do + cargo check --locked --release -p "$crate" + done + break + fi + group=$(( group + 1 )) +done diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index 2e403c10d4520..906c1bcbe242e 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -90,9 +90,6 @@ build-linux-substrate: variables: # this variable gets overriden by "rusty-cachier environment inject", use the value as default CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target" - needs: - - job: cargo-check-subkey - artifacts: false before_script: - mkdir -p ./artifacts/subkey - !reference [.rusty-cachier, before_script] diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 710fbe6227f46..c38eac45d7ba4 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -133,24 +133,8 @@ node-bench-regression-guard: --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA' after_script: [""] -cargo-check-subkey: - stage: test - extends: - - .docker-env - - .test-refs - - .pipeline-stopper-artifacts - script: - - rusty-cachier snapshot create - - cd ./bin/utils/subkey - - SKIP_WASM_BUILD=1 time cargo check --locked --release - - rusty-cachier cache upload - cargo-check-try-runtime: stage: test - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: cargo-check-subkey - artifacts: false extends: - .docker-env - .test-refs @@ -393,6 +377,9 @@ test-full-crypto-feature: test-wasmer-sandbox: stage: test + needs: + - job: cargo-check-wasmer-sandbox + artifacts: false extends: - .docker-env - .test-refs-wasmer-sandbox @@ -409,18 +396,6 @@ test-wasmer-sandbox: - time cargo nextest run --locked --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} - if [ ${CI_NODE_INDEX} == 1 ]; then rusty-cachier cache upload; fi -cargo-check-macos: - stage: test - extends: .test-refs-no-trigger - before_script: - - !reference [.rust-info-script, script] - variables: - SKIP_WASM_BUILD: 1 - script: - - time cargo check --locked --release - tags: - - osx - check-rustdoc: stage: test variables: @@ -435,3 +410,39 @@ check-rustdoc: - rusty-cachier snapshot create - time cargo +nightly doc --locked --workspace --all-features --verbose --no-deps - rusty-cachier cache upload + +cargo-check-each-crate: + stage: test + extends: + - .docker-env + - .test-refs + - .collect-artifacts + - .pipeline-stopper-artifacts + variables: + # $CI_JOB_NAME is set manually so that rusty-cachier can share the cache for all + # "cargo-check-each-crate I/N" jobs + CI_JOB_NAME: cargo-check-each-crate + script: + - rusty-cachier snapshot create + - time ./scripts/ci/gitlab/check-each-crate.sh "$CI_NODE_INDEX" "$CI_NODE_TOTAL" + # need to update cache only from one job + - if [ "$CI_NODE_INDEX" == 1 ]; then rusty-cachier cache upload; fi + parallel: 2 + +cargo-check-each-crate-macos: + stage: test + extends: + - .test-refs + - .collect-artifacts + - .pipeline-stopper-artifacts + before_script: + - !reference [.rust-info-script, script] + - !reference [.pipeline-stopper-vars, script] + variables: + SKIP_WASM_BUILD: 1 + script: + # TODO: enable rusty-cachier once it supports Mac + # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available + - time ./scripts/ci/gitlab/check-each-crate.sh 1 1 + tags: + - osx