From 9f525f9835bf1cb1fac0f15002c7f586a1ada7f4 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Thu, 31 Aug 2023 14:00:25 +0200 Subject: [PATCH 1/9] [ci] Remove runtime-benchmarks from tests --- .gitlab/pipeline/test.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index c81750d49f92..afb3079c650e 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -34,7 +34,7 @@ test-linux-stable: --locked \ --release \ --no-fail-fast \ - --features runtime-benchmarks,try-runtime,experimental \ + --features try-runtime,experimental \ --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} # run runtime-api tests with `enable-staging-api` feature on the 1st node - if [ ${CI_NODE_INDEX} == 1 ]; then time cargo nextest run -p sp-api-test --features enable-staging-api; fi @@ -45,6 +45,22 @@ test-linux-oldkernel-stable: tags: - oldkernel-vm +# https://github.com/paritytech/ci_cd/issues/864 +test-linux-stable-runtime-benchmarks: + stage: test + extends: + - .docker-env + - .common-refs + - .run-immediately + - .pipeline-stopper-artifacts + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - cargo nextest run --features runtime-benchmarks benchmarking + # can be used to run all tests # test-linux-stable-all: # stage: test From 2212cd62eb393ef98c997a54ad1f57dff0699b6f Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Mon, 4 Sep 2023 09:53:49 +0200 Subject: [PATCH 2/9] Update .gitlab/pipeline/test.yml Co-authored-by: Oliver Tale-Yazdi --- .gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index afb3079c650e..454549a0de63 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -59,7 +59,7 @@ test-linux-stable-runtime-benchmarks: # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - - cargo nextest run --features runtime-benchmarks benchmarking + - cargo nextest run --features runtime-benchmarks benchmarking --locked -q --profile testnet # can be used to run all tests # test-linux-stable-all: From c1291ad8ff339caf92b84bbad96bd442c21cf1d8 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 4 Sep 2023 09:55:22 +0200 Subject: [PATCH 3/9] remove ing --- .gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 454549a0de63..fb52d7b52f2a 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -59,7 +59,7 @@ test-linux-stable-runtime-benchmarks: # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - - cargo nextest run --features runtime-benchmarks benchmarking --locked -q --profile testnet + - cargo nextest run --features runtime-benchmarks benchmark --locked -q --profile testnet # can be used to run all tests # test-linux-stable-all: From 1003386a46af37626ec7e3b6838d14a722a93913 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 4 Sep 2023 10:25:48 +0200 Subject: [PATCH 4/9] move benchmark tests to additional tests --- .gitlab/pipeline/test.yml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index fb52d7b52f2a..85b4f123a4cb 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -45,22 +45,6 @@ test-linux-oldkernel-stable: tags: - oldkernel-vm -# https://github.com/paritytech/ci_cd/issues/864 -test-linux-stable-runtime-benchmarks: - stage: test - extends: - - .docker-env - - .common-refs - - .run-immediately - - .pipeline-stopper-artifacts - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - script: - - cargo nextest run --features runtime-benchmarks benchmark --locked -q --profile testnet - # can be used to run all tests # test-linux-stable-all: # stage: test @@ -108,7 +92,8 @@ test-linux-stable-additional-tests: --locked \ --release \ --features runtime-benchmarks,try-runtime - allow_failure: true + # https://github.com/paritytech/ci_cd/issues/864 + - time cargo nextest run --features runtime-benchmarks benchmark --locked -q --profile testnet # these ones can be really slow so it's better to run them separately test-linux-stable-slow: From 3bc9f208026f4ef27c11cb571c776a966a99352b Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 4 Sep 2023 11:26:49 +0200 Subject: [PATCH 5/9] rm -q option --- .gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 85b4f123a4cb..8d7d1671cbbd 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -93,7 +93,7 @@ test-linux-stable-additional-tests: --release \ --features runtime-benchmarks,try-runtime # https://github.com/paritytech/ci_cd/issues/864 - - time cargo nextest run --features runtime-benchmarks benchmark --locked -q --profile testnet + - time cargo nextest run --features runtime-benchmarks benchmark --locked --profile testnet # these ones can be really slow so it's better to run them separately test-linux-stable-slow: From 7b77087d17184f6d626e6ae8e27ab0d649efc733 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 4 Sep 2023 12:13:46 +0200 Subject: [PATCH 6/9] try release profile --- .gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 8d7d1671cbbd..95f22d0e3836 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -93,7 +93,7 @@ test-linux-stable-additional-tests: --release \ --features runtime-benchmarks,try-runtime # https://github.com/paritytech/ci_cd/issues/864 - - time cargo nextest run --features runtime-benchmarks benchmark --locked --profile testnet + - time cargo nextest run --features runtime-benchmarks benchmark --locked --release # these ones can be really slow so it's better to run them separately test-linux-stable-slow: From d2619fd69d8e36512503f0ced038c5fa9cc87e1a Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 4 Sep 2023 13:40:18 +0200 Subject: [PATCH 7/9] use testnet profile --- .gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 95f22d0e3836..dfcd31223f0a 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -93,7 +93,7 @@ test-linux-stable-additional-tests: --release \ --features runtime-benchmarks,try-runtime # https://github.com/paritytech/ci_cd/issues/864 - - time cargo nextest run --features runtime-benchmarks benchmark --locked --release + - time cargo nextest run --features runtime-benchmarks benchmark --locked --cargo-profile testnet # these ones can be really slow so it's better to run them separately test-linux-stable-slow: From b9508dbbe879c103c7260519e79ca492b99fe145 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 4 Sep 2023 15:51:28 +0200 Subject: [PATCH 8/9] move to a separate job --- .gitlab/pipeline/test.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index dfcd31223f0a..be4ad76c52c4 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -95,6 +95,22 @@ test-linux-stable-additional-tests: # https://github.com/paritytech/ci_cd/issues/864 - time cargo nextest run --features runtime-benchmarks benchmark --locked --cargo-profile testnet +# https://github.com/paritytech/ci_cd/issues/864 +test-linux-stable-runtime-benchmarks: + stage: test + extends: + - .docker-env + - .common-refs + - .run-immediately + - .pipeline-stopper-artifacts + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo nextest run --features runtime-benchmarks benchmark --locked --cargo-profile testnet + # these ones can be really slow so it's better to run them separately test-linux-stable-slow: stage: test From 1594e2704508c6e2095d691f7897c749df3ee2e4 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Mon, 4 Sep 2023 16:09:34 +0200 Subject: [PATCH 9/9] rm dup --- .gitlab/pipeline/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index be4ad76c52c4..d128cb099021 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -92,8 +92,6 @@ test-linux-stable-additional-tests: --locked \ --release \ --features runtime-benchmarks,try-runtime - # https://github.com/paritytech/ci_cd/issues/864 - - time cargo nextest run --features runtime-benchmarks benchmark --locked --cargo-profile testnet # https://github.com/paritytech/ci_cd/issues/864 test-linux-stable-runtime-benchmarks: