diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml
index c81750d49f92..1d3ec43f4717 100644
--- a/.gitlab/pipeline/test.yml
+++ b/.gitlab/pipeline/test.yml
@@ -34,7 +34,7 @@ test-linux-stable:
         --locked \
         --release \
         --no-fail-fast \
-        --features runtime-benchmarks,try-runtime,experimental \
+        --features try-runtime,experimental \
         --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
     # run runtime-api tests with `enable-staging-api` feature on the 1st node
     - if [ ${CI_NODE_INDEX} == 1 ]; then time cargo nextest run -p sp-api-test --features enable-staging-api; fi
@@ -67,11 +67,13 @@ test-linux-oldkernel-stable:
 #         --locked \
 #         --release \
 #         --no-fail-fast \
-#         --features runtime-benchmarks,try-runtime \
+#         --features try-runtime \
 #         --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
 #     # todo: add flacky-test collector
 
 # for some reasons these tests fail if run with all tests
+# We also include running the autogenerated benchmark tests here
+# as we do not want to have runtime-benchmarks feature enabled in general.
 test-linux-stable-additional-tests:
   stage: test
   extends:
@@ -87,7 +89,7 @@ test-linux-stable-additional-tests:
   script:
     - |
       time cargo nextest run \
-        -E 'test(receive_rate_limit_is_enforced) + test(benchmark_block_works)' \
+        -E 'test(receive_rate_limit_is_enforced) + test(benchmark_block_works) + test(benchmark)' \
         --workspace \
         --locked \
         --release \
@@ -116,7 +118,7 @@ test-linux-stable-slow:
         --workspace \
         --locked \
         --release \
-        --features runtime-benchmarks,try-runtime
+        --features try-runtime
   allow_failure: true
 
 # takes about 1,5h without cache
@@ -139,7 +141,7 @@ test-linux-stable-slow:
 #     - time cargo test --workspace
 #       --locked
 #       --profile testnet
-#       --features=runtime-benchmarks,runtime-metrics,try-runtime --
+#       --features=runtime-metrics,try-runtime --
 #       --skip upgrade_version_checks_should_work
 
 test-doc: