[JENKINS-57425] Add profile to run benchmarks only when the benchmark property is set

README.md

@@ -58,6 +58,18 @@ If you had a `jar:test-jar` execution, delete it and add to `properties`:
 You can configure your plugin to treat every commit as a release candidate.
 See [Incrementals](https://github.com/jenkinsci/incrementals-tools) for details.
 
+## Running Benchmarks
+
+To run JMH benchmarks from JUnit tests, you must run you must activate the `benchmark`
+profile. For example:
+```bash
+mvn -Dbenchmark test
+```
+When the `benchmark` property is set, no tests apart from JMH benchmarks will be run.
+The names of the classes containing the benchmark runners should either begin with or
+end with the the word `Benchmark`. For example, `FooBenchmark` and `BenchmarkFoo` will
+be detected when using `-Dbenchmark`, however, `FooBar` will be ignored.
+
 ## Baselines
 
 It is handy to be able to select different Jenkins baselines with a Maven profile.

pom.xml

@@ -1493,5 +1493,28 @@
         </plugins>
       </build>
     </profile>
+    <!-- Profile to run benchmarks when the `benchmark` property is set -->
+    <profile>
+      <id>jmh-benchmark</id>
+      <activation>
+        <property>
+          <name>benchmark</name>
+        </property>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <includes>
+                <include>**/*Benchmark.java</include>
+                <include>**/Benchmark*.java</include>
+              </includes>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
   </profiles>
 </project>

src/it/benchmark/invoker.properties

@@ -0,0 +1 @@
+invoker.goals=-Dbenchmark clean test

src/it/benchmark/pom.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.jenkins-ci.plugins</groupId>
+        <artifactId>plugin</artifactId>
+        <version>@project.version@</version>
+        <relativePath/>
+    </parent>
+    <groupId>org.jenkins-ci.plugins.its</groupId>
+    <artifactId>benchmark-it</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+    <properties>
+        <jenkins.version>2.60.3</jenkins.version>
+        <java.level>8</java.level>
+    </properties>
+    <repositories>
+        <repository>
+            <id>repo.jenkins-ci.org</id>
+            <url>https://repo.jenkins-ci.org/public/</url>
+        </repository>
+    </repositories>
+    <pluginRepositories>
+        <pluginRepository>
+            <id>repo.jenkins-ci.org</id>
+            <url>https://repo.jenkins-ci.org/public/</url>
+        </pluginRepository>
+    </pluginRepositories>
+</project>

src/it/benchmark/postbuild.groovy

@@ -0,0 +1,4 @@
+// check if the benchmark was run
+def file = new File(basedir, 'target/benchmark-run')
+assert file.exists()
+file.deleteDir()

src/it/benchmark/src/test/java/benchmark/DontRunMeTest.java

@@ -0,0 +1,10 @@
+package benchmark;
+
+import org.junit.Test;
+
+public class DontRunMeTest {
+    @Test
+    public void dontRunThis() throws Exception {
+        throw new Exception("Normal tests should not be run with benchmarks.");
+    }
+}

src/it/benchmark/src/test/java/benchmark/TestBenchmark.java

@@ -0,0 +1,16 @@
+package benchmark;
+
+import org.junit.Test;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class TestBenchmark {
+    @Test
+    public void runThis() throws Exception {
+        // create a temporary folder to show that this method has executed
+        Path path = Paths.get("target/benchmark-run");
+        Files.createDirectories(path);
+    }
+}