Merge pull request #26 from quarkus-qe/feature/add-flaky-summary-repo…

…rt-test Test Flaky summary report
quarkus-qe · Sep 25, 2024 · 3583af5 · 3583af5
2 parents 61eeb8f + eb8fa63
commit 3583af5
Show file tree

Hide file tree

Showing 3 changed files with 130 additions and 6 deletions.
diff --git a/src/main/java/io/quarkus/qe/reporter/flakyrun/summary/FlakyRunSummaryReporter.java b/src/main/java/io/quarkus/qe/reporter/flakyrun/summary/FlakyRunSummaryReporter.java
@@ -20,17 +20,18 @@
 import static java.util.stream.Collectors.groupingBy;
 
 public class FlakyRunSummaryReporter {
+    public static final String TEST_BASE_DIR = FlakyRunSummaryReporter.class.getSimpleName() + ".test-base-dir";
     public static final String FLAKY_SUMMARY_REPORT = "flaky-summary-report.json";
+    public static final String CI_BUILD_NUMBER = "flaky-report-ci-build-number";
+    public static final String DAY_RETENTION = "day-retention";
     private static final Path CURRENT_DIR = Path.of(".");
     private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
-    private static final String DAY_RETENTION = "day-retention";
     private static final String MAX_FLAKES_PER_TEST = "max-flakes-per-test";
     private static final String PREVIOUS_SUMMARY_REPORT_PATH = "previous-summary-report-path";
     private static final String NEW_SUMMARY_REPORT_PATH = "new-summary-report-path";
     private static final String NEW_FLAKY_REPORT_PATH = "new-flaky-report-path";
     private static final String EQUALS = "=";
     private static final String CI_JOB_NAME = "flaky-report-ci-job-name";
-    private static final String CI_BUILD_NUMBER = "flaky-report-ci-build-number";
     private final int dayRetention;
     private final int maxFlakesPerTest;
     private final Path newBuildReportPath;
@@ -42,9 +43,15 @@ public class FlakyRunSummaryReporter {
     public FlakyRunSummaryReporter(String[] args) {
         int dayRetention = 30;
         int maxFlakesPerTest = 50;
-        Path newBuildReportPath = CURRENT_DIR.resolve(FlakyRunReporter.FLAKY_RUN_REPORT);
-        Path previousSummaryReportPath = CURRENT_DIR.resolve(FLAKY_SUMMARY_REPORT);
-        Path newSummaryReportPath = CURRENT_DIR.resolve(FLAKY_SUMMARY_REPORT);
+        final Path baseDir;
+        if (System.getProperty(TEST_BASE_DIR) != null) {
+            baseDir = Path.of(System.getProperty(TEST_BASE_DIR));
+        } else {
+            baseDir = CURRENT_DIR;
+        }
+        Path newBuildReportPath = baseDir.resolve(FlakyRunReporter.FLAKY_RUN_REPORT);
+        Path previousSummaryReportPath = baseDir.resolve(FLAKY_SUMMARY_REPORT);
+        Path newSummaryReportPath = baseDir.resolve(FLAKY_SUMMARY_REPORT);
         String ciJobName = "";
         int ciJobBuildNumber = -1;
         for (String arg : args) {

diff --git a/src/test/java/io/quarkus/qe/reporter/flakyrun/FlakyRunReportTest.java b/src/test/java/io/quarkus/qe/reporter/flakyrun/FlakyRunReportTest.java
@@ -1,5 +1,6 @@
 package io.quarkus.qe.reporter.flakyrun;
 
+import io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter;
 import org.apache.maven.shared.invoker.DefaultInvocationRequest;
 import org.apache.maven.shared.invoker.DefaultInvoker;
 import org.apache.maven.shared.invoker.InvocationRequest;
@@ -21,6 +22,11 @@
 import java.util.Properties;
 
 import static io.quarkus.qe.reporter.flakyrun.reporter.FlakyRunReporter.FLAKY_RUN_REPORT;
+import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.CI_BUILD_NUMBER;
+import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.DAY_RETENTION;
+import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.FLAKY_SUMMARY_REPORT;
+import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.TEST_BASE_DIR;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class FlakyRunReportTest {
@@ -47,10 +53,34 @@ public void test() throws IOException, MavenInvocationException {
         }
 
         assertGeneratedFlakyRunReport();
+        assertFlakyRunSummary();
+    }
+
+    private void assertFlakyRunSummary() throws IOException {
+        // use old summary I downloaded from Jenkins, if format changes, it needs to change as well
+        var oldSummary = new File("src/test/resources/flaky-summary-report.json");
+        var summaryTarget = new File(TARGET_FLAKY_TEST_DIR, "target/" + FLAKY_SUMMARY_REPORT);
+        FileUtils.copyFile(oldSummary, summaryTarget);
+        var previousValue = Files.readString(summaryTarget.toPath());
+        assertTrue(previousValue.contains("PicocliDevIT.verifyGreetingCommandOutputsExpectedMessage"), previousValue);
+        assertFalse(previousValue.contains("FlakyTest.testFlaky"), previousValue);
+
+        System.setProperty(TEST_BASE_DIR, getFlakyRunReportFile().getParent());
+        // making it maximal day retention because the old message needs to be valid for this test to pass
+        var expectedBuildNumber = "987654321";
+        new FlakyRunSummaryReporter(
+                new String[] { CI_BUILD_NUMBER + "=" + expectedBuildNumber, DAY_RETENTION + "=" + Integer.MAX_VALUE })
+                        .createReport();
+
+        // now assert the old summary and new flaky run report were merged
+        var newValue = Files.readString(summaryTarget.toPath());
+        assertTrue(newValue.contains("PicocliDevIT.verifyGreetingCommandOutputsExpectedMessage"), newValue);
+        assertTrue(newValue.contains("FlakyTest.testFlaky"), newValue);
+        assertTrue(newValue.contains(expectedBuildNumber), newValue);
     }
 
     private static void assertGeneratedFlakyRunReport() throws IOException {
-        var flakyRunReport = new File(TARGET_FLAKY_TEST_DIR, "target/" + FLAKY_RUN_REPORT);
+        var flakyRunReport = getFlakyRunReportFile();
         assertTrue(flakyRunReport.exists(),
                 "Flaky Run report wasn't generated, '%s' does not exist".formatted(flakyRunReport));
         var reportContent = Files.readString(flakyRunReport.toPath());
@@ -63,6 +93,10 @@ private static void assertGeneratedFlakyRunReport() throws IOException {
         assertReportContent(reportContent, "dateTime");
     }
 
+    private static File getFlakyRunReportFile() {
+        return new File(TARGET_FLAKY_TEST_DIR, "target/" + FLAKY_RUN_REPORT);
+    }
+
     private static void assertReportContent(String reportContent, String expectedPortion) {
         assertTrue(reportContent.contains(expectedPortion), reportContent);
     }