Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test Flaky summary report #26

Merged
merged 1 commit into from
Sep 25, 2024
Merged

Test Flaky summary report #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions src/main/java/io/quarkus/qe/reporter/flakyrun/summary/FlakyRunSummaryReporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,18 @@
import static java.util.stream.Collectors.groupingBy;

public class FlakyRunSummaryReporter {
public static final String TEST_BASE_DIR = FlakyRunSummaryReporter.class.getSimpleName() + ".test-base-dir";
public static final String FLAKY_SUMMARY_REPORT = "flaky-summary-report.json";
public static final String CI_BUILD_NUMBER = "flaky-report-ci-build-number";
public static final String DAY_RETENTION = "day-retention";
private static final Path CURRENT_DIR = Path.of(".");
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
private static final String DAY_RETENTION = "day-retention";
private static final String MAX_FLAKES_PER_TEST = "max-flakes-per-test";
private static final String PREVIOUS_SUMMARY_REPORT_PATH = "previous-summary-report-path";
private static final String NEW_SUMMARY_REPORT_PATH = "new-summary-report-path";
private static final String NEW_FLAKY_REPORT_PATH = "new-flaky-report-path";
private static final String EQUALS = "=";
private static final String CI_JOB_NAME = "flaky-report-ci-job-name";
private static final String CI_BUILD_NUMBER = "flaky-report-ci-build-number";
private final int dayRetention;
private final int maxFlakesPerTest;
private final Path newBuildReportPath;
Expand All @@ -42,9 +43,15 @@ public class FlakyRunSummaryReporter {
public FlakyRunSummaryReporter(String[] args) {
int dayRetention = 30;
int maxFlakesPerTest = 50;
Path newBuildReportPath = CURRENT_DIR.resolve(FlakyRunReporter.FLAKY_RUN_REPORT);
Path previousSummaryReportPath = CURRENT_DIR.resolve(FLAKY_SUMMARY_REPORT);
Path newSummaryReportPath = CURRENT_DIR.resolve(FLAKY_SUMMARY_REPORT);
final Path baseDir;
if (System.getProperty(TEST_BASE_DIR) != null) {
baseDir = Path.of(System.getProperty(TEST_BASE_DIR));
} else {
baseDir = CURRENT_DIR;
}
Path newBuildReportPath = baseDir.resolve(FlakyRunReporter.FLAKY_RUN_REPORT);
Path previousSummaryReportPath = baseDir.resolve(FLAKY_SUMMARY_REPORT);
Path newSummaryReportPath = baseDir.resolve(FLAKY_SUMMARY_REPORT);
String ciJobName = "";
int ciJobBuildNumber = -1;
for (String arg : args) {
Expand Down
36 changes: 35 additions & 1 deletion src/test/java/io/quarkus/qe/reporter/flakyrun/FlakyRunReportTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.quarkus.qe.reporter.flakyrun;

import io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter;
import org.apache.maven.shared.invoker.DefaultInvocationRequest;
import org.apache.maven.shared.invoker.DefaultInvoker;
import org.apache.maven.shared.invoker.InvocationRequest;
Expand All @@ -21,6 +22,11 @@
import java.util.Properties;

import static io.quarkus.qe.reporter.flakyrun.reporter.FlakyRunReporter.FLAKY_RUN_REPORT;
import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.CI_BUILD_NUMBER;
import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.DAY_RETENTION;
import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.FLAKY_SUMMARY_REPORT;
import static io.quarkus.qe.reporter.flakyrun.summary.FlakyRunSummaryReporter.TEST_BASE_DIR;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;

public class FlakyRunReportTest {
Expand All @@ -47,10 +53,34 @@ public void test() throws IOException, MavenInvocationException {
}

assertGeneratedFlakyRunReport();
assertFlakyRunSummary();
}

private void assertFlakyRunSummary() throws IOException {
// use old summary I downloaded from Jenkins, if format changes, it needs to change as well
var oldSummary = new File("src/test/resources/flaky-summary-report.json");
var summaryTarget = new File(TARGET_FLAKY_TEST_DIR, "target/" + FLAKY_SUMMARY_REPORT);
FileUtils.copyFile(oldSummary, summaryTarget);
var previousValue = Files.readString(summaryTarget.toPath());
assertTrue(previousValue.contains("PicocliDevIT.verifyGreetingCommandOutputsExpectedMessage"), previousValue);
assertFalse(previousValue.contains("FlakyTest.testFlaky"), previousValue);

System.setProperty(TEST_BASE_DIR, getFlakyRunReportFile().getParent());
// making it maximal day retention because the old message needs to be valid for this test to pass
var expectedBuildNumber = "987654321";
new FlakyRunSummaryReporter(
new String[] { CI_BUILD_NUMBER + "=" + expectedBuildNumber, DAY_RETENTION + "=" + Integer.MAX_VALUE })
.createReport();

// now assert the old summary and new flaky run report were merged
var newValue = Files.readString(summaryTarget.toPath());
assertTrue(newValue.contains("PicocliDevIT.verifyGreetingCommandOutputsExpectedMessage"), newValue);
assertTrue(newValue.contains("FlakyTest.testFlaky"), newValue);
assertTrue(newValue.contains(expectedBuildNumber), newValue);
}

private static void assertGeneratedFlakyRunReport() throws IOException {
var flakyRunReport = new File(TARGET_FLAKY_TEST_DIR, "target/" + FLAKY_RUN_REPORT);
var flakyRunReport = getFlakyRunReportFile();
assertTrue(flakyRunReport.exists(),
"Flaky Run report wasn't generated, '%s' does not exist".formatted(flakyRunReport));
var reportContent = Files.readString(flakyRunReport.toPath());
Expand All @@ -63,6 +93,10 @@ private static void assertGeneratedFlakyRunReport() throws IOException {
assertReportContent(reportContent, "dateTime");
}

private static File getFlakyRunReportFile() {
return new File(TARGET_FLAKY_TEST_DIR, "target/" + FLAKY_RUN_REPORT);
}

private static void assertReportContent(String reportContent, String expectedPortion) {
assertTrue(reportContent.contains(expectedPortion), reportContent);
}
Expand Down
Loading
Loading