ARCH-2011 - Review items

- Add teardown step to neutralize the failing status check - Rename dlls in test files to make status check easier to identify - Update expected-test-results.md with new names - Add note to readme about the overwritten test-results.md
im-open · Feb 26, 2024 · 04c1e14 · 04c1e14
1 parent a7d1946
commit 04c1e14
Show file tree

Hide file tree

Showing 8 changed files with 114 additions and 46 deletions.
diff --git a/.github/workflows/build-and-review-pr.yml b/.github/workflows/build-and-review-pr.yml
@@ -14,6 +14,7 @@ on:
   # and complication we will request those changes be manually made by the PR author.
   pull_request:
     types: [opened, synchronize, reopened]
+  push:
   # paths:
   #   Do not include specific paths here.  We always want this build to run and produce a
   #   status check which are branch protection rules can use.  If this is skipped because of
@@ -764,23 +765,23 @@ jobs:
             const expectedBody = fs.readFileSync('./test-results.md', 'utf8');
 
             const expectedValues = {
-              name: 'status check - dotnet unit tests (widget.tests)',
+              name: 'status check - dotnet unit tests (purposely.failing.tests)',
               status: 'completed',
               conclusion: 'neutral',
-              title: 'WIDGET.TESTS',
+              title: 'PURPOSELY.FAILING.TESTS',
               text: expectedBody
             };
-            const actualCheck = actualStatusChecks.find(check => check.name.includes('widget'));
+            const actualCheck = actualStatusChecks.find(check => check.name.includes('purposely'));
             assertStatusCheckMatchesExpectations(core, actualCheck, expectedValues);
 
       - name: '-------------------------------------------------------------------------------------------------------------'
         run: echo ""
-      - name: '                                    TEST 17 - STATUS CHECK - ACK FAILURES                                    '
+      - name: '                                    TEST 17 - STATUS CHECK - ALLOW FAILURES                                    '
         run: echo ""
 
       - name: 17 - When process-dotnet-test-results is called with test failures & ignore-test-failures=false
         if: always()
-        id: ack-failures
+        id: allow-failures
         uses: ./
         with:
           github-token: '${{ secrets.GITHUB_TOKEN }}'
@@ -792,15 +793,15 @@ jobs:
 
       - name: 17 - Then the action outcome should be success
         if: always()
-        run: ./test/assert-values-match.sh --name "step outcome" --expected "success" --actual "${{ steps.ack-failures.outcome }}"
+        run: ./test/assert-values-match.sh --name "step outcome" --expected "success" --actual "${{ steps.allow-failures.outcome }}"
 
       - name: 17 - And the status-check-ids output should be populated
         if: always()
-        run: ./test/assert-value-is-not-empty.sh --name "status-check-ids output" --value "${{ steps.ack-failures.outputs.status-check-ids }}"
+        run: ./test/assert-value-is-not-empty.sh --name "status-check-ids output" --value "${{ steps.allow-failures.outputs.status-check-ids }}"
 
       - name: 17 - And the test-outcome output should be Failed
         if: always()
-        run: ./test/assert-values-match.sh --name "test-outcome output" --expected "Failed" --actual "${{ steps.ack-failures.outputs.test-outcome }}"
+        run: ./test/assert-values-match.sh --name "test-outcome output" --expected "Failed" --actual "${{ steps.allow-failures.outputs.test-outcome }}"
 
       - name: 17 - And the status check should match the inputs
         if: always()
@@ -811,22 +812,37 @@ jobs:
             const assertStatusCheckExists = require('./test/assert-status-checks-exist.js');
             const assertStatusCheckMatchesExpectations = require('./test/assert-status-check-matches-expectations.js');
 
-            const checkIds = '${{ steps.ack-failures.outputs.status-check-ids }}';
+            const checkIds = '${{ steps.allow-failures.outputs.status-check-ids }}';
             const actualStatusChecks = await assertStatusCheckExists(github, core, checkIds);
             const expectedBody = fs.readFileSync('./test-results.md', 'utf8');
 
             const expectedValues = {
-              name: 'status check - dotnet unit tests (widget.tests)',
+              name: 'status check - dotnet unit tests (purposely.failing.tests)',
               status: 'completed',
               conclusion: 'failure',
-              title: 'WIDGET.TESTS',
+              title: 'PURPOSELY.FAILING.TESTS',
               text: expectedBody
             };
             const actualCheck = actualStatusChecks.find(check => check.name.includes('widget'));
             assertStatusCheckMatchesExpectations(core, actualCheck, expectedValues);
 
       - name: '-------------------------------------------------------------------------------------------------------------'
         run: echo ""
+      - name: '                                                   TEARDOWN                                                  '
+        run: echo ""
+
+      - name: Teardown - Modify failing Status Check conclusion
+        if: always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const updateFailingStatusCheck = require('./test/update-failing-status-check.js');
+
+            await updateFailingStatusCheck(github, core, '${{ steps.allow-failures.outputs.status-check-ids }}');
+
+      - name: '-------------------------------------------------------------------------------------------------------------'
+        run: echo ""
 
   test-pr-comments:
     runs-on: ubuntu-latest

diff --git a/README.md b/README.md
@@ -48,6 +48,8 @@ GitHub does have a size limitation of 65535 characters for a Status Check body o
 
 If you have multiple workflows triggered by the same `pull_request` or `push` event, GitHub creates one checksuite for that commit. The checksuite gets assigned to one of the workflows randomly and all status checks for that commit are reported to that checksuite. That means if there are multiple workflows with the same trigger, your status checks for this action may show on a different workflow run than the run that executed this action.
 
+Currently when a results file is created, the contents are written to `./test-results.md`.  If this action is called multiple times in one job, the file is overwritten each time.  If the contents are needed, they should be retrieved before the next invocation of this action.
+
 ## Action Outputs
 
 ### Pull Request Comment

diff --git a/test/files/failing-test/expected-test-results.md b/test/files/failing-test/expected-test-results.md
@@ -1,5 +1,5 @@
 
-# WIDGET.TESTS
+# PURPOSELY.FAILING.TESTS
 
 ![Generic badge](https://img.shields.io/badge/1/4-FAILED-red.svg)
 <details>
@@ -49,11 +49,11 @@
   </table>
 </details>
 <details>
-  <summary>:x: Widget.Tests.MathTests.OnePlusOneShouldNotEqualFive</summary>
+  <summary>:x: Purposely.Failing.Tests.MathTests.OnePlusOneShouldNotEqualFive</summary>
   <table>
     <tr>
       <th>Name:</th>
-      <td><code>Widget.Tests.MathTests.OnePlusOneShouldNotEqualFive</code></td>
+      <td><code>Purposely.Failing.Tests.MathTests.OnePlusOneShouldNotEqualFive</code></td>
     </tr>
     <tr>
       <th>Outcome:</th>
@@ -73,11 +73,11 @@
     </tr>
     <tr>
       <th>Code Base</th>
-      <td><code>C:\code\widget\bin\Release\net6.0\Widget.Tests.dll</code></td>
+      <td><code>C:\code\widget\bin\Release\net6.0\Purposely.Failing.Tests.dll</code></td>
     </tr>
     <tr>
       <th>Class Name</th>
-      <td><code>Widget.Tests.MathTests</code></td>
+      <td><code>Purposely.Failing.Tests.MathTests</code></td>
     </tr>
     <tr>
       <th>Method Name</th>

diff --git a/test/files/failing-test/failure.trx b/test/files/failing-test/failure.trx
@@ -5,36 +5,36 @@
     <Deployment runDeploymentRoot="myuser_MY-MACHINE_2024-02-21_13_43_46" />
   </TestSettings>
   <Results>
-    <UnitTestResult executionId="9ded150c-bff5-4b72-8799-9bc97925a61a" testId="9af6d3c7-0160-1c91-e2c9-1f3b63cd4ca6" testName="Widget.Tests.MathTests.AddShouldReturnCorrectValue(number1: 960, number2: 4, expectedValue: 964)" computerName="MY-MACHINE" duration="00:00:00.0000344" startTime="2024-02-21T13:43:46.3594854-07:00" endTime="2024-02-21T13:43:46.3594855-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="9ded150c-bff5-4b72-8799-9bc97925a61a" />
-    <UnitTestResult executionId="1da09059-c2e3-488d-9b7b-ce42d8098ef3" testId="be281ffb-a4d6-67ab-d9af-941232c31283" testName="Widget.Tests.MathTests.AddShouldReturnCorrectValue(number1: 1, number2: 2, expectedValue: 3)" computerName="MY-MACHINE" duration="00:00:00.0000121" startTime="2024-02-21T13:43:46.3595538-07:00" endTime="2024-02-21T13:43:46.3595539-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="1da09059-c2e3-488d-9b7b-ce42d8098ef3" />
-    <UnitTestResult executionId="d0645693-6e82-4808-9e27-06ce699950af" testId="9631a370-9720-44c9-70b2-11e51e32ba9f" testName="Widget.Tests.MathTests.AddShouldReturnCorrectValue(number1: 1, number2: 1, expectedValue: 2)" computerName="MY-MACHINE" duration="00:00:00.0077834" startTime="2024-02-21T13:43:46.3551315-07:00" endTime="2024-02-21T13:43:46.3551534-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="d0645693-6e82-4808-9e27-06ce699950af" />
-    <UnitTestResult executionId="49f53981-a391-4a96-b0d1-dfc1f2efbd58" testId="09917f6f-ce84-0a6f-b407-bec561fda4c1" testName="Widget.Tests.MathTests.OnePlusOneShouldNotEqualFive" computerName="MY-MACHINE" duration="00:00:00.0065475" startTime="2024-02-21T13:43:46.3841020-07:00" endTime="2024-02-21T13:43:46.3841023-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Failed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="49f53981-a391-4a96-b0d1-dfc1f2efbd58">
+    <UnitTestResult executionId="9ded150c-bff5-4b72-8799-9bc97925a61a" testId="9af6d3c7-0160-1c91-e2c9-1f3b63cd4ca6" testName="Purposely.Failing.Tests.MathTests.AddShouldReturnCorrectValue(number1: 960, number2: 4, expectedValue: 964)" computerName="MY-MACHINE" duration="00:00:00.0000344" startTime="2024-02-21T13:43:46.3594854-07:00" endTime="2024-02-21T13:43:46.3594855-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="9ded150c-bff5-4b72-8799-9bc97925a61a" />
+    <UnitTestResult executionId="1da09059-c2e3-488d-9b7b-ce42d8098ef3" testId="be281ffb-a4d6-67ab-d9af-941232c31283" testName="Purposely.Failing.Tests.MathTests.AddShouldReturnCorrectValue(number1: 1, number2: 2, expectedValue: 3)" computerName="MY-MACHINE" duration="00:00:00.0000121" startTime="2024-02-21T13:43:46.3595538-07:00" endTime="2024-02-21T13:43:46.3595539-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="1da09059-c2e3-488d-9b7b-ce42d8098ef3" />
+    <UnitTestResult executionId="d0645693-6e82-4808-9e27-06ce699950af" testId="9631a370-9720-44c9-70b2-11e51e32ba9f" testName="Purposely.Failing.Tests.MathTests.AddShouldReturnCorrectValue(number1: 1, number2: 1, expectedValue: 2)" computerName="MY-MACHINE" duration="00:00:00.0077834" startTime="2024-02-21T13:43:46.3551315-07:00" endTime="2024-02-21T13:43:46.3551534-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="d0645693-6e82-4808-9e27-06ce699950af" />
+    <UnitTestResult executionId="49f53981-a391-4a96-b0d1-dfc1f2efbd58" testId="09917f6f-ce84-0a6f-b407-bec561fda4c1" testName="Purposely.Failing.Tests.MathTests.OnePlusOneShouldNotEqualFive" computerName="MY-MACHINE" duration="00:00:00.0065475" startTime="2024-02-21T13:43:46.3841020-07:00" endTime="2024-02-21T13:43:46.3841023-07:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Failed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="49f53981-a391-4a96-b0d1-dfc1f2efbd58">
       <Output>
         <ErrorInfo>
           <Message>Assert.NotEqual() Failure: Values are equal&#xD;
 Expected: Not 2&#xD;
 Actual:       2</Message>
-          <StackTrace>   at Widget.Tests.MathTests.OnePlusOneShouldNotEqualFive() in C:\code\widget\MathTests.cs:line 19</StackTrace>
+          <StackTrace>   at Purposely.Failing.Tests.MathTests.OnePlusOneShouldNotEqualFive() in C:\code\widget\MathTests.cs:line 19</StackTrace>
         </ErrorInfo>
       </Output>
     </UnitTestResult>
   </Results>
   <TestDefinitions>
-    <UnitTest name="Widget.Tests.MathTests.AddShouldReturnCorrectValue(number1: 1, number2: 2, expectedValue: 3)" storage="c:\code\widget\bin\release\net6.0\widget.tests.dll" id="be281ffb-a4d6-67ab-d9af-941232c31283">
+    <UnitTest name="Purposely.Failing.Tests.MathTests.AddShouldReturnCorrectValue(number1: 1, number2: 2, expectedValue: 3)" storage="c:\code\widget\bin\release\net6.0\purposely.failing.tests.dll" id="be281ffb-a4d6-67ab-d9af-941232c31283">
       <Execution id="1da09059-c2e3-488d-9b7b-ce42d8098ef3" />
-      <TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Widget.Tests.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Widget.Tests.MathTests" name="AddShouldReturnCorrectValue" />
+      <TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Purposely.Failing.tests.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Purposely.Failing.Tests.MathTests" name="AddShouldReturnCorrectValue" />
     </UnitTest>
-    <UnitTest name="Widget.Tests.MathTests.AddShouldReturnCorrectValue(number1: 960, number2: 4, expectedValue: 964)" storage="c:\code\widget\bin\release\net6.0\widget.tests.dll" id="9af6d3c7-0160-1c91-e2c9-1f3b63cd4ca6">
+    <UnitTest name="Purposely.Failing.Tests.MathTests.AddShouldReturnCorrectValue(number1: 960, number2: 4, expectedValue: 964)" storage="c:\code\widget\bin\release\net6.0\purposely.failing.tests.dll" id="9af6d3c7-0160-1c91-e2c9-1f3b63cd4ca6">
       <Execution id="9ded150c-bff5-4b72-8799-9bc97925a61a" />
-      <TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Widget.Tests.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Widget.Tests.MathTests" name="AddShouldReturnCorrectValue" />
+      <TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Purposely.Failing.tests.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Purposely.Failing.Tests.MathTests" name="AddShouldReturnCorrectValue" />
     </UnitTest>
-    <UnitTest name="Widget.Tests.MathTests.AddShouldReturnCorrectValue(number1: 1, number2: 1, expectedValue: 2)" storage="c:\code\widget\bin\release\net6.0\widget.tests.dll" id="9631a370-9720-44c9-70b2-11e51e32ba9f">
+    <UnitTest name="Purposely.Failing.Tests.MathTests.AddShouldReturnCorrectValue(number1: 1, number2: 1, expectedValue: 2)" storage="c:\code\widget\bin\release\net6.0\purposely.failing.tests.dll" id="9631a370-9720-44c9-70b2-11e51e32ba9f">
       <Execution id="d0645693-6e82-4808-9e27-06ce699950af" />
-      <TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Widget.Tests.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Widget.Tests.MathTests" name="AddShouldReturnCorrectValue" />
+      <TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Purposely.Failing.tests.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Purposely.Failing.Tests.MathTests" name="AddShouldReturnCorrectValue" />
     </UnitTest>
-    <UnitTest name="Widget.Tests.MathTests.OnePlusOneShouldNotEqualFive" storage="c:\code\widget\bin\release\net6.0\widget.tests.dll" id="09917f6f-ce84-0a6f-b407-bec561fda4c1">
+    <UnitTest name="Purposely.Failing.Tests.MathTests.OnePlusOneShouldNotEqualFive" storage="c:\code\widget\bin\release\net6.0\purposely.failing.tests.dll" id="09917f6f-ce84-0a6f-b407-bec561fda4c1">
       <Execution id="49f53981-a391-4a96-b0d1-dfc1f2efbd58" />
-      <TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Widget.Tests.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Widget.Tests.MathTests" name="OnePlusOneShouldNotEqualFive" />
+      <TestMethod codeBase="C:\code\widget\bin\Release\net6.0\Purposely.Failing.tests.dll" adapterTypeName="executor://xunit/VsTestRunner2/netcoreapp" className="Purposely.Failing.Tests.MathTests" name="OnePlusOneShouldNotEqualFive" />
     </UnitTest>
   </TestDefinitions>
   <TestEntries>
@@ -51,7 +51,7 @@ Actual:       2</Message>
     <Counters total="4" executed="4" passed="3" failed="1" error="0" timeout="0" aborted="0" inconclusive="0" passedButRunAborted="0" notRunnable="0" notExecuted="0" disconnected="0" warning="0" completed="0" inProgress="0" pending="0" />
     <RunInfos>
       <RunInfo computerName="MY-MACHINE" outcome="Error" timestamp="2024-02-21T13:43:46.3794356-07:00">
-        <Text>[xUnit.net 00:00:00.26]     Widget.Tests.MathTests.OnePlusOneShouldNotEqualFive [FAIL]</Text>
+        <Text>[xUnit.net 00:00:00.26]     Purposely.Failing.Tests.MathTests.OnePlusOneShouldNotEqualFive [FAIL]</Text>
       </RunInfo>
     </RunInfos>
   </ResultSummary>