TW-79582 Failed test report for automatic test retries

Merge-request: TC-MR-8638
Merged-by: Vladislav Ma-iu-shan <Vladislav.Ma-iu-shan@jetbrains.com>

TeamCity.VSTest.TestLogger.Tests/FailedTestsReportWriterTests.cs

@@ -0,0 +1,59 @@
+namespace TeamCity.VSTest.TestLogger.Tests;
+
+using System;
+using System.Text;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+using Moq;
+using Xunit;
+
+public class FailedTestsReportWriterTests
+{
+    private readonly Mock<IOptions> _optionsMock = new();
+    private readonly Mock<IBytesWriterFactory> _bytesWriterFactoryMock = new();
+    private readonly Mock<IBytesWriter> _bytesWriterMock = new();
+
+    public FailedTestsReportWriterTests()
+    {
+        _bytesWriterFactoryMock
+            .Setup(x => x.Create(It.IsAny<string>()))
+            .Returns(_bytesWriterMock.Object);
+    }
+
+    [Fact]
+    public void ShouldNotReportWhenDisabled()
+    {
+        // Arrange
+        _optionsMock
+            .SetupGet(x => x.FailedTestsReportSavePath)
+            .Returns(string.Empty);
+        var writer = new FailedTestsReportWriter(_optionsMock.Object, _bytesWriterFactoryMock.Object);
+
+        // Act
+        writer.ReportFailedTest(CreateTestCase("Test"));
+
+        // Assert
+        _bytesWriterFactoryMock.Verify(x => x.Create(It.IsAny<string>()), Times.Never);
+    }
+
+    [Fact]
+    public void ShouldNotReportSameParameterizedTestTwice()
+    {
+        // Arrange
+        _optionsMock
+            .SetupGet(x => x.FailedTestsReportSavePath)
+            .Returns("path-to-report");
+        var writer = new FailedTestsReportWriter(_optionsMock.Object, _bytesWriterFactoryMock.Object);
+
+        // Act
+        writer.ReportFailedTest(CreateTestCase("SameTest(1)"));
+        writer.ReportFailedTest(CreateTestCase("SameTest(2)"));
+
+        // Assert
+        var expected = Encoding.UTF8.GetBytes("SameTest" + Environment.NewLine);
+        _bytesWriterMock.Verify(x => x.Write(expected), Times.Once);
+        _bytesWriterMock.Verify(x => x.Flush(), Times.Once);
+        _bytesWriterMock.VerifyNoOtherCalls();
+    }
+
+    private static TestCase CreateTestCase(string testName) => new(testName, new Uri("executor://NUnit3TestExecutor"), "Tests");
+}

TeamCity.VSTest.TestLogger.Tests/MessageHandlerTests.cs

@@ -36,9 +36,10 @@ public MessageHandlerTests()
         testNameProvider.Setup(i => i.GetTestName(It.IsAny<string>(), It.IsAny<string>())).Returns<string, string>((fullyQualifiedName, _) => fullyQualifiedName);
 
         var eventRegistry = new Mock<IEventRegistry>();
+        var failedTestsWriter = new Mock<IFailedTestsReportWriter>();
 
         var root = new Root(_lines);
-        _events = new MessageHandler(root, _suiteNameProvider.Object, _attachments.Object, testNameProvider.Object, eventRegistry.Object);
+        _events = new MessageHandler(root, _suiteNameProvider.Object, _attachments.Object, testNameProvider.Object, eventRegistry.Object, failedTestsWriter.Object);
     }
 
     private static TestResultEventArgs CreateTestResult(

TeamCity.VSTest.TestLogger/BytesWriter.cs

@@ -1,9 +1,8 @@
 namespace TeamCity.VSTest.TestLogger;
 
-using System;
 using System.IO;
 
-internal class BytesWriter(string fileName) : IBytesWriter, IDisposable
+internal class BytesWriter(string fileName) : IBytesWriter
 {
     private FileStream? _stream;
     private BinaryWriter? _writer;
@@ -16,11 +15,14 @@ public void Write(byte[] bytes)
 
     public void Flush()
     {
-        EnsureOpened();
-        _writer!.Flush();
+        _writer?.Flush();
     }
 
-    public void Dispose() => _stream?.Dispose();
+    public void Dispose()
+    {
+        _writer?.Flush();
+        _stream?.Dispose();
+    }
 
     private void EnsureOpened()
     {

TeamCity.VSTest.TestLogger/BytesWriterFactory.cs

@@ -0,0 +1,6 @@
+namespace TeamCity.VSTest.TestLogger;
+
+internal class BytesWriterFactory : IBytesWriterFactory
+{
+    public IBytesWriter Create(string fileName) => new BytesWriter(fileName);
+}

TeamCity.VSTest.TestLogger/FailedTestsReportWriter.cs

@@ -0,0 +1,83 @@
+namespace TeamCity.VSTest.TestLogger;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+
+internal class FailedTestsReportWriter : IFailedTestsReportWriter
+{
+    private readonly IOptions _options;
+    private readonly IBytesWriterFactory _bytesWriterFactory;
+    private readonly IBytesWriter? _reportWriter;
+    private readonly HashSet<string> _reportedTests;
+
+    public FailedTestsReportWriter(IOptions options, IBytesWriterFactory bytesWriterFactory)
+    {
+        _options = options;
+        _bytesWriterFactory = bytesWriterFactory;
+        _reportWriter = InitFileMessageWriter();
+        _reportedTests = new HashSet<string>();
+    }
+
+
+    private bool EnsureFailedTestsFileSavePathDirectoryExists()
+    {
+        if (string.IsNullOrEmpty(_options.FailedTestsReportSavePath))
+            return false;
+
+        try
+        {
+            if (!Directory.Exists(_options.FailedTestsReportSavePath))
+                Directory.CreateDirectory(_options.FailedTestsReportSavePath);
+
+            return true;
+        }
+        catch
+        {
+            return false;
+        }
+    }
+
+    public void ReportFailedTest(TestCase testCase)
+    {
+        if (_reportWriter == null)
+            return;
+
+        if (string.IsNullOrWhiteSpace(testCase.FullyQualifiedName))
+            return;
+
+        var testName = GetTestNameForRetry(testCase.FullyQualifiedName);
+        if (!_reportedTests.Add(testName))
+            return;
+
+        var bytesToWrite = Encoding.UTF8.GetBytes(testName + Environment.NewLine);
+        _reportWriter.Write(bytesToWrite);
+        _reportWriter.Flush();
+    }
+
+    /// <summary>
+    /// For MSTest and XUnit FullyQualifiedName is supported as is
+    /// In case of NUnit we have to remove arguments from FullyQualifiedName
+    /// </summary>
+    private string GetTestNameForRetry(string fullyQualifiedName)
+    {
+        var name = fullyQualifiedName.Trim();
+        var argsPosition = name.IndexOf("(", StringComparison.Ordinal);
+        var hasArgs = argsPosition >= 0;
+        return hasArgs ? name.Substring(0, argsPosition) : name;
+    }
+
+    private IBytesWriter? InitFileMessageWriter()
+    {
+        if (!EnsureFailedTestsFileSavePathDirectoryExists())
+            return null;
+
+        var messagesFilePath = Path.Combine(_options.FailedTestsReportSavePath,
+            Guid.NewGuid().ToString("n")) + ".txt";
+        return _bytesWriterFactory.Create(messagesFilePath);
+    }
+
+    public void Dispose() => _reportWriter?.Dispose();
+}

TeamCity.VSTest.TestLogger/IBytesWriter.cs

@@ -1,6 +1,8 @@
 namespace TeamCity.VSTest.TestLogger;
 
-internal interface IBytesWriter
+using System;
+
+internal interface IBytesWriter : IDisposable
 {
     void Write(byte[] bytes);
 

TeamCity.VSTest.TestLogger/IBytesWriterFactory.cs

@@ -0,0 +1,6 @@
+namespace TeamCity.VSTest.TestLogger;
+
+internal interface IBytesWriterFactory
+{
+    IBytesWriter Create(string fileName);
+}

TeamCity.VSTest.TestLogger/IFailedTestsReportWriter.cs

@@ -0,0 +1,9 @@
+namespace TeamCity.VSTest.TestLogger;
+
+using System;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+
+internal interface IFailedTestsReportWriter : IDisposable
+{
+    void ReportFailedTest(TestCase testName);
+}

TeamCity.VSTest.TestLogger/IMessageWriter.cs

@@ -1,8 +1,8 @@
 namespace TeamCity.VSTest.TestLogger;
 
-internal interface IMessageWriter
+using System;
+
+internal interface IMessageWriter : IDisposable
 {
     void Write(string message);
-
-    void Flush();
 }

TeamCity.VSTest.TestLogger/IOptions.cs

@@ -9,7 +9,9 @@ internal interface IOptions
 
     string RootFlowId { get; }
 
-    public string ServiceMessagesFileSavePath { get; }
+    string ServiceMessagesFileSavePath { get; }
+
+    string FailedTestsReportSavePath { get; }
 
     bool FallbackToStdOutTestReporting { get; }
 

TeamCity.VSTest.TestLogger/IoCConfiguration.cs

@@ -36,6 +36,8 @@ public IEnumerable<IToken> Apply(IMutableContainer container)
                 .Bind<IServiceMessageUpdater>().Tag(typeof(TestInfoUpdater)).As(Singleton).To<TestInfoUpdater>()
                 .Bind<IAttachments>().As(Singleton).To<Attachments>()
                 .Bind<ITestAttachmentPathResolver>().As(Singleton).To<TestAttachmentPathResolver>()
+                .Bind<IBytesWriterFactory>().As(Singleton).To<BytesWriterFactory>()
+                .Bind<IFailedTestsReportWriter>().As(Singleton).To<FailedTestsReportWriter>()
                 .Bind<IMessageWriter>().As(Singleton).To(ctx => new MessageWriterFactory(ctx.Container.Inject<Options>()).GetMessageWriter())
                 .Bind<ITeamCityWriter>().To(
                     ctx => ctx.Container.Inject<ITeamCityServiceMessages>().CreateWriter(

TeamCity.VSTest.TestLogger/MessageHandler.cs

@@ -12,6 +12,7 @@ internal class MessageHandler : IMessageHandler
     private readonly IAttachments _attachments;
     private readonly ITestNameProvider _testNameProvider;
     private readonly IEventRegistry _eventRegistry;
+    private readonly IFailedTestsReportWriter _failedTestsReportWriter;
     private ITeamCityWriter? _flowWriter;
     private ITeamCityWriter? _blockWriter;
 
@@ -22,17 +23,20 @@ internal MessageHandler(
         ISuiteNameProvider suiteNameProvider,
         IAttachments attachments,
         ITestNameProvider testNameProvider,
-        IEventRegistry eventRegistry)
+        IEventRegistry eventRegistry,
+        IFailedTestsReportWriter failedTestsReportWriter)
     {
         _rootWriter = rootWriter ?? throw new ArgumentNullException(nameof(rootWriter));
         _suiteNameProvider = suiteNameProvider ?? throw new ArgumentNullException(nameof(suiteNameProvider));
         _attachments = attachments ?? throw new ArgumentNullException(nameof(attachments));
         _testNameProvider = testNameProvider ?? throw new ArgumentNullException(nameof(testNameProvider));
         _eventRegistry = eventRegistry ?? throw new ArgumentNullException(nameof(eventRegistry));
+        _failedTestsReportWriter = failedTestsReportWriter ?? throw new ArgumentNullException(nameof(failedTestsReportWriter));
     }
 
     public void OnTestRunMessage(TestRunMessageEventArgs? ev)
-    { }
+    {
+    }
 
     public void OnTestResult(TestResultEventArgs? ev)
     {
@@ -49,7 +53,7 @@ public void OnTestResult(TestResultEventArgs? ev)
         {
             testName = testCase.Id.ToString();
         }
-            
+
         if (!string.IsNullOrEmpty(suiteName))
         {
             testName = suiteName + ": " + testName;
@@ -94,6 +98,7 @@ public void OnTestResult(TestResultEventArgs? ev)
 
                 case TestOutcome.Failed:
                     testWriter.WriteFailed(result.ErrorMessage ?? string.Empty, result.ErrorStackTrace ?? string.Empty);
+                    _failedTestsReportWriter.ReportFailedTest(testCase);
                     break;
 
                 case TestOutcome.Skipped:
@@ -127,5 +132,6 @@ public void OnTestRunComplete()
         _blockWriter?.Dispose();
         _flowWriter?.Dispose();
         _rootWriter.Dispose();
+        _failedTestsReportWriter.Dispose();
     }
 }

TeamCity.VSTest.TestLogger/MessageWriters/FileMessageWriter.cs

@@ -12,8 +12,8 @@ public void Write(string message)
         messageBytesWriter.Write(messageBytes);
     }
 
-    public void Flush()
+    public void Dispose()
     {
-        messageBytesWriter.Flush();
+        messageBytesWriter.Dispose();
     }
 }

TeamCity.VSTest.TestLogger/MessageWriters/StdOutMessageWriter.cs

@@ -10,5 +10,5 @@ public void Write(string message)
         Console.WriteLine(messageToWrite);
     }
 
-    public void Flush() {}
+    public void Dispose() {}
 }

TeamCity.VSTest.TestLogger/MessageWriters/StdOutMessageWriterWithBackup.cs

@@ -32,9 +32,9 @@ public void Write(string message)
                 _allowServiceMessageBackup = false;
             }
         }
-            
+
         Console.Write(messageToWrite);
     }
 
-    public void Flush() { }
+    public void Dispose() { }
 }

TeamCity.VSTest.TestLogger/Options.cs

@@ -20,6 +20,7 @@ internal class Options : IOptions
     private static readonly string ServiceMessagesBackupSourceVal;
     private static readonly string ServiceMessagesBackupPathVal;
     private static readonly bool MetadataEnableVal;
+    private static readonly string FailedTestsReportSavePathVal;
 
     static Options()
     {
@@ -54,6 +55,7 @@ static Options()
 
         AllowExperimentalVal = GetBool(GetEnvironmentVariable("TEAMCITY_LOGGER_ALLOW_EXPERIMENTAL"), true);
         MetadataEnableVal = GetBool(GetEnvironmentVariable("TEAMCITY_DOTNET_TEST_METADATA_ENABLE"), true);
+        FailedTestsReportSavePathVal = GetEnvironmentVariable("TEAMCITY_FAILED_TESTS_REPORTING_PATH");
 
         Environment.SetEnvironmentVariable("TEAMCITY_PROJECT_NAME", string.Empty);
     }
@@ -77,6 +79,8 @@ static Options()
     public bool AllowExperimental => AllowExperimentalVal;
 
     public bool MetadataEnable => MetadataEnableVal;
+
+    public string FailedTestsReportSavePath => FailedTestsReportSavePathVal;
 
     public TeamCityVersion TestMetadataSupportVersion => TestMetadataSupportVersionVal;
 

TeamCity.VSTest.TestLogger/ServiceLocatorNet35.cs

@@ -33,7 +33,8 @@ public ServiceLocatorNet35()
                 new SuiteNameProvider(),
                 new Attachments(this, idGenerator, teamCityWriter, testAttachmentPathResolver),
                 new TestNameProvider(),
-                eventContext);
+                eventContext,
+                new FailedTestsReportWriter(this, new BytesWriterFactory()));
         }
 
         public IMessageWriter MessageWriter { get; }

TeamCity.VSTest.TestLogger/TeamCityTestLogger.cs

@@ -46,7 +46,7 @@ public void Initialize(TestLoggerEvents events, string testRunDirectory)
         events.TestRunComplete += (_, _) =>
         {
             _handler.OnTestRunComplete();
-            _messageWriter.Flush();
+            _messageWriter.Dispose();
         };
 
 #if (NET35 || NET40)