Skip to content

Commit

Permalink
Merge pull request #8726 from dfederm/project-cache-vnext
Browse files Browse the repository at this point in the history 
This change add "cache add" functionality to project caching.

Today project caching exposes a hook to plugins for "I'm about to build this thing, do you want to take over instead?" and that's it. This change adds a few more hooks which report file accesses and processes (via Detours) and also for when a project finishes building. This allows a plugin to collect the file accesses and add entries to the cache for future replayability.
  • Loading branch information
@rainersigwald
rainersigwald authored Aug 29, 2023
2 parents 5958b59 + a67e9ac commit 462a917
Show file tree
Hide file tree
Showing 73 changed files with 2,607 additions and 51 deletions.
1 change: 1 addition & 0 deletions NuGet.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
<add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
<add key="dotnet8" value="https://dnceng.pkgs.visualstudio.com/public/_packaging/dotnet8/nuget/v3/index.json" />
<add key="dotnet8-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8-transport/nuget/v3/index.json" />
<add key="BuildXL" value="https://pkgs.dev.azure.com/ms/BuildXL/_packaging/BuildXL/nuget/v3/index.json" />
</packageSources>
<disabledPackageSources />
</configuration>
3 changes: 3 additions & 0 deletions eng/dependabot/Packages.props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
<PackageVersion Include="LargeAddressAware" Version="1.0.5" />
<PackageVersion Update="LargeAddressAware" Condition="'$(LargeAddressAwareVersion)' != ''" Version="$(LargeAddressAwareVersion)" />

<PackageVersion Include="Microsoft.BuildXL.Processes" Version="0.1.0-20230727.4.2" />
<PackageVersion Update="Microsoft.BuildXL.Processes" Condition="'$(BuildXLProcessesVersion)' != ''" Version="$(BuildXLProcessesVersion)" />

<PackageVersion Include="Microsoft.VisualStudio.Setup.Configuration.Interop" Version="3.2.2146" PrivateAssets="All" />
<PackageVersion Update="Microsoft.VisualStudio.Setup.Configuration.Interop" Condition="'$(MicrosoftVisualStudioSetupConfigurationInteropVersion)' != ''" Version="$(MicrosoftVisualStudioSetupConfigurationInteropVersion)" PrivateAssets="All" />

Expand Down
103 changes: 90 additions & 13 deletions src/Build.UnitTests/BackEnd/TaskHostTaskComplete_Tests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Collections.Generic;
using Microsoft.Build.BackEnd;
using Microsoft.Build.Framework;
using Microsoft.Build.Framework.FileAccess;
using Microsoft.Build.Shared;
using Microsoft.Build.Utilities;
using Xunit;
Expand All @@ -25,21 +26,67 @@ public class TaskHostTaskComplete_Tests
[Fact]
public void TestConstructors()
{
TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), null);
TaskHostTaskComplete complete2 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Failure), null);
TaskHostTaskComplete complete3 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringInitialization, new ArgumentOutOfRangeException()), null);
TaskHostTaskComplete complete4 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringExecution, new ArgumentNullException()), null);
#if FEATURE_REPORTFILEACCESSES
var fileAccessData = new List<FileAccessData>()
{
new FileAccessData(
ReportedFileOperation.CreateFile,
RequestedAccess.Read,
0,
0,
DesiredAccess.GENERIC_READ,
FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL,
"foo",
null,
true),
};
#endif

_ = new TaskHostTaskComplete(
new OutOfProcTaskHostTaskResult(TaskCompleteType.Success),
#if FEATURE_REPORTFILEACCESSES
fileAccessData,
#endif
null);
_ = new TaskHostTaskComplete(
new OutOfProcTaskHostTaskResult(TaskCompleteType.Failure),
#if FEATURE_REPORTFILEACCESSES
fileAccessData,
#endif
null);
_ = new TaskHostTaskComplete(
new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringInitialization,
new ArgumentOutOfRangeException()),
#if FEATURE_REPORTFILEACCESSES
fileAccessData,
#endif
null);
_ = new TaskHostTaskComplete(
new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringExecution, new ArgumentNullException()),
#if FEATURE_REPORTFILEACCESSES
fileAccessData,
#endif
null);

IDictionary<string, object> parameters = new Dictionary<string, object>();
TaskHostTaskComplete complete5 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null);
_ = new TaskHostTaskComplete(
new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters),
#if FEATURE_REPORTFILEACCESSES
null,
#endif
null);

IDictionary<string, object> parameters2 = new Dictionary<string, object>();
parameters2.Add("Text", "Hello!");
parameters2.Add("MyBoolValue", true);
parameters2.Add("MyITaskItem", new TaskItem("ABC"));
parameters2.Add("ItemArray", new ITaskItem[] { new TaskItem("DEF"), new TaskItem("GHI"), new TaskItem("JKL") });

TaskHostTaskComplete complete6 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters2), null);
_ = new TaskHostTaskComplete(
new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters2),
#if FEATURE_REPORTFILEACCESSES
null,
#endif
null);
}

/// <summary>
Expand All @@ -60,7 +107,12 @@ public void TestInvalidConstructors()
[Fact]
public void TestTranslationWithNullDictionary()
{
TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), null);
TaskHostTaskComplete complete = new(
new OutOfProcTaskHostTaskResult(TaskCompleteType.Success),
#if FEATURE_REPORTFILEACCESSES
null,
#endif
null);

((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator());
INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
Expand All @@ -78,7 +130,12 @@ public void TestTranslationWithNullDictionary()
[Fact]
public void TestTranslationWithEmptyDictionary()
{
TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, new Dictionary<string, object>()), null);
TaskHostTaskComplete complete = new(
new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, new Dictionary<string, object>()),
#if FEATURE_REPORTFILEACCESSES
null,
#endif
null);

((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator());
INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
Expand All @@ -99,7 +156,12 @@ public void TestTranslationWithValueTypesInDictionary()
IDictionary<string, object> parameters = new Dictionary<string, object>();
parameters.Add("Text", "Foo");
parameters.Add("BoolValue", false);
TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null);
TaskHostTaskComplete complete = new(
new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters),
#if FEATURE_REPORTFILEACCESSES
null,
#endif
null);

((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator());
INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
Expand All @@ -121,7 +183,12 @@ public void TestTranslationWithITaskItemInDictionary()
{
IDictionary<string, object> parameters = new Dictionary<string, object>();
parameters.Add("TaskItemValue", new TaskItem("Foo"));
TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null);
TaskHostTaskComplete complete = new(
new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters),
#if FEATURE_REPORTFILEACCESSES
null,
#endif
null);

((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator());
INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
Expand All @@ -142,7 +209,12 @@ public void TestTranslationWithITaskItemArrayInDictionary()
{
IDictionary<string, object> parameters = new Dictionary<string, object>();
parameters.Add("TaskItemArrayValue", new ITaskItem[] { new TaskItem("Foo"), new TaskItem("Baz") });
TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null);
TaskHostTaskComplete complete = new(
new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters),
#if FEATURE_REPORTFILEACCESSES
null,
#endif
null);

((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator());
INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
Expand All @@ -168,7 +240,12 @@ private void AssertInvalidConstructorThrows(Type expectedExceptionType, TaskComp

try
{
TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(taskResult, taskOutputParameters, taskException, taskExceptionMessage, taskExceptionMessageArgs), buildProcessEnvironment);
TaskHostTaskComplete complete = new(
new OutOfProcTaskHostTaskResult(taskResult, taskOutputParameters, taskException, taskExceptionMessage, taskExceptionMessageArgs),
#if FEATURE_REPORTFILEACCESSES
null,
#endif
buildProcessEnvironment);
}
catch (Exception e)
{
Expand Down
97 changes: 97 additions & 0 deletions src/Build/BackEnd/BuildManager/BuildManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
using Microsoft.Build.Exceptions;
using Microsoft.Build.Experimental;
using Microsoft.Build.Experimental.ProjectCache;
using Microsoft.Build.FileAccesses;
using Microsoft.Build.Framework;
using Microsoft.Build.Framework.Telemetry;
using Microsoft.Build.Graph;
Expand Down Expand Up @@ -558,6 +559,19 @@ public void BeginBuild(BuildParameters parameters)
_buildParameters.OutputResultsCacheFile = FileUtilities.NormalizePath("msbuild-cache");
}

#if FEATURE_REPORTFILEACCESSES
if (_buildParameters.ReportFileAccesses)
{
// To properly report file access, we need to disable the in-proc node which won't be detoured.
_buildParameters.DisableInProcNode = true;

// Node reuse must be disabled as future builds will not be able to listen to events raised by detours.
_buildParameters.EnableNodeReuse = false;

_componentFactories.ReplaceFactory(BuildComponentType.NodeLauncher, DetouredNodeLauncher.CreateComponent);
}
#endif

// Initialize components.
_nodeManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.NodeManager) as INodeManager;

Expand All @@ -572,9 +586,17 @@ public void BeginBuild(BuildParameters parameters)

InitializeCaches();

#if FEATURE_REPORTFILEACCESSES
var fileAccessManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager) as IFileAccessManager;
#endif

_projectCacheService = new ProjectCacheService(
this,
loggingService,
#if FEATURE_REPORTFILEACCESSES
fileAccessManager,
#endif
_configCache,
_buildParameters.ProjectCacheDescriptor);

_taskHostNodeManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.TaskHostNodeManager) as INodeManager;
Expand All @@ -584,7 +606,9 @@ public void BeginBuild(BuildParameters parameters)
_nodeManager.RegisterPacketHandler(NodePacketType.BuildRequestConfiguration, BuildRequestConfiguration.FactoryForDeserialization, this);
_nodeManager.RegisterPacketHandler(NodePacketType.BuildRequestConfigurationResponse, BuildRequestConfigurationResponse.FactoryForDeserialization, this);
_nodeManager.RegisterPacketHandler(NodePacketType.BuildResult, BuildResult.FactoryForDeserialization, this);
_nodeManager.RegisterPacketHandler(NodePacketType.FileAccessReport, FileAccessReport.FactoryForDeserialization, this);
_nodeManager.RegisterPacketHandler(NodePacketType.NodeShutdown, NodeShutdown.FactoryForDeserialization, this);
_nodeManager.RegisterPacketHandler(NodePacketType.ProcessReport, ProcessReport.FactoryForDeserialization, this);
_nodeManager.RegisterPacketHandler(NodePacketType.ResolveSdkRequest, SdkResolverRequest.FactoryForDeserialization, SdkResolverService as INodePacketHandler);
_nodeManager.RegisterPacketHandler(NodePacketType.ResourceRequest, ResourceRequest.FactoryForDeserialization, this);

Expand Down Expand Up @@ -1564,6 +1588,16 @@ private void ProcessPacket(int node, INodePacket packet)
HandleNodeShutdown(node, shutdownPacket);
break;

case NodePacketType.FileAccessReport:
FileAccessReport fileAccessReport = ExpectPacketType<FileAccessReport>(packet, NodePacketType.FileAccessReport);
HandleFileAccessReport(node, fileAccessReport);
break;

case NodePacketType.ProcessReport:
ProcessReport processReport = ExpectPacketType<ProcessReport>(packet, NodePacketType.ProcessReport);
HandleProcessReport(node, processReport);
break;

default:
ErrorUtilities.ThrowInternalError("Unexpected packet received by BuildManager: {0}", packet.Type);
break;
Expand Down Expand Up @@ -2371,6 +2405,39 @@ private void HandleResult(int node, BuildResult result)
configuration.ProjectTargets ??= result.ProjectTargets;
}

// Only report results to the project cache services if it's the result for a build submission.
// Note that graph builds create a submission for each node in the graph, so each node in the graph will be
// handled here. This intentionally mirrors the behavior for cache requests, as it doesn't make sense to
// report for projects which aren't going to be requested. Ideally, *any* request could be handled, but that
// would require moving the cache service interactions to the Scheduler.
if (_buildSubmissions.TryGetValue(result.SubmissionId, out BuildSubmission buildSubmission))
{
// The result may be associated with the build submission due to it being the submission which
// caused the build, but not the actual request which was originally used with the build submission.
// ie. it may be a dependency of the "root-level" project which is associated with this submission, which
// isn't what we're looking for. Ensure only the actual submission's request is considered.
if (buildSubmission.BuildRequest != null
&& buildSubmission.BuildRequest.ConfigurationId == configuration.ConfigurationId
&& _projectCacheService.ShouldUseCache(configuration))
{
BuildEventContext buildEventContext = _projectStartedEvents.TryGetValue(result.SubmissionId, out BuildEventArgs buildEventArgs)
? buildEventArgs.BuildEventContext
: new BuildEventContext(result.SubmissionId, node, configuration.Project?.EvaluationId ?? BuildEventContext.InvalidEvaluationId, configuration.ConfigurationId, BuildEventContext.InvalidProjectContextId, BuildEventContext.InvalidTargetId, BuildEventContext.InvalidTaskId);
try
{
_projectCacheService.HandleBuildResultAsync(configuration, result, buildEventContext, _executionCancellationTokenSource.Token).Wait();
}
catch (AggregateException ex) when (ex.InnerExceptions.All(inner => inner is OperationCanceledException))
{
// The build is being cancelled. Swallow any exceptions related specifically to cancellation.
}
catch (OperationCanceledException)
{
// The build is being cancelled. Swallow any exceptions related specifically to cancellation.
}
}
}

IEnumerable<ScheduleResponse> response = _scheduler.ReportResult(node, result);
PerformSchedulingActions(response);
}
Expand Down Expand Up @@ -2437,6 +2504,36 @@ private void HandleNodeShutdown(int node, NodeShutdown shutdownPacket)
CheckForActiveNodesAndCleanUpSubmissions();
}

/// <summary>
/// Report the received <paramref name="fileAccessReport"/> to the file access manager.
/// </summary>
/// <param name="nodeId">The id of the node from which the <paramref name="fileAccessReport"/> was received.</param>
/// <param name="fileAccessReport">The file access report.</param>
private void HandleFileAccessReport(int nodeId, FileAccessReport fileAccessReport)
{
#if FEATURE_REPORTFILEACCESSES
if (_buildParameters.ReportFileAccesses)
{
((FileAccessManager)((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager)).ReportFileAccess(fileAccessReport.FileAccessData, nodeId);
}
#endif
}

/// <summary>
/// Report the received <paramref name="processReport"/> to the file access manager.
/// </summary>
/// <param name="nodeId">The id of the node from which the <paramref name="processReport"/> was received.</param>
/// <param name="processReport">The process data report.</param>
private void HandleProcessReport(int nodeId, ProcessReport processReport)
{
#if FEATURE_REPORTFILEACCESSES
if (_buildParameters.ReportFileAccesses)
{
((FileAccessManager)((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager)).ReportProcess(processReport.ProcessData, nodeId);
}
#endif
}

/// <summary>
/// If there are no more active nodes, cleans up any remaining submissions.
/// </summary>
Expand Down
Loading

0 comments on commit 462a917

Please sign in to comment.