From 919b2e546eef37483a3c081bb25d93de0e0ee8e8 Mon Sep 17 00:00:00 2001 From: David Federman Date: Fri, 5 Aug 2022 11:52:57 -0700 Subject: [PATCH 01/21] Add -reportFileAccesses command line param --- .../CommandLineSwitches_Tests.cs | 1 + src/MSBuild/CommandLineSwitches.cs | 2 ++ src/MSBuild/Resources/Strings.resx | 23 ++++++++++++++- src/MSBuild/Resources/xlf/Strings.cs.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.de.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.es.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.fr.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.it.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.ja.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.ko.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.pl.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.pt-BR.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.ru.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.tr.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.zh-Hans.xlf | 28 +++++++++++++++++++ src/MSBuild/Resources/xlf/Strings.zh-Hant.xlf | 28 +++++++++++++++++++ src/MSBuild/XMake.cs | 12 ++++++++ 17 files changed, 401 insertions(+), 1 deletion(-) diff --git a/src/MSBuild.UnitTests/CommandLineSwitches_Tests.cs b/src/MSBuild.UnitTests/CommandLineSwitches_Tests.cs index 551114540e1..92b8f1087e3 100644 --- a/src/MSBuild.UnitTests/CommandLineSwitches_Tests.cs +++ b/src/MSBuild.UnitTests/CommandLineSwitches_Tests.cs @@ -1051,6 +1051,7 @@ public void InvalidToolsVersionErrors() question: false, inputResultsCaches: null, outputResultsCache: null, + reportFileAccesses: false, commandLine: null); } finally diff --git a/src/MSBuild/CommandLineSwitches.cs b/src/MSBuild/CommandLineSwitches.cs index 5b2f7e67fb4..8a328ea479b 100644 --- a/src/MSBuild/CommandLineSwitches.cs +++ b/src/MSBuild/CommandLineSwitches.cs @@ -105,6 +105,7 @@ internal enum ParameterizedSwitch GraphBuild, InputResultsCaches, OutputResultsCache, + ReportFileAccesses, LowPriority, Question, DetailedSummary, @@ -262,6 +263,7 @@ internal ParameterizedSwitchInfo( new ParameterizedSwitchInfo( new string[] { "graphbuild", "graph" }, ParameterizedSwitch.GraphBuild, null, true, null, true, false), new ParameterizedSwitchInfo( new string[] { "inputResultsCaches", "irc" }, ParameterizedSwitch.InputResultsCaches, null, true, null, true, true), new ParameterizedSwitchInfo( new string[] { "outputResultsCache", "orc" }, ParameterizedSwitch.OutputResultsCache, "DuplicateOutputResultsCache", false, null, true, true), + new ParameterizedSwitchInfo( new string[] { "reportfileaccesses" }, ParameterizedSwitch.ReportFileAccesses, null, false, null, true, false), new ParameterizedSwitchInfo( new string[] { "lowpriority", "low" }, ParameterizedSwitch.LowPriority, null, false, null, true, false), new ParameterizedSwitchInfo( new string[] { "question", "q" }, ParameterizedSwitch.Question, null, false, null, true, false), new ParameterizedSwitchInfo( new string[] { "detailedsummary", "ds" }, ParameterizedSwitch.DetailedSummary, null, false, null, true, false), diff --git a/src/MSBuild/Resources/Strings.resx b/src/MSBuild/Resources/Strings.resx index 7ae84b6c31a..20628db0449 100644 --- a/src/MSBuild/Resources/Strings.resx +++ b/src/MSBuild/Resources/Strings.resx @@ -865,6 +865,18 @@ LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + MSBUILD : Configuration error MSB1043: The application could not start. {0} @@ -1278,6 +1290,15 @@ MSBUILD : error MSB1049: The {0} parameter must be specified {StrBegin="MSBUILD : error MSB1049: "} + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. + + @@ -1477,7 +1498,7 @@ diff --git a/src/MSBuild/Resources/xlf/Strings.cs.xlf b/src/MSBuild/Resources/xlf/Strings.cs.xlf index 76859a03807..faea43a7530 100644 --- a/src/MSBuild/Resources/xlf/Strings.cs.xlf +++ b/src/MSBuild/Resources/xlf/Strings.cs.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.de.xlf b/src/MSBuild/Resources/xlf/Strings.de.xlf index 17a94be7d44..d99c33a60ce 100644 --- a/src/MSBuild/Resources/xlf/Strings.de.xlf +++ b/src/MSBuild/Resources/xlf/Strings.de.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.es.xlf b/src/MSBuild/Resources/xlf/Strings.es.xlf index ae3869eaa4a..2fd176941c0 100644 --- a/src/MSBuild/Resources/xlf/Strings.es.xlf +++ b/src/MSBuild/Resources/xlf/Strings.es.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.fr.xlf b/src/MSBuild/Resources/xlf/Strings.fr.xlf index 5e37b7d1274..767167db4ce 100644 --- a/src/MSBuild/Resources/xlf/Strings.fr.xlf +++ b/src/MSBuild/Resources/xlf/Strings.fr.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.it.xlf b/src/MSBuild/Resources/xlf/Strings.it.xlf index fec55df1f73..8923bf0c074 100644 --- a/src/MSBuild/Resources/xlf/Strings.it.xlf +++ b/src/MSBuild/Resources/xlf/Strings.it.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.ja.xlf b/src/MSBuild/Resources/xlf/Strings.ja.xlf index bbb6e17898b..1b22d2853e1 100644 --- a/src/MSBuild/Resources/xlf/Strings.ja.xlf +++ b/src/MSBuild/Resources/xlf/Strings.ja.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.ko.xlf b/src/MSBuild/Resources/xlf/Strings.ko.xlf index f495f72c8d2..46b81da176f 100644 --- a/src/MSBuild/Resources/xlf/Strings.ko.xlf +++ b/src/MSBuild/Resources/xlf/Strings.ko.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.pl.xlf b/src/MSBuild/Resources/xlf/Strings.pl.xlf index 8b338f18881..2fd4b559d27 100644 --- a/src/MSBuild/Resources/xlf/Strings.pl.xlf +++ b/src/MSBuild/Resources/xlf/Strings.pl.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.pt-BR.xlf b/src/MSBuild/Resources/xlf/Strings.pt-BR.xlf index 69de85dedb9..626e4a4c41a 100644 --- a/src/MSBuild/Resources/xlf/Strings.pt-BR.xlf +++ b/src/MSBuild/Resources/xlf/Strings.pt-BR.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.ru.xlf b/src/MSBuild/Resources/xlf/Strings.ru.xlf index c97ef13092d..873bcb284b6 100644 --- a/src/MSBuild/Resources/xlf/Strings.ru.xlf +++ b/src/MSBuild/Resources/xlf/Strings.ru.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.tr.xlf b/src/MSBuild/Resources/xlf/Strings.tr.xlf index c5754c1ba8c..48824cf4d0f 100644 --- a/src/MSBuild/Resources/xlf/Strings.tr.xlf +++ b/src/MSBuild/Resources/xlf/Strings.tr.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.zh-Hans.xlf b/src/MSBuild/Resources/xlf/Strings.zh-Hans.xlf index 3a8735047a6..6ec2ee3c314 100644 --- a/src/MSBuild/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/MSBuild/Resources/xlf/Strings.zh-Hans.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/Resources/xlf/Strings.zh-Hant.xlf b/src/MSBuild/Resources/xlf/Strings.zh-Hant.xlf index e257ba50130..cfec9ba80a5 100644 --- a/src/MSBuild/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/MSBuild/Resources/xlf/Strings.zh-Hant.xlf @@ -91,6 +91,34 @@ LOCALIZATION: "MSBuild" should not be localized. LOCALIZATION: "-question" and "-q" should not be localized. LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + -reportFileAccesses[:True|False] + Causes MSBuild to report file accesses to any configured + project cache plugins. + + This flag is experimental and may not work as intended. + + + LOCALIZATION: "-reportFileAccesses" should not be localized. + LOCALIZATION: None of the lines should be longer than a standard width console window, eg 80 chars. + + + + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + MSBUILD : error MSB1063: Report file accesses value is not valid. {0} + + {StrBegin="MSBUILD : error MSB1063: "} + UE: This message does not need in-line parameters because the exception takes care of displaying the invalid arg. + This error is shown when a user specifies a value that is not equivalent to Boolean.TrueString or Boolean.FalseString. + LOCALIZATION: The prefix "MSBUILD : error MSBxxxx:" should not be localized. diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 06c53027f78..49829e2c1a4 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -707,6 +707,7 @@ public static ExitType Execute( string[] inputResultsCaches = null; string outputResultsCache = null; bool question = false; + bool reportFileAccesses = false; GatherAllSwitches(commandLine, out var switchesFromAutoResponseFile, out var switchesNotFromAutoResponseFile, out _); bool buildCanBeInvoked = ProcessCommandLineSwitches( @@ -741,6 +742,7 @@ public static ExitType Execute( ref graphBuildOptions, ref inputResultsCaches, ref outputResultsCache, + ref reportFileAccesses, ref lowPriority, ref question, recursing: false, @@ -816,6 +818,7 @@ public static ExitType Execute( question, inputResultsCaches, outputResultsCache, + reportFileAccesses, commandLine)) { exitType = ExitType.BuildError; @@ -1137,6 +1140,7 @@ internal static bool BuildProject( bool question, string[] inputResultsCaches, string outputResultsCache, + bool reportFileAccesses, #if FEATURE_GET_COMMANDLINE string commandLine) #else @@ -2259,6 +2263,7 @@ private static bool ProcessCommandLineSwitches( ref GraphBuildOptions graphBuild, ref string[] inputResultsCaches, ref string outputResultsCache, + ref bool reportFileAccesses, ref bool lowPriority, ref bool question, bool recursing, @@ -2313,6 +2318,11 @@ private static bool ProcessCommandLineSwitches( // leave priority where it was. catch (Win32Exception) { } + if (commandLineSwitches.IsParameterizedSwitchSet(CommandLineSwitches.ParameterizedSwitch.ReportFileAccesses)) + { + reportFileAccesses = ProcessBooleanSwitch(commandLineSwitches[CommandLineSwitches.ParameterizedSwitch.ReportFileAccesses], defaultValue: true, resourceName: ""); + } + // if help switch is set (regardless of switch errors), show the help message and ignore the other switches if (commandLineSwitches[CommandLineSwitches.ParameterlessSwitch.Help]) { @@ -2375,6 +2385,7 @@ private static bool ProcessCommandLineSwitches( ref graphBuild, ref inputResultsCaches, ref outputResultsCache, + ref reportFileAccesses, ref lowPriority, ref question, recursing: true, @@ -4128,6 +4139,7 @@ private static void ShowHelpMessage() Console.WriteLine(AssemblyResources.GetString("HelpMessage_InputCachesFiles")); Console.WriteLine(AssemblyResources.GetString("HelpMessage_OutputCacheFile")); Console.WriteLine(AssemblyResources.GetString("HelpMessage_36_GraphBuildSwitch")); + Console.WriteLine(AssemblyResources.GetString("HelpMessage_42_ReportFileAccessesSwitch")); Console.WriteLine(AssemblyResources.GetString("HelpMessage_39_LowPrioritySwitch")); Console.WriteLine(AssemblyResources.GetString("HelpMessage_41_QuestionSwitch")); Console.WriteLine(AssemblyResources.GetString("HelpMessage_7_ResponseFile")); From b7a0ad5283a002831faf73afb48c19e5e21bba33 Mon Sep 17 00:00:00 2001 From: David Federman Date: Fri, 5 Aug 2022 14:06:56 -0700 Subject: [PATCH 02/21] Add ReportFileAccesses to BuildParameters --- .../BackEnd/BuildManager/BuildParameters.cs | 26 +++++++++++++++++++ src/MSBuild/XMake.cs | 1 + 2 files changed, 27 insertions(+) diff --git a/src/Build/BackEnd/BuildManager/BuildParameters.cs b/src/Build/BackEnd/BuildManager/BuildParameters.cs index 8d7a8268648..325c67d1001 100644 --- a/src/Build/BackEnd/BuildManager/BuildParameters.cs +++ b/src/Build/BackEnd/BuildManager/BuildParameters.cs @@ -220,6 +220,8 @@ public class BuildParameters : ITranslatable private string _outputResultsCacheFile; + private bool _reportFileAccesses; + /// /// Constructor for those who intend to set all properties themselves. /// @@ -303,6 +305,7 @@ internal BuildParameters(BuildParameters other, bool resetEnvironment = false) _projectIsolationMode = other.ProjectIsolationMode; _inputResultsCacheFiles = other._inputResultsCacheFiles; _outputResultsCacheFile = other._outputResultsCacheFile; + _reportFileAccesses = other._reportFileAccesses; DiscardBuildResults = other.DiscardBuildResults; LowPriority = other.LowPriority; Question = other.Question; @@ -801,6 +804,28 @@ public string OutputResultsCacheFile set => _outputResultsCacheFile = value; } + /// + /// Gets or sets a value indicating whether file accesses should be reported to any configured project cache plugins. + /// + public bool ReportFileAccesses + { + get => _reportFileAccesses; + set + { + _reportFileAccesses = value; + + // TODO dfederm: What if either of these are set after ReportFileAccesses is? Do we need to move this elsewhere? + if (_reportFileAccesses) + { + // To properly report file access, we need to disable the in-proc node which won't be detoured. + DisableInProcNode = true; + + // Node reuse must be disabled as future builds will not be able to listen to events raised by detours. + EnableNodeReuse = false; + } + } + } + /// /// Determines whether MSBuild will save the results of builds after EndBuild to speed up future builds. /// @@ -885,6 +910,7 @@ void ITranslatable.Translate(ITranslator translator) translator.Translate(ref _interactive); translator.Translate(ref _question); translator.TranslateEnum(ref _projectIsolationMode, (int)_projectIsolationMode); + translator.Translate(ref _reportFileAccesses); // ProjectRootElementCache is not transmitted. // ResetCaches is not transmitted. diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 49829e2c1a4..48fd815f93e 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -1332,6 +1332,7 @@ internal static bool BuildProject( parameters.InputResultsCacheFiles = inputResultsCaches; parameters.OutputResultsCacheFile = outputResultsCache; parameters.Question = question; + parameters.ReportFileAccesses = reportFileAccesses; // Propagate the profiler flag into the project load settings so the evaluator // can pick it up From effbcb2c31af0a3a4cac4a75417f98532e9c9e20 Mon Sep 17 00:00:00 2001 From: David Federman Date: Tue, 9 Aug 2022 09:40:46 -0700 Subject: [PATCH 03/21] Implement FileAccessManager --- .../BuildComponentFactoryCollection.cs | 3 + .../FileAccesses/FileAccessManager.cs | 126 +++++++++ .../FileAccesses/IFileAccessManager.cs | 21 ++ .../BackEnd/Components/IBuildComponentHost.cs | 5 + .../Components/Scheduler/IScheduler.cs | 5 + .../BackEnd/Components/Scheduler/Scheduler.cs | 14 + src/Build/Microsoft.Build.csproj | 4 +- src/Framework/FileAccess/DesiredAccess.cs | 109 ++++++++ src/Framework/FileAccess/FileAccessData.cs | 32 +++ .../FileAccess/FlagsAndAttributes.cs | 182 +++++++++++++ src/Framework/FileAccess/ProcessData.cs | 32 +++ .../FileAccess/ReportedFileOperation.cs | 257 ++++++++++++++++++ src/Framework/FileAccess/RequestedAccess.cs | 54 ++++ src/Framework/NativeMethods.cs | 2 +- 14 files changed, 844 insertions(+), 2 deletions(-) create mode 100644 src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs create mode 100644 src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs create mode 100644 src/Framework/FileAccess/DesiredAccess.cs create mode 100644 src/Framework/FileAccess/FileAccessData.cs create mode 100644 src/Framework/FileAccess/FlagsAndAttributes.cs create mode 100644 src/Framework/FileAccess/ProcessData.cs create mode 100644 src/Framework/FileAccess/ReportedFileOperation.cs create mode 100644 src/Framework/FileAccess/RequestedAccess.cs diff --git a/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs b/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs index 5ea3ee2bde1..938e5e2183d 100644 --- a/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs +++ b/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using Microsoft.Build.BackEnd.Components.Caching; using Microsoft.Build.BackEnd.SdkResolution; +using Microsoft.Build.FileAccesses; using Microsoft.Build.Shared; #nullable disable @@ -80,6 +81,8 @@ public void RegisterDefaultFactories() // SDK resolution _componentEntriesByType[BuildComponentType.SdkResolverService] = new BuildComponentEntry(BuildComponentType.SdkResolverService, MainNodeSdkResolverService.CreateComponent, CreationPattern.Singleton); + + _componentEntriesByType[BuildComponentType.FileAccessManager] = new BuildComponentEntry(BuildComponentType.FileAccessManager, FileAccessManager.CreateComponent, CreationPattern.Singleton); } /// diff --git a/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs b/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs new file mode 100644 index 00000000000..0c80b6d9539 --- /dev/null +++ b/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using Microsoft.Build.BackEnd; +using Microsoft.Build.Framework.FileAccess; +using Microsoft.Build.Shared; + +namespace Microsoft.Build.FileAccesses +{ + internal sealed class FileAccessManager : IFileAccessManager, IBuildComponent + { + private record Handlers(Action FileAccessHander, Action ProcessHandler); + + private IScheduler? _scheduler; + private IConfigCache? _configCache; + + private object _handlersWriteLock = new object(); + private Handlers[] _handlers = Array.Empty(); + + public static IBuildComponent CreateComponent(BuildComponentType type) + { + ErrorUtilities.VerifyThrowArgumentOutOfRange(type == BuildComponentType.FileAccessManager, nameof(type)); + return new FileAccessManager(); + } + + public void InitializeComponent(IBuildComponentHost host) + { + _scheduler = host.GetComponent(BuildComponentType.Scheduler) as IScheduler; + _configCache = host.GetComponent(BuildComponentType.ConfigCache) as IConfigCache; + } + + public void ShutdownComponent() + { + _scheduler = null; + _configCache = null; + } + + public void ReportFileAccess(FileAccessData fileAccessData, int nodeId) + { + BuildRequest? buildRequest = GetBuildRequest(nodeId); + if (buildRequest != null) + { + Handlers[] localHandlers = _handlers; + foreach (Handlers handlers in localHandlers) + { + handlers.FileAccessHander.Invoke(buildRequest, fileAccessData); + } + } + } + + public void ReportProcess(ProcessData processData, int nodeId) + { + BuildRequest? buildRequest = GetBuildRequest(nodeId); + if (buildRequest != null) + { + Handlers[] localHandlers = _handlers; + foreach (Handlers handlers in localHandlers) + { + handlers.ProcessHandler.Invoke(buildRequest, processData); + } + } + } + + public HandlerRegistration RegisterHandlers(Action fileAccessHandler, Action processHandler) + { + lock (_handlersWriteLock) + { + Handlers[] newHandlers = new Handlers[_handlers.Length + 1]; + _handlers.CopyTo(newHandlers, 0); + + Handlers addedHandlers = new(fileAccessHandler, processHandler); + newHandlers[_handlers.Length] = addedHandlers; + + _handlers = newHandlers; + + return new HandlerRegistration(() => UnregisterHandlers(addedHandlers)); + } + } + + private void UnregisterHandlers(Handlers handlersToRemove) + { + lock (_handlersWriteLock) + { + Handlers[] newHandlers = new Handlers[_handlers.Length - 1]; + int newHandlersIdx = 0; + for (int handlersIdx = 0; handlersIdx < _handlers.Length; handlersIdx++) + { + if (_handlers[handlersIdx] != handlersToRemove) + { + newHandlers[newHandlersIdx] = _handlers[handlersIdx]; + newHandlersIdx++; + } + } + + _handlers = newHandlers; + } + } + + private BuildRequest? GetBuildRequest(int nodeId) + { + ErrorUtilities.VerifyThrow( + _scheduler != null && _configCache != null, + "Component has not been initialized"); + + // Note: If the node isn't executing anything it may be accessing binaries required to run, eg. the MSBuild binaries + return _scheduler!.GetExecutingRequestByNode(nodeId); + } + + internal readonly struct HandlerRegistration : IDisposable + { + private readonly Action _unregisterAction; + + public HandlerRegistration(Action unregisterAction) + { + _unregisterAction = unregisterAction; + } + + public void Dispose() + { + _unregisterAction(); + } + } + } +} diff --git a/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs b/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs new file mode 100644 index 00000000000..2834759f55b --- /dev/null +++ b/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Build.BackEnd; +using Microsoft.Build.Framework.FileAccess; + +namespace Microsoft.Build.FileAccesses +{ + internal interface IFileAccessManager + { + void ReportFileAccess(FileAccessData fileAccessData, int nodeId); + + void ReportProcess(ProcessData processData, int nodeId); + + // Note: HandlerRegistration is exposed directly instead of IDisposable to avoid boxing. + FileAccessManager.HandlerRegistration RegisterHandlers( + Action fileAccessHandler, + Action processHandler); + } +} diff --git a/src/Build/BackEnd/Components/IBuildComponentHost.cs b/src/Build/BackEnd/Components/IBuildComponentHost.cs index bcbc7eac430..1711b9d27af 100644 --- a/src/Build/BackEnd/Components/IBuildComponentHost.cs +++ b/src/Build/BackEnd/Components/IBuildComponentHost.cs @@ -130,6 +130,11 @@ internal enum BuildComponentType /// The SDK resolution service. /// SdkResolverService, + + /// + /// The component which is the sink for file access reports and forwards reports to other components. + /// + FileAccessManager, } /// diff --git a/src/Build/BackEnd/Components/Scheduler/IScheduler.cs b/src/Build/BackEnd/Components/Scheduler/IScheduler.cs index d66e50d62c1..84e22a9c67c 100644 --- a/src/Build/BackEnd/Components/Scheduler/IScheduler.cs +++ b/src/Build/BackEnd/Components/Scheduler/IScheduler.cs @@ -33,6 +33,11 @@ internal interface IScheduler : IBuildComponent /// A positive configuration id if one exists in the plan, 0 otherwise. int GetConfigurationIdFromPlan(string configurationPath); + /// + /// Retrieves the request executing on a node. + /// + BuildRequest GetExecutingRequestByNode(int nodeId); + /// /// Reports to the scheduler that a request is blocked. /// diff --git a/src/Build/BackEnd/Components/Scheduler/Scheduler.cs b/src/Build/BackEnd/Components/Scheduler/Scheduler.cs index bedccfe03cd..da2592baeb4 100644 --- a/src/Build/BackEnd/Components/Scheduler/Scheduler.cs +++ b/src/Build/BackEnd/Components/Scheduler/Scheduler.cs @@ -283,6 +283,20 @@ public int GetConfigurationIdFromPlan(string configPath) return _schedulingPlan.GetConfigIdForPath(configPath); } + /// + /// Retrieves the request executing on a node. + /// + public BuildRequest GetExecutingRequestByNode(int nodeId) + { + if (!_schedulingData.IsNodeWorking(nodeId)) + { + return null; + } + + SchedulableRequest request = _schedulingData.GetExecutingRequestByNode(nodeId); + return request.BuildRequest; + } + /// /// Reports that the specified request has become blocked and cannot proceed. /// diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index 6e6725c439d..fab357ad8ec 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -1,4 +1,4 @@ - + @@ -156,6 +156,8 @@ + + diff --git a/src/Framework/FileAccess/DesiredAccess.cs b/src/Framework/FileAccess/DesiredAccess.cs new file mode 100644 index 00000000000..a205acf9d03 --- /dev/null +++ b/src/Framework/FileAccess/DesiredAccess.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Framework.FileAccess +{ + /// + /// The requested access to the file or device. + /// + /// + /// See https://learn.microsoft.com/en-us/windows/win32/fileio/file-access-rights-constants for a full list of values. + /// + [Flags] + + // TODO dshepelev: Fix suppression. +#pragma warning disable CS3009 // Base type is not CLS-compliant. + public enum DesiredAccess : uint +#pragma warning restore CS3009 // Base type is not CLS-compliant. + { + /// + /// For a directory, the right to list the contents of the directory. + /// + FILE_LIST_DIRECTORY = 0x00000001, + + /// + /// For a directory, the right to create a file in the directory. + /// + FILE_ADD_FILE = 0x00000002, + + /// + /// For a directory, the right to create a subdirectory. + /// + FILE_ADD_SUBDIRECTORY = 0x00000004, + + /// + /// The right to read extended file attributes. + /// + FILE_READ_EA = 0x00000008, + + /// + /// Right to delete an object. + /// + DELETE = 0x00010000, + + /// + /// Right to wait on a handle. + /// + SYNCHRONIZE = 0x00100000, + + /// + /// For a file object, the right to append data to the file. (For local files, write operations will not overwrite existing + /// data if this flag is specified without .) For a directory object, the right to create a subdirectory + /// (). + /// + FILE_APPEND_DATA = 0x00000004, + + /// + /// The right to write extended file attributes. + /// + FILE_WRITE_EA = 0x00000010, + + /// + /// For a native code file, the right to execute the file. This access right given to scripts may cause the script to be executable, depending on the script interpreter. + /// + FILE_EXECUTE = 0x00000020, + + /// + /// For a directory, the right to delete a directory and all the files it contains, including read-only files. + /// + FILE_DELETE_CHILD = 0x00000040, + + /// + /// The right to read file attributes. + /// + FILE_READ_ATTRIBUTES = 0x00000080, + + /// + /// The right to write file attributes. + /// + FILE_WRITE_ATTRIBUTES = 0x00000100, + + /// + /// For a file object, the right to write data to the file. For a directory object, the right to create a file in the + /// directory (). + /// + FILE_WRITE_DATA = 0x00000002, + + /// + /// All possible access rights. + /// + GENERIC_ALL = 0x10000000, + + /// + /// Execute access. + /// + GENERIC_EXECUTE = 0x20000000, + + /// + /// Write access. + /// + GENERIC_WRITE = 0x40000000, + + /// + /// Read access. + /// + GENERIC_READ = 0x80000000, + } +} diff --git a/src/Framework/FileAccess/FileAccessData.cs b/src/Framework/FileAccess/FileAccessData.cs new file mode 100644 index 00000000000..e8b37a69f70 --- /dev/null +++ b/src/Framework/FileAccess/FileAccessData.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Build.Framework.FileAccess +{ + /// + /// File access data. + /// + /// The operation that performed the file access. + /// The requested access. + /// The process id. + /// The error code of the operation. + /// The desired access. + /// The file flags and attributes. + /// The path being accessed. + /// The process arguments. + /// Whether the file access is augmented. + public readonly record struct FileAccessData( + ReportedFileOperation Operation, + RequestedAccess RequestedAccess, + + // TODO dshepelev: Fix suppression. +#pragma warning disable CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. + uint ProcessId, + uint Error, +#pragma warning restore CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. + DesiredAccess DesiredAccess, + FlagsAndAttributes FlagsAndAttributes, + string Path, + string? ProcessArgs, + bool IsAnAugmentedFileAccess); +} diff --git a/src/Framework/FileAccess/FlagsAndAttributes.cs b/src/Framework/FileAccess/FlagsAndAttributes.cs new file mode 100644 index 00000000000..b386c2119f4 --- /dev/null +++ b/src/Framework/FileAccess/FlagsAndAttributes.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Framework.FileAccess +{ + /// + /// The file or device attributes and flags. + /// + [Flags] + + // TODO dshepelev: Fix suppression. +#pragma warning disable CS3009 // Base type is not CLS-compliant. + public enum FlagsAndAttributes : uint +#pragma warning restore CS3009 // Base type is not CLS-compliant. + { + /// + /// The file is read only. Applications can read the file but cannot write to or delete it. + /// + FILE_ATTRIBUTE_READONLY = 0x00000001, + + /// + /// The file is hidden. Do not include it in an ordinary directory listing. + /// + FILE_ATTRIBUTE_HIDDEN = 0x00000002, + + /// + /// The file is part of or used exclusively by an operating system. + /// + FILE_ATTRIBUTE_SYSTEM = 0x00000004, + + /// + /// The path is a directory. + /// + FILE_ATTRIBUTE_DIRECTORY = 0x00000010, + + /// + /// The file should be archived. Applications use this attribute to mark files for backup or removal. + /// + FILE_ATTRIBUTE_ARCHIVE = 0x00000020, + + /// + /// The file does not have other attributes set. This attribute is valid only if used alone. + /// + FILE_ATTRIBUTE_NORMAL = 0x00000080, + + /// + /// The file is being used for temporary storage. + /// + /// + /// For more information, see the Caching Behavior section of this topic. + /// + FILE_ATTRIBUTE_TEMPORARY = 0x00000100, + + /// + /// The data of a file is not immediately available. This attribute indicates that file data is physically moved to offline + /// storage. This attribute is used by Remote Storage, the hierarchical storage management software. Applications should + /// not arbitrarily change this attribute. + /// + FILE_ATTRIBUTE_OFFLINE = 0x00001000, + + /// + /// The file or directory is encrypted. For a file, this means that all data in the file is encrypted. For a directory, + /// this means that encryption is the default for newly created files and subdirectories. For more information, see File + /// Encryption. + /// + /// + /// This flag has no effect if is also specified. + /// This flag is not supported on Home, Home Premium, Starter, or ARM editions of Windows. + /// + FILE_ATTRIBUTE_ENCRYPED = 0x00004000, + + /// + /// The file data is requested, but it should continue to be located in remote storage. It should not be transported back + /// to local storage. This flag is for use by remote storage systems. + /// + FILE_FLAG_OPEN_NO_RECALL = 0x00100000, + + /// + /// Normal reparse point processing will not occur; CreateFile will attempt to open the reparse point. When a file is + /// opened, a file handle is returned, whether or not the filter that controls the reparse point is operational. + /// + /// + /// This flag cannot be used with the CREATE_ALWAYS flag. + /// If the file is not a reparse point, then this flag is ignored. + /// For more information, see the Remarks section. + /// + FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000, + + /// + /// The file or device is being opened with session awareness. If this flag is not specified, then per-session devices + /// (such as a redirected USB device) cannot be opened by processes running in session 0. This flag has no effect for + /// callers not in session 0. This flag is supported only on server editions of Windows. + /// + /// + /// Windows Server 2008 R2, Windows Server 2008, and Windows Server 2003: This flag is not supported before Windows Server + /// 2012. + /// + FILE_FLAG_SESSION_AWARE = 0x00800000, + + /// + /// Access will occur according to POSIX rules. This includes allowing multiple files with names, differing only in case, + /// for file systems that support that naming. Use care when using this option, because files created with this flag may + /// not be accessible by applications that are written for MS-DOS or 16-bit Windows. + /// + FILE_FLAG_POSIX_SEMANTICS = 0x01000000, + + /// + /// The file is being opened or created for a backup or restore operation. The system ensures that the calling process + /// overrides file security checks when the process has SE_BACKUP_NAME and SE_RESTORE_NAME privileges. For more + /// information, see Changing Privileges in a Token. + /// + /// + /// You must set this flag to obtain a handle to a directory. A directory handle can be passed to some functions instead of + /// a file handle. For more information, see the Remarks section. + /// + FILE_FLAG_BACKUP_SEMANTICS = 0x02000000, + + /// + /// The file is to be deleted immediately after all of its handles are closed, which includes the specified handle and any + /// other open or duplicated handles. + /// + /// + /// If there are existing open handles to a file, the call fails unless they were all opened with the FILE_SHARE_DELETE + /// share mode. + /// Subsequent open requests for the file fail, unless the FILE_SHARE_DELETE share mode is specified. + /// + FILE_FLAG_DELETE_ON_CLOSE = 0x04000000, + + /// + /// Access is intended to be sequential from beginning to end. The system can use this as a hint to optimize file caching. + /// + /// + /// This flag should not be used if read-behind (that is, reverse scans) will be used. + /// This flag has no effect if the file system does not support cached I/O and . + /// For more information, see the Caching Behavior section of this topic. + /// + FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000, + + /// + /// Access is intended to be random. The system can use this as a hint to optimize file caching. + /// + /// + /// This flag has no effect if the file system does not support cached I/O and . + /// For more information, see the Caching Behavior section of this topic. + /// + FILE_FLAG_RANDOM_ACCESS = 0x10000000, + + /// + /// The file or device is being opened with no system caching for data reads and writes. This flag does not affect hard + /// disk caching or memory mapped files. + /// + /// + /// There are strict requirements for successfully working with files opened with CreateFile using this + /// flag; for details, see File Buffering. + /// + FILE_FLAG_NO_BUFFERING = 0x20000000, + + /// + /// The file or device is being opened or created for asynchronous I/O. + /// + /// + /// When subsequent I/O operations are completed on this handle, the event specified in the OVERLAPPED structure will be + /// set to the signaled state. + /// If this flag is specified, the file can be used for simultaneous read and write operations. + /// If this flag is not specified, then I/O operations are serialized, even if the calls to the read and write functions + /// specify an OVERLAPPED structure. + /// For information about considerations when using a file handle created with this flag, see the Synchronous and + /// Asynchronous I/O Handles section of this topic. + /// + FILE_FLAG_OVERLAPPED = 0x40000000, + + /// + /// Write operations will not go through any intermediate cache; they will go directly to disk. + /// + /// + /// For additional information, see the Caching Behavior section of this topic. + /// + FILE_FLAG_WRITE_THROUGH = 0x80000000, + } +} diff --git a/src/Framework/FileAccess/ProcessData.cs b/src/Framework/FileAccess/ProcessData.cs new file mode 100644 index 00000000000..18e1f57366a --- /dev/null +++ b/src/Framework/FileAccess/ProcessData.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Framework.FileAccess +{ + /// + /// Process data. + /// + /// The process name. + /// The process id. + /// The parent process id. + /// The creation date time. + /// The exit date time. + /// The exit code. + public readonly record struct ProcessData( + string ProcessName, + + // TODO dshepelev: Fix suppression. +#pragma warning disable CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. + uint ProcessId, + uint ParentProcessId, +#pragma warning restore CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. + DateTime CreationDateTime, + DateTime ExitDateTime, + + // TODO dshepelev: Fix suppression. +#pragma warning disable CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. + uint ExitCode); +#pragma warning restore CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. +} diff --git a/src/Framework/FileAccess/ReportedFileOperation.cs b/src/Framework/FileAccess/ReportedFileOperation.cs new file mode 100644 index 00000000000..205fc7f5b31 --- /dev/null +++ b/src/Framework/FileAccess/ReportedFileOperation.cs @@ -0,0 +1,257 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Build.Framework.FileAccess +{ + /// + /// Which operation resulted in a reported file access. + /// + public enum ReportedFileOperation : byte + { + /// + /// Unknown operation. + /// + Unknown = 0, + + /// + /// CreateFile. + /// + CreateFile, + + /// + /// CreateProcess. + /// + CreateProcess, + + /// + /// GetFileAttributes. + /// + GetFileAttributes, + + /// + /// GetFileAttributesEx. + /// + GetFileAttributesEx, + + /// + /// Process forked. + /// + Process, + + /// + /// FindFirstFileEx. + /// + /// + /// FindFirstFile also indicates this op, since we implement it in terms of FindFirstFileEx. + /// + FindFirstFileEx, + + /// + /// FindNextFile. + /// + FindNextFile, + + /// + /// CreateDirectory. + /// + CreateDirectory, + + /// + /// DeleteFile. + /// + DeleteFile, + + /// + /// MoveFile (source; read and deleted). + /// + MoveFileSource, + + /// + /// MoveFile (destination; written). + /// + MoveFileDestination, + + /// + /// SetFileInformationByHandleSource (source; read and deleted). + /// + SetFileInformationByHandleSource, + + /// + /// SetFileInformationByHandleDest (destination; written). + /// + SetFileInformationByHandleDest, + + /// + /// ZwSetRenameInformationFileSource (source; read and deleted). + /// + ZwSetRenameInformationFileSource, + + /// + /// ZwSetRenameInformationFileDest (destination; written). + /// + ZwSetRenameInformationFileDest, + + /// + /// ZwSetLinkInformationFileDest. + /// + ZwSetLinkInformationFile, + + /// + /// ZwSetDispositionInformationFile (delete-on-close; deleted). + /// + ZwSetDispositionInformationFile, + + /// + /// ZwSetModeInformationFile (delete-on-close; deleted). + /// + ZwSetModeInformationFile, + + /// + /// ZwSetFileNameInformationFile (source; read and written). + /// + ZwSetFileNameInformationFileSource, + + /// + /// ZwSetFileNameInformationFile (destination; written). + /// + ZwSetFileNameInformationFileDest, + + /// + /// CopyFile (source; read). + /// + CopyFileSource, + + /// + /// CopyFile (destination; written). + /// + CopyFileDestination, + + /// + /// CreateHardLink (source; read). + /// + CreateHardLinkSource, + + /// + /// CreateHardLink (destination; written). + /// + CreateHardLinkDestination, + + /// + /// RemoveDirectory. + /// + RemoveDirectory, + + /// + /// RemoveDirectory (source; written). + /// + RemoveDirectorySource, + + /// + /// NtQueryDirectoryFile. + /// + NtQueryDirectoryFile, + + /// + /// ZwQueryDirectoryFile. + /// + ZwQueryDirectoryFile, + + /// + /// NtCreateFile. + /// + NtCreateFile, + + /// + /// ZwCreateFile. + /// + ZwCreateFile, + + /// + /// ZwOpenFile. + /// + ZwOpenFile, + + /// + /// This is a quasi operation. We issue this + /// report when Detours is changing file open + /// request with Read/Write access to Read access only. + /// + ChangedReadWriteToReadAccess, + + /// + /// This is a quasi operation. The sandbox issues this only when FileAccessPolicy.OverrideAllowWriteForExistingFiles is set, representing + /// that an allow for write check was performed for a given path for the first time (in the scope of a process, another process in the same process + /// tree may also report this for the same path). + /// + FirstAllowWriteCheckInProcess, + + /// + /// This operation used to indicate to the engine by the Linux sandbox that a process being executed statically links libc + /// and may have missing file observations. + /// + StaticallyLinkedProcess, + + /// + /// Access of reparse point target. + /// + ReparsePointTarget, + + /// + /// Access of reparse point target, cached by Detours. + /// + ReparsePointTargetCached, + + /// + /// Access checks for source of CreateSymbolicLink API. + /// + CreateSymbolicLinkSource, + + /// + /// Access check for MoveFileWithgProgress source target. + /// + MoveFileWithProgressSource, + + /// + /// Access check for MoveFileWithProgress dest target. + /// + MoveFileWithProgressDest, + + /// + /// Multiple operations lumped into one. + /// + MultipleOperations, + + /// + /// Process exited. + /// + ProcessExit, + + #region Operation Names Reported by BuildXLSandbox (macOS sandbox implementation) + MacLookup, + MacReadlink, + MacVNodeCreate, + KAuthMoveSource, + KAuthMoveDest, + KAuthCreateHardlinkSource, + KAuthCreateHardlinkDest, + KAuthCopySource, + KAuthCopyDest, + KAuthDeleteDir, + KAuthDeleteFile, + KAuthOpenDir, + KAuthReadFile, + KAuthCreateDir, + KAuthWriteFile, + KAuthClose, + KAuthCloseModified, + KAuthGetAttributes, + KAuthVNodeExecute, + KAuthVNodeWrite, + KAuthVNodeRead, + KAuthVNodeProbe, + MacVNodeWrite, + MacVNodeCloneSource, + MacVNodeCloneDest, + #endregion + } +} diff --git a/src/Framework/FileAccess/RequestedAccess.cs b/src/Framework/FileAccess/RequestedAccess.cs new file mode 100644 index 00000000000..d86831f2979 --- /dev/null +++ b/src/Framework/FileAccess/RequestedAccess.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Build.Framework.FileAccess +{ + /// + /// Level of access requested by a reported file operation. + /// + [Flags] + public enum RequestedAccess : byte + { + /// + /// No access requested. + /// + None = 0, + + /// + /// Read access requested. + /// + Read = 1, + + /// + /// Write access requested. + /// + Write = 2, + + /// + /// Metadata-only probe access requested (e.g. ). + /// + Probe = 4, + + /// + /// Directory enumeration access requested (on the directory itself; immediate children will be enumerated). + /// + Enumerate = 8, + + /// + /// Metadata-only probe access requested; probed as part of a directory enumeration (e.g. ). + /// + EnumerationProbe = 16, + + /// + /// Both read and write access requested. + /// + ReadWrite = Read | Write, + + /// + /// All defined access levels requested. + /// + All = Read | Write | Probe | Enumerate | EnumerationProbe, + } +} diff --git a/src/Framework/NativeMethods.cs b/src/Framework/NativeMethods.cs index c71d9f49ff6..6e2a8439320 100644 --- a/src/Framework/NativeMethods.cs +++ b/src/Framework/NativeMethods.cs @@ -1300,7 +1300,7 @@ internal static int GetParentProcessId(int processId) // using (var r = FileUtilities.OpenRead("/proc/" + processId + "/stat")) // and could be again when FileUtilities moves to Framework - using var fileStream = new FileStream("/proc/" + processId + "/stat", FileMode.Open, FileAccess.Read); + using var fileStream = new FileStream("/proc/" + processId + "/stat", FileMode.Open, System.IO.FileAccess.Read); using StreamReader r = new(fileStream); line = r.ReadLine(); From 837f8db50e8af7ddf81aaab55d953bbb5faf8279 Mon Sep 17 00:00:00 2001 From: David Federman Date: Mon, 15 May 2023 09:35:10 -0700 Subject: [PATCH 04/21] Make NodeLauncher a BuildComponent --- .../BuildComponentFactoryCollection.cs | 1 + .../Components/Communications/INodeLauncher.cs | 12 ++++++++++++ .../Components/Communications/NodeLauncher.cs | 16 +++++++++++++++- .../Communications/NodeProviderOutOfProcBase.cs | 2 +- .../BackEnd/Components/IBuildComponentHost.cs | 5 +++++ src/Build/Microsoft.Build.csproj | 1 + 6 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 src/Build/BackEnd/Components/Communications/INodeLauncher.cs diff --git a/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs b/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs index 938e5e2183d..d8406cf2b94 100644 --- a/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs +++ b/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs @@ -62,6 +62,7 @@ public void RegisterDefaultFactories() _componentEntriesByType[BuildComponentType.NodeManager] = new BuildComponentEntry(BuildComponentType.NodeManager, NodeManager.CreateComponent, CreationPattern.Singleton); _componentEntriesByType[BuildComponentType.TaskHostNodeManager] = new BuildComponentEntry(BuildComponentType.TaskHostNodeManager, TaskHostNodeManager.CreateComponent, CreationPattern.Singleton); + _componentEntriesByType[BuildComponentType.NodeLauncher] = new BuildComponentEntry(BuildComponentType.NodeLauncher, NodeLauncher.CreateComponent, CreationPattern.Singleton); _componentEntriesByType[BuildComponentType.InProcNodeProvider] = new BuildComponentEntry(BuildComponentType.InProcNodeProvider, NodeProviderInProc.CreateComponent, CreationPattern.Singleton); _componentEntriesByType[BuildComponentType.OutOfProcNodeProvider] = new BuildComponentEntry(BuildComponentType.OutOfProcNodeProvider, NodeProviderOutOfProc.CreateComponent, CreationPattern.Singleton); _componentEntriesByType[BuildComponentType.OutOfProcTaskHostNodeProvider] = new BuildComponentEntry(BuildComponentType.OutOfProcTaskHostNodeProvider, NodeProviderOutOfProcTaskHost.CreateComponent, CreationPattern.Singleton); diff --git a/src/Build/BackEnd/Components/Communications/INodeLauncher.cs b/src/Build/BackEnd/Components/Communications/INodeLauncher.cs new file mode 100644 index 00000000000..e1c424889cc --- /dev/null +++ b/src/Build/BackEnd/Components/Communications/INodeLauncher.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; + +namespace Microsoft.Build.BackEnd +{ + internal interface INodeLauncher + { + Process Start(string msbuildLocation, string commandLineArgs); + } +} diff --git a/src/Build/BackEnd/Components/Communications/NodeLauncher.cs b/src/Build/BackEnd/Components/Communications/NodeLauncher.cs index 611c4ca68c9..375dbf2bae6 100644 --- a/src/Build/BackEnd/Components/Communications/NodeLauncher.cs +++ b/src/Build/BackEnd/Components/Communications/NodeLauncher.cs @@ -16,8 +16,22 @@ namespace Microsoft.Build.BackEnd { - internal class NodeLauncher + internal sealed class NodeLauncher : INodeLauncher, IBuildComponent { + public static IBuildComponent CreateComponent(BuildComponentType type) + { + ErrorUtilities.VerifyThrowArgumentOutOfRange(type == BuildComponentType.NodeLauncher, nameof(type)); + return new NodeLauncher(); + } + + public void InitializeComponent(IBuildComponentHost host) + { + } + + public void ShutdownComponent() + { + } + /// /// Creates a new MSBuild process /// diff --git a/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs b/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs index 7346954557d..a822dbaedb7 100644 --- a/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs +++ b/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs @@ -332,7 +332,7 @@ bool StartNewNode(int nodeId) } #endif // Create the node process - NodeLauncher nodeLauncher = new NodeLauncher(); + INodeLauncher nodeLauncher = (INodeLauncher)_componentHost.GetComponent(BuildComponentType.NodeLauncher); Process msbuildProcess = nodeLauncher.Start(msbuildLocation, commandLineArgs); _processesToIgnore.TryAdd(GetProcessesToIgnoreKey(hostHandshake, msbuildProcess.Id), default); diff --git a/src/Build/BackEnd/Components/IBuildComponentHost.cs b/src/Build/BackEnd/Components/IBuildComponentHost.cs index 1711b9d27af..be9fa43f5e8 100644 --- a/src/Build/BackEnd/Components/IBuildComponentHost.cs +++ b/src/Build/BackEnd/Components/IBuildComponentHost.cs @@ -135,6 +135,11 @@ internal enum BuildComponentType /// The component which is the sink for file access reports and forwards reports to other components. /// FileAccessManager, + + /// + /// The component which launches new MSBuild nodes. + /// + NodeLauncher, } /// diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index fab357ad8ec..30f4252206d 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -362,6 +362,7 @@ + From 2340b2f205a2517cb2766433e1ef72a01cfdfc97 Mon Sep 17 00:00:00 2001 From: David Federman Date: Thu, 11 Aug 2022 10:04:13 -0700 Subject: [PATCH 05/21] Launch nodes using BXL's SandboxedProcess --- NuGet.config | 2 + eng/Versions.props | 2 +- eng/dependabot/Packages.props | 3 + .../BackEnd/BuildManager/BuildManager.cs | 5 + src/Build/BackEnd/Client/MSBuildClient.cs | 2 +- .../Communications/DetouredNodeLauncher.cs | 223 ++++++++++++++++++ .../Communications/INodeLauncher.cs | 2 +- .../Components/Communications/NodeLauncher.cs | 4 +- .../NodeProviderOutOfProcBase.cs | 2 +- .../RetrievableEntryHashSet/HashSet.cs | 2 + src/Build/Microsoft.Build.csproj | 5 +- 11 files changed, 245 insertions(+), 7 deletions(-) create mode 100644 src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs diff --git a/NuGet.config b/NuGet.config index 9b4c6f6759c..bda8e534907 100644 --- a/NuGet.config +++ b/NuGet.config @@ -6,6 +6,8 @@ + + diff --git a/eng/Versions.props b/eng/Versions.props index 49f6658c4ab..c3a952dddc8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -39,7 +39,7 @@ 7.0.0 7.0.0 7.0.0 - 5.0.0 + 6.0.0-preview.5.21301.5 7.0.0 diff --git a/eng/dependabot/Packages.props b/eng/dependabot/Packages.props index 14986a790d2..2d95f0a68b6 100644 --- a/eng/dependabot/Packages.props +++ b/eng/dependabot/Packages.props @@ -13,6 +13,9 @@ + + + diff --git a/src/Build/BackEnd/BuildManager/BuildManager.cs b/src/Build/BackEnd/BuildManager/BuildManager.cs index 9787ffe4a5c..4fb434a2d6f 100644 --- a/src/Build/BackEnd/BuildManager/BuildManager.cs +++ b/src/Build/BackEnd/BuildManager/BuildManager.cs @@ -557,6 +557,11 @@ public void BeginBuild(BuildParameters parameters) _buildParameters.OutputResultsCacheFile = FileUtilities.NormalizePath("msbuild-cache"); } + if (_buildParameters.ReportFileAccesses) + { + _componentFactories.ReplaceFactory(BuildComponentType.NodeLauncher, DetouredNodeLauncher.CreateComponent); + } + // Initialize components. _nodeManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.NodeManager) as INodeManager; diff --git a/src/Build/BackEnd/Client/MSBuildClient.cs b/src/Build/BackEnd/Client/MSBuildClient.cs index 693912475d0..e0782c3fbf3 100644 --- a/src/Build/BackEnd/Client/MSBuildClient.cs +++ b/src/Build/BackEnd/Client/MSBuildClient.cs @@ -473,7 +473,7 @@ private bool TryLaunchServer() }; NodeLauncher nodeLauncher = new NodeLauncher(); CommunicationsUtilities.Trace("Starting Server..."); - Process msbuildProcess = nodeLauncher.Start(_msbuildLocation, string.Join(" ", msBuildServerOptions)); + Process msbuildProcess = nodeLauncher.Start(_msbuildLocation, string.Join(" ", msBuildServerOptions), nodeId: 0); CommunicationsUtilities.Trace("Server started with PID: {0}", msbuildProcess?.Id); } catch (Exception ex) diff --git a/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs b/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs new file mode 100644 index 00000000000..bd5819d53ac --- /dev/null +++ b/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs @@ -0,0 +1,223 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using BuildXL.Processes; +using BuildXL.Utilities.Core; +using Microsoft.Build.Exceptions; +using Microsoft.Build.FileAccesses; +using Microsoft.Build.Internal; +using Microsoft.Build.Shared; +using Microsoft.Build.Shared.FileSystem; + +#nullable disable + +namespace Microsoft.Build.BackEnd +{ + internal sealed class DetouredNodeLauncher : INodeLauncher, IBuildComponent + { + private readonly List _sandboxedProcesses = new(); + + private IFileAccessManager _fileAccessManager; + + public static IBuildComponent CreateComponent(BuildComponentType type) + { + ErrorUtilities.VerifyThrowArgumentOutOfRange(type == BuildComponentType.NodeLauncher, nameof(type)); + return new DetouredNodeLauncher(); + } + + public void InitializeComponent(IBuildComponentHost host) + { + _fileAccessManager = (IFileAccessManager)host.GetComponent(BuildComponentType.FileAccessManager); + } + + public void ShutdownComponent() + { + _fileAccessManager = null; + + foreach (ISandboxedProcess sandboxedProcess in _sandboxedProcesses) + { + sandboxedProcess.Dispose(); + } + + _sandboxedProcesses.Clear(); + } + + /// + /// Creates a new MSBuild process + /// + public Process Start(string msbuildLocation, string commandLineArgs, int nodeId) + { + // Should always have been set already. + ErrorUtilities.VerifyThrowInternalLength(msbuildLocation, nameof(msbuildLocation)); + + ErrorUtilities.VerifyThrowInternalNull(_fileAccessManager, nameof(_fileAccessManager)); + + if (!FileSystems.Default.FileExists(msbuildLocation)) + { + throw new BuildAbortedException(ResourceUtilities.FormatResourceStringStripCodeAndKeyword("CouldNotFindMSBuildExe", msbuildLocation)); + } + + // Repeat the executable name as the first token of the command line because the command line + // parser logic expects it and will otherwise skip the first argument + commandLineArgs = $"\"{msbuildLocation}\" {commandLineArgs}"; + + CommunicationsUtilities.Trace("Launching node from {0}", msbuildLocation); + + string exeName = msbuildLocation; + +#if RUNTIME_TYPE_NETCORE || MONO + // Mono automagically uses the current mono, to execute a managed assembly + if (!NativeMethodsShared.IsMono) + { + // Run the child process with the same host as the currently-running process. + exeName = CurrentHost.GetCurrentHost(); + } +#endif + + var eventListener = new DetoursEventListener(_fileAccessManager, nodeId); + eventListener.SetMessageHandlingFlags(MessageHandlingFlags.DebugMessageNotify | MessageHandlingFlags.FileAccessNotify | MessageHandlingFlags.ProcessDataNotify | MessageHandlingFlags.ProcessDetoursStatusNotify); + + var info = new SandboxedProcessInfo( + fileStorage: null, // Don't write stdout/stderr to files + fileName: exeName, + disableConHostSharing: false, + detoursEventListener: eventListener, + createJobObjectForCurrentProcess: false) + { + SandboxKind = SandboxKind.Default, + PipDescription = "MSBuild", + PipSemiStableHash = 0, + Arguments = commandLineArgs, + EnvironmentVariables = EnvironmentalBuildParameters.Instance, + MaxLengthInMemory = 0, // Don't buffer any output + }; + + // FileAccessManifest.AddScope is used to define the list of files which the running process is allowed to access and what kinds of file accesses are allowed + // Tracker internally uses AbsolutePath.Invalid to represent the root, just like Unix '/' root. + // this code allows all types of accesses for all files + info.FileAccessManifest.AddScope( + AbsolutePath.Invalid, + FileAccessPolicy.MaskNothing, + FileAccessPolicy.AllowAll | FileAccessPolicy.ReportAccess); + + // Support shared compilation + info.FileAccessManifest.ChildProcessesToBreakawayFromSandbox = new string[] { NativeMethodsShared.IsWindows ? "VBCSCompiler.exe" : "VBCSCompiler" }; + info.FileAccessManifest.MonitorChildProcesses = true; + info.FileAccessManifest.IgnoreReparsePoints = true; + info.FileAccessManifest.UseExtraThreadToDrainNtClose = false; + info.FileAccessManifest.UseLargeNtClosePreallocatedList = true; + info.FileAccessManifest.LogProcessData = true; + + // needed for logging process arguments when a new process is invoked; see DetoursEventListener.cs + info.FileAccessManifest.ReportProcessArgs = true; + + // By default, Domino sets the timestamp of all input files to January 1, 1970 + // This breaks some tools like Robocopy which will not copy a file to the destination if the file exists at the destination and has a timestamp that is more recent than the source file + info.FileAccessManifest.NormalizeReadTimestamps = false; + + // If a process exits but its child processes survive, Tracker waits 30 seconds by default to wait for this process to exit. + // This slows down C++ builds in which mspdbsrv.exe doesn't exit when it's parent exits. Set this time to 0. + info.NestedProcessTerminationTimeout = TimeSpan.Zero; + + ISandboxedProcess sp = SandboxedProcessFactory.StartAsync(info, forceSandboxing: false).GetAwaiter().GetResult(); + lock (_sandboxedProcesses) + { + _sandboxedProcesses.Add(sp); + } + + CommunicationsUtilities.Trace("Successfully launched {1} node with PID {0}", sp.ProcessId, exeName); + return Process.GetProcessById(sp.ProcessId); + } + + private sealed class EnvironmentalBuildParameters : BuildParameters.IBuildParameters + { + private readonly Dictionary _envVars; + + private EnvironmentalBuildParameters() + { + var envVars = new Dictionary(); + foreach (DictionaryEntry baseVar in Environment.GetEnvironmentVariables()) + { + envVars.Add((string)baseVar.Key, (string)baseVar.Value); + } + + _envVars = envVars; + } + + private EnvironmentalBuildParameters(Dictionary envVars) + { + _envVars = envVars; + } + + public static EnvironmentalBuildParameters Instance { get; } = new EnvironmentalBuildParameters(); + + public string this[string key] => _envVars[key]; + + public BuildParameters.IBuildParameters Select(IEnumerable keys) + => new EnvironmentalBuildParameters(keys.ToDictionary(key => key, key => _envVars[key])); + + public BuildParameters.IBuildParameters Override(IEnumerable> parameters) + { + var copy = new Dictionary(_envVars); + foreach (KeyValuePair param in parameters) + { + copy[param.Key] = param.Value; + } + + return new EnvironmentalBuildParameters(copy); + } + + public IReadOnlyDictionary ToDictionary() => _envVars; + + public bool ContainsKey(string key) => _envVars.ContainsKey(key); + } + + private sealed class DetoursEventListener : IDetoursEventListener + { + private readonly IFileAccessManager _fileAccessManager; + private readonly int _nodeId; + + public DetoursEventListener(IFileAccessManager fileAccessManager, int nodeId) + { + _fileAccessManager = fileAccessManager; + _nodeId = nodeId; + } + + public override void HandleDebugMessage(DebugData debugData) + { + } + + public override void HandleFileAccess(FileAccessData fileAccessData) => _fileAccessManager.ReportFileAccess( + new Framework.FileAccess.FileAccessData( + (Framework.FileAccess.ReportedFileOperation)fileAccessData.Operation, + (Framework.FileAccess.RequestedAccess)fileAccessData.RequestedAccess, + fileAccessData.ProcessId, + fileAccessData.Error, + (Framework.FileAccess.DesiredAccess)fileAccessData.DesiredAccess, + (Framework.FileAccess.FlagsAndAttributes)fileAccessData.FlagsAndAttributes, + fileAccessData.Path, + fileAccessData.ProcessArgs, + fileAccessData.IsAnAugmentedFileAccess), + _nodeId); + + public override void HandleProcessData(ProcessData processData) => _fileAccessManager.ReportProcess( + new Framework.FileAccess.ProcessData( + processData.ProcessName, + processData.ProcessId, + processData.ParentProcessId, + processData.CreationDateTime, + processData.ExitDateTime, + processData.ExitCode), + _nodeId); + + public override void HandleProcessDetouringStatus(ProcessDetouringStatusData data) + { + } + } + } +} diff --git a/src/Build/BackEnd/Components/Communications/INodeLauncher.cs b/src/Build/BackEnd/Components/Communications/INodeLauncher.cs index e1c424889cc..c409c856c0b 100644 --- a/src/Build/BackEnd/Components/Communications/INodeLauncher.cs +++ b/src/Build/BackEnd/Components/Communications/INodeLauncher.cs @@ -7,6 +7,6 @@ namespace Microsoft.Build.BackEnd { internal interface INodeLauncher { - Process Start(string msbuildLocation, string commandLineArgs); + Process Start(string msbuildLocation, string commandLineArgs, int nodeId); } } diff --git a/src/Build/BackEnd/Components/Communications/NodeLauncher.cs b/src/Build/BackEnd/Components/Communications/NodeLauncher.cs index 375dbf2bae6..ddec6d79279 100644 --- a/src/Build/BackEnd/Components/Communications/NodeLauncher.cs +++ b/src/Build/BackEnd/Components/Communications/NodeLauncher.cs @@ -35,7 +35,7 @@ public void ShutdownComponent() /// /// Creates a new MSBuild process /// - public Process Start(string msbuildLocation, string commandLineArgs) + public Process Start(string msbuildLocation, string commandLineArgs, int nodeId) { // Disable MSBuild server for a child process. // In case of starting msbuild server it prevents an infinite recurson. In case of starting msbuild node we also do not want this variable to be set. @@ -195,7 +195,7 @@ private Process StartInternal(string msbuildLocation, string commandLineArgs) } } - private Process DisableMSBuildServer(Func func) + private static Process DisableMSBuildServer(Func func) { string useMSBuildServerEnvVarValue = Environment.GetEnvironmentVariable(Traits.UseMSBuildServerEnvVarName); try diff --git a/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs b/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs index a822dbaedb7..86849eadf2e 100644 --- a/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs +++ b/src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs @@ -333,7 +333,7 @@ bool StartNewNode(int nodeId) #endif // Create the node process INodeLauncher nodeLauncher = (INodeLauncher)_componentHost.GetComponent(BuildComponentType.NodeLauncher); - Process msbuildProcess = nodeLauncher.Start(msbuildLocation, commandLineArgs); + Process msbuildProcess = nodeLauncher.Start(msbuildLocation, commandLineArgs, nodeId); _processesToIgnore.TryAdd(GetProcessesToIgnoreKey(hostHandshake, msbuildProcess.Id), default); // Note, when running under IMAGEFILEEXECUTIONOPTIONS registry key to debug, the process ID diff --git a/src/Build/Collections/RetrievableEntryHashSet/HashSet.cs b/src/Build/Collections/RetrievableEntryHashSet/HashSet.cs index eeedd7c4c1b..d4d170a32f1 100644 --- a/src/Build/Collections/RetrievableEntryHashSet/HashSet.cs +++ b/src/Build/Collections/RetrievableEntryHashSet/HashSet.cs @@ -38,6 +38,8 @@ nor unit test it directly. #nullable disable +#pragma warning disable RA001 // Do not use System.Diagnostics.Contract class. + namespace Microsoft.Build.Collections { /// diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index 30f4252206d..be87cad1d65 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -32,6 +32,7 @@ + @@ -39,6 +40,8 @@ + + @@ -51,7 +54,6 @@ - @@ -155,6 +157,7 @@ + From f963b7f39d44566e5f8741b6fc5a0d9ad53dfbca Mon Sep 17 00:00:00 2001 From: David Federman Date: Thu, 11 Aug 2022 12:49:22 -0700 Subject: [PATCH 06/21] Add new interfaces to ProjectCachePluginBase --- .../ProjectCache/FileAccessContext.cs | 26 +++++++++++++++++++ .../ProjectCache/ProjectCachePluginBase.cs | 26 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 src/Build/BackEnd/Components/ProjectCache/FileAccessContext.cs diff --git a/src/Build/BackEnd/Components/ProjectCache/FileAccessContext.cs b/src/Build/BackEnd/Components/ProjectCache/FileAccessContext.cs new file mode 100644 index 00000000000..5ff62cfb514 --- /dev/null +++ b/src/Build/BackEnd/Components/ProjectCache/FileAccessContext.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace Microsoft.Build.Experimental.ProjectCache +{ + public readonly struct FileAccessContext + { + public FileAccessContext( + string projectFullPath, + IReadOnlyDictionary globalProperties, + IReadOnlyList targets) + { + ProjectFullPath = projectFullPath; + GlobalProperties = globalProperties; + Targets = targets; + } + + public string ProjectFullPath { get; } + + public IReadOnlyDictionary GlobalProperties { get; } + + public IReadOnlyList Targets { get; } + } +} diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCachePluginBase.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCachePluginBase.cs index 2bf479c6055..07e725c703a 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCachePluginBase.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCachePluginBase.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Execution; +using Microsoft.Build.Framework.FileAccess; namespace Microsoft.Build.Experimental.ProjectCache { @@ -39,5 +40,30 @@ public abstract Task GetCacheResultAsync( /// Errors are checked via . /// public abstract Task EndBuildAsync(PluginLoggerBase logger, CancellationToken cancellationToken); + + /// + /// Called for each file access from an MSBuild node or one of its children. + /// + public virtual void HandleFileAccess(FileAccessContext fileAccessContext, FileAccessData fileAccessData) + { + } + + /// + /// Called for each new child process created by an MSBuild node or one of its children. + /// + public virtual void HandleProcess(FileAccessContext fileAccessContext, ProcessData processData) + { + } + + /// + /// Called when a build request finishes execution. This provides an opportunity for the plugin to take action on the + /// aggregated file access reports from . + /// Errors are checked via . + /// + public virtual Task HandleProjectFinishedAsync( + FileAccessContext fileAccessContext, + BuildResult buildResult, + PluginLoggerBase logger, + CancellationToken cancellationToken) => Task.CompletedTask; } } From bb9f587960b685e06337023d84b08ad7f1d11f3a Mon Sep 17 00:00:00 2001 From: David Federman Date: Mon, 29 Aug 2022 14:46:54 -0700 Subject: [PATCH 07/21] Register and call plugin's file access handlers --- .../BackEnd/BuildManager/BuildManager.cs | 36 +++++ .../ProjectCache/ProjectCacheService.cs | 135 +++++++++++++++++- src/Build/Resources/Strings.resx | 3 + src/Build/Resources/xlf/Strings.cs.xlf | 5 + src/Build/Resources/xlf/Strings.de.xlf | 5 + src/Build/Resources/xlf/Strings.es.xlf | 5 + src/Build/Resources/xlf/Strings.fr.xlf | 5 + src/Build/Resources/xlf/Strings.it.xlf | 5 + src/Build/Resources/xlf/Strings.ja.xlf | 5 + src/Build/Resources/xlf/Strings.ko.xlf | 5 + src/Build/Resources/xlf/Strings.pl.xlf | 5 + src/Build/Resources/xlf/Strings.pt-BR.xlf | 5 + src/Build/Resources/xlf/Strings.ru.xlf | 5 + src/Build/Resources/xlf/Strings.tr.xlf | 5 + src/Build/Resources/xlf/Strings.zh-Hans.xlf | 5 + src/Build/Resources/xlf/Strings.zh-Hant.xlf | 5 + src/Framework/MSBuildEventSource.cs | 12 ++ 17 files changed, 247 insertions(+), 4 deletions(-) diff --git a/src/Build/BackEnd/BuildManager/BuildManager.cs b/src/Build/BackEnd/BuildManager/BuildManager.cs index 4fb434a2d6f..7ffb491eee4 100644 --- a/src/Build/BackEnd/BuildManager/BuildManager.cs +++ b/src/Build/BackEnd/BuildManager/BuildManager.cs @@ -25,6 +25,7 @@ using Microsoft.Build.Exceptions; using Microsoft.Build.Experimental; using Microsoft.Build.Experimental.ProjectCache; +using Microsoft.Build.FileAccesses; using Microsoft.Build.Framework; using Microsoft.Build.Framework.Telemetry; using Microsoft.Build.Graph; @@ -572,9 +573,12 @@ public void BeginBuild(BuildParameters parameters) InitializeCaches(); + var fileAccessManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager) as IFileAccessManager; _projectCacheService = new ProjectCacheService( this, loggingService, + fileAccessManager, + _configCache, _buildParameters.ProjectCacheDescriptor); _taskHostNodeManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.TaskHostNodeManager) as INodeManager; @@ -2371,6 +2375,38 @@ private void HandleResult(int node, BuildResult result) configuration.ProjectTargets ??= result.ProjectTargets; } + // Only report results to the project cache services if it's the result for a build submission. + // Note that graph builds create a submission for each node in the graph, so each node in the graph will be + // handled here. This intentionally mirrors the behavior for cache requests, as it doesn't make sense to + // report for projects which aren't going to be requested. Ideally, *any* request could be handled, but that + // would require moving the cache service interactions to the Scheduler. + if (_buildSubmissions.TryGetValue(result.SubmissionId, out BuildSubmission buildSubmission)) + { + // The result may be associated with the build submission due to it being the submission which + // caused the build, but not the actual request which was used with the build submission. Ensure + // only the actual submission's request is considered. + if (buildSubmission.BuildRequest != null + && buildSubmission.BuildRequest.ConfigurationId == configuration.ConfigurationId + && _projectCacheService.ShouldUseCache(configuration)) + { + BuildEventContext buildEventContext = _projectStartedEvents.TryGetValue(result.SubmissionId, out BuildEventArgs buildEventArgs) + ? buildEventArgs.BuildEventContext + : new BuildEventContext(result.SubmissionId, node, configuration.Project?.EvaluationId ?? BuildEventContext.InvalidEvaluationId, configuration.ConfigurationId, BuildEventContext.InvalidProjectContextId, BuildEventContext.InvalidTargetId, BuildEventContext.InvalidTaskId); + try + { + _projectCacheService.HandleBuildResultAsync(configuration, result, buildEventContext, _executionCancellationTokenSource.Token).Wait(); + } + catch (AggregateException ex) when (ex.InnerExceptions.All(inner => inner is OperationCanceledException)) + { + // The build is being cancelled. Swallow any exceptions related specifically to cancellation. + } + catch (OperationCanceledException) + { + // The build is being cancelled. Swallow any exceptions related specifically to cancellation. + } + } + } + IEnumerable response = _scheduler.ReportResult(node, result); PerformSchedulingActions(response); } diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 9d1cefe040d..54506dbc2a0 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -16,6 +16,7 @@ using Microsoft.Build.Construction; using Microsoft.Build.Eventing; using Microsoft.Build.Execution; +using Microsoft.Build.FileAccesses; using Microsoft.Build.FileSystem; using Microsoft.Build.Framework; using Microsoft.Build.Graph; @@ -34,16 +35,25 @@ internal sealed class ProjectCacheService : IAsyncDisposable private readonly BuildManager _buildManager; private readonly ILoggingService _loggingService; + private readonly IFileAccessManager _fileAccessManager; + private readonly IConfigCache _configCache; private readonly ProjectCacheDescriptor? _globalProjectCacheDescriptor; private readonly ConcurrentDictionary>> _projectCachePlugins = new(ProjectCacheDescriptorEqualityComparer.Instance); + // Helps to avoid excessive allocation since BuildRequestConfiguration doesn't expose global properties in a way the plugins can consume (PropertyDictionary vs IReadOnlyDictionary). + private readonly ConcurrentDictionary> _globalPropertiesPerConfiguration = new(); + private bool _isVsScenario; private bool _isDisposed; - private record struct ProjectCachePlugin(string Name, ProjectCachePluginBase? Instance, ExceptionDispatchInfo? InitializationException = null); + private record struct ProjectCachePlugin( + string Name, + ProjectCachePluginBase? Instance, + FileAccessManager.HandlerRegistration? HandlerRegistration, + ExceptionDispatchInfo? InitializationException = null); /// /// An instanatiable version of MSBuildFileSystemBase not overriding any methods, @@ -61,10 +71,14 @@ private DefaultMSBuildFileSystem() public ProjectCacheService( BuildManager buildManager, ILoggingService loggingService, + IFileAccessManager fileAccessManager, + IConfigCache configCache, ProjectCacheDescriptor? globalProjectCacheDescriptor) { _buildManager = buildManager; _loggingService = loggingService; + _fileAccessManager = fileAccessManager; + _configCache = configCache; _globalProjectCacheDescriptor = globalProjectCacheDescriptor; } @@ -187,7 +201,7 @@ private async Task CreateAndInitializePluginAsync( } catch (Exception e) { - return new ProjectCachePlugin(pluginTypeName, Instance: null, ExceptionDispatchInfo.Capture(e)); + return new ProjectCachePlugin(pluginTypeName, Instance: null, HandlerRegistration: null, ExceptionDispatchInfo.Capture(e)); } finally { @@ -218,11 +232,25 @@ await pluginInstance.BeginBuildAsync( ProjectCacheException.ThrowForErrorLoggedInsideTheProjectCache("ProjectCacheInitializationFailed"); } - return new ProjectCachePlugin(pluginTypeName, pluginInstance); + FileAccessManager.HandlerRegistration handlerRegistration = _fileAccessManager.RegisterHandlers( + (buildRequest, fileAccessData) => + { + // TODO: Filter out projects which do not configure this plugin + FileAccessContext fileAccessContext = GetFileAccessContext(buildRequest); + pluginInstance.HandleFileAccess(fileAccessContext, fileAccessData); + }, + (buildRequest, processData) => + { + // TODO: Filter out projects which do not configure this plugin + FileAccessContext fileAccessContext = GetFileAccessContext(buildRequest); + pluginInstance.HandleProcess(fileAccessContext, processData); + }); + + return new ProjectCachePlugin(pluginTypeName, pluginInstance, handlerRegistration); } catch (Exception e) { - return new ProjectCachePlugin(pluginTypeName, Instance: null, ExceptionDispatchInfo.Capture(e)); + return new ProjectCachePlugin(pluginTypeName, Instance: null, HandlerRegistration: null, ExceptionDispatchInfo.Capture(e)); } finally { @@ -230,6 +258,27 @@ await pluginInstance.BeginBuildAsync( } } + private FileAccessContext GetFileAccessContext(BuildRequest buildRequest) + { + BuildRequestConfiguration configuration = _configCache[buildRequest.ConfigurationId]; + IReadOnlyDictionary globalProperties = GetGlobalProperties(configuration); + return new FileAccessContext(configuration.ProjectFullPath, globalProperties, buildRequest.Targets); + } + + private IReadOnlyDictionary GetGlobalProperties(BuildRequestConfiguration configuration) + => _globalPropertiesPerConfiguration.GetOrAdd( + configuration, + static configuration => + { + Dictionary globalProperties = new(configuration.GlobalProperties.Count, StringComparer.OrdinalIgnoreCase); + foreach (ProjectPropertyInstance property in configuration.GlobalProperties) + { + globalProperties.Add(property.Name, property.EvaluatedValue); + } + + return globalProperties; + }); + private static ProjectCachePluginBase GetPluginInstanceFromType(Type pluginType) { try @@ -587,6 +636,79 @@ static IReadOnlyCollection GenerateGraphEntryPointsFromS } } + public async Task HandleBuildResultAsync( + BuildRequestConfiguration requestConfiguration, + BuildResult buildResult, + BuildEventContext buildEventContext, + CancellationToken cancellationToken) + { + ErrorUtilities.VerifyThrowInternalNull(requestConfiguration.Project, nameof(requestConfiguration.Project)); + + if (_projectCachePlugins.IsEmpty) + { + return; + } + + // Filter to plugins which apply to the project, if any + List projectCacheDescriptors = GetProjectCacheDescriptors(requestConfiguration.Project).ToList(); + if (projectCacheDescriptors.Count == 0) + { + return; + } + + + IReadOnlyDictionary globalProperties = GetGlobalProperties(requestConfiguration); + + List targets = buildResult.ResultsByTarget.Keys.ToList(); + string? targetNames = string.Join(", ", targets); + + FileAccessContext fileAccessContext = new(requestConfiguration.ProjectFullPath, globalProperties, targets); + + var buildEventFileInfo = new BuildEventFileInfo(requestConfiguration.ProjectFullPath); + var pluginLogger = new LoggingServiceToPluginLoggerAdapter( + _loggingService, + buildEventContext, + buildEventFileInfo); + + Task[] tasks = new Task[projectCacheDescriptors.Count]; + int idx = 0; + foreach (ProjectCacheDescriptor projectCacheDescriptor in projectCacheDescriptors) + { + tasks[idx++] = Task.Run( + async () => + { + ProjectCachePlugin plugin = await _projectCachePlugins[projectCacheDescriptor].Value; + + // Rethrow any initialization exception. + plugin.InitializationException?.Throw(); + + ErrorUtilities.VerifyThrow(plugin.Instance != null, "Plugin '{0}' instance is null", plugin.Name); + + MSBuildEventSource.Log.ProjectCacheHandleBuildResultStart(plugin.Name, fileAccessContext.ProjectFullPath, targetNames); + try + { + await plugin.Instance!.HandleProjectFinishedAsync(fileAccessContext, buildResult, pluginLogger, cancellationToken); + } + catch (Exception e) when (e is not ProjectCacheException) + { + HandlePluginException(e, nameof(ProjectCachePluginBase.HandleProjectFinishedAsync)); + } + finally + { + MSBuildEventSource.Log.ProjectCacheHandleBuildResultStop(plugin.Name, fileAccessContext.ProjectFullPath, targetNames); + } + }, + cancellationToken); + } + + await Task.WhenAll(tasks).ConfigureAwait(false); + + if (pluginLogger.HasLoggedErrors) + { + ProjectCacheException.ThrowForErrorLoggedInsideTheProjectCache("ProjectCacheHandleBuildResultFailed", fileAccessContext.ProjectFullPath); + } + } + public async ValueTask DisposeAsync() { if (_isDisposed) @@ -624,6 +746,11 @@ public async ValueTask DisposeAsync() return; } + if (plugin.HandlerRegistration.HasValue) + { + plugin.HandlerRegistration.Value.Dispose(); + } + MSBuildEventSource.Log.ProjectCacheEndBuildStart(plugin.Name); try { diff --git a/src/Build/Resources/Strings.resx b/src/Build/Resources/Strings.resx index 5a8f8640a74..564caf33095 100644 --- a/src/Build/Resources/Strings.resx +++ b/src/Build/Resources/Strings.resx @@ -1902,6 +1902,9 @@ Utilization: {0} Average Utilization: {1:###.0} MSB4268: The project cache failed to shut down properly. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4270: No project cache plugins found in assembly "{0}". Expected one. diff --git a/src/Build/Resources/xlf/Strings.cs.xlf b/src/Build/Resources/xlf/Strings.cs.xlf index 0f52b70e2b8..4df1c74126c 100644 --- a/src/Build/Resources/xlf/Strings.cs.xlf +++ b/src/Build/Resources/xlf/Strings.cs.xlf @@ -254,6 +254,11 @@ MSB4273: Mezipaměť projektu vyvolala neošetřenou výjimku z metody {0}. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). Přístup do mezipaměti projektu pro „{0}“ (výchozí cíle). diff --git a/src/Build/Resources/xlf/Strings.de.xlf b/src/Build/Resources/xlf/Strings.de.xlf index 363edd349b7..d6c171a3dfe 100644 --- a/src/Build/Resources/xlf/Strings.de.xlf +++ b/src/Build/Resources/xlf/Strings.de.xlf @@ -254,6 +254,11 @@ MSB4273: Der Projektcache hat über die Methode {0} eine unbehandelte Ausnahme ausgelöst. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). Projektcachetreffer für „{0}“ (Standardziele). diff --git a/src/Build/Resources/xlf/Strings.es.xlf b/src/Build/Resources/xlf/Strings.es.xlf index b614cd5d41a..21c0297c4b9 100644 --- a/src/Build/Resources/xlf/Strings.es.xlf +++ b/src/Build/Resources/xlf/Strings.es.xlf @@ -254,6 +254,11 @@ MSB4273: la caché del proyecto inició una excepción no controlada desde el método {0}. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). Acierto de caché de proyecto para "{0}" (destinos predeterminados). diff --git a/src/Build/Resources/xlf/Strings.fr.xlf b/src/Build/Resources/xlf/Strings.fr.xlf index c38b649d037..4bd1e6f2989 100644 --- a/src/Build/Resources/xlf/Strings.fr.xlf +++ b/src/Build/Resources/xlf/Strings.fr.xlf @@ -254,6 +254,11 @@ MSB4273: le cache de projet a levé une exception non gérée à partir de la méthode {0}. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). Le cache de projet a été atteint pour « {0} » (cibles par défaut). diff --git a/src/Build/Resources/xlf/Strings.it.xlf b/src/Build/Resources/xlf/Strings.it.xlf index 4c4932631a8..3ff49b5aa8e 100644 --- a/src/Build/Resources/xlf/Strings.it.xlf +++ b/src/Build/Resources/xlf/Strings.it.xlf @@ -254,6 +254,11 @@ MSB4273: la cache del progetto ha generato un'eccezione non gestita dal metodo {0}. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). Riscontro nella cache del progetto per "{0}" (destinazioni predefinite). diff --git a/src/Build/Resources/xlf/Strings.ja.xlf b/src/Build/Resources/xlf/Strings.ja.xlf index 0ab12ba89e3..289deb0cc2e 100644 --- a/src/Build/Resources/xlf/Strings.ja.xlf +++ b/src/Build/Resources/xlf/Strings.ja.xlf @@ -254,6 +254,11 @@ MSB4273: プロジェクト キャッシュが {0} メソッドで処理されていない例外が返されました。 + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). "{0}" のプロジェクト キャッシュ ヒット (既定のターゲット)。 diff --git a/src/Build/Resources/xlf/Strings.ko.xlf b/src/Build/Resources/xlf/Strings.ko.xlf index 8101d90c436..992147bfe58 100644 --- a/src/Build/Resources/xlf/Strings.ko.xlf +++ b/src/Build/Resources/xlf/Strings.ko.xlf @@ -254,6 +254,11 @@ MSB4273: 프로젝트 캐시는 {0} 메서드에서 처리되지 않은 예외를 발생시켰습니다. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). "{0}"(기본 대상)에 대한 프로젝트 캐시 적중입니다. diff --git a/src/Build/Resources/xlf/Strings.pl.xlf b/src/Build/Resources/xlf/Strings.pl.xlf index 334e24ff11e..85c79075179 100644 --- a/src/Build/Resources/xlf/Strings.pl.xlf +++ b/src/Build/Resources/xlf/Strings.pl.xlf @@ -254,6 +254,11 @@ MSB4273: pamięć podręczna projektu zgłosiła nieobsługiwany wyjątek z metody {0}. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). Trafienie pamięci podręcznej projektu dla „{0}” (domyślne elementy docelowe). diff --git a/src/Build/Resources/xlf/Strings.pt-BR.xlf b/src/Build/Resources/xlf/Strings.pt-BR.xlf index 1231fa2b51b..b87e7d369d8 100644 --- a/src/Build/Resources/xlf/Strings.pt-BR.xlf +++ b/src/Build/Resources/xlf/Strings.pt-BR.xlf @@ -254,6 +254,11 @@ MSB4273: O cache do projeto lançou uma exceção sem tratamento do método {0}. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). Acerto de cache do projeto para "{0}" (destinos padrão). diff --git a/src/Build/Resources/xlf/Strings.ru.xlf b/src/Build/Resources/xlf/Strings.ru.xlf index 2858ea61d4b..065230c8537 100644 --- a/src/Build/Resources/xlf/Strings.ru.xlf +++ b/src/Build/Resources/xlf/Strings.ru.xlf @@ -254,6 +254,11 @@ MSB4273: в кэше проектов возникло необработанное исключение из метода {0}. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). Попадание в кэше проекта для "{0}" (целевые объекты по умолчанию). diff --git a/src/Build/Resources/xlf/Strings.tr.xlf b/src/Build/Resources/xlf/Strings.tr.xlf index dda58e54d67..6603883eff4 100644 --- a/src/Build/Resources/xlf/Strings.tr.xlf +++ b/src/Build/Resources/xlf/Strings.tr.xlf @@ -254,6 +254,11 @@ MSB4273: Proje önbelleği {0} yönteminden yakalanamayan özel durum oluşturdu. + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). "{0}" (varsayılan hedefler) için proje önbelleği isabeti. diff --git a/src/Build/Resources/xlf/Strings.zh-Hans.xlf b/src/Build/Resources/xlf/Strings.zh-Hans.xlf index 8a2d41f5a59..19ef6bae0b4 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hans.xlf @@ -254,6 +254,11 @@ MSB4273: 项目缓存从 {0} 方法引发了未经处理的异常。 + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). 项目缓存命中 "{0}" (默认目标)。 diff --git a/src/Build/Resources/xlf/Strings.zh-Hant.xlf b/src/Build/Resources/xlf/Strings.zh-Hant.xlf index 3bae314d441..6da2dfaa105 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hant.xlf @@ -254,6 +254,11 @@ MSB4273: 專案快取從 {0} 方法擲回未處理的例外狀況。 + + MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling build result for the following project: {0}. + + Project cache hit for "{0}" (default targets). "{0}" 的專案快取命中 (預設目標)。 diff --git a/src/Framework/MSBuildEventSource.cs b/src/Framework/MSBuildEventSource.cs index 29e9e67abfb..a5dbe84a3ee 100644 --- a/src/Framework/MSBuildEventSource.cs +++ b/src/Framework/MSBuildEventSource.cs @@ -661,6 +661,18 @@ public void MSBuildServerBuildStop(string commandLine, int countOfConsoleMessage { WriteEvent(90, commandLine, countOfConsoleMessages, sumSizeOfConsoleMessages, clientExitType, serverExitType); } + + [Event(91, Keywords = Keywords.All)] + public void ProjectCacheHandleBuildResultStart(string pluginTypeName, string projectPath, string targets) + { + WriteEvent(91, pluginTypeName, projectPath, targets); + } + + [Event(92, Keywords = Keywords.All)] + public void ProjectCacheHandleBuildResultStop(string pluginTypeName, string projectPath, string targets) + { + WriteEvent(92, pluginTypeName, projectPath, targets); + } #endregion } } From 796ec8f7633191465fe9e63b7eec9a13c6a711b5 Mon Sep 17 00:00:00 2001 From: David Federman Date: Tue, 29 Nov 2022 16:46:14 -0800 Subject: [PATCH 08/21] Synchronize node communication with file access reporting --- .../FileAccesses/FileAccessManager.cs | 56 +++++++++++++++++-- .../FileAccesses/IFileAccessManager.cs | 3 + .../ProjectCache/ProjectCacheService.cs | 6 ++ src/Build/BackEnd/Node/OutOfProcNode.cs | 6 ++ 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs b/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs index 0c80b6d9539..3b8bf0326ce 100644 --- a/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs +++ b/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs @@ -2,8 +2,11 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Concurrent; +using System.IO; using System.Threading; using Microsoft.Build.BackEnd; +using Microsoft.Build.Execution; using Microsoft.Build.Framework.FileAccess; using Microsoft.Build.Shared; @@ -13,12 +16,20 @@ internal sealed class FileAccessManager : IFileAccessManager, IBuildComponent { private record Handlers(Action FileAccessHander, Action ProcessHandler); + // In order to synchronize between the node communication and the file access reporting, a special file access + // is used to mark when the file accesses should be considered complete. Only after both this special file access is seen + // and the build result is reported can plugins be notified about project completion. + private static readonly string FileAccessCompletionPrefix = BuildParameters.StartupDirectory[0] + @":\{MSBuildFileAccessCompletion}\"; + private IScheduler? _scheduler; private IConfigCache? _configCache; private object _handlersWriteLock = new object(); private Handlers[] _handlers = Array.Empty(); + // Keyed on global request id + private readonly ConcurrentDictionary _fileAccessCompletionWaitHandles = new(); + public static IBuildComponent CreateComponent(BuildComponentType type) { ErrorUtilities.VerifyThrowArgumentOutOfRange(type == BuildComponentType.FileAccessManager, nameof(type)); @@ -35,17 +46,33 @@ public void ShutdownComponent() { _scheduler = null; _configCache = null; + _fileAccessCompletionWaitHandles.Clear(); } public void ReportFileAccess(FileAccessData fileAccessData, int nodeId) { - BuildRequest? buildRequest = GetBuildRequest(nodeId); - if (buildRequest != null) + string fileAccessPath = fileAccessData.Path; + + // Intercept and avoid forwarding the file access completion + if (fileAccessPath.StartsWith(FileAccessCompletionPrefix, StringComparison.Ordinal)) { - Handlers[] localHandlers = _handlers; - foreach (Handlers handlers in localHandlers) + // Parse out the global request id. Note, this must match what NotifyFileAccessCompletion does. + int globalRequestId = int.Parse(fileAccessPath.Substring(FileAccessCompletionPrefix.Length)); + + ManualResetEventSlim handle = _fileAccessCompletionWaitHandles.GetOrAdd(globalRequestId, static _ => new ManualResetEventSlim()); + handle.Set(); + } + else + { + // Forward the file access to handlers. + BuildRequest? buildRequest = GetBuildRequest(nodeId); + if (buildRequest != null) { - handlers.FileAccessHander.Invoke(buildRequest, fileAccessData); + Handlers[] localHandlers = _handlers; + foreach (Handlers handlers in localHandlers) + { + handlers.FileAccessHander.Invoke(buildRequest, fileAccessData); + } } } } @@ -98,6 +125,25 @@ private void UnregisterHandlers(Handlers handlersToRemove) } } + public static void NotifyFileAccessCompletion(int globalRequestId) + { + // Make a dummy file access to use as a notification that the file accesses should be completed for a project. + string filePath = FileAccessCompletionPrefix + globalRequestId.ToString(); + _ = File.Exists(filePath); + } + + public void WaitForFileAccessReportCompletion(int globalRequestId, CancellationToken cancellationToken) + { + ManualResetEventSlim handle = _fileAccessCompletionWaitHandles.GetOrAdd(globalRequestId, static _ => new ManualResetEventSlim()); + if (!handle.IsSet) + { + handle.Wait(cancellationToken); + } + + // Try to keep the collection clean. A request should not need to be completed twice. + _fileAccessCompletionWaitHandles.TryRemove(globalRequestId, out _); + } + private BuildRequest? GetBuildRequest(int nodeId) { ErrorUtilities.VerifyThrow( diff --git a/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs b/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs index 2834759f55b..5772ad93d21 100644 --- a/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs +++ b/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Threading; using Microsoft.Build.BackEnd; using Microsoft.Build.Framework.FileAccess; @@ -17,5 +18,7 @@ internal interface IFileAccessManager FileAccessManager.HandlerRegistration RegisterHandlers( Action fileAccessHandler, Action processHandler); + + void WaitForFileAccessReportCompletion(int globalRequestId, CancellationToken cancellationToken); } } diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 54506dbc2a0..7a9c31cb310 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -34,6 +34,7 @@ internal sealed class ProjectCacheService : IAsyncDisposable private static HashSet s_projectSpecificPropertyNames = new(StringComparer.OrdinalIgnoreCase) { "TargetFramework", "Configuration", "Platform", "TargetPlatform", "OutputType" }; private readonly BuildManager _buildManager; + private readonly IBuildComponentHost _componentHost; private readonly ILoggingService _loggingService; private readonly IFileAccessManager _fileAccessManager; private readonly IConfigCache _configCache; @@ -76,6 +77,7 @@ public ProjectCacheService( ProjectCacheDescriptor? globalProjectCacheDescriptor) { _buildManager = buildManager; + _componentHost = buildManager; _loggingService = loggingService; _fileAccessManager = fileAccessManager; _configCache = configCache; @@ -656,6 +658,10 @@ public async Task HandleBuildResultAsync( return; } + if (_componentHost.BuildParameters.ReportFileAccesses) + { + _fileAccessManager.WaitForFileAccessReportCompletion(buildResult.GlobalRequestId, cancellationToken); + } IReadOnlyDictionary globalProperties = GetGlobalProperties(requestConfiguration); diff --git a/src/Build/BackEnd/Node/OutOfProcNode.cs b/src/Build/BackEnd/Node/OutOfProcNode.cs index c0c3531ff34..ab9f554f82b 100644 --- a/src/Build/BackEnd/Node/OutOfProcNode.cs +++ b/src/Build/BackEnd/Node/OutOfProcNode.cs @@ -14,6 +14,7 @@ using Microsoft.Build.BackEnd.Logging; using Microsoft.Build.BackEnd.SdkResolution; using Microsoft.Build.Evaluation; +using Microsoft.Build.FileAccesses; using Microsoft.Build.Framework; using Microsoft.Build.Internal; using Microsoft.Build.Shared; @@ -368,6 +369,11 @@ private void OnRequestComplete(BuildRequest request, BuildResult result) { _nodeEndpoint.SendData(result); } + + if (_buildParameters.ReportFileAccesses) + { + FileAccessManager.NotifyFileAccessCompletion(result.GlobalRequestId); + } } /// From 9df8ec89783a96d0eeb2327f72e173b938162b83 Mon Sep 17 00:00:00 2001 From: David Federman Date: Mon, 12 Dec 2022 13:07:31 -0800 Subject: [PATCH 09/21] Exclude file accesses under TempFileDirectory --- .../BackEnd/Components/FileAccesses/FileAccessManager.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs b/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs index 3b8bf0326ce..fbafcab1c16 100644 --- a/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs +++ b/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs @@ -26,6 +26,7 @@ private record Handlers(Action FileAccessHander, A private object _handlersWriteLock = new object(); private Handlers[] _handlers = Array.Empty(); + private string? _tempDirectory; // Keyed on global request id private readonly ConcurrentDictionary _fileAccessCompletionWaitHandles = new(); @@ -40,12 +41,14 @@ public void InitializeComponent(IBuildComponentHost host) { _scheduler = host.GetComponent(BuildComponentType.Scheduler) as IScheduler; _configCache = host.GetComponent(BuildComponentType.ConfigCache) as IConfigCache; + _tempDirectory = FileUtilities.EnsureNoTrailingSlash(FileUtilities.TempFileDirectory); } public void ShutdownComponent() { _scheduler = null; _configCache = null; + _tempDirectory = null; _fileAccessCompletionWaitHandles.Clear(); } @@ -62,6 +65,12 @@ public void ReportFileAccess(FileAccessData fileAccessData, int nodeId) ManualResetEventSlim handle = _fileAccessCompletionWaitHandles.GetOrAdd(globalRequestId, static _ => new ManualResetEventSlim()); handle.Set(); } + else if (_tempDirectory != null && fileAccessPath.StartsWith(_tempDirectory)) + { + // Ignore the temp directory as these are related to internal MSBuild functionality and not always directly related to the execution of the project itself, + // so should not be exposed to handlers. + return; + } else { // Forward the file access to handlers. From 9094e33ed1df066ad52ccdb7f0e1ff22ac3fd8a2 Mon Sep 17 00:00:00 2001 From: David Federman Date: Wed, 14 Dec 2022 16:14:36 -0800 Subject: [PATCH 10/21] Do not allow task yielding when reporting file accesses --- .../BackEnd/Components/RequestBuilder/TaskHost.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs b/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs index 3e0ea3b7abf..94215412f9d 100644 --- a/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs +++ b/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs @@ -343,6 +343,12 @@ public BuildEngineResult BuildProjectFilesInParallel(string[] projectFileNames, /// public void Yield() { + // If file accesses are being reported we should not yield as file access will be attributed to the wrong project. + if (_host.BuildParameters.ReportFileAccesses) + { + return; + } + lock (_callbackMonitor) { IRequestBuilderCallback builderCallback = _requestEntry.Builder as IRequestBuilderCallback; @@ -364,6 +370,12 @@ public void Reacquire() // to release explicitly granted cores when reacquiring the node may lead to deadlocks. ReleaseAllCores(); + // If file accesses are being reported yielding is a no-op so reacquire should be too. + if (_host.BuildParameters.ReportFileAccesses) + { + return; + } + lock (_callbackMonitor) { IRequestBuilderCallback builderCallback = _requestEntry.Builder as IRequestBuilderCallback; From 8e3b92fa98c75b6b4d0851e023e7a43911574d65 Mon Sep 17 00:00:00 2001 From: David Federman Date: Tue, 28 Feb 2023 14:01:05 -0800 Subject: [PATCH 11/21] Implement OutOfProcNodeFileAccessManager --- .../BackEnd/BuildManager/BuildManager.cs | 28 ++++ .../FileAccesses/FileAccessReport.cs | 27 ++++ .../OutOfProcNodeFileAccessManager.cs | 61 ++++++++ .../Components/FileAccesses/ProcessReport.cs | 27 ++++ src/Build/BackEnd/Node/OutOfProcNode.cs | 5 +- src/Build/Microsoft.Build.csproj | 5 +- src/Shared/BinaryTranslator.cs | 131 ++++++++++++++++++ src/Shared/INodePacket.cs | 10 ++ src/Shared/ITranslator.cs | 17 +++ 9 files changed, 308 insertions(+), 3 deletions(-) create mode 100644 src/Build/BackEnd/Components/FileAccesses/FileAccessReport.cs create mode 100644 src/Build/BackEnd/Components/FileAccesses/OutOfProcNodeFileAccessManager.cs create mode 100644 src/Build/BackEnd/Components/FileAccesses/ProcessReport.cs diff --git a/src/Build/BackEnd/BuildManager/BuildManager.cs b/src/Build/BackEnd/BuildManager/BuildManager.cs index 7ffb491eee4..ee52bb67ab1 100644 --- a/src/Build/BackEnd/BuildManager/BuildManager.cs +++ b/src/Build/BackEnd/BuildManager/BuildManager.cs @@ -588,7 +588,9 @@ public void BeginBuild(BuildParameters parameters) _nodeManager.RegisterPacketHandler(NodePacketType.BuildRequestConfiguration, BuildRequestConfiguration.FactoryForDeserialization, this); _nodeManager.RegisterPacketHandler(NodePacketType.BuildRequestConfigurationResponse, BuildRequestConfigurationResponse.FactoryForDeserialization, this); _nodeManager.RegisterPacketHandler(NodePacketType.BuildResult, BuildResult.FactoryForDeserialization, this); + _nodeManager.RegisterPacketHandler(NodePacketType.FileAccessReport, FileAccessReport.FactoryForDeserialization, this); _nodeManager.RegisterPacketHandler(NodePacketType.NodeShutdown, NodeShutdown.FactoryForDeserialization, this); + _nodeManager.RegisterPacketHandler(NodePacketType.ProcessReport, ProcessReport.FactoryForDeserialization, this); _nodeManager.RegisterPacketHandler(NodePacketType.ResolveSdkRequest, SdkResolverRequest.FactoryForDeserialization, SdkResolverService as INodePacketHandler); _nodeManager.RegisterPacketHandler(NodePacketType.ResourceRequest, ResourceRequest.FactoryForDeserialization, this); @@ -1568,6 +1570,16 @@ private void ProcessPacket(int node, INodePacket packet) HandleNodeShutdown(node, shutdownPacket); break; + case NodePacketType.FileAccessReport: + FileAccessReport fileAccessReport = ExpectPacketType(packet, NodePacketType.FileAccessReport); + HandleFileAccessReport(node, fileAccessReport); + break; + + case NodePacketType.ProcessReport: + ProcessReport processReport = ExpectPacketType(packet, NodePacketType.ProcessReport); + HandleProcessReport(node, processReport); + break; + default: ErrorUtilities.ThrowInternalError("Unexpected packet received by BuildManager: {0}", packet.Type); break; @@ -2473,6 +2485,22 @@ private void HandleNodeShutdown(int node, NodeShutdown shutdownPacket) CheckForActiveNodesAndCleanUpSubmissions(); } + /// + /// Report the received to the . + /// + /// The id of the node from which the was received. + /// The file access to report to the . + private void HandleFileAccessReport(int nodeId, FileAccessReport fileAccessReport) => + ((FileAccessManager)((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager)).ReportFileAccess(fileAccessReport.FileAccessData, nodeId); + + /// + /// Report the received to the . + /// + /// The id of the node from which the was received. + /// The process data to report to the . + private void HandleProcessReport(int nodeId, ProcessReport processReport) => + ((FileAccessManager)((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager)).ReportProcess(processReport.ProcessData, nodeId); + /// /// If there are no more active nodes, cleans up any remaining submissions. /// diff --git a/src/Build/BackEnd/Components/FileAccesses/FileAccessReport.cs b/src/Build/BackEnd/Components/FileAccesses/FileAccessReport.cs new file mode 100644 index 00000000000..f69b6fd1580 --- /dev/null +++ b/src/Build/BackEnd/Components/FileAccesses/FileAccessReport.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Build.BackEnd; +using Microsoft.Build.Framework.FileAccess; + +namespace Microsoft.Build.FileAccesses +{ + internal sealed class FileAccessReport : INodePacket + { + private FileAccessData _fileAccessData; + + internal FileAccessReport(FileAccessData fileAccessData) => _fileAccessData = fileAccessData; + + private FileAccessReport(ITranslator translator) => Translate(translator); + + /// + public NodePacketType Type => NodePacketType.FileAccessReport; + + /// + public void Translate(ITranslator translator) => translator.Translate(ref _fileAccessData); + + internal FileAccessData FileAccessData => _fileAccessData; + + internal static INodePacket FactoryForDeserialization(ITranslator translator) => new FileAccessReport(translator); + } +} diff --git a/src/Build/BackEnd/Components/FileAccesses/OutOfProcNodeFileAccessManager.cs b/src/Build/BackEnd/Components/FileAccesses/OutOfProcNodeFileAccessManager.cs new file mode 100644 index 00000000000..ea1f42f1c79 --- /dev/null +++ b/src/Build/BackEnd/Components/FileAccesses/OutOfProcNodeFileAccessManager.cs @@ -0,0 +1,61 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using Microsoft.Build.BackEnd; +using Microsoft.Build.Framework.FileAccess; +using Microsoft.Build.Shared; + +namespace Microsoft.Build.FileAccesses +{ + /// + /// Reports file accesses and process data to the in-proc node. + /// + internal sealed class OutOfProcNodeFileAccessManager : IFileAccessManager, IBuildComponent + { + /// + /// The to report file accesses and process + /// data to the in-proc node. + /// + private readonly Action _sendPacket; + + private OutOfProcNodeFileAccessManager(Action sendPacket) => _sendPacket = sendPacket; + + public static IBuildComponent CreateComponent(BuildComponentType type, Action sendPacket) + { + ErrorUtilities.VerifyThrowArgumentOutOfRange(type == BuildComponentType.FileAccessManager, nameof(type)); + return new OutOfProcNodeFileAccessManager(sendPacket); + } + + public void InitializeComponent(IBuildComponentHost host) + { + } + + public void ShutdownComponent() + { + } + + /// + /// Reports a file access to the in-proc node. + /// + /// The file access to report to the in-proc node. + /// The id of the reporting out-of-proc node. + public void ReportFileAccess(FileAccessData fileAccessData, int nodeId) => _sendPacket(new FileAccessReport(fileAccessData)); + + /// + /// Reports process data to the in-proc node. + /// + /// The process data to report to the in-proc node. + /// The id of the reporting out-of-proc node. + public void ReportProcess(ProcessData processData, int nodeId) => _sendPacket(new ProcessReport(processData)); + + public FileAccessManager.HandlerRegistration RegisterHandlers( + Action fileAccessHandler, + Action processHandler) => + throw new NotImplementedException("This method should not be called in OOP nodes."); + + public void WaitForFileAccessReportCompletion(int globalRequestId, CancellationToken cancellationToken) => + throw new NotImplementedException("This method should not be called in OOP nodes."); + } +} diff --git a/src/Build/BackEnd/Components/FileAccesses/ProcessReport.cs b/src/Build/BackEnd/Components/FileAccesses/ProcessReport.cs new file mode 100644 index 00000000000..89bf533ed86 --- /dev/null +++ b/src/Build/BackEnd/Components/FileAccesses/ProcessReport.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Build.BackEnd; +using Microsoft.Build.Framework.FileAccess; + +namespace Microsoft.Build.FileAccesses +{ + internal sealed class ProcessReport : INodePacket + { + private ProcessData _processData; + + internal ProcessReport(ProcessData processData) => _processData = processData; + + private ProcessReport(ITranslator translator) => Translate(translator); + + /// + public NodePacketType Type => NodePacketType.ProcessReport; + + internal ProcessData ProcessData => _processData; + + internal static INodePacket FactoryForDeserialization(ITranslator translator) => new ProcessReport(translator); + + /// + public void Translate(ITranslator translator) => translator.Translate(ref _processData); + } +} diff --git a/src/Build/BackEnd/Node/OutOfProcNode.cs b/src/Build/BackEnd/Node/OutOfProcNode.cs index ab9f554f82b..64723e04a65 100644 --- a/src/Build/BackEnd/Node/OutOfProcNode.cs +++ b/src/Build/BackEnd/Node/OutOfProcNode.cs @@ -153,10 +153,11 @@ public OutOfProcNode() // Create a factory for the out-of-proc SDK resolver service which can pass our SendPacket delegate to be used for sending packets to the main node OutOfProcNodeSdkResolverServiceFactory sdkResolverServiceFactory = new OutOfProcNodeSdkResolverServiceFactory(SendPacket); - ((IBuildComponentHost)this).RegisterFactory(BuildComponentType.SdkResolverService, sdkResolverServiceFactory.CreateInstance); - _sdkResolverService = (this as IBuildComponentHost).GetComponent(BuildComponentType.SdkResolverService) as ISdkResolverService; + ((IBuildComponentHost)this).RegisterFactory( + BuildComponentType.FileAccessManager, + (componentType) => OutOfProcNodeFileAccessManager.CreateComponent(componentType, SendPacket)); if (s_projectRootElementCacheBase == null) { diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index be87cad1d65..fab8d491882 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -159,8 +159,11 @@ - + + + + diff --git a/src/Shared/BinaryTranslator.cs b/src/Shared/BinaryTranslator.cs index 055d34bb141..96cf4aa1671 100644 --- a/src/Shared/BinaryTranslator.cs +++ b/src/Shared/BinaryTranslator.cs @@ -397,6 +397,83 @@ public void Translate(ref BuildEventContext value) _reader.ReadInt32()); } + /// + public void Translate(ref FileAccessData fileAccessData) + { + ReportedFileOperation reportedFileOperation = default; + RequestedAccess requestedAccess = default; + uint processId = default; + uint error = default; + DesiredAccess desiredAccess = default; + FlagsAndAttributes flagsAndAttributes = default; + string path = default; +#nullable enable + string? processArgs = default; +#nullable disable + bool isAnAugmentedFileAccess = default; + TranslateEnum(ref reportedFileOperation, (int)reportedFileOperation); + TranslateEnum(ref requestedAccess, (int)requestedAccess); + Translate(ref processId); + Translate(ref error); + TranslateEnum(ref desiredAccess, (int)desiredAccess); + TranslateEnum(ref flagsAndAttributes, (int)flagsAndAttributes); + Translate(ref path); + Translate(ref processArgs); + Translate(ref isAnAugmentedFileAccess); + fileAccessData = new FileAccessData( + reportedFileOperation, + requestedAccess, + processId, + error, + desiredAccess, + flagsAndAttributes, + path, + processArgs, + isAnAugmentedFileAccess); + } + + /// + public void Translate(ref List fileAccessDataList) + { + if (!TranslateNullable(fileAccessDataList)) + { + return; + } + + int count = default; + Translate(ref count); + fileAccessDataList = new List(count); + for (int i = 0; i < count; i++) + { + FileAccessData fileAccessData = default; + Translate(ref fileAccessData); + fileAccessDataList.Add(fileAccessData); + } + } + + /// + public void Translate(ref ProcessData processData) + { + string processName = default; + uint processId = default; + uint parentProcessId = default; + DateTime creationDateTime = default; + DateTime exitDateTime = default; + uint exitCode = default; + Translate(ref processName); + Translate(ref processId); + Translate(ref parentProcessId); + Translate(ref creationDateTime); + Translate(ref exitDateTime); + Translate(ref exitCode); + processData = new ProcessData( + processName, + processId, + parentProcessId, + creationDateTime, + exitDateTime, + exitCode); + } #endif /// @@ -1054,6 +1131,60 @@ public void Translate(ref BuildEventContext value) _writer.Write(value.TaskId); } + /// + public void Translate(ref FileAccessData fileAccessData) + { + ReportedFileOperation reportedFileOperation = fileAccessData.Operation; + RequestedAccess requestedAccess = fileAccessData.RequestedAccess; + uint processId = fileAccessData.ProcessId; + uint error = fileAccessData.Error; + DesiredAccess desiredAccess = fileAccessData.DesiredAccess; + FlagsAndAttributes flagsAndAttributes = fileAccessData.FlagsAndAttributes; + string path = fileAccessData.Path; +#nullable enable + string? processArgs = fileAccessData.ProcessArgs; +#nullable disable + bool isAnAugmentedFileAccess = fileAccessData.IsAnAugmentedFileAccess; + TranslateEnum(ref reportedFileOperation, (int)reportedFileOperation); + TranslateEnum(ref requestedAccess, (int)requestedAccess); + Translate(ref processId); + Translate(ref error); + TranslateEnum(ref desiredAccess, (int)desiredAccess); + TranslateEnum(ref flagsAndAttributes, (int)flagsAndAttributes); + Translate(ref path); + Translate(ref processArgs); + Translate(ref isAnAugmentedFileAccess); + } + + /// + public void Translate(ref List fileAccessDataList) + { + if (!TranslateNullable(fileAccessDataList)) + { + return; + } + + int count = fileAccessDataList.Count; + Translate(ref count); + fileAccessDataList.ForEach(fileAccessData => Translate(ref fileAccessData)); + } + + /// + public void Translate(ref ProcessData processData) + { + string processName = processData.ProcessName; + uint processId = processData.ProcessId; + uint parentProcessId = processData.ParentProcessId; + DateTime creationDateTime = processData.CreationDateTime; + DateTime exitDateTime = processData.ExitDateTime; + uint exitCode = processData.ExitCode; + Translate(ref processName); + Translate(ref processId); + Translate(ref parentProcessId); + Translate(ref creationDateTime); + Translate(ref exitDateTime); + Translate(ref exitCode); + } #endif /// diff --git a/src/Shared/INodePacket.cs b/src/Shared/INodePacket.cs index cb89889c3ac..4b077475510 100644 --- a/src/Shared/INodePacket.cs +++ b/src/Shared/INodePacket.cs @@ -213,6 +213,16 @@ internal enum NodePacketType : byte /// Keep this enum value constant intact as this is part of contract with dotnet CLI /// ServerNodeBuildCancel = 0xF3, + + /// + /// Message sent from a node reporting a file access. + /// + FileAccessReport, + + /// + /// Message sent from a node reporting process data. + /// + ProcessReport, } #endregion diff --git a/src/Shared/ITranslator.cs b/src/Shared/ITranslator.cs index abe32f8d96f..abd68da3e0f 100644 --- a/src/Shared/ITranslator.cs +++ b/src/Shared/ITranslator.cs @@ -228,6 +228,23 @@ BinaryWriter Writer /// The context to be translated. void Translate(ref BuildEventContext value); + /// + /// Translates . + /// + /// The to translate. + void Translate(ref FileAccessData fileAccessData); + + /// + /// Translates . + /// + /// The file accesses to translate. + void Translate(ref List fileAccessDataList); + + /// + /// Translates . + /// + /// The to translate. + void Translate(ref ProcessData processData); #endif /// From 823739c6c1bc7a9861e8970ad9508563f4b67394 Mon Sep 17 00:00:00 2001 From: Dmitriy Shepelev Date: Thu, 23 Feb 2023 18:52:53 -0500 Subject: [PATCH 12/21] Enable tasks to report file accesses --- .../BackEnd/TaskHostTaskComplete_Tests.cs | 39 ++++++++++++------- .../Components/RequestBuilder/TaskHost.cs | 25 ++++++++---- .../Instance/TaskFactories/TaskHostTask.cs | 8 ++++ src/Framework/EngineServices.cs | 21 ++++++++-- src/MSBuild/OutOfProcTaskHostNode.cs | 35 ++++++++++++++--- src/Shared/BinaryTranslator.cs | 9 +++++ src/Shared/ITranslator.cs | 9 +++++ src/Shared/TaskHostTaskComplete.cs | 36 +++++++++++++++-- 8 files changed, 149 insertions(+), 33 deletions(-) diff --git a/src/Build.UnitTests/BackEnd/TaskHostTaskComplete_Tests.cs b/src/Build.UnitTests/BackEnd/TaskHostTaskComplete_Tests.cs index 5c6bf006b2c..a37cba2e08e 100644 --- a/src/Build.UnitTests/BackEnd/TaskHostTaskComplete_Tests.cs +++ b/src/Build.UnitTests/BackEnd/TaskHostTaskComplete_Tests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using Microsoft.Build.BackEnd; using Microsoft.Build.Framework; +using Microsoft.Build.Framework.FileAccess; using Microsoft.Build.Shared; using Microsoft.Build.Utilities; using Xunit; @@ -25,21 +26,33 @@ public class TaskHostTaskComplete_Tests [Fact] public void TestConstructors() { - TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), null); - TaskHostTaskComplete complete2 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Failure), null); - TaskHostTaskComplete complete3 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringInitialization, new ArgumentOutOfRangeException()), null); - TaskHostTaskComplete complete4 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringExecution, new ArgumentNullException()), null); + var fileAccessData = new List() + { + new FileAccessData( + ReportedFileOperation.CreateFile, + RequestedAccess.Read, + 0, + 0, + DesiredAccess.GENERIC_READ, + FlagsAndAttributes.FILE_ATTRIBUTE_NORMAL, + "foo", + null, + true), + }; + _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), fileAccessData, null); + _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Failure), fileAccessData, null); + _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringInitialization, new ArgumentOutOfRangeException()), fileAccessData, null); + _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringExecution, new ArgumentNullException()), fileAccessData, null); IDictionary parameters = new Dictionary(); - TaskHostTaskComplete complete5 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null); + _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), default, null); IDictionary parameters2 = new Dictionary(); parameters2.Add("Text", "Hello!"); parameters2.Add("MyBoolValue", true); parameters2.Add("MyITaskItem", new TaskItem("ABC")); parameters2.Add("ItemArray", new ITaskItem[] { new TaskItem("DEF"), new TaskItem("GHI"), new TaskItem("JKL") }); - - TaskHostTaskComplete complete6 = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters2), null); + _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters2), default, null); } /// @@ -60,7 +73,7 @@ public void TestInvalidConstructors() [Fact] public void TestTranslationWithNullDictionary() { - TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), null); + TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), default, null); ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator()); INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator()); @@ -78,7 +91,7 @@ public void TestTranslationWithNullDictionary() [Fact] public void TestTranslationWithEmptyDictionary() { - TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, new Dictionary()), null); + TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, new Dictionary()), default, null); ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator()); INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator()); @@ -99,7 +112,7 @@ public void TestTranslationWithValueTypesInDictionary() IDictionary parameters = new Dictionary(); parameters.Add("Text", "Foo"); parameters.Add("BoolValue", false); - TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null); + TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), default, null); ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator()); INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator()); @@ -121,7 +134,7 @@ public void TestTranslationWithITaskItemInDictionary() { IDictionary parameters = new Dictionary(); parameters.Add("TaskItemValue", new TaskItem("Foo")); - TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null); + TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), default, null); ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator()); INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator()); @@ -142,7 +155,7 @@ public void TestTranslationWithITaskItemArrayInDictionary() { IDictionary parameters = new Dictionary(); parameters.Add("TaskItemArrayValue", new ITaskItem[] { new TaskItem("Foo"), new TaskItem("Baz") }); - TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), null); + TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), default, null); ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator()); INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator()); @@ -168,7 +181,7 @@ private void AssertInvalidConstructorThrows(Type expectedExceptionType, TaskComp try { - TaskHostTaskComplete complete = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(taskResult, taskOutputParameters, taskException, taskExceptionMessage, taskExceptionMessageArgs), buildProcessEnvironment); + TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(taskResult, taskOutputParameters, taskException, taskExceptionMessage, taskExceptionMessageArgs), default, buildProcessEnvironment); } catch (Exception e) { diff --git a/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs b/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs index 94215412f9d..120d7459bc5 100644 --- a/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs +++ b/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs @@ -6,22 +6,24 @@ using System.Collections.Generic; using System.Globalization; #if FEATURE_APPDOMAIN -using System.Runtime.Remoting.Lifetime; using System.Runtime.Remoting; +using System.Runtime.Remoting.Lifetime; #endif +using System.Diagnostics; +using System.Reflection; using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.BackEnd.Components.Caching; +using Microsoft.Build.Collections; +using Microsoft.Build.Eventing; +using Microsoft.Build.Execution; +using Microsoft.Build.FileAccesses; using Microsoft.Build.Framework; +using Microsoft.Build.Framework.FileAccess; using Microsoft.Build.Shared; -using Microsoft.Build.Execution; -using System.Diagnostics; -using Microsoft.Build.Collections; using ElementLocation = Microsoft.Build.Construction.ElementLocation; using TaskItem = Microsoft.Build.Execution.ProjectItemInstance.TaskItem; using TaskLoggingContext = Microsoft.Build.BackEnd.Logging.TaskLoggingContext; -using System.Threading.Tasks; -using Microsoft.Build.BackEnd.Components.Caching; -using System.Reflection; -using Microsoft.Build.Eventing; #nullable disable @@ -932,6 +934,13 @@ public override bool LogsMessagesOfImportance(MessageImportance importance) /// public override bool IsTaskInputLoggingEnabled => _taskHost._host.BuildParameters.LogTaskInputs; + + /// + public override void ReportFileAccess(FileAccessData fileAccessData) + { + IBuildComponentHost buildComponentHost = _taskHost._host; + ((IFileAccessManager)buildComponentHost.GetComponent(BuildComponentType.FileAccessManager)).ReportFileAccess(fileAccessData, buildComponentHost.BuildParameters.NodeId); + } } public EngineServices EngineServices { get; } diff --git a/src/Build/Instance/TaskFactories/TaskHostTask.cs b/src/Build/Instance/TaskFactories/TaskHostTask.cs index 0b874696321..41615587951 100644 --- a/src/Build/Instance/TaskFactories/TaskHostTask.cs +++ b/src/Build/Instance/TaskFactories/TaskHostTask.cs @@ -9,7 +9,9 @@ using System.Threading; using Microsoft.Build.BackEnd.Logging; using Microsoft.Build.Exceptions; +using Microsoft.Build.FileAccesses; using Microsoft.Build.Framework; +using Microsoft.Build.Framework.FileAccess; using Microsoft.Build.Internal; using Microsoft.Build.Shared; @@ -433,6 +435,12 @@ private void HandlePacket(INodePacket packet, out bool taskFinished) /// private void HandleTaskHostTaskComplete(TaskHostTaskComplete taskHostTaskComplete) { + foreach (FileAccessData fileAccessData in taskHostTaskComplete.FileAccessData) + { + ((IFileAccessManager)_buildComponentHost.GetComponent(BuildComponentType.FileAccessManager)) + .ReportFileAccess(fileAccessData, _buildComponentHost.BuildParameters.NodeId); + } + // If it crashed, or if it failed, it didn't succeed. _taskExecutionSucceeded = taskHostTaskComplete.TaskResult == TaskCompleteType.Success ? true : false; diff --git a/src/Framework/EngineServices.cs b/src/Framework/EngineServices.cs index 271bc7d33c7..8d61e37c032 100644 --- a/src/Framework/EngineServices.cs +++ b/src/Framework/EngineServices.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using Microsoft.Build.Framework.FileAccess; namespace Microsoft.Build.Framework { @@ -21,10 +22,18 @@ public abstract class EngineServices public const int Version1 = 1; /// - /// An explicit version of this class. Must be incremented whenever new members are added. Derived classes should override - /// the property to return the version actually being implemented. + /// Includes . /// - public virtual int Version => Version1; // Not updated since we have not shipped 17.0 yet. This comment is meant to bypass RequiredVersionBumps check in build.ps1 for PR #470646. If the changes in the file are cosmetic, change PR# in this comment to silence the build error on CI build. + public const int Version2 = 2; + + /// + /// Gets an explicit version of this class. + /// + /// + /// Must be incremented whenever new members are added. Derived classes should override + /// the property to return the version actually being implemented. + /// + public virtual int Version => Version2; /// /// Returns if the given message importance is not guaranteed to be ignored by registered loggers. @@ -45,5 +54,11 @@ public abstract class EngineServices /// This is a performance optimization allowing tasks to skip expensive double-logging. /// public virtual bool IsTaskInputLoggingEnabled => throw new NotImplementedException(); + + /// + /// Reports a file access from a task. + /// + /// The file access to report. + public virtual void ReportFileAccess(FileAccessData fileAccessData) => throw new NotImplementedException(); } } diff --git a/src/MSBuild/OutOfProcTaskHostNode.cs b/src/MSBuild/OutOfProcTaskHostNode.cs index f1f7c3b7ffe..5f81ccf28a9 100644 --- a/src/MSBuild/OutOfProcTaskHostNode.cs +++ b/src/MSBuild/OutOfProcTaskHostNode.cs @@ -5,14 +5,16 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Globalization; -using System.Threading; +using System.IO; using System.Reflection; - +using System.Threading; using Microsoft.Build.BackEnd; using Microsoft.Build.Execution; using Microsoft.Build.Framework; +#if !CLR2COMPATIBILITY +using Microsoft.Build.Framework.FileAccess; +#endif using Microsoft.Build.Internal; using Microsoft.Build.Shared; #if FEATURE_APPDOMAIN @@ -163,6 +165,11 @@ internal class OutOfProcTaskHostNode : /// The task object cache. /// private RegisteredTaskObjectCacheBase _registeredTaskObjectCache; + + /// + /// The file accesses reported by the most recently completed task. + /// + private List _fileAccessData; #endif /// @@ -193,6 +200,7 @@ public OutOfProcTaskHostNode() #if !CLR2COMPATIBILITY EngineServices = new EngineServicesImpl(this); + _fileAccessData = new List(); #endif } @@ -531,6 +539,9 @@ public override bool IsTaskInputLoggingEnabled return _taskHost._currentConfiguration.IsTaskInputLoggingEnabled; } } + + /// + public override void ReportFileAccess(FileAccessData fileAccessData) => _taskHost._fileAccessData.Add(fileAccessData); } public EngineServices EngineServices { get; } @@ -936,8 +947,11 @@ private void RunTask(object state) lock (_taskCompleteLock) { _taskCompletePacket = new TaskHostTaskComplete( - taskResult, - currentEnvironment); + taskResult, +#if !CLR2COMPATIBILITY + _fileAccessData, +#endif + currentEnvironment); } #if FEATURE_APPDOMAIN @@ -956,11 +970,20 @@ private void RunTask(object state) lock (_taskCompleteLock) { // Create a minimal taskCompletePacket to carry the exception so that the TaskHostTask does not hang while waiting - _taskCompletePacket = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedAfterExecution, e), null); + _taskCompletePacket = new TaskHostTaskComplete( + new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedAfterExecution, e), +#if !CLR2COMPATIBILITY + _fileAccessData, +#endif + null); } } finally { +#if !CLR2COMPATIBILITY + _fileAccessData = new List(); +#endif + // Call CleanupTask to unload any domains and other necessary cleanup in the taskWrapper _taskWrapper.CleanupTask(); diff --git a/src/Shared/BinaryTranslator.cs b/src/Shared/BinaryTranslator.cs index 96cf4aa1671..8d02092203a 100644 --- a/src/Shared/BinaryTranslator.cs +++ b/src/Shared/BinaryTranslator.cs @@ -8,6 +8,9 @@ using System.IO; using System.Runtime.Serialization.Formatters.Binary; using Microsoft.Build.Framework; +#if !CLR2COMPATIBILITY +using Microsoft.Build.Framework.FileAccess; +#endif using Microsoft.Build.Shared; #nullable disable @@ -148,6 +151,9 @@ public void Translate(ref int value) value = _reader.ReadInt32(); } + /// + public void Translate(ref uint unsignedInteger) => unsignedInteger = _reader.ReadUInt32(); + /// /// Translates an array. /// @@ -906,6 +912,9 @@ public void Translate(ref int value) _writer.Write(value); } + /// + public void Translate(ref uint unsignedInteger) => _writer.Write(unsignedInteger); + /// /// Translates an array. /// diff --git a/src/Shared/ITranslator.cs b/src/Shared/ITranslator.cs index abd68da3e0f..e3daab57216 100644 --- a/src/Shared/ITranslator.cs +++ b/src/Shared/ITranslator.cs @@ -6,6 +6,9 @@ using System.Globalization; using System.IO; using Microsoft.Build.Framework; +#if !CLR2COMPATIBILITY +using Microsoft.Build.Framework.FileAccess; +#endif #nullable disable @@ -128,6 +131,12 @@ BinaryWriter Writer /// The value to be translated. void Translate(ref int value); + /// + /// Translates an unsigned integer. + /// + /// The unsigned integer to translate. + void Translate(ref uint unsignedInteger); + /// /// Translates an array. /// diff --git a/src/Shared/TaskHostTaskComplete.cs b/src/Shared/TaskHostTaskComplete.cs index faad07dd31e..4ef63f32117 100644 --- a/src/Shared/TaskHostTaskComplete.cs +++ b/src/Shared/TaskHostTaskComplete.cs @@ -4,6 +4,9 @@ using System; using System.Collections.Generic; using System.Diagnostics; +#if !CLR2COMPATIBILITY +using Microsoft.Build.Framework.FileAccess; +#endif using Microsoft.Build.Shared; #nullable disable @@ -49,6 +52,10 @@ internal enum TaskCompleteType /// internal class TaskHostTaskComplete : INodePacket { +#if !CLR2COMPATIBILITY + private List _fileAccessData; +#endif + /// /// Result of the task's execution. /// @@ -83,11 +90,17 @@ internal class TaskHostTaskComplete : INodePacket private Dictionary _buildProcessEnvironment = null; /// - /// Constructor + /// Initializes a new instance of the class. /// - /// Result of the task's execution. + /// The result of the task's execution. + /// The file accesses reported by the task. /// The build process environment as it was at the end of the task's execution. - public TaskHostTaskComplete(OutOfProcTaskHostTaskResult result, IDictionary buildProcessEnvironment) + public TaskHostTaskComplete( + OutOfProcTaskHostTaskResult result, +#if !CLR2COMPATIBILITY + List fileAccessData, +#endif + IDictionary buildProcessEnvironment) { ErrorUtilities.VerifyThrowInternalNull(result, nameof(result)); @@ -95,6 +108,9 @@ public TaskHostTaskComplete(OutOfProcTaskHostTaskResult result, IDictionary + /// Gets the file accesses reported by the task. + /// + public List FileAccessData + { + [DebuggerStepThrough] + get => _fileAccessData; + } +#endif + /// /// Translates the packet to/from binary form. /// @@ -213,6 +240,9 @@ public void Translate(ITranslator translator) translator.Translate(ref _taskExceptionMessageArgs); translator.TranslateDictionary(ref _taskOutputParameters, StringComparer.OrdinalIgnoreCase, TaskParameter.FactoryForDeserialization); translator.TranslateDictionary(ref _buildProcessEnvironment, StringComparer.OrdinalIgnoreCase); +#if !CLR2COMPATIBILITY + translator.Translate(ref _fileAccessData); +#endif } /// From 5af7105bb2bd86215afb2a57b352857ad4a67e08 Mon Sep 17 00:00:00 2001 From: David Federman Date: Mon, 26 Jun 2023 20:33:42 -0700 Subject: [PATCH 13/21] Fix UTs --- .../ProjectCache/ProjectCacheService.cs | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 7a9c31cb310..1320d8cadea 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -357,6 +357,12 @@ public bool ShouldUseCache(BuildRequestConfiguration buildRequestConfiguration) return false; } + // We need to retrieve the configuration if it's already loaded in order to access the Project property below. + if (buildRequestConfiguration.IsCached) + { + buildRequestConfiguration.RetrieveFromCache(); + } + // Check if there are any project cache items defined in the project return GetProjectCacheDescriptors(buildRequestConfiguration.Project).Any(); } @@ -651,6 +657,12 @@ public async Task HandleBuildResultAsync( return; } + // We need to retrieve the configuration if it's already loaded in order to access the Project property below. + if (requestConfiguration.IsCached) + { + requestConfiguration.RetrieveFromCache(); + } + // Filter to plugins which apply to the project, if any List projectCacheDescriptors = GetProjectCacheDescriptors(requestConfiguration.Project).ToList(); if (projectCacheDescriptors.Count == 0) @@ -683,7 +695,14 @@ public async Task HandleBuildResultAsync( tasks[idx++] = Task.Run( async () => { - ProjectCachePlugin plugin = await _projectCachePlugins[projectCacheDescriptor].Value; + if (!_projectCachePlugins.TryGetValue(projectCacheDescriptor, out Lazy> pluginLazyTask)) + { + // The plugin might not be in the collection if it was never initialized, which can happen if there are multiple plugins + // and the first one(s) always handles the cache request so the subsequent one(s) never get lazy initialized. + return; + } + + ProjectCachePlugin plugin = await pluginLazyTask.Value; // Rethrow any initialization exception. plugin.InitializationException?.Throw(); From 1b8b2b8f31764cd36ffeebefd52f288d8a1adf09 Mon Sep 17 00:00:00 2001 From: David Federman Date: Mon, 26 Jun 2023 20:57:47 -0700 Subject: [PATCH 14/21] Fix nullability --- .../BackEnd/Components/ProjectCache/ProjectCacheService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 1320d8cadea..e5cf607b54b 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -695,7 +695,7 @@ public async Task HandleBuildResultAsync( tasks[idx++] = Task.Run( async () => { - if (!_projectCachePlugins.TryGetValue(projectCacheDescriptor, out Lazy> pluginLazyTask)) + if (!_projectCachePlugins.TryGetValue(projectCacheDescriptor, out Lazy>? pluginLazyTask)) { // The plugin might not be in the collection if it was never initialized, which can happen if there are multiple plugins // and the first one(s) always handles the cache request so the subsequent one(s) never get lazy initialized. From 9861d878bf987eb92c4e105a6593d6895176351f Mon Sep 17 00:00:00 2001 From: David Federman Date: Thu, 3 Aug 2023 10:13:03 -0700 Subject: [PATCH 15/21] Condition behind a feature flag to target windows-only for now --- eng/dependabot/Packages.props | 2 +- .../BackEnd/TaskHostTaskComplete_Tests.cs | 88 ++++++++++++++++--- .../BackEnd/BuildManager/BuildManager.cs | 24 ++++- .../BackEnd/BuildManager/BuildParameters.cs | 2 + .../Communications/DetouredNodeLauncher.cs | 2 + .../ProjectCache/ProjectCacheService.cs | 2 + .../Components/RequestBuilder/TaskHost.cs | 15 +++- src/Build/BackEnd/Node/OutOfProcNode.cs | 2 + .../Instance/TaskFactories/TaskHostTask.cs | 2 + src/Build/Microsoft.Build.csproj | 2 +- src/Directory.BeforeCommon.targets | 5 ++ src/Framework/FileAccess/FileAccessData.cs | 2 - .../CommandLineSwitches_Tests.cs | 2 + src/MSBuild/CommandLineSwitches.cs | 4 + src/MSBuild/OutOfProcTaskHostNode.cs | 35 +++++--- src/MSBuild/XMake.cs | 18 ++++ src/Shared/TaskHostTaskComplete.cs | 13 +-- 17 files changed, 179 insertions(+), 41 deletions(-) diff --git a/eng/dependabot/Packages.props b/eng/dependabot/Packages.props index 631ed03029d..43ecce95500 100644 --- a/eng/dependabot/Packages.props +++ b/eng/dependabot/Packages.props @@ -13,7 +13,7 @@ - + diff --git a/src/Build.UnitTests/BackEnd/TaskHostTaskComplete_Tests.cs b/src/Build.UnitTests/BackEnd/TaskHostTaskComplete_Tests.cs index a37cba2e08e..9fba141c3a3 100644 --- a/src/Build.UnitTests/BackEnd/TaskHostTaskComplete_Tests.cs +++ b/src/Build.UnitTests/BackEnd/TaskHostTaskComplete_Tests.cs @@ -26,6 +26,7 @@ public class TaskHostTaskComplete_Tests [Fact] public void TestConstructors() { +#if FEATURE_REPORTFILEACCESSES var fileAccessData = new List() { new FileAccessData( @@ -39,20 +40,53 @@ public void TestConstructors() null, true), }; - _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), fileAccessData, null); - _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Failure), fileAccessData, null); - _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringInitialization, new ArgumentOutOfRangeException()), fileAccessData, null); - _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringExecution, new ArgumentNullException()), fileAccessData, null); +#endif + + _ = new TaskHostTaskComplete( + new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), +#if FEATURE_REPORTFILEACCESSES + fileAccessData, +#endif + null); + _ = new TaskHostTaskComplete( + new OutOfProcTaskHostTaskResult(TaskCompleteType.Failure), +#if FEATURE_REPORTFILEACCESSES + fileAccessData, +#endif + null); + _ = new TaskHostTaskComplete( + new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringInitialization, + new ArgumentOutOfRangeException()), +#if FEATURE_REPORTFILEACCESSES + fileAccessData, +#endif + null); + _ = new TaskHostTaskComplete( + new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedDuringExecution, new ArgumentNullException()), +#if FEATURE_REPORTFILEACCESSES + fileAccessData, +#endif + null); IDictionary parameters = new Dictionary(); - _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), default, null); + _ = new TaskHostTaskComplete( + new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), +#if FEATURE_REPORTFILEACCESSES + null, +#endif + null); IDictionary parameters2 = new Dictionary(); parameters2.Add("Text", "Hello!"); parameters2.Add("MyBoolValue", true); parameters2.Add("MyITaskItem", new TaskItem("ABC")); parameters2.Add("ItemArray", new ITaskItem[] { new TaskItem("DEF"), new TaskItem("GHI"), new TaskItem("JKL") }); - _ = new TaskHostTaskComplete(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters2), default, null); + _ = new TaskHostTaskComplete( + new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters2), +#if FEATURE_REPORTFILEACCESSES + null, +#endif + null); } /// @@ -73,7 +107,12 @@ public void TestInvalidConstructors() [Fact] public void TestTranslationWithNullDictionary() { - TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), default, null); + TaskHostTaskComplete complete = new( + new OutOfProcTaskHostTaskResult(TaskCompleteType.Success), +#if FEATURE_REPORTFILEACCESSES + null, +#endif + null); ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator()); INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator()); @@ -91,7 +130,12 @@ public void TestTranslationWithNullDictionary() [Fact] public void TestTranslationWithEmptyDictionary() { - TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, new Dictionary()), default, null); + TaskHostTaskComplete complete = new( + new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, new Dictionary()), +#if FEATURE_REPORTFILEACCESSES + null, +#endif + null); ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator()); INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator()); @@ -112,7 +156,12 @@ public void TestTranslationWithValueTypesInDictionary() IDictionary parameters = new Dictionary(); parameters.Add("Text", "Foo"); parameters.Add("BoolValue", false); - TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), default, null); + TaskHostTaskComplete complete = new( + new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), +#if FEATURE_REPORTFILEACCESSES + null, +#endif + null); ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator()); INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator()); @@ -134,7 +183,12 @@ public void TestTranslationWithITaskItemInDictionary() { IDictionary parameters = new Dictionary(); parameters.Add("TaskItemValue", new TaskItem("Foo")); - TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), default, null); + TaskHostTaskComplete complete = new( + new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), +#if FEATURE_REPORTFILEACCESSES + null, +#endif + null); ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator()); INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator()); @@ -155,7 +209,12 @@ public void TestTranslationWithITaskItemArrayInDictionary() { IDictionary parameters = new Dictionary(); parameters.Add("TaskItemArrayValue", new ITaskItem[] { new TaskItem("Foo"), new TaskItem("Baz") }); - TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), default, null); + TaskHostTaskComplete complete = new( + new OutOfProcTaskHostTaskResult(TaskCompleteType.Success, parameters), +#if FEATURE_REPORTFILEACCESSES + null, +#endif + null); ((ITranslatable)complete).Translate(TranslationHelpers.GetWriteTranslator()); INodePacket packet = TaskHostTaskComplete.FactoryForDeserialization(TranslationHelpers.GetReadTranslator()); @@ -181,7 +240,12 @@ private void AssertInvalidConstructorThrows(Type expectedExceptionType, TaskComp try { - TaskHostTaskComplete complete = new(new OutOfProcTaskHostTaskResult(taskResult, taskOutputParameters, taskException, taskExceptionMessage, taskExceptionMessageArgs), default, buildProcessEnvironment); + TaskHostTaskComplete complete = new( + new OutOfProcTaskHostTaskResult(taskResult, taskOutputParameters, taskException, taskExceptionMessage, taskExceptionMessageArgs), +#if FEATURE_REPORTFILEACCESSES + null, +#endif + buildProcessEnvironment); } catch (Exception e) { diff --git a/src/Build/BackEnd/BuildManager/BuildManager.cs b/src/Build/BackEnd/BuildManager/BuildManager.cs index 18f106be224..f8a5acf81fc 100644 --- a/src/Build/BackEnd/BuildManager/BuildManager.cs +++ b/src/Build/BackEnd/BuildManager/BuildManager.cs @@ -559,10 +559,12 @@ public void BeginBuild(BuildParameters parameters) _buildParameters.OutputResultsCacheFile = FileUtilities.NormalizePath("msbuild-cache"); } +#if FEATURE_REPORTFILEACCESSES if (_buildParameters.ReportFileAccesses) { _componentFactories.ReplaceFactory(BuildComponentType.NodeLauncher, DetouredNodeLauncher.CreateComponent); } +#endif // Initialize components. _nodeManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.NodeManager) as INodeManager; @@ -2491,16 +2493,30 @@ private void HandleNodeShutdown(int node, NodeShutdown shutdownPacket) /// /// The id of the node from which the was received. /// The file access to report to the . - private void HandleFileAccessReport(int nodeId, FileAccessReport fileAccessReport) => - ((FileAccessManager)((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager)).ReportFileAccess(fileAccessReport.FileAccessData, nodeId); + private void HandleFileAccessReport(int nodeId, FileAccessReport fileAccessReport) + { +#if FEATURE_REPORTFILEACCESSES + if (_buildParameters.ReportFileAccesses) + { + ((FileAccessManager)((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager)).ReportFileAccess(fileAccessReport.FileAccessData, nodeId); + } +#endif + } /// /// Report the received to the . /// /// The id of the node from which the was received. /// The process data to report to the . - private void HandleProcessReport(int nodeId, ProcessReport processReport) => - ((FileAccessManager)((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager)).ReportProcess(processReport.ProcessData, nodeId); + private void HandleProcessReport(int nodeId, ProcessReport processReport) + { +#if FEATURE_REPORTFILEACCESSES + if (_buildParameters.ReportFileAccesses) + { + ((FileAccessManager)((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager)).ReportProcess(processReport.ProcessData, nodeId); + } +#endif + } /// /// If there are no more active nodes, cleans up any remaining submissions. diff --git a/src/Build/BackEnd/BuildManager/BuildParameters.cs b/src/Build/BackEnd/BuildManager/BuildParameters.cs index 325c67d1001..c692db04664 100644 --- a/src/Build/BackEnd/BuildManager/BuildParameters.cs +++ b/src/Build/BackEnd/BuildManager/BuildParameters.cs @@ -804,6 +804,7 @@ public string OutputResultsCacheFile set => _outputResultsCacheFile = value; } +#if FEATURE_REPORTFILEACCESSES /// /// Gets or sets a value indicating whether file accesses should be reported to any configured project cache plugins. /// @@ -825,6 +826,7 @@ public bool ReportFileAccesses } } } +#endif /// /// Determines whether MSBuild will save the results of builds after EndBuild to speed up future builds. diff --git a/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs b/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs index bd5819d53ac..f690ea66926 100644 --- a/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs +++ b/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#if FEATURE_REPORTFILEACCESSES using System; using System.Collections; using System.Collections.Generic; @@ -221,3 +222,4 @@ public override void HandleProcessDetouringStatus(ProcessDetouringStatusData dat } } } +#endif diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 2ad37ac4e71..8e90f5a0933 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -670,10 +670,12 @@ public async Task HandleBuildResultAsync( return; } +#if FEATURE_REPORTFILEACCESSES if (_componentHost.BuildParameters.ReportFileAccesses) { _fileAccessManager.WaitForFileAccessReportCompletion(buildResult.GlobalRequestId, cancellationToken); } +#endif IReadOnlyDictionary globalProperties = GetGlobalProperties(requestConfiguration); diff --git a/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs b/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs index 1487217710e..3457749e0cb 100644 --- a/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs +++ b/src/Build/BackEnd/Components/RequestBuilder/TaskHost.cs @@ -345,11 +345,13 @@ public BuildEngineResult BuildProjectFilesInParallel(string[] projectFileNames, /// public void Yield() { +#if FEATURE_REPORTFILEACCESSES // If file accesses are being reported we should not yield as file access will be attributed to the wrong project. if (_host.BuildParameters.ReportFileAccesses) { return; } +#endif lock (_callbackMonitor) { @@ -372,11 +374,13 @@ public void Reacquire() // to release explicitly granted cores when reacquiring the node may lead to deadlocks. ReleaseAllCores(); +#if FEATURE_REPORTFILEACCESSES // If file accesses are being reported yielding is a no-op so reacquire should be too. if (_host.BuildParameters.ReportFileAccesses) { return; } +#endif lock (_callbackMonitor) { @@ -391,7 +395,7 @@ public void Reacquire() } } - #endregion +#endregion #region IBuildEngine Members @@ -938,14 +942,19 @@ public override bool LogsMessagesOfImportance(MessageImportance importance) /// public override void ReportFileAccess(FileAccessData fileAccessData) { +#if FEATURE_REPORTFILEACCESSES IBuildComponentHost buildComponentHost = _taskHost._host; - ((IFileAccessManager)buildComponentHost.GetComponent(BuildComponentType.FileAccessManager)).ReportFileAccess(fileAccessData, buildComponentHost.BuildParameters.NodeId); + if (buildComponentHost.BuildParameters.ReportFileAccesses) + { + ((IFileAccessManager)buildComponentHost.GetComponent(BuildComponentType.FileAccessManager)).ReportFileAccess(fileAccessData, buildComponentHost.BuildParameters.NodeId); + } +#endif } } public EngineServices EngineServices { get; } - #endregion +#endregion /// /// Called by the internal MSBuild task. diff --git a/src/Build/BackEnd/Node/OutOfProcNode.cs b/src/Build/BackEnd/Node/OutOfProcNode.cs index 277505e3e3e..3ea39e10808 100644 --- a/src/Build/BackEnd/Node/OutOfProcNode.cs +++ b/src/Build/BackEnd/Node/OutOfProcNode.cs @@ -372,10 +372,12 @@ private void OnRequestComplete(BuildRequest request, BuildResult result) _nodeEndpoint.SendData(result); } +#if FEATURE_REPORTFILEACCESSES if (_buildParameters.ReportFileAccesses) { FileAccessManager.NotifyFileAccessCompletion(result.GlobalRequestId); } +#endif } /// diff --git a/src/Build/Instance/TaskFactories/TaskHostTask.cs b/src/Build/Instance/TaskFactories/TaskHostTask.cs index 41615587951..f4b3e0e4e70 100644 --- a/src/Build/Instance/TaskFactories/TaskHostTask.cs +++ b/src/Build/Instance/TaskFactories/TaskHostTask.cs @@ -435,11 +435,13 @@ private void HandlePacket(INodePacket packet, out bool taskFinished) /// private void HandleTaskHostTaskComplete(TaskHostTaskComplete taskHostTaskComplete) { +#if FEATURE_REPORTFILEACCESSES foreach (FileAccessData fileAccessData in taskHostTaskComplete.FileAccessData) { ((IFileAccessManager)_buildComponentHost.GetComponent(BuildComponentType.FileAccessManager)) .ReportFileAccess(fileAccessData, _buildComponentHost.BuildParameters.NodeId); } +#endif // If it crashed, or if it failed, it didn't succeed. _taskExecutionSucceeded = taskHostTaskComplete.TaskResult == TaskCompleteType.Success ? true : false; diff --git a/src/Build/Microsoft.Build.csproj b/src/Build/Microsoft.Build.csproj index 264e6b431fd..ebe695775a2 100644 --- a/src/Build/Microsoft.Build.csproj +++ b/src/Build/Microsoft.Build.csproj @@ -40,7 +40,7 @@ - + diff --git a/src/Directory.BeforeCommon.targets b/src/Directory.BeforeCommon.targets index d136a2607c9..0352fbf056b 100644 --- a/src/Directory.BeforeCommon.targets +++ b/src/Directory.BeforeCommon.targets @@ -117,4 +117,9 @@ true + + $(DefineConstants);FEATURE_REPORTFILEACCESSES + true + + diff --git a/src/Framework/FileAccess/FileAccessData.cs b/src/Framework/FileAccess/FileAccessData.cs index e8b37a69f70..b61af0a7fcb 100644 --- a/src/Framework/FileAccess/FileAccessData.cs +++ b/src/Framework/FileAccess/FileAccessData.cs @@ -18,8 +18,6 @@ namespace Microsoft.Build.Framework.FileAccess public readonly record struct FileAccessData( ReportedFileOperation Operation, RequestedAccess RequestedAccess, - - // TODO dshepelev: Fix suppression. #pragma warning disable CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. uint ProcessId, uint Error, diff --git a/src/MSBuild.UnitTests/CommandLineSwitches_Tests.cs b/src/MSBuild.UnitTests/CommandLineSwitches_Tests.cs index 1e372fc0ebb..34fd3268722 100644 --- a/src/MSBuild.UnitTests/CommandLineSwitches_Tests.cs +++ b/src/MSBuild.UnitTests/CommandLineSwitches_Tests.cs @@ -1051,7 +1051,9 @@ public void InvalidToolsVersionErrors() question: false, inputResultsCaches: null, outputResultsCache: null, +#if FEATURE_REPORTFILEACCESSES reportFileAccesses: false, +#endif commandLine: null); } finally diff --git a/src/MSBuild/CommandLineSwitches.cs b/src/MSBuild/CommandLineSwitches.cs index 8206d47ffa2..313c95c0c29 100644 --- a/src/MSBuild/CommandLineSwitches.cs +++ b/src/MSBuild/CommandLineSwitches.cs @@ -105,7 +105,9 @@ internal enum ParameterizedSwitch GraphBuild, InputResultsCaches, OutputResultsCache, +#if FEATURE_REPORTFILEACCESSES ReportFileAccesses, +#endif LowPriority, Question, DetailedSummary, @@ -264,7 +266,9 @@ internal ParameterizedSwitchInfo( new ParameterizedSwitchInfo( new string[] { "graphbuild", "graph" }, ParameterizedSwitch.GraphBuild, null, true, null, true, false), new ParameterizedSwitchInfo( new string[] { "inputResultsCaches", "irc" }, ParameterizedSwitch.InputResultsCaches, null, true, null, true, true), new ParameterizedSwitchInfo( new string[] { "outputResultsCache", "orc" }, ParameterizedSwitch.OutputResultsCache, "DuplicateOutputResultsCache", false, null, true, true), +#if FEATURE_REPORTFILEACCESSES new ParameterizedSwitchInfo( new string[] { "reportfileaccesses" }, ParameterizedSwitch.ReportFileAccesses, null, false, null, true, false), +#endif new ParameterizedSwitchInfo( new string[] { "lowpriority", "low" }, ParameterizedSwitch.LowPriority, null, false, null, true, false), new ParameterizedSwitchInfo( new string[] { "question", "q" }, ParameterizedSwitch.Question, null, false, null, true, false), new ParameterizedSwitchInfo( new string[] { "detailedsummary", "ds" }, ParameterizedSwitch.DetailedSummary, null, false, null, true, false), diff --git a/src/MSBuild/OutOfProcTaskHostNode.cs b/src/MSBuild/OutOfProcTaskHostNode.cs index 9d9d721368b..e8f698ef26b 100644 --- a/src/MSBuild/OutOfProcTaskHostNode.cs +++ b/src/MSBuild/OutOfProcTaskHostNode.cs @@ -165,7 +165,9 @@ internal class OutOfProcTaskHostNode : /// The task object cache. /// private RegisteredTaskObjectCacheBase _registeredTaskObjectCache; +#endif +#if FEATURE_REPORTFILEACCESSES /// /// The file accesses reported by the most recently completed task. /// @@ -200,6 +202,8 @@ public OutOfProcTaskHostNode() #if !CLR2COMPATIBILITY EngineServices = new EngineServicesImpl(this); +#endif +#if FEATURE_REPORTFILEACCESSES _fileAccessData = new List(); #endif } @@ -541,24 +545,29 @@ public override bool IsTaskInputLoggingEnabled } /// - public override void ReportFileAccess(FileAccessData fileAccessData) => _taskHost._fileAccessData.Add(fileAccessData); + public override void ReportFileAccess(FileAccessData fileAccessData) + { +#if FEATURE_REPORTFILEACCESSES + _taskHost._fileAccessData.Add(fileAccessData); +#endif + } } public EngineServices EngineServices { get; } - #endregion +#endregion #endif - #region INodePacketFactory Members + #region INodePacketFactory Members - /// - /// Registers the specified handler for a particular packet type. - /// - /// The packet type. - /// The factory for packets of the specified type. - /// The handler to be called when packets of the specified type are received. - public void RegisterPacketHandler(NodePacketType packetType, NodePacketFactoryMethod factory, INodePacketHandler handler) + /// + /// Registers the specified handler for a particular packet type. + /// + /// The packet type. + /// The factory for packets of the specified type. + /// The handler to be called when packets of the specified type are received. + public void RegisterPacketHandler(NodePacketType packetType, NodePacketFactoryMethod factory, INodePacketHandler handler) { _packetFactory.RegisterPacketHandler(packetType, factory, handler); } @@ -948,7 +957,7 @@ private void RunTask(object state) { _taskCompletePacket = new TaskHostTaskComplete( taskResult, -#if !CLR2COMPATIBILITY +#if FEATURE_REPORTFILEACCESSES _fileAccessData, #endif currentEnvironment); @@ -972,7 +981,7 @@ private void RunTask(object state) // Create a minimal taskCompletePacket to carry the exception so that the TaskHostTask does not hang while waiting _taskCompletePacket = new TaskHostTaskComplete( new OutOfProcTaskHostTaskResult(TaskCompleteType.CrashedAfterExecution, e), -#if !CLR2COMPATIBILITY +#if FEATURE_REPORTFILEACCESSES _fileAccessData, #endif null); @@ -980,7 +989,7 @@ private void RunTask(object state) } finally { -#if !CLR2COMPATIBILITY +#if FEATURE_REPORTFILEACCESSES _fileAccessData = new List(); #endif diff --git a/src/MSBuild/XMake.cs b/src/MSBuild/XMake.cs index 5e726201061..7179e2498c1 100644 --- a/src/MSBuild/XMake.cs +++ b/src/MSBuild/XMake.cs @@ -707,7 +707,9 @@ public static ExitType Execute( string[] inputResultsCaches = null; string outputResultsCache = null; bool question = false; +#if FEATURE_REPORTFILEACCESSES bool reportFileAccesses = false; +#endif GatherAllSwitches(commandLine, out var switchesFromAutoResponseFile, out var switchesNotFromAutoResponseFile, out _); bool buildCanBeInvoked = ProcessCommandLineSwitches( @@ -742,7 +744,9 @@ public static ExitType Execute( ref graphBuildOptions, ref inputResultsCaches, ref outputResultsCache, +#if FEATURE_REPORTFILEACCESSES ref reportFileAccesses, +#endif ref lowPriority, ref question, recursing: false, @@ -818,7 +822,9 @@ public static ExitType Execute( question, inputResultsCaches, outputResultsCache, +#if FEATURE_REPORTFILEACCESSES reportFileAccesses, +#endif commandLine)) { exitType = ExitType.BuildError; @@ -1140,7 +1146,9 @@ internal static bool BuildProject( bool question, string[] inputResultsCaches, string outputResultsCache, +#if FEATURE_REPORTFILEACCESSES bool reportFileAccesses, +#endif #if FEATURE_GET_COMMANDLINE string commandLine) #else @@ -1332,7 +1340,9 @@ internal static bool BuildProject( parameters.InputResultsCacheFiles = inputResultsCaches; parameters.OutputResultsCacheFile = outputResultsCache; parameters.Question = question; +#if FEATURE_REPORTFILEACCESSES parameters.ReportFileAccesses = reportFileAccesses; +#endif // Propagate the profiler flag into the project load settings so the evaluator // can pick it up @@ -2264,7 +2274,9 @@ private static bool ProcessCommandLineSwitches( ref GraphBuildOptions graphBuild, ref string[] inputResultsCaches, ref string outputResultsCache, +#if FEATURE_REPORTFILEACCESSES ref bool reportFileAccesses, +#endif ref bool lowPriority, ref bool question, bool recursing, @@ -2319,10 +2331,12 @@ private static bool ProcessCommandLineSwitches( // leave priority where it was. catch (Win32Exception) { } +#if FEATURE_REPORTFILEACCESSES if (commandLineSwitches.IsParameterizedSwitchSet(CommandLineSwitches.ParameterizedSwitch.ReportFileAccesses)) { reportFileAccesses = ProcessBooleanSwitch(commandLineSwitches[CommandLineSwitches.ParameterizedSwitch.ReportFileAccesses], defaultValue: true, resourceName: ""); } +#endif // if help switch is set (regardless of switch errors), show the help message and ignore the other switches if (commandLineSwitches[CommandLineSwitches.ParameterlessSwitch.Help]) @@ -2386,7 +2400,9 @@ private static bool ProcessCommandLineSwitches( ref graphBuild, ref inputResultsCaches, ref outputResultsCache, +#if FEATURE_REPORTFILEACCESSES ref reportFileAccesses, +#endif ref lowPriority, ref question, recursing: true, @@ -4163,7 +4179,9 @@ private static void ShowHelpMessage() Console.WriteLine(AssemblyResources.GetString("HelpMessage_InputCachesFiles")); Console.WriteLine(AssemblyResources.GetString("HelpMessage_OutputCacheFile")); Console.WriteLine(AssemblyResources.GetString("HelpMessage_36_GraphBuildSwitch")); +#if FEATURE_REPORTFILEACCESSES Console.WriteLine(AssemblyResources.GetString("HelpMessage_42_ReportFileAccessesSwitch")); +#endif Console.WriteLine(AssemblyResources.GetString("HelpMessage_39_LowPrioritySwitch")); Console.WriteLine(AssemblyResources.GetString("HelpMessage_41_QuestionSwitch")); Console.WriteLine(AssemblyResources.GetString("HelpMessage_7_ResponseFile")); diff --git a/src/Shared/TaskHostTaskComplete.cs b/src/Shared/TaskHostTaskComplete.cs index 4ef63f32117..599b5bfe9db 100644 --- a/src/Shared/TaskHostTaskComplete.cs +++ b/src/Shared/TaskHostTaskComplete.cs @@ -52,7 +52,7 @@ internal enum TaskCompleteType /// internal class TaskHostTaskComplete : INodePacket { -#if !CLR2COMPATIBILITY +#if FEATURE_REPORTFILEACCESSES private List _fileAccessData; #endif @@ -89,15 +89,18 @@ internal class TaskHostTaskComplete : INodePacket /// private Dictionary _buildProcessEnvironment = null; + +#pragma warning disable CS1572 // XML comment has a param tag, but there is no parameter by that name. Justification: xmldoc doesn't seem to interact well with #ifdef of params. /// /// Initializes a new instance of the class. /// /// The result of the task's execution. /// The file accesses reported by the task. /// The build process environment as it was at the end of the task's execution. +#pragma warning restore CS1572 // XML comment has a param tag, but there is no parameter by that name public TaskHostTaskComplete( OutOfProcTaskHostTaskResult result, -#if !CLR2COMPATIBILITY +#if FEATURE_REPORTFILEACCESSES List fileAccessData, #endif IDictionary buildProcessEnvironment) @@ -108,7 +111,7 @@ public TaskHostTaskComplete( _taskException = result.TaskException; _taskExceptionMessage = result.ExceptionMessage; _taskExceptionMessageArgs = result.ExceptionMessageArgs; -#if !CLR2COMPATIBILITY +#if FEATURE_REPORTFILEACCESSES _fileAccessData = fileAccessData; #endif @@ -217,7 +220,7 @@ public NodePacketType Type get { return NodePacketType.TaskHostTaskComplete; } } -#if !CLR2COMPATIBILITY +#if FEATURE_REPORTFILEACCESSES /// /// Gets the file accesses reported by the task. /// @@ -240,7 +243,7 @@ public void Translate(ITranslator translator) translator.Translate(ref _taskExceptionMessageArgs); translator.TranslateDictionary(ref _taskOutputParameters, StringComparer.OrdinalIgnoreCase, TaskParameter.FactoryForDeserialization); translator.TranslateDictionary(ref _buildProcessEnvironment, StringComparer.OrdinalIgnoreCase); -#if !CLR2COMPATIBILITY +#if FEATURE_REPORTFILEACCESSES translator.Translate(ref _fileAccessData); #endif } From 784c010f5bcd5d44bd73405cfed0ede1f0d3b8f9 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 23 Aug 2023 16:26:02 +0200 Subject: [PATCH 16/21] Update SourceBuild Baseline --- eng/SourceBuildPrebuiltBaseline.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml index 15a527720b7..2010068a409 100644 --- a/eng/SourceBuildPrebuiltBaseline.xml +++ b/eng/SourceBuildPrebuiltBaseline.xml @@ -12,6 +12,9 @@ + + + From f5e7122e5fd9990df6bdd7a80ff47e293585da39 Mon Sep 17 00:00:00 2001 From: Jan Krivanek Date: Wed, 23 Aug 2023 16:27:09 +0200 Subject: [PATCH 17/21] Remove nuget.org nuget feed --- NuGet.config | 1 - 1 file changed, 1 deletion(-) diff --git a/NuGet.config b/NuGet.config index 704a7117678..d7187b4d64a 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,6 @@ - From 7aa4e3f4bb2889c1a04fec87ef50fa73bb857e6b Mon Sep 17 00:00:00 2001 From: David Federman Date: Thu, 24 Aug 2023 08:33:18 -0700 Subject: [PATCH 18/21] Revert change to System.Security.Principal.Windows --- eng/SourceBuildPrebuiltBaseline.xml | 3 --- eng/Versions.props | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml index 2010068a409..15a527720b7 100644 --- a/eng/SourceBuildPrebuiltBaseline.xml +++ b/eng/SourceBuildPrebuiltBaseline.xml @@ -12,9 +12,6 @@ - - - diff --git a/eng/Versions.props b/eng/Versions.props index a8cf4404488..19d43220e2d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -39,7 +39,7 @@ 7.0.0 7.0.0 7.0.0 - 6.0.0-preview.5.21301.5 + 5.0.0 7.0.0 From cf5d6c455809bf13b05e29741dc23fbb47520108 Mon Sep 17 00:00:00 2001 From: Rainer Sigwald Date: Thu, 24 Aug 2023 11:41:38 -0500 Subject: [PATCH 19/21] Mark types as non-CLS-compliant Instead of suppressing warnings, explicitly declare CLSCompliant(false). --- .../Components/ProjectCache/ProjectCachePluginBase.cs | 3 +++ src/Framework/EngineServices.cs | 1 + src/Framework/FileAccess/DesiredAccess.cs | 4 +--- src/Framework/FileAccess/FileAccessData.cs | 5 +++-- src/Framework/FileAccess/FlagsAndAttributes.cs | 5 +---- src/Framework/FileAccess/ProcessData.cs | 8 +------- 6 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCachePluginBase.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCachePluginBase.cs index 07e725c703a..eb55d482ba0 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCachePluginBase.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCachePluginBase.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Execution; @@ -44,6 +45,7 @@ public abstract Task GetCacheResultAsync( /// /// Called for each file access from an MSBuild node or one of its children. /// + [CLSCompliant(false)] public virtual void HandleFileAccess(FileAccessContext fileAccessContext, FileAccessData fileAccessData) { } @@ -51,6 +53,7 @@ public virtual void HandleFileAccess(FileAccessContext fileAccessContext, FileAc /// /// Called for each new child process created by an MSBuild node or one of its children. /// + [CLSCompliant(false)] public virtual void HandleProcess(FileAccessContext fileAccessContext, ProcessData processData) { } diff --git a/src/Framework/EngineServices.cs b/src/Framework/EngineServices.cs index 8d61e37c032..853663d4e5b 100644 --- a/src/Framework/EngineServices.cs +++ b/src/Framework/EngineServices.cs @@ -59,6 +59,7 @@ public abstract class EngineServices /// Reports a file access from a task. /// /// The file access to report. + [CLSCompliant(false)] public virtual void ReportFileAccess(FileAccessData fileAccessData) => throw new NotImplementedException(); } } diff --git a/src/Framework/FileAccess/DesiredAccess.cs b/src/Framework/FileAccess/DesiredAccess.cs index a205acf9d03..1bb2e4bb492 100644 --- a/src/Framework/FileAccess/DesiredAccess.cs +++ b/src/Framework/FileAccess/DesiredAccess.cs @@ -12,11 +12,9 @@ namespace Microsoft.Build.Framework.FileAccess /// See https://learn.microsoft.com/en-us/windows/win32/fileio/file-access-rights-constants for a full list of values. /// [Flags] + [CLSCompliant(false)] - // TODO dshepelev: Fix suppression. -#pragma warning disable CS3009 // Base type is not CLS-compliant. public enum DesiredAccess : uint -#pragma warning restore CS3009 // Base type is not CLS-compliant. { /// /// For a directory, the right to list the contents of the directory. diff --git a/src/Framework/FileAccess/FileAccessData.cs b/src/Framework/FileAccess/FileAccessData.cs index b61af0a7fcb..2f95ce9d471 100644 --- a/src/Framework/FileAccess/FileAccessData.cs +++ b/src/Framework/FileAccess/FileAccessData.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; + namespace Microsoft.Build.Framework.FileAccess { /// @@ -15,13 +17,12 @@ namespace Microsoft.Build.Framework.FileAccess /// The path being accessed. /// The process arguments. /// Whether the file access is augmented. + [CLSCompliant(false)] public readonly record struct FileAccessData( ReportedFileOperation Operation, RequestedAccess RequestedAccess, -#pragma warning disable CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. uint ProcessId, uint Error, -#pragma warning restore CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. DesiredAccess DesiredAccess, FlagsAndAttributes FlagsAndAttributes, string Path, diff --git a/src/Framework/FileAccess/FlagsAndAttributes.cs b/src/Framework/FileAccess/FlagsAndAttributes.cs index b386c2119f4..00e0b9a5c2d 100644 --- a/src/Framework/FileAccess/FlagsAndAttributes.cs +++ b/src/Framework/FileAccess/FlagsAndAttributes.cs @@ -9,11 +9,8 @@ namespace Microsoft.Build.Framework.FileAccess /// The file or device attributes and flags. /// [Flags] - - // TODO dshepelev: Fix suppression. -#pragma warning disable CS3009 // Base type is not CLS-compliant. + [CLSCompliant(false)] public enum FlagsAndAttributes : uint -#pragma warning restore CS3009 // Base type is not CLS-compliant. { /// /// The file is read only. Applications can read the file but cannot write to or delete it. diff --git a/src/Framework/FileAccess/ProcessData.cs b/src/Framework/FileAccess/ProcessData.cs index 18e1f57366a..aa7a17b57cd 100644 --- a/src/Framework/FileAccess/ProcessData.cs +++ b/src/Framework/FileAccess/ProcessData.cs @@ -14,19 +14,13 @@ namespace Microsoft.Build.Framework.FileAccess /// The creation date time. /// The exit date time. /// The exit code. + [CLSCompliant(false)] public readonly record struct ProcessData( string ProcessName, - // TODO dshepelev: Fix suppression. -#pragma warning disable CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. uint ProcessId, uint ParentProcessId, -#pragma warning restore CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. DateTime CreationDateTime, DateTime ExitDateTime, - - // TODO dshepelev: Fix suppression. -#pragma warning disable CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. uint ExitCode); -#pragma warning restore CS3001, CS3003 // Argument type is not CLS-compliant; Type is not CLS-compliant. } From 2dac5e18c4f79cc86882931cd0571445a3da2c37 Mon Sep 17 00:00:00 2001 From: David Federman Date: Thu, 24 Aug 2023 16:25:09 -0700 Subject: [PATCH 20/21] PR comments --- .../BackEnd/BuildManager/BuildManager.cs | 16 ++++++- .../BackEnd/BuildManager/BuildParameters.cs | 15 +------ .../BuildComponentFactoryCollection.cs | 2 + .../Communications/DetouredNodeLauncher.cs | 12 ++---- .../FileAccesses/FileAccessManager.cs | 11 ++++- .../FileAccesses/IFileAccessManager.cs | 4 +- .../OutOfProcNodeFileAccessManager.cs | 2 + .../BackEnd/Components/IBuildComponentHost.cs | 2 + .../ProjectCache/ProjectCacheService.cs | 42 +++++++++++++++++-- src/Build/BackEnd/Node/OutOfProcNode.cs | 3 ++ .../RetrievableEntryHashSet/HashSet.cs | 3 ++ .../Instance/TaskFactories/TaskHostTask.cs | 9 ++-- src/Build/Resources/Strings.resx | 2 +- src/Build/Resources/xlf/Strings.cs.xlf | 4 +- src/Build/Resources/xlf/Strings.de.xlf | 4 +- src/Build/Resources/xlf/Strings.es.xlf | 4 +- src/Build/Resources/xlf/Strings.fr.xlf | 4 +- src/Build/Resources/xlf/Strings.it.xlf | 4 +- src/Build/Resources/xlf/Strings.ja.xlf | 4 +- src/Build/Resources/xlf/Strings.ko.xlf | 4 +- src/Build/Resources/xlf/Strings.pl.xlf | 4 +- src/Build/Resources/xlf/Strings.pt-BR.xlf | 4 +- src/Build/Resources/xlf/Strings.ru.xlf | 4 +- src/Build/Resources/xlf/Strings.tr.xlf | 4 +- src/Build/Resources/xlf/Strings.zh-Hans.xlf | 4 +- src/Build/Resources/xlf/Strings.zh-Hant.xlf | 4 +- src/Framework/BinaryTranslator.cs | 8 +--- src/Framework/FileAccess/DesiredAccess.cs | 7 +++- .../FileAccess/FlagsAndAttributes.cs | 6 +++ src/Framework/FileAccess/ProcessData.cs | 1 - .../FileAccess/ReportedFileOperation.cs | 6 +++ src/Framework/FileAccess/RequestedAccess.cs | 6 +++ src/MSBuild/OutOfProcTaskHostNode.cs | 21 ++++------ src/Shared/INodePacket.cs | 20 ++++----- 34 files changed, 159 insertions(+), 91 deletions(-) diff --git a/src/Build/BackEnd/BuildManager/BuildManager.cs b/src/Build/BackEnd/BuildManager/BuildManager.cs index f8a5acf81fc..d5b65f9d28f 100644 --- a/src/Build/BackEnd/BuildManager/BuildManager.cs +++ b/src/Build/BackEnd/BuildManager/BuildManager.cs @@ -562,6 +562,12 @@ public void BeginBuild(BuildParameters parameters) #if FEATURE_REPORTFILEACCESSES if (_buildParameters.ReportFileAccesses) { + // To properly report file access, we need to disable the in-proc node which won't be detoured. + _buildParameters.DisableInProcNode = true; + + // Node reuse must be disabled as future builds will not be able to listen to events raised by detours. + _buildParameters.EnableNodeReuse = false; + _componentFactories.ReplaceFactory(BuildComponentType.NodeLauncher, DetouredNodeLauncher.CreateComponent); } #endif @@ -576,11 +582,16 @@ public void BeginBuild(BuildParameters parameters) InitializeCaches(); +#if FEATURE_REPORTFILEACCESSES var fileAccessManager = ((IBuildComponentHost)this).GetComponent(BuildComponentType.FileAccessManager) as IFileAccessManager; +#endif + _projectCacheService = new ProjectCacheService( this, loggingService, +#if FEATURE_REPORTFILEACCESSES fileAccessManager, +#endif _configCache, _buildParameters.ProjectCacheDescriptor); @@ -2398,8 +2409,9 @@ private void HandleResult(int node, BuildResult result) if (_buildSubmissions.TryGetValue(result.SubmissionId, out BuildSubmission buildSubmission)) { // The result may be associated with the build submission due to it being the submission which - // caused the build, but not the actual request which was used with the build submission. Ensure - // only the actual submission's request is considered. + // caused the build, but not the actual request which was originally used with the build submission. + // ie. it may be a dependency of the "root-level" project which is associated with this submission, which + // isn't what we're looking for. Ensure only the actual submission's request is considered. if (buildSubmission.BuildRequest != null && buildSubmission.BuildRequest.ConfigurationId == configuration.ConfigurationId && _projectCacheService.ShouldUseCache(configuration)) diff --git a/src/Build/BackEnd/BuildManager/BuildParameters.cs b/src/Build/BackEnd/BuildManager/BuildParameters.cs index c692db04664..582532e5795 100644 --- a/src/Build/BackEnd/BuildManager/BuildParameters.cs +++ b/src/Build/BackEnd/BuildManager/BuildParameters.cs @@ -811,20 +811,7 @@ public string OutputResultsCacheFile public bool ReportFileAccesses { get => _reportFileAccesses; - set - { - _reportFileAccesses = value; - - // TODO dfederm: What if either of these are set after ReportFileAccesses is? Do we need to move this elsewhere? - if (_reportFileAccesses) - { - // To properly report file access, we need to disable the in-proc node which won't be detoured. - DisableInProcNode = true; - - // Node reuse must be disabled as future builds will not be able to listen to events raised by detours. - EnableNodeReuse = false; - } - } + set => _reportFileAccesses = value; } #endif diff --git a/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs b/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs index d8406cf2b94..c4d543c87da 100644 --- a/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs +++ b/src/Build/BackEnd/Components/BuildComponentFactoryCollection.cs @@ -83,7 +83,9 @@ public void RegisterDefaultFactories() // SDK resolution _componentEntriesByType[BuildComponentType.SdkResolverService] = new BuildComponentEntry(BuildComponentType.SdkResolverService, MainNodeSdkResolverService.CreateComponent, CreationPattern.Singleton); +#if FEATURE_REPORTFILEACCESSES _componentEntriesByType[BuildComponentType.FileAccessManager] = new BuildComponentEntry(BuildComponentType.FileAccessManager, FileAccessManager.CreateComponent, CreationPattern.Singleton); +#endif } /// diff --git a/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs b/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs index f690ea66926..445b8caf217 100644 --- a/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs +++ b/src/Build/BackEnd/Components/Communications/DetouredNodeLauncher.cs @@ -71,13 +71,9 @@ public Process Start(string msbuildLocation, string commandLineArgs, int nodeId) string exeName = msbuildLocation; -#if RUNTIME_TYPE_NETCORE || MONO - // Mono automagically uses the current mono, to execute a managed assembly - if (!NativeMethodsShared.IsMono) - { - // Run the child process with the same host as the currently-running process. - exeName = CurrentHost.GetCurrentHost(); - } +#if RUNTIME_TYPE_NETCORE + // Run the child process with the same host as the currently-running process. + exeName = CurrentHost.GetCurrentHost(); #endif var eventListener = new DetoursEventListener(_fileAccessManager, nodeId); @@ -117,7 +113,7 @@ public Process Start(string msbuildLocation, string commandLineArgs, int nodeId) // needed for logging process arguments when a new process is invoked; see DetoursEventListener.cs info.FileAccessManifest.ReportProcessArgs = true; - // By default, Domino sets the timestamp of all input files to January 1, 1970 + // By default, BuildXL sets the timestamp of all input files to January 1, 1970 // This breaks some tools like Robocopy which will not copy a file to the destination if the file exists at the destination and has a timestamp that is more recent than the source file info.FileAccessManifest.NormalizeReadTimestamps = false; diff --git a/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs b/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs index fbafcab1c16..efbe32a0f64 100644 --- a/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs +++ b/src/Build/BackEnd/Components/FileAccesses/FileAccessManager.cs @@ -1,9 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#if FEATURE_REPORTFILEACCESSES using System; using System.Collections.Concurrent; using System.IO; +using System.Runtime.Versioning; using System.Threading; using Microsoft.Build.BackEnd; using Microsoft.Build.Execution; @@ -19,6 +21,7 @@ private record Handlers(Action FileAccessHander, A // In order to synchronize between the node communication and the file access reporting, a special file access // is used to mark when the file accesses should be considered complete. Only after both this special file access is seen // and the build result is reported can plugins be notified about project completion. + // NOTE! This is currently Windows-specific and will need to change once this feature is opened up to more scenarios. private static readonly string FileAccessCompletionPrefix = BuildParameters.StartupDirectory[0] + @":\{MSBuildFileAccessCompletion}\"; private IScheduler? _scheduler; @@ -67,8 +70,8 @@ public void ReportFileAccess(FileAccessData fileAccessData, int nodeId) } else if (_tempDirectory != null && fileAccessPath.StartsWith(_tempDirectory)) { - // Ignore the temp directory as these are related to internal MSBuild functionality and not always directly related to the execution of the project itself, - // so should not be exposed to handlers. + // Ignore MSBuild's temp directory as these are related to internal MSBuild functionality and not always directly related to the execution of the project itself, + // so should not be exposed to handlers. Note that this is not %TEMP% but instead a subdir under %TEMP% which is only expected to be used by MSBuild. return; } else @@ -134,6 +137,9 @@ private void UnregisterHandlers(Handlers handlersToRemove) } } + // The [SupportedOSPlatform] attribute is a safeguard to ensure that the comment on FileAccessCompletionPrefix regarding being Windows-only gets addressed. + // [SupportedOSPlatform] doesn't apply to fields, so using it here as a reasonable proxy. + [SupportedOSPlatform("windows")] public static void NotifyFileAccessCompletion(int globalRequestId) { // Make a dummy file access to use as a notification that the file accesses should be completed for a project. @@ -179,3 +185,4 @@ public void Dispose() } } } +#endif diff --git a/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs b/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs index 5772ad93d21..3dd724afef3 100644 --- a/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs +++ b/src/Build/BackEnd/Components/FileAccesses/IFileAccessManager.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#if FEATURE_REPORTFILEACCESSES using System; using System.Threading; using Microsoft.Build.BackEnd; @@ -14,7 +15,7 @@ internal interface IFileAccessManager void ReportProcess(ProcessData processData, int nodeId); - // Note: HandlerRegistration is exposed directly instead of IDisposable to avoid boxing. + // Note: The return type of FileAccessManager.HandlerRegistration is exposed directly instead of IDisposable to avoid boxing. FileAccessManager.HandlerRegistration RegisterHandlers( Action fileAccessHandler, Action processHandler); @@ -22,3 +23,4 @@ FileAccessManager.HandlerRegistration RegisterHandlers( void WaitForFileAccessReportCompletion(int globalRequestId, CancellationToken cancellationToken); } } +#endif diff --git a/src/Build/BackEnd/Components/FileAccesses/OutOfProcNodeFileAccessManager.cs b/src/Build/BackEnd/Components/FileAccesses/OutOfProcNodeFileAccessManager.cs index ea1f42f1c79..80255059350 100644 --- a/src/Build/BackEnd/Components/FileAccesses/OutOfProcNodeFileAccessManager.cs +++ b/src/Build/BackEnd/Components/FileAccesses/OutOfProcNodeFileAccessManager.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#if FEATURE_REPORTFILEACCESSES using System; using System.Threading; using Microsoft.Build.BackEnd; @@ -59,3 +60,4 @@ public void WaitForFileAccessReportCompletion(int globalRequestId, CancellationT throw new NotImplementedException("This method should not be called in OOP nodes."); } } +#endif diff --git a/src/Build/BackEnd/Components/IBuildComponentHost.cs b/src/Build/BackEnd/Components/IBuildComponentHost.cs index be9fa43f5e8..5ae9d947906 100644 --- a/src/Build/BackEnd/Components/IBuildComponentHost.cs +++ b/src/Build/BackEnd/Components/IBuildComponentHost.cs @@ -131,10 +131,12 @@ internal enum BuildComponentType /// SdkResolverService, +#if FEATURE_REPORTFILEACCESSES /// /// The component which is the sink for file access reports and forwards reports to other components. /// FileAccessManager, +#endif /// /// The component which launches new MSBuild nodes. diff --git a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs index 8e90f5a0933..0960e200e33 100644 --- a/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs +++ b/src/Build/BackEnd/Components/ProjectCache/ProjectCacheService.cs @@ -36,7 +36,9 @@ internal sealed class ProjectCacheService : IAsyncDisposable private readonly BuildManager _buildManager; private readonly IBuildComponentHost _componentHost; private readonly ILoggingService _loggingService; +#if FEATURE_REPORTFILEACCESSES private readonly IFileAccessManager _fileAccessManager; +#endif private readonly IConfigCache _configCache; private readonly ProjectCacheDescriptor? _globalProjectCacheDescriptor; @@ -53,7 +55,9 @@ internal sealed class ProjectCacheService : IAsyncDisposable private record struct ProjectCachePlugin( string Name, ProjectCachePluginBase? Instance, +#if FEATURE_REPORTFILEACCESSES FileAccessManager.HandlerRegistration? HandlerRegistration, +#endif ExceptionDispatchInfo? InitializationException = null); /// @@ -72,14 +76,18 @@ private DefaultMSBuildFileSystem() public ProjectCacheService( BuildManager buildManager, ILoggingService loggingService, +#if FEATURE_REPORTFILEACCESSES IFileAccessManager fileAccessManager, +#endif IConfigCache configCache, ProjectCacheDescriptor? globalProjectCacheDescriptor) { _buildManager = buildManager; _componentHost = buildManager; _loggingService = loggingService; +#if FEATURE_REPORTFILEACCESSES _fileAccessManager = fileAccessManager; +#endif _configCache = configCache; _globalProjectCacheDescriptor = globalProjectCacheDescriptor; } @@ -203,7 +211,13 @@ private async Task CreateAndInitializePluginAsync( } catch (Exception e) { - return new ProjectCachePlugin(pluginTypeName, Instance: null, HandlerRegistration: null, ExceptionDispatchInfo.Capture(e)); + return new ProjectCachePlugin( + pluginTypeName, + Instance: null, +#if FEATURE_REPORTFILEACCESSES + HandlerRegistration: null, +#endif + ExceptionDispatchInfo.Capture(e)); } finally { @@ -234,7 +248,11 @@ await pluginInstance.BeginBuildAsync( ProjectCacheException.ThrowForErrorLoggedInsideTheProjectCache("ProjectCacheInitializationFailed"); } - FileAccessManager.HandlerRegistration handlerRegistration = _fileAccessManager.RegisterHandlers( +#if FEATURE_REPORTFILEACCESSES + FileAccessManager.HandlerRegistration? handlerRegistration = null; + if (_componentHost.BuildParameters.ReportFileAccesses) + { + handlerRegistration = _fileAccessManager.RegisterHandlers( (buildRequest, fileAccessData) => { // TODO: Filter out projects which do not configure this plugin @@ -247,12 +265,26 @@ await pluginInstance.BeginBuildAsync( FileAccessContext fileAccessContext = GetFileAccessContext(buildRequest); pluginInstance.HandleProcess(fileAccessContext, processData); }); + } +#endif - return new ProjectCachePlugin(pluginTypeName, pluginInstance, handlerRegistration); + return new ProjectCachePlugin( + pluginTypeName, + pluginInstance, +#if FEATURE_REPORTFILEACCESSES + handlerRegistration, +#endif + InitializationException: null); } catch (Exception e) { - return new ProjectCachePlugin(pluginTypeName, Instance: null, HandlerRegistration: null, ExceptionDispatchInfo.Capture(e)); + return new ProjectCachePlugin( + pluginTypeName, + Instance: null, +#if FEATURE_REPORTFILEACCESSES + HandlerRegistration: null, +#endif + ExceptionDispatchInfo.Capture(e)); } finally { @@ -773,10 +805,12 @@ public async ValueTask DisposeAsync() return; } +#if FEATURE_REPORTFILEACCESSES if (plugin.HandlerRegistration.HasValue) { plugin.HandlerRegistration.Value.Dispose(); } +#endif MSBuildEventSource.Log.ProjectCacheEndBuildStart(plugin.Name); try diff --git a/src/Build/BackEnd/Node/OutOfProcNode.cs b/src/Build/BackEnd/Node/OutOfProcNode.cs index 58a1c4a7a6f..c5d8282d5bb 100644 --- a/src/Build/BackEnd/Node/OutOfProcNode.cs +++ b/src/Build/BackEnd/Node/OutOfProcNode.cs @@ -156,9 +156,12 @@ public OutOfProcNode() OutOfProcNodeSdkResolverServiceFactory sdkResolverServiceFactory = new OutOfProcNodeSdkResolverServiceFactory(SendPacket); ((IBuildComponentHost)this).RegisterFactory(BuildComponentType.SdkResolverService, sdkResolverServiceFactory.CreateInstance); _sdkResolverService = (this as IBuildComponentHost).GetComponent(BuildComponentType.SdkResolverService) as ISdkResolverService; + +#if FEATURE_REPORTFILEACCESSES ((IBuildComponentHost)this).RegisterFactory( BuildComponentType.FileAccessManager, (componentType) => OutOfProcNodeFileAccessManager.CreateComponent(componentType, SendPacket)); +#endif if (s_projectRootElementCacheBase == null) { diff --git a/src/Build/Collections/RetrievableEntryHashSet/HashSet.cs b/src/Build/Collections/RetrievableEntryHashSet/HashSet.cs index 21bfc210bc1..fb954982e34 100644 --- a/src/Build/Collections/RetrievableEntryHashSet/HashSet.cs +++ b/src/Build/Collections/RetrievableEntryHashSet/HashSet.cs @@ -32,6 +32,9 @@ #nullable disable +// The BuildXL package causes an indirect dependency on the RuntimeContracts package, which adds an analyzer which forbids the use of System.Diagnostics.Contract. +// So effectively if your dependencies use RuntimeContracts, it attempts to force itself on your as well. +// See: https://github.com/SergeyTeplyakov/RuntimeContracts/issues/12 #pragma warning disable RA001 // Do not use System.Diagnostics.Contract class. namespace Microsoft.Build.Collections diff --git a/src/Build/Instance/TaskFactories/TaskHostTask.cs b/src/Build/Instance/TaskFactories/TaskHostTask.cs index f4b3e0e4e70..4fb2fe61f8d 100644 --- a/src/Build/Instance/TaskFactories/TaskHostTask.cs +++ b/src/Build/Instance/TaskFactories/TaskHostTask.cs @@ -436,10 +436,13 @@ private void HandlePacket(INodePacket packet, out bool taskFinished) private void HandleTaskHostTaskComplete(TaskHostTaskComplete taskHostTaskComplete) { #if FEATURE_REPORTFILEACCESSES - foreach (FileAccessData fileAccessData in taskHostTaskComplete.FileAccessData) + if (taskHostTaskComplete.FileAccessData.Count > 0) { - ((IFileAccessManager)_buildComponentHost.GetComponent(BuildComponentType.FileAccessManager)) - .ReportFileAccess(fileAccessData, _buildComponentHost.BuildParameters.NodeId); + IFileAccessManager fileAccessManager = ((IFileAccessManager)_buildComponentHost.GetComponent(BuildComponentType.FileAccessManager)); + foreach (FileAccessData fileAccessData in taskHostTaskComplete.FileAccessData) + { + fileAccessManager.ReportFileAccess(fileAccessData, _buildComponentHost.BuildParameters.NodeId); + } } #endif diff --git a/src/Build/Resources/Strings.resx b/src/Build/Resources/Strings.resx index ae7133c5857..77ba4bf1928 100644 --- a/src/Build/Resources/Strings.resx +++ b/src/Build/Resources/Strings.resx @@ -1907,7 +1907,7 @@ Utilization: {0} Average Utilization: {1:###.0} MSB4268: The project cache failed to shut down properly. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. MSB4270: No project cache plugins found in assembly "{0}". Expected one. diff --git a/src/Build/Resources/xlf/Strings.cs.xlf b/src/Build/Resources/xlf/Strings.cs.xlf index 3f65b88955b..58322b4d71c 100644 --- a/src/Build/Resources/xlf/Strings.cs.xlf +++ b/src/Build/Resources/xlf/Strings.cs.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.de.xlf b/src/Build/Resources/xlf/Strings.de.xlf index c30384189db..9df8c02b4e4 100644 --- a/src/Build/Resources/xlf/Strings.de.xlf +++ b/src/Build/Resources/xlf/Strings.de.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.es.xlf b/src/Build/Resources/xlf/Strings.es.xlf index c5948e2a812..6f6a8accdd3 100644 --- a/src/Build/Resources/xlf/Strings.es.xlf +++ b/src/Build/Resources/xlf/Strings.es.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.fr.xlf b/src/Build/Resources/xlf/Strings.fr.xlf index 2f69c6eefee..fb485ad66d5 100644 --- a/src/Build/Resources/xlf/Strings.fr.xlf +++ b/src/Build/Resources/xlf/Strings.fr.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.it.xlf b/src/Build/Resources/xlf/Strings.it.xlf index 3ada9c0cf41..90e465e9b9c 100644 --- a/src/Build/Resources/xlf/Strings.it.xlf +++ b/src/Build/Resources/xlf/Strings.it.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.ja.xlf b/src/Build/Resources/xlf/Strings.ja.xlf index 7144e1ad301..b5bd9c4b976 100644 --- a/src/Build/Resources/xlf/Strings.ja.xlf +++ b/src/Build/Resources/xlf/Strings.ja.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.ko.xlf b/src/Build/Resources/xlf/Strings.ko.xlf index ffc49aa8594..0119bf3cf53 100644 --- a/src/Build/Resources/xlf/Strings.ko.xlf +++ b/src/Build/Resources/xlf/Strings.ko.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.pl.xlf b/src/Build/Resources/xlf/Strings.pl.xlf index 64671e3b796..0a42102a49e 100644 --- a/src/Build/Resources/xlf/Strings.pl.xlf +++ b/src/Build/Resources/xlf/Strings.pl.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.pt-BR.xlf b/src/Build/Resources/xlf/Strings.pt-BR.xlf index 986884982d0..68b22a9abcb 100644 --- a/src/Build/Resources/xlf/Strings.pt-BR.xlf +++ b/src/Build/Resources/xlf/Strings.pt-BR.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.ru.xlf b/src/Build/Resources/xlf/Strings.ru.xlf index 957a43c74d8..5c11764c3ca 100644 --- a/src/Build/Resources/xlf/Strings.ru.xlf +++ b/src/Build/Resources/xlf/Strings.ru.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.tr.xlf b/src/Build/Resources/xlf/Strings.tr.xlf index 51385eef3e8..199be842787 100644 --- a/src/Build/Resources/xlf/Strings.tr.xlf +++ b/src/Build/Resources/xlf/Strings.tr.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.zh-Hans.xlf b/src/Build/Resources/xlf/Strings.zh-Hans.xlf index 2d94fc55ad7..78cef9bde28 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hans.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Build/Resources/xlf/Strings.zh-Hant.xlf b/src/Build/Resources/xlf/Strings.zh-Hant.xlf index 45ce25a25b7..da21593b766 100644 --- a/src/Build/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/Build/Resources/xlf/Strings.zh-Hant.xlf @@ -265,8 +265,8 @@ - MSB4269: The project cache failed while handling build result for the following project: {0}. - MSB4269: The project cache failed while handling build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. + MSB4269: The project cache failed while handling a build result for the following project: {0}. diff --git a/src/Framework/BinaryTranslator.cs b/src/Framework/BinaryTranslator.cs index 8c923902aef..207390427d6 100644 --- a/src/Framework/BinaryTranslator.cs +++ b/src/Framework/BinaryTranslator.cs @@ -437,9 +437,7 @@ public void Translate(ref FileAccessData fileAccessData) DesiredAccess desiredAccess = default; FlagsAndAttributes flagsAndAttributes = default; string path = default; -#nullable enable - string? processArgs = default; -#nullable disable + string processArgs = default; bool isAnAugmentedFileAccess = default; TranslateEnum(ref reportedFileOperation, (int)reportedFileOperation); TranslateEnum(ref requestedAccess, (int)requestedAccess); @@ -1205,9 +1203,7 @@ public void Translate(ref FileAccessData fileAccessData) DesiredAccess desiredAccess = fileAccessData.DesiredAccess; FlagsAndAttributes flagsAndAttributes = fileAccessData.FlagsAndAttributes; string path = fileAccessData.Path; -#nullable enable - string? processArgs = fileAccessData.ProcessArgs; -#nullable disable + string processArgs = fileAccessData.ProcessArgs; bool isAnAugmentedFileAccess = fileAccessData.IsAnAugmentedFileAccess; TranslateEnum(ref reportedFileOperation, (int)reportedFileOperation); TranslateEnum(ref requestedAccess, (int)requestedAccess); diff --git a/src/Framework/FileAccess/DesiredAccess.cs b/src/Framework/FileAccess/DesiredAccess.cs index 1bb2e4bb492..056c74d6f19 100644 --- a/src/Framework/FileAccess/DesiredAccess.cs +++ b/src/Framework/FileAccess/DesiredAccess.cs @@ -5,6 +5,12 @@ namespace Microsoft.Build.Framework.FileAccess { + /* + * Implementation note: This is a copy of BuildXL.Processes.DesiredAccess. + * The purpose of the copy is because this is part of the public MSBuild API and it's not desirable to + * expose BuildXL types directly. + */ + /// /// The requested access to the file or device. /// @@ -13,7 +19,6 @@ namespace Microsoft.Build.Framework.FileAccess /// [Flags] [CLSCompliant(false)] - public enum DesiredAccess : uint { /// diff --git a/src/Framework/FileAccess/FlagsAndAttributes.cs b/src/Framework/FileAccess/FlagsAndAttributes.cs index 00e0b9a5c2d..8b01f48106f 100644 --- a/src/Framework/FileAccess/FlagsAndAttributes.cs +++ b/src/Framework/FileAccess/FlagsAndAttributes.cs @@ -5,6 +5,12 @@ namespace Microsoft.Build.Framework.FileAccess { + /* + * Implementation note: This is a copy of BuildXL.Processes.FlagsAndAttributes. + * The purpose of the copy is because this is part of the public MSBuild API and it's not desirable to + * expose BuildXL types directly. + */ + /// /// The file or device attributes and flags. /// diff --git a/src/Framework/FileAccess/ProcessData.cs b/src/Framework/FileAccess/ProcessData.cs index aa7a17b57cd..aa4c8ab873c 100644 --- a/src/Framework/FileAccess/ProcessData.cs +++ b/src/Framework/FileAccess/ProcessData.cs @@ -17,7 +17,6 @@ namespace Microsoft.Build.Framework.FileAccess [CLSCompliant(false)] public readonly record struct ProcessData( string ProcessName, - uint ProcessId, uint ParentProcessId, DateTime CreationDateTime, diff --git a/src/Framework/FileAccess/ReportedFileOperation.cs b/src/Framework/FileAccess/ReportedFileOperation.cs index 205fc7f5b31..22e90c56cea 100644 --- a/src/Framework/FileAccess/ReportedFileOperation.cs +++ b/src/Framework/FileAccess/ReportedFileOperation.cs @@ -3,6 +3,12 @@ namespace Microsoft.Build.Framework.FileAccess { + /* + * Implementation note: This is a copy of BuildXL.Processes.ReportedFileOperation. + * The purpose of the copy is because this is part of the public MSBuild API and it's not desirable to + * expose BuildXL types directly. + */ + /// /// Which operation resulted in a reported file access. /// diff --git a/src/Framework/FileAccess/RequestedAccess.cs b/src/Framework/FileAccess/RequestedAccess.cs index d86831f2979..b522571460f 100644 --- a/src/Framework/FileAccess/RequestedAccess.cs +++ b/src/Framework/FileAccess/RequestedAccess.cs @@ -5,6 +5,12 @@ namespace Microsoft.Build.Framework.FileAccess { + /* + * Implementation note: This is a copy of BuildXL.Processes.RequestedAccess. + * The purpose of the copy is because this is part of the public MSBuild API and it's not desirable to + * expose BuildXL types directly. + */ + /// /// Level of access requested by a reported file operation. /// diff --git a/src/MSBuild/OutOfProcTaskHostNode.cs b/src/MSBuild/OutOfProcTaskHostNode.cs index 39be08f4b80..64ab30c8d6d 100644 --- a/src/MSBuild/OutOfProcTaskHostNode.cs +++ b/src/MSBuild/OutOfProcTaskHostNode.cs @@ -171,7 +171,7 @@ internal class OutOfProcTaskHostNode : /// /// The file accesses reported by the most recently completed task. /// - private List _fileAccessData; + private List _fileAccessData = new List(); #endif /// @@ -202,9 +202,6 @@ public OutOfProcTaskHostNode() #if !CLR2COMPATIBILITY EngineServices = new EngineServicesImpl(this); -#endif -#if FEATURE_REPORTFILEACCESSES - _fileAccessData = new List(); #endif } @@ -559,15 +556,15 @@ public override void ReportFileAccess(FileAccessData fileAccessData) #endif - #region INodePacketFactory Members + #region INodePacketFactory Members - /// - /// Registers the specified handler for a particular packet type. - /// - /// The packet type. - /// The factory for packets of the specified type. - /// The handler to be called when packets of the specified type are received. - public void RegisterPacketHandler(NodePacketType packetType, NodePacketFactoryMethod factory, INodePacketHandler handler) + /// + /// Registers the specified handler for a particular packet type. + /// + /// The packet type. + /// The factory for packets of the specified type. + /// The handler to be called when packets of the specified type are received. + public void RegisterPacketHandler(NodePacketType packetType, NodePacketFactoryMethod factory, INodePacketHandler handler) { _packetFactory.RegisterPacketHandler(packetType, factory, handler); } diff --git a/src/Shared/INodePacket.cs b/src/Shared/INodePacket.cs index 4b077475510..52d335944af 100644 --- a/src/Shared/INodePacket.cs +++ b/src/Shared/INodePacket.cs @@ -190,6 +190,16 @@ internal enum NodePacketType : byte /// ResourceResponse, + /// + /// Message sent from a node reporting a file access. + /// + FileAccessReport, + + /// + /// Message sent from a node reporting process data. + /// + ProcessReport, + /// /// Command in form of MSBuild command line for server node - MSBuild Server. /// Keep this enum value constant intact as this is part of contract with dotnet CLI @@ -213,16 +223,6 @@ internal enum NodePacketType : byte /// Keep this enum value constant intact as this is part of contract with dotnet CLI /// ServerNodeBuildCancel = 0xF3, - - /// - /// Message sent from a node reporting a file access. - /// - FileAccessReport, - - /// - /// Message sent from a node reporting process data. - /// - ProcessReport, } #endregion From a67e9acb7e2dc365cb2269f775ed16dc61b6531c Mon Sep 17 00:00:00 2001 From: David Federman Date: Mon, 28 Aug 2023 12:36:00 -0700 Subject: [PATCH 21/21] Fix build --- src/Build/BackEnd/BuildManager/BuildManager.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Build/BackEnd/BuildManager/BuildManager.cs b/src/Build/BackEnd/BuildManager/BuildManager.cs index d5b65f9d28f..4490bd07609 100644 --- a/src/Build/BackEnd/BuildManager/BuildManager.cs +++ b/src/Build/BackEnd/BuildManager/BuildManager.cs @@ -2501,10 +2501,10 @@ private void HandleNodeShutdown(int node, NodeShutdown shutdownPacket) } /// - /// Report the received to the . + /// Report the received to the file access manager. /// /// The id of the node from which the was received. - /// The file access to report to the . + /// The file access report. private void HandleFileAccessReport(int nodeId, FileAccessReport fileAccessReport) { #if FEATURE_REPORTFILEACCESSES @@ -2516,10 +2516,10 @@ private void HandleFileAccessReport(int nodeId, FileAccessReport fileAccessRepor } /// - /// Report the received to the . + /// Report the received to the file access manager. /// /// The id of the node from which the was received. - /// The process data to report to the . + /// The process data report. private void HandleProcessReport(int nodeId, ProcessReport processReport) { #if FEATURE_REPORTFILEACCESSES