From 24cf85ba6021da97f2273f8df550d2ca7edc4962 Mon Sep 17 00:00:00 2001 From: nojaf Date: Mon, 19 Feb 2024 14:04:24 +0100 Subject: [PATCH 01/13] Initial attempt for GetProjectSnapshotFromScript --- src/Compiler/Service/BackgroundCompiler.fs | 48 +++++++ src/Compiler/Service/BackgroundCompiler.fsi | 14 ++ src/Compiler/Service/TransparentCompiler.fs | 151 +++++++++++++++++++- 3 files changed, 208 insertions(+), 5 deletions(-) diff --git a/src/Compiler/Service/BackgroundCompiler.fs b/src/Compiler/Service/BackgroundCompiler.fs index 54bea5584ad..d66847b5c17 100644 --- a/src/Compiler/Service/BackgroundCompiler.fs +++ b/src/Compiler/Service/BackgroundCompiler.fs @@ -123,6 +123,20 @@ type internal IBackgroundCompiler = userOpName: string -> Async + abstract GetProjectSnapshotFromScript: + fileName: string * + sourceText: ISourceTextNew * + previewEnabled: bool option * + loadedTimeStamp: System.DateTime option * + otherFlags: string array option * + useFsiAuxLib: bool option * + useSdkRefs: bool option * + sdkDirOverride: string option * + assumeDotNetFramework: bool option * + optionsStamp: int64 option * + userOpName: string -> + Async + abstract member GetSemanticClassificationForFile: fileName: string * options: FSharpProjectOptions * userOpName: string -> NodeCode @@ -1595,6 +1609,40 @@ type internal BackgroundCompiler userOpName ) + member _.GetProjectSnapshotFromScript + ( + fileName: string, + sourceText: ISourceTextNew, + previewEnabled: bool option, + loadedTimeStamp: DateTime option, + otherFlags: string array option, + useFsiAuxLib: bool option, + useSdkRefs: bool option, + sdkDirOverride: string option, + assumeDotNetFramework: bool option, + optionsStamp: int64 option, + userOpName: string + ) : Async = + async { + let! options, diagnostics = + self.GetProjectOptionsFromScript( + fileName, + sourceText, + previewEnabled, + loadedTimeStamp, + otherFlags, + useFsiAuxLib, + useSdkRefs, + sdkDirOverride, + assumeDotNetFramework, + optionsStamp, + userOpName + ) + + let! snapshot = FSharpProjectSnapshot.FromOptions(options, DocumentSource.FileSystem) + return snapshot, diagnostics + } + member _.GetSemanticClassificationForFile ( fileName: string, diff --git a/src/Compiler/Service/BackgroundCompiler.fsi b/src/Compiler/Service/BackgroundCompiler.fsi index 6d63c3b93e9..cd7c163a0d1 100644 --- a/src/Compiler/Service/BackgroundCompiler.fsi +++ b/src/Compiler/Service/BackgroundCompiler.fsi @@ -102,6 +102,20 @@ type internal IBackgroundCompiler = userOpName: string -> Async + abstract GetProjectSnapshotFromScript: + fileName: string * + sourceText: ISourceTextNew * + previewEnabled: bool option * + loadedTimeStamp: System.DateTime option * + otherFlags: string array option * + useFsiAuxLib: bool option * + useSdkRefs: bool option * + sdkDirOverride: string option * + assumeDotNetFramework: bool option * + optionsStamp: int64 option * + userOpName: string -> + Async + abstract GetSemanticClassificationForFile: fileName: string * snapshot: FSharpProjectSnapshot * userOpName: string -> NodeCode diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index 19d383d00bd..a84659661e7 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -384,10 +384,11 @@ type internal TransparentCompiler (useSdkRefs: bool option) (sdkDirOverride: string option) (assumeDotNetFramework: bool option) - (projectSnapshot: ProjectSnapshot) + (fileKey: ICacheKey) + (otherOptions: string list) = caches.ScriptClosure.Get( - projectSnapshot.FileKey fileName, + fileKey, node { let useFsiAuxLib = defaultArg useFsiAuxLib true let useSdkRefs = defaultArg useSdkRefs true @@ -396,7 +397,7 @@ type internal TransparentCompiler let applyCompilerOptions tcConfig = let fsiCompilerOptions = GetCoreFsiCompilerOptions tcConfig - ParseCompilerOptions(ignore, fsiCompilerOptions, projectSnapshot.OtherOptions) + ParseCompilerOptions(ignore, fsiCompilerOptions, otherOptions) let closure = LoadClosure.ComputeClosureOfScriptText( @@ -665,7 +666,8 @@ type internal TransparentCompiler None None None - projectSnapshot + (projectSnapshot.FileKey fsxFile.FileName) + projectSnapshot.OtherOptions return (Some closure) } @@ -1521,7 +1523,8 @@ type internal TransparentCompiler (Some tcConfig.useSdkRefs) tcConfig.sdkDirOverride (Some tcConfig.assumeDotNetFramework) - projectSnapshot + (projectSnapshot.FileKey fileName) + projectSnapshot.OtherOptions let typedResults = FSharpCheckFileResults.Make( @@ -2111,6 +2114,144 @@ type internal TransparentCompiler userOpName ) + member this.GetProjectSnapshotFromScript + ( + fileName: string, + sourceText: ISourceTextNew, + previewEnabled: bool option, + loadedTimeStamp: DateTime option, + otherFlags: string array option, + useFsiAuxLib: bool option, + useSdkRefs: bool option, + sdkDirOverride: string option, + assumeDotNetFramework: bool option, + optionsStamp: int64 option, + userOpName: string + ) : Async = + use _ = + Activity.start + "BackgroundCompiler.GetProjectOptionsFromScript" + [| Activity.Tags.fileName, fileName; Activity.Tags.userOpName, userOpName |] + + cancellable { + let previewEnabled = defaultArg previewEnabled false + let! ct = Cancellable.token () + use _ = Cancellable.UsingToken(ct) + + let extraFlags = + if previewEnabled then + [| "--langversion:preview" |] + else + [||] + + let otherFlags = defaultArg otherFlags extraFlags + use diagnostics = new DiagnosticsScope(otherFlags |> Array.contains "--flaterrors") + + let useSimpleResolution = + otherFlags |> Array.exists (fun x -> x = "--simpleresolution") + + let loadedTimeStamp = defaultArg loadedTimeStamp DateTime.MaxValue // Not 'now', we don't want to force reloading + let projectFileName = fileName + ".fsproj" + + let currentSourceFile = + FSharpFileSnapshot.Create( + fileName, + sourceText.GetHashCode().ToString(), + fun () -> Task.FromResult(SourceTextNew.ofISourceText sourceText) + ) + + let cacheKey: ICacheKey = + let hash = + Md5Hasher.empty + |> Md5Hasher.addString fileName + |> Md5Hasher.addString (string sourceText) + + { new ICacheKey with + member _.GetLabel() = + $"{fileName} ({shortPath projectFileName})" + + member _.GetKey() = fileName, (projectFileName, "") + member _.GetVersion() = hash |> Md5Hasher.toString + } + + // TODO: this is most likely wrong from the Cancellable/NodeCode/Async side of things + // the NodeCode needs to be a Cancellable thing... + let loadClosure = + ComputeScriptClosure + fileName + sourceText + FSharpCheckerResultsSettings.defaultFSharpBinariesDir + useSimpleResolution + useFsiAuxLib + useSdkRefs + sdkDirOverride + assumeDotNetFramework + cacheKey + (Array.toList otherFlags) + |> fun n -> NodeCode.RunImmediate(n, ct) + + let otherFlags = + [ + yield "--noframework" + yield "--warn:3" + yield! otherFlags + for code, _ in loadClosure.NoWarns do + yield "--nowarn:" + code + ] + + let sourceFiles = + loadClosure.SourceFiles + |> List.map (fun (sf, _) -> + if sf = fileName then + currentSourceFile + else + FSharpFileSnapshot.CreateFromFileSystem sf) + + let references = + loadClosure.References + |> List.map (fun (r, _) -> + let lastModified = FileSystem.GetLastWriteTimeShim r + + { + Path = r + LastModified = lastModified + }) + + let snapshot = + FSharpProjectSnapshot.Create( + fileName + ".fsproj", + None, + sourceFiles, + references, + otherFlags, + List.empty, + false, + true, + loadedTimeStamp, + Some(FSharpUnresolvedReferencesSet(loadClosure.UnresolvedReferences)), + loadClosure.OriginalLoadReferences, + optionsStamp + ) + + // I would expect the closure to be cached in this case. + // scriptClosureCache.Set(AnyCallerThread, options, loadClosure) // Save the full load closure for later correlation. + + let diags = + loadClosure.LoadClosureRootFileDiagnostics + |> List.map (fun (exn, isError) -> + FSharpDiagnostic.CreateFromException( + exn, + isError, + range.Zero, + false, + otherFlags |> List.contains "--flaterrors", + None + )) + + return snapshot, (diags @ diagnostics.Diagnostics) + } + |> Cancellable.toAsync + member this.GetSemanticClassificationForFile(fileName: string, snapshot: FSharpProjectSnapshot, userOpName: string) = node { ignore userOpName From b5d98eae40a1d8e9f4258240add92628c83fa2fc Mon Sep 17 00:00:00 2001 From: nojaf Date: Mon, 19 Feb 2024 14:15:43 +0100 Subject: [PATCH 02/13] Add GetProjectSnapshotFromScript to service. --- src/Compiler/Service/service.fs | 31 +++++++++++++++++++ src/Compiler/Service/service.fsi | 27 ++++++++++++++++ ...ervice.SurfaceArea.netstandard20.debug.bsl | 1 + ...vice.SurfaceArea.netstandard20.release.bsl | 1 + 4 files changed, 60 insertions(+) diff --git a/src/Compiler/Service/service.fs b/src/Compiler/Service/service.fs index 82b43827e1f..94dce6286b5 100644 --- a/src/Compiler/Service/service.fs +++ b/src/Compiler/Service/service.fs @@ -557,6 +557,37 @@ type FSharpChecker userOpName ) + /// For a given script file, get the ProjectSnapshot implied by the #load closure + member _.GetProjectSnapshotFromScript + ( + fileName, + source, + ?previewEnabled, + ?loadedTimeStamp, + ?otherFlags, + ?useFsiAuxLib, + ?useSdkRefs, + ?assumeDotNetFramework, + ?sdkDirOverride, + ?optionsStamp: int64, + ?userOpName: string + ) = + let userOpName = defaultArg userOpName "Unknown" + + backgroundCompiler.GetProjectSnapshotFromScript( + fileName, + source, + previewEnabled, + loadedTimeStamp, + otherFlags, + useFsiAuxLib, + useSdkRefs, + sdkDirOverride, + assumeDotNetFramework, + optionsStamp, + userOpName + ) + member _.GetProjectOptionsFromCommandLineArgs(projectFileName, argv, ?loadedTimeStamp, ?isInteractive, ?isEditing) = let isEditing = defaultArg isEditing false let isInteractive = defaultArg isInteractive false diff --git a/src/Compiler/Service/service.fsi b/src/Compiler/Service/service.fsi index b8e87b805f8..21fd1aaad93 100644 --- a/src/Compiler/Service/service.fsi +++ b/src/Compiler/Service/service.fsi @@ -251,6 +251,33 @@ type public FSharpChecker = ?userOpName: string -> Async + /// Used to differentiate between scripts, to consider each script a separate project. Also used in formatted error messages. + /// The source for the file. + /// Is the preview compiler enabled. + /// Indicates when the script was loaded into the editing environment, + /// so that an 'unload' and 'reload' action will cause the script to be considered as a new project, + /// so that references are re-resolved. + /// Other flags for compilation. + /// Add a default reference to the FSharp.Compiler.Interactive.Settings library. + /// Use the implicit references from the .NET SDK. + /// Set up compilation and analysis for .NET Framework scripts. + /// Override the .NET SDK used for default references. + /// An optional unique stamp for the options. + /// An optional string used for tracing compiler operations associated with this request. + member GetProjectSnapshotFromScript: + fileName: string * + source: ISourceTextNew * + ?previewEnabled: bool * + ?loadedTimeStamp: DateTime * + ?otherFlags: string[] * + ?useFsiAuxLib: bool * + ?useSdkRefs: bool * + ?assumeDotNetFramework: bool * + ?sdkDirOverride: string * + ?optionsStamp: int64 * + ?userOpName: string -> + Async + /// Get the FSharpProjectOptions implied by a set of command line arguments. /// /// Used to differentiate between projects and for the base directory of the project. diff --git a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.debug.bsl b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.debug.bsl index e526351effd..69fb762ea84 100644 --- a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.debug.bsl +++ b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.debug.bsl @@ -2067,6 +2067,7 @@ FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileAnswer]] ParseAndCheckFileInProject(System.String, Int32, FSharp.Compiler.Text.ISourceText, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults]] GetBackgroundCheckResultsForFileInProject(System.String, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpProjectOptions,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectOptionsFromScript(System.String, FSharp.Compiler.Text.ISourceText, Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) +FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.ProjectSnapshot+FSharpProjectSnapshot,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectSnapshotFromScript(System.String, FSharp.Compiler.Text.ISourceTextNew, Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Diagnostics.FSharpDiagnostic[],System.Int32]] Compile(System.String[], Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Text.Range,FSharp.Compiler.Text.Range][]] MatchBraces(System.String, FSharp.Compiler.Text.ISourceText, FSharp.Compiler.CodeAnalysis.FSharpParsingOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Text.Range,FSharp.Compiler.Text.Range][]] MatchBraces(System.String, System.String, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) diff --git a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.release.bsl b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.release.bsl index e526351effd..69fb762ea84 100644 --- a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.release.bsl +++ b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.release.bsl @@ -2067,6 +2067,7 @@ FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileAnswer]] ParseAndCheckFileInProject(System.String, Int32, FSharp.Compiler.Text.ISourceText, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults]] GetBackgroundCheckResultsForFileInProject(System.String, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpProjectOptions,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectOptionsFromScript(System.String, FSharp.Compiler.Text.ISourceText, Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) +FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.ProjectSnapshot+FSharpProjectSnapshot,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectSnapshotFromScript(System.String, FSharp.Compiler.Text.ISourceTextNew, Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Diagnostics.FSharpDiagnostic[],System.Int32]] Compile(System.String[], Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Text.Range,FSharp.Compiler.Text.Range][]] MatchBraces(System.String, FSharp.Compiler.Text.ISourceText, FSharp.Compiler.CodeAnalysis.FSharpParsingOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Text.Range,FSharp.Compiler.Text.Range][]] MatchBraces(System.String, System.String, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) From 3933b66046453e5f823c13b589c14a305f3937b5 Mon Sep 17 00:00:00 2001 From: nojaf Date: Tue, 20 Feb 2024 14:14:38 +0100 Subject: [PATCH 03/13] Address feedback from code review. --- src/Compiler/Service/TransparentCompiler.fs | 62 +++++++++------------ 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index a84659661e7..777936f8d3b 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -384,11 +384,10 @@ type internal TransparentCompiler (useSdkRefs: bool option) (sdkDirOverride: string option) (assumeDotNetFramework: bool option) - (fileKey: ICacheKey) - (otherOptions: string list) + (projectSnapshot: ProjectSnapshot) = caches.ScriptClosure.Get( - fileKey, + projectSnapshot.FileKey fileName, node { let useFsiAuxLib = defaultArg useFsiAuxLib true let useSdkRefs = defaultArg useSdkRefs true @@ -397,7 +396,7 @@ type internal TransparentCompiler let applyCompilerOptions tcConfig = let fsiCompilerOptions = GetCoreFsiCompilerOptions tcConfig - ParseCompilerOptions(ignore, fsiCompilerOptions, otherOptions) + ParseCompilerOptions(ignore, fsiCompilerOptions, projectSnapshot.OtherOptions) let closure = LoadClosure.ComputeClosureOfScriptText( @@ -666,8 +665,7 @@ type internal TransparentCompiler None None None - (projectSnapshot.FileKey fsxFile.FileName) - projectSnapshot.OtherOptions + projectSnapshot return (Some closure) } @@ -1523,8 +1521,7 @@ type internal TransparentCompiler (Some tcConfig.useSdkRefs) tcConfig.sdkDirOverride (Some tcConfig.assumeDotNetFramework) - (projectSnapshot.FileKey fileName) - projectSnapshot.OtherOptions + projectSnapshot let typedResults = FSharpCheckFileResults.Make( @@ -2133,9 +2130,9 @@ type internal TransparentCompiler "BackgroundCompiler.GetProjectOptionsFromScript" [| Activity.Tags.fileName, fileName; Activity.Tags.userOpName, userOpName |] - cancellable { + async { let previewEnabled = defaultArg previewEnabled false - let! ct = Cancellable.token () + let! ct = Async.CancellationToken use _ = Cancellable.UsingToken(ct) let extraFlags = @@ -2154,29 +2151,26 @@ type internal TransparentCompiler let projectFileName = fileName + ".fsproj" let currentSourceFile = - FSharpFileSnapshot.Create( - fileName, - sourceText.GetHashCode().ToString(), - fun () -> Task.FromResult(SourceTextNew.ofISourceText sourceText) - ) - - let cacheKey: ICacheKey = - let hash = - Md5Hasher.empty - |> Md5Hasher.addString fileName - |> Md5Hasher.addString (string sourceText) - - { new ICacheKey with - member _.GetLabel() = - $"{fileName} ({shortPath projectFileName})" + FSharpFileSnapshot.Create(fileName, sourceText.GetHashCode().ToString(), (fun () -> Task.FromResult sourceText)) - member _.GetKey() = fileName, (projectFileName, "") - member _.GetVersion() = hash |> Md5Hasher.toString - } + // We create a intermediate snapshot with the first file only to reuse the cache key logic. + let loadCloseSnapshot = + FSharpProjectSnapshot.Create( + projectFileName, + None, + [ currentSourceFile ], + List.empty, + List.ofArray otherFlags, + List.empty, + false, + true, + loadedTimeStamp, + None, + List.empty, + optionsStamp + ) - // TODO: this is most likely wrong from the Cancellable/NodeCode/Async side of things - // the NodeCode needs to be a Cancellable thing... - let loadClosure = + let! loadClosure = ComputeScriptClosure fileName sourceText @@ -2186,9 +2180,8 @@ type internal TransparentCompiler useSdkRefs sdkDirOverride assumeDotNetFramework - cacheKey - (Array.toList otherFlags) - |> fun n -> NodeCode.RunImmediate(n, ct) + loadCloseSnapshot.ProjectSnapshot + |> Async.AwaitNodeCode let otherFlags = [ @@ -2250,7 +2243,6 @@ type internal TransparentCompiler return snapshot, (diags @ diagnostics.Diagnostics) } - |> Cancellable.toAsync member this.GetSemanticClassificationForFile(fileName: string, snapshot: FSharpProjectSnapshot, userOpName: string) = node { From d1cdbd7e5ae903ca369481c03fc8fb458966c90f Mon Sep 17 00:00:00 2001 From: nojaf Date: Tue, 20 Feb 2024 14:46:41 +0100 Subject: [PATCH 04/13] Assert both return the same results. --- src/Compiler/Service/TransparentCompiler.fs | 14 +++++++++----- .../FSharpChecker/TransparentCompiler.fs | 19 ++++++++++++++++++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index 777936f8d3b..28b94f1c20f 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -2131,9 +2131,13 @@ type internal TransparentCompiler [| Activity.Tags.fileName, fileName; Activity.Tags.userOpName, userOpName |] async { + // Use the same default as the background compiler. + let useFsiAuxLib = defaultArg useFsiAuxLib true + let useSdkRefs = defaultArg useSdkRefs true let previewEnabled = defaultArg previewEnabled false - let! ct = Async.CancellationToken - use _ = Cancellable.UsingToken(ct) + + // Do we assume .NET Framework references for scripts? + let assumeDotNetFramework = defaultArg assumeDotNetFramework true let extraFlags = if previewEnabled then @@ -2176,10 +2180,10 @@ type internal TransparentCompiler sourceText FSharpCheckerResultsSettings.defaultFSharpBinariesDir useSimpleResolution - useFsiAuxLib - useSdkRefs + (Some useFsiAuxLib) + (Some useSdkRefs) sdkDirOverride - assumeDotNetFramework + (Some assumeDotNetFramework) loadCloseSnapshot.ProjectSnapshot |> Async.AwaitNodeCode diff --git a/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs b/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs index cbc8e7690fe..2adb305a2ea 100644 --- a/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs +++ b/tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs @@ -3,6 +3,7 @@ open System.Collections.Concurrent open System.Diagnostics open FSharp.Compiler.CodeAnalysis +open FSharp.Compiler.Text open Internal.Utilities.Collections open FSharp.Compiler.CodeAnalysis.TransparentCompiler open Internal.Utilities.Library.Extras @@ -894,4 +895,20 @@ let ``LoadClosure for script is recomputed after changes`` () = |> Seq.map (fun (k, g) -> k, g |> Seq.map fst |> Seq.toList) |> Map - Assert.Equal([Weakened; Requested; Started; Finished; Weakened; Requested; Started; Finished], closureComputations["FileFirst.fs"]) \ No newline at end of file + Assert.Equal([Weakened; Requested; Started; Finished; Weakened; Requested; Started; Finished], closureComputations["FileFirst.fs"]) + +[] +let ``Background compiler and Transparent compiler return the same options`` () = + async { + let backgroundChecker = FSharpChecker.Create(useTransparentCompiler = false) + let transparentChecker = FSharpChecker.Create(useTransparentCompiler = true) + let scriptName = Path.Combine(__SOURCE_DIRECTORY__, "script.fsx") + let content = SourceTextNew.ofString "" + + let! backgroundSnapshot, backgroundDiags = backgroundChecker.GetProjectSnapshotFromScript(scriptName, content) + let! transparentSnapshot, transparentDiags = transparentChecker.GetProjectSnapshotFromScript(scriptName, content) + Assert.Empty(backgroundDiags) + Assert.Empty(transparentDiags) + Assert.Equal(backgroundSnapshot.OtherOptions, transparentSnapshot.OtherOptions) + Assert.Equal(backgroundSnapshot.ReferencesOnDisk, transparentSnapshot.ReferencesOnDisk) + } From 30a97e8e1bb573c3400f022aff623d9bf440fbf5 Mon Sep 17 00:00:00 2001 From: nojaf Date: Tue, 20 Feb 2024 17:47:05 +0100 Subject: [PATCH 05/13] Remove old comment --- src/Compiler/Service/TransparentCompiler.fs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index 28b94f1c20f..5dbc28abc6b 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -2230,9 +2230,6 @@ type internal TransparentCompiler optionsStamp ) - // I would expect the closure to be cached in this case. - // scriptClosureCache.Set(AnyCallerThread, options, loadClosure) // Save the full load closure for later correlation. - let diags = loadClosure.LoadClosureRootFileDiagnostics |> List.map (fun (exn, isError) -> From c9130d247ebd91a6cbe9dd662fddc7d53c6583a7 Mon Sep 17 00:00:00 2001 From: nojaf Date: Tue, 20 Feb 2024 18:19:11 +0100 Subject: [PATCH 06/13] Construct cache key in ComputeScriptClosure --- src/Compiler/Service/TransparentCompiler.fs | 26 +++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index 5dbc28abc6b..b8c113ad16b 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -386,13 +386,31 @@ type internal TransparentCompiler (assumeDotNetFramework: bool option) (projectSnapshot: ProjectSnapshot) = + let useFsiAuxLib = defaultArg useFsiAuxLib true + let useSdkRefs = defaultArg useSdkRefs true + let assumeDotNetFramework = defaultArg assumeDotNetFramework false + + let key = + { new ICacheKey with + member _.GetKey() = + $"ScriptClosure%s{fileName}%b{useFsiAuxLib}%b{useSdkRefs}%b{assumeDotNetFramework}", projectSnapshot.Identifier + + member _.GetLabel() = $"ScriptClosure for %s{fileName}" + + member _.GetVersion() = + Md5Hasher.empty + |> Md5Hasher.addStrings [| fileName; string (source.GetHashCode()) |] + |> Md5Hasher.addBool useFsiAuxLib + |> Md5Hasher.addBool useFsiAuxLib + |> Md5Hasher.addBool useSdkRefs + |> Md5Hasher.addBool assumeDotNetFramework + |> Md5Hasher.toString + } + caches.ScriptClosure.Get( - projectSnapshot.FileKey fileName, + key, node { - let useFsiAuxLib = defaultArg useFsiAuxLib true - let useSdkRefs = defaultArg useSdkRefs true let reduceMemoryUsage = ReduceMemoryFlag.Yes - let assumeDotNetFramework = defaultArg assumeDotNetFramework false let applyCompilerOptions tcConfig = let fsiCompilerOptions = GetCoreFsiCompilerOptions tcConfig From 13fc86cfdb233fb19cf7c019af0e684b6075fc46 Mon Sep 17 00:00:00 2001 From: nojaf Date: Wed, 21 Feb 2024 09:15:52 +0100 Subject: [PATCH 07/13] Include stamp --- src/Compiler/Service/TransparentCompiler.fs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index b8c113ad16b..d4e8c3a1ca8 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -399,7 +399,14 @@ type internal TransparentCompiler member _.GetVersion() = Md5Hasher.empty - |> Md5Hasher.addStrings [| fileName; string (source.GetHashCode()) |] + |> Md5Hasher.addStrings + [| + fileName + string (source.GetHashCode()) + match projectSnapshot.Stamp with + | None -> () + | Some stamp -> string stamp + |] |> Md5Hasher.addBool useFsiAuxLib |> Md5Hasher.addBool useFsiAuxLib |> Md5Hasher.addBool useSdkRefs From 039c38cb86bb532a1ff350c2ebd7a0b6952ef215 Mon Sep 17 00:00:00 2001 From: nojaf Date: Wed, 21 Feb 2024 09:21:52 +0100 Subject: [PATCH 08/13] Don't take projectSnapshot as input for ComputeScriptClosure --- src/Compiler/Service/TransparentCompiler.fs | 41 ++++++++------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index d4e8c3a1ca8..9652e2834cc 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -384,7 +384,9 @@ type internal TransparentCompiler (useSdkRefs: bool option) (sdkDirOverride: string option) (assumeDotNetFramework: bool option) - (projectSnapshot: ProjectSnapshot) + (projectIdentifier: ProjectIdentifier) + (otherOptions: string list) + (stamp: int64 option) = let useFsiAuxLib = defaultArg useFsiAuxLib true let useSdkRefs = defaultArg useSdkRefs true @@ -392,9 +394,7 @@ type internal TransparentCompiler let key = { new ICacheKey with - member _.GetKey() = - $"ScriptClosure%s{fileName}%b{useFsiAuxLib}%b{useSdkRefs}%b{assumeDotNetFramework}", projectSnapshot.Identifier - + member _.GetKey() = fileName, projectIdentifier member _.GetLabel() = $"ScriptClosure for %s{fileName}" member _.GetVersion() = @@ -403,7 +403,7 @@ type internal TransparentCompiler [| fileName string (source.GetHashCode()) - match projectSnapshot.Stamp with + match stamp with | None -> () | Some stamp -> string stamp |] @@ -421,7 +421,7 @@ type internal TransparentCompiler let applyCompilerOptions tcConfig = let fsiCompilerOptions = GetCoreFsiCompilerOptions tcConfig - ParseCompilerOptions(ignore, fsiCompilerOptions, projectSnapshot.OtherOptions) + ParseCompilerOptions(ignore, fsiCompilerOptions, otherOptions) let closure = LoadClosure.ComputeClosureOfScriptText( @@ -690,7 +690,9 @@ type internal TransparentCompiler None None None - projectSnapshot + projectSnapshot.Identifier + projectSnapshot.OtherOptions + projectSnapshot.Stamp return (Some closure) } @@ -1546,7 +1548,9 @@ type internal TransparentCompiler (Some tcConfig.useSdkRefs) tcConfig.sdkDirOverride (Some tcConfig.assumeDotNetFramework) - projectSnapshot + projectSnapshot.Identifier + projectSnapshot.OtherOptions + projectSnapshot.Stamp let typedResults = FSharpCheckFileResults.Make( @@ -2182,23 +2186,6 @@ type internal TransparentCompiler let currentSourceFile = FSharpFileSnapshot.Create(fileName, sourceText.GetHashCode().ToString(), (fun () -> Task.FromResult sourceText)) - // We create a intermediate snapshot with the first file only to reuse the cache key logic. - let loadCloseSnapshot = - FSharpProjectSnapshot.Create( - projectFileName, - None, - [ currentSourceFile ], - List.empty, - List.ofArray otherFlags, - List.empty, - false, - true, - loadedTimeStamp, - None, - List.empty, - optionsStamp - ) - let! loadClosure = ComputeScriptClosure fileName @@ -2209,7 +2196,9 @@ type internal TransparentCompiler (Some useSdkRefs) sdkDirOverride (Some assumeDotNetFramework) - loadCloseSnapshot.ProjectSnapshot + (projectFileName, fileName) + (List.ofArray otherFlags) + optionsStamp |> Async.AwaitNodeCode let otherFlags = From 1e1319f0fd6292a80724f471da72ccb4000de9f6 Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 22 Feb 2024 13:17:20 +0100 Subject: [PATCH 09/13] Add other flags --- src/Compiler/Service/TransparentCompiler.fs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index 9652e2834cc..f70a043bb64 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -377,7 +377,7 @@ type internal TransparentCompiler let ComputeScriptClosure (fileName: string) - (source: ISourceText) + (source: ISourceTextNew) (defaultFSharpBinariesDir: string) (useSimpleResolution: bool) (useFsiAuxLib: bool option) @@ -401,8 +401,8 @@ type internal TransparentCompiler Md5Hasher.empty |> Md5Hasher.addStrings [| - fileName - string (source.GetHashCode()) + yield! otherOptions + System.Text.Encoding.UTF8.GetString(Seq.toArray (source.GetChecksum())) match stamp with | None -> () | Some stamp -> string stamp @@ -1541,7 +1541,7 @@ type internal TransparentCompiler let! loadClosure = ComputeScriptClosure fileName - file.Source + (SourceTextNew.ofISourceText file.Source) tcConfig.fsharpBinariesDir tcConfig.useSimpleResolution (Some tcConfig.useFsiAuxLib) From de4baed27f6ddb9f8fcd52e4dcbe901b7f774140 Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 22 Feb 2024 13:45:35 +0100 Subject: [PATCH 10/13] Add checksum directly. --- src/Compiler/Service/TransparentCompiler.fs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index f70a043bb64..634a3ebe9cf 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -1,6 +1,7 @@ namespace FSharp.Compiler.CodeAnalysis.TransparentCompiler open System +open System.Linq open System.Collections.Generic open System.Runtime.CompilerServices open System.Diagnostics @@ -407,6 +408,7 @@ type internal TransparentCompiler | None -> () | Some stamp -> string stamp |] + |> Md5Hasher.addBytes (source.GetChecksum().ToArray()) |> Md5Hasher.addBool useFsiAuxLib |> Md5Hasher.addBool useFsiAuxLib |> Md5Hasher.addBool useSdkRefs From 8bfcb32efa81e05e27ad8559b39bf8e2c9742807 Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 22 Feb 2024 13:46:30 +0100 Subject: [PATCH 11/13] Remove duplicate checksum --- src/Compiler/Service/TransparentCompiler.fs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index 634a3ebe9cf..3a0f7877be2 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -403,7 +403,6 @@ type internal TransparentCompiler |> Md5Hasher.addStrings [| yield! otherOptions - System.Text.Encoding.UTF8.GetString(Seq.toArray (source.GetChecksum())) match stamp with | None -> () | Some stamp -> string stamp From 57f7e79e0b06312183ed05bd6990b07d85387763 Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 22 Feb 2024 14:17:42 +0100 Subject: [PATCH 12/13] Use ISourceTextNew for Source --- src/Compiler/Service/FSharpProjectSnapshot.fs | 2 +- src/Compiler/Service/TransparentCompiler.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Compiler/Service/FSharpProjectSnapshot.fs b/src/Compiler/Service/FSharpProjectSnapshot.fs index a5ae2447db6..7ab252ef73b 100644 --- a/src/Compiler/Service/FSharpProjectSnapshot.fs +++ b/src/Compiler/Service/FSharpProjectSnapshot.fs @@ -123,7 +123,7 @@ type FSharpFileSnapshot(FileName: string, Version: string, GetSource: unit -> Ta /// A source file snapshot with loaded source text. type internal FSharpFileSnapshotWithSource - (FileName: string, SourceHash: ImmutableArray, Source: ISourceText, IsLastCompiland: bool, IsExe: bool) = + (FileName: string, SourceHash: ImmutableArray, Source: ISourceTextNew, IsLastCompiland: bool, IsExe: bool) = let version = lazy (SourceHash.ToBuilder().ToArray()) let stringVersion = lazy (version.Value |> BitConverter.ToString) diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index 3a0f7877be2..e85488e7738 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -1542,7 +1542,7 @@ type internal TransparentCompiler let! loadClosure = ComputeScriptClosure fileName - (SourceTextNew.ofISourceText file.Source) + file.Source tcConfig.fsharpBinariesDir tcConfig.useSimpleResolution (Some tcConfig.useFsiAuxLib) From c8693233ce72d768be57886352cfcfd0f08bd243 Mon Sep 17 00:00:00 2001 From: nojaf Date: Thu, 22 Feb 2024 14:17:51 +0100 Subject: [PATCH 13/13] Mark API as experimental --- src/Compiler/Service/service.fsi | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Compiler/Service/service.fsi b/src/Compiler/Service/service.fsi index 21fd1aaad93..cb8272b0f69 100644 --- a/src/Compiler/Service/service.fsi +++ b/src/Compiler/Service/service.fsi @@ -264,6 +264,7 @@ type public FSharpChecker = /// Override the .NET SDK used for default references. /// An optional unique stamp for the options. /// An optional string used for tracing compiler operations associated with this request. + [] member GetProjectSnapshotFromScript: fileName: string * source: ISourceTextNew *