diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 296b6f982989..ea6c2dc32f23 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -399,7 +399,7 @@ jobs: publishRunAttachments: true testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName) - - publish: '$(sourcesPath)/artifacts/${{ parameters.architecture }}/Release/' + - publish: '$(sourcesPath)/artifacts/packages/Release/' artifact: $(Agent.JobName)_Artifacts displayName: Publish Artifacts condition: succeededOrFailed() diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index fabe4be01827..1e0967404015 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -92,6 +92,9 @@ $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'obj')) $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'bin')) $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'log', '$(Configuration)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'packages', '$(Configuration)')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsPackagesDir)', 'NonShipping')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsPackagesDir)', 'Shipping')) $([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'eng')) @@ -168,14 +171,7 @@ $(NuGetPackageRoot) $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers')) - - - $([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', '$(TargetArchitecture)', '$(Configuration)')) - $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', '$(TargetArchitecture)', '$(Configuration)')) - - $([MSBuild]::NormalizeDirectory('$(SharedIntermediateOutputPath)', 'blob-feed')) - $([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'packages')) - $([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'assets')) + $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols')) $([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt')) $([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored')) diff --git a/src/SourceBuild/content/Directory.Build.targets b/src/SourceBuild/content/Directory.Build.targets index b8488418c6da..a1f0cb2adafb 100644 --- a/src/SourceBuild/content/Directory.Build.targets +++ b/src/SourceBuild/content/Directory.Build.targets @@ -7,7 +7,7 @@ dotnet-sdk- - + - - - - - - - - diff --git a/src/SourceBuild/content/eng/build.sourcebuild.targets b/src/SourceBuild/content/eng/build.sourcebuild.targets index 21875b5ead9e..d9b77280e2e5 100644 --- a/src/SourceBuild/content/eng/build.sourcebuild.targets +++ b/src/SourceBuild/content/eng/build.sourcebuild.targets @@ -15,43 +15,15 @@ - - - - - - - - - - $([System.IO.Path]::GetFileName('%(SymbolsTarball.Identity)')) - $(Filename.Split('.')[1]) - $(ArtifactsTmpDir)Symbols - $(UnifiedSymbolsLayout)/$(RepositoryName) - - - - - - - - + DependsOnTargets="DetermineSourceBuiltSdkVersion"> - $(SharedOutputPath)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension) + $(ArtifactsShippingPackagesDir)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension) + @@ -64,19 +36,19 @@ AfterTargets="Build" DependsOnTargets="RepackageSymbols"> - + $(ArtifactsTmpDir)SdkSymbols - $(SharedOutputPath)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension) + $(ArtifactsShippingPackagesDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension) $(ArtifactsTmpDir)Sdk %(SdkTarballItem.Identity) + WorkingDirectory="$(ArtifactsShippingPackagesDir)" /> - + - - - - + PoisonReportOutputFilePath="$(PoisonUsageReportFile)" /> @@ -155,8 +122,8 @@ - - + + @@ -203,17 +170,18 @@ DependsOnTargets="DetermineSourceBuiltSdkVersion" Condition="'@(SmokeTestsPrereqs->Count())' != '0'"> - $(SharedOutputPath)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) + $(ArtifactsShippingPackagesDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) $(SmokeTestsArtifactsDir)prereq-packages/ - + - + - $(SharedOutputPath)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) + $(ArtifactsShippingPackagesDir)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) $(ResultingPrebuiltPackagesDir) - + + - + public string OverrideTempPath { get; set; } - /// - /// Array of files containing lists of non-shipping packages - /// - public ITaskItem[] NonShippingPackagesListFiles { get; set; } - private static readonly string[] ZipFileExtensions = { ".zip", @@ -186,7 +181,6 @@ public override bool Execute() /// List of poisoned packages and files found and reasons for each internal IEnumerable GetPoisonedFiles(IEnumerable initialCandidates, string catalogedPackagesFilePath, string markerFileName) { - IEnumerable nonShippingPackages = GetAllNonShippingPackages(); IEnumerable catalogedPackages = ReadCatalog(catalogedPackagesFilePath); var poisons = new List(); var candidateQueue = new Queue(initialCandidates.Select(candidate => @@ -209,12 +203,6 @@ internal IEnumerable GetPoisonedFiles(IEnumerable ini { Log.LogMessage($"Zip or NuPkg file to check: {candidate.ExtractedPath}"); - // Skip non-shipping packages - if (nonShippingPackages.Contains(Path.GetFileName(candidate.ExtractedPath), StringComparer.OrdinalIgnoreCase)) - { - continue; - } - var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(candidate.ExtractedPath)); PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, candidate, markerFileName, tempCheckingDir, candidateQueue); if (result != null) @@ -237,21 +225,6 @@ internal IEnumerable GetPoisonedFiles(IEnumerable ini return poisons; } - private IEnumerable GetAllNonShippingPackages() - { - if (NonShippingPackagesListFiles != null) - { - return NonShippingPackagesListFiles - .SelectMany(item => File.ReadAllLines(item.ItemSpec)) - .Distinct() - .ToList(); - } - else - { - return Enumerable.Empty(); - } - } - private static PoisonedFileEntry CheckSingleFile(IEnumerable catalogedPackages, CandidateFileEntry candidate) { // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.props b/src/SourceBuild/content/repo-projects/Directory.Build.props index 27ea825ba014..5089d4ac307e 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.props +++ b/src/SourceBuild/content/repo-projects/Directory.Build.props @@ -17,10 +17,11 @@ $([MSBuild]::NormalizeDirectory('$(SrcDir)', '$(RepositoryName)')) - $(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).props - $(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Current.props - $(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Previous.props - $(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Snapshot.props + $([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'PackageVersions')) + $(PackageVersionsRootDir)PackageVersions.$(RepositoryName).props + $(PackageVersionsRootDir)PackageVersions.$(RepositoryName).Current.props + $(PackageVersionsRootDir)PackageVersions.$(RepositoryName).Previous.props + $(PackageVersionsRootDir)PackageVersions.$(RepositoryName).Snapshot.props DependenciesOnly $(ProjectDirectory)global.json diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets index 5807aeb3e864..3c30984108bf 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.targets +++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets @@ -28,6 +28,8 @@ $(PackageReportDir)all-project-assets-json-files.zip $(PackageReportDir)prodcon-build.xml $([MSBuild]::NormalizePath('$(ProjectDirectory)', 'artifacts', 'RepoManifest.xml')) + + $(UnifiedSymbolsLayout)$(RepositoryName) @@ -41,6 +43,8 @@ + + @@ -78,9 +82,14 @@ Inputs="$(MSBuildProjectFullPath)" Outputs="$(BaseIntermediateOutputPath)UpdateNuGetConfig.complete"> - source-built + + source-built + + source-built-transport + $(SourceBuiltShippingNuGetSourceName);SourceBuiltNonShippingNuGetSourceName + + ExtraSources - $(SourceBuiltNuGetSourceName) $(SourceBuildSources);$(ExtraSourcesNuGetSourceName) @@ -129,8 +138,12 @@ Condition="'$(DotNetBuildSourceOnly)' == 'true'" /> + SourceName="$(SourceBuiltShippingNuGetSourceName)" + SourcePath="$(ArtifactsShippingPackagesDir)" /> + + - <_CurrentSourceBuiltPackages Include="$(SourceBuiltPackagesPath)*.nupkg" - Exclude="$(SourceBuiltPackagesPath)*.symbols.nupkg" /> + <_CurrentSourceBuiltPackages Include="$(ArtifactsShippingPackagesDir)*.nupkg; + $(ArtifactsNonShippingPackagesDir)*.nupkg" + Condition="!$([System.String]::Copy('%(Identity)').EndsWith('.symbols.nupkg'))" /> <_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" /> @@ -276,6 +290,18 @@ StopOnFirstFailure="true" /> + + + + + + + + + + + @@ -307,7 +333,8 @@ + $(ArtifactsShippingPackagesDir); + $(ArtifactsNonShippingPackagesDir)" /> - - - - + Outputs="$(BaseIntermediateOutputPath)CopyRepoArtifacts.complete"> - - - + + + - - <_NupkgsDestination>$(SourceBuiltPackagesPath) - <_NupkgsDestination Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir) - + + + $(ArtifactsNonShippingPackagesDir) + $(ReferencePackagesDir) + + + + $(ArtifactsShippingPackagesDir) + $(ReferencePackagesDir) + + - + - - - - - - - - - $(PackageListsDir)$(NonShippingPackagesListPrefix)$(RepositoryName).lst - + + - + + - + - + <_InnerPackageCacheFiles Include="$(ProjectDirectory)artifacts/sb/package-cache/**/*" /> + SkipUnchangedFiles="true" + Condition="'@(_InnerPackageCacheFiles)' != ''" /> + Condition="'$(IsUtilityProject)' != 'true' and + '$(CleanWhileBuilding)' == 'true' and + Exists('$(ProjectDirectory)artifacts')"> $([MSBuild]::NormalizeDirectory('$(ProjectDirectory)', 'artifacts', 'buildLogs')) $([MSBuild]::NormalizeDirectory('$(ProjectDirectory)', 'artifacts', 'buildObj')) @@ -463,7 +480,7 @@ Inputs="$(MSBuildProjectFullPath)" Outputs="$(BaseIntermediateOutputPath)ExtractToolPackage.complete"> - <_ToolPackagesRoot Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(SourceBuiltPackagesPath) + <_ToolPackagesRoot Condition="'$(RepositoryName)' != 'source-build-reference-packages'">$(ArtifactsNonShippingPackagesDir) <_ToolPackagesRoot Condition="'$(RepositoryName)' == 'source-build-reference-packages'">$(ReferencePackagesDir) @@ -502,15 +519,29 @@ + + + + + + + + + - + @@ -536,17 +567,16 @@ - - - - - - - - - - + - + @@ -564,7 +594,7 @@ - + diff --git a/src/SourceBuild/content/repo-projects/installer.proj b/src/SourceBuild/content/repo-projects/installer.proj index bcc0fe5f7534..66e458c71292 100644 --- a/src/SourceBuild/content/repo-projects/installer.proj +++ b/src/SourceBuild/content/repo-projects/installer.proj @@ -29,7 +29,7 @@ $(BuildArgs) /p:CoreSetupRid=$(TargetRid) - $(BuildArgs) /p:PublicBaseURL=file:%2F%2F$(SourceBuiltAssetsDir) + $(BuildArgs) /p:PublicBaseURL=file:%2F%2F$(ArtifactsShippingPackagesDir) $(BuildArgs) /p:UsePortableLinuxSharedFramework=false diff --git a/src/SourceBuild/content/repo-projects/package-source-build.proj b/src/SourceBuild/content/repo-projects/package-source-build.proj index 822361892091..95311acf9374 100644 --- a/src/SourceBuild/content/repo-projects/package-source-build.proj +++ b/src/SourceBuild/content/repo-projects/package-source-build.proj @@ -14,43 +14,53 @@ - - - - $(SourceBuiltPackagesPath)SourceBuildReferencePackages/ + $([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', '$(SourceBuiltArtifactsTarballName)')) + $([MSBuild]::NormalizeDirectory('$(SourceBuildTarballStagingDir)', 'SourceBuildReferencePackages')) + + + - + + DestinationFiles="@(SourceBuildReferencePackagesNupkgFiles -> '$(SourceBuildReferencePackagesDestination)%(Filename)%(Extension)')" + Condition="'@(SourceBuildReferencePackagesNupkgFiles)' != ''" /> + + + - $(SharedOutputPath)$(SourceBuiltArtifactsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) + $(ArtifactsShippingPackagesDir)$(SourceBuiltArtifactsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension) .version - + - + + WorkingDirectory="$(SourceBuildTarballStagingDir)" /> + + diff --git a/src/SourceBuild/content/repo-projects/source-build-reference-packages.proj b/src/SourceBuild/content/repo-projects/source-build-reference-packages.proj index cff1f387aab1..f9aa99180c07 100644 --- a/src/SourceBuild/content/repo-projects/source-build-reference-packages.proj +++ b/src/SourceBuild/content/repo-projects/source-build-reference-packages.proj @@ -3,7 +3,7 @@