Skip to content

Commit

Permalink
Enable Vertical build PoC as a more first-class concept (#14274)
Browse files Browse the repository at this point in the history
Co-authored-by: Viktor Hofer <viktor.hofer@microsoft.com>
  • Loading branch information
mmitche and ViktorHofer authored Dec 11, 2023
1 parent abe3fdc commit 0a0217f
Show file tree
Hide file tree
Showing 13 changed files with 165 additions and 20 deletions.
3 changes: 3 additions & 0 deletions eng/common/build.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Param(
[switch] $pack,
[switch] $publish,
[switch] $clean,
[switch] $verticalBuild,
[switch][Alias('bl')]$binaryLog,
[switch][Alias('nobl')]$excludeCIBinarylog,
[switch] $ci,
Expand Down Expand Up @@ -58,6 +59,7 @@ function Print-Usage() {
Write-Host " -sign Sign build outputs"
Write-Host " -publish Publish artifacts (e.g. symbols)"
Write-Host " -clean Clean the solution"
Write-Host " -verticalBuild Run in 'vertical build' infra mode."
Write-Host ""

Write-Host "Advanced settings:"
Expand Down Expand Up @@ -120,6 +122,7 @@ function Build {
/p:Deploy=$deploy `
/p:Test=$test `
/p:Pack=$pack `
/p:ArcadeBuildVertical=$verticalBuild `
/p:IntegrationTest=$integrationTest `
/p:PerformanceTest=$performanceTest `
/p:Sign=$sign `
Expand Down
8 changes: 8 additions & 0 deletions eng/common/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
restore=false
build=false
source_build=false
vertical_build=false
rebuild=false
test=false
integration_test=false
Expand Down Expand Up @@ -129,6 +130,12 @@ while [[ $# > 0 ]]; do
restore=true
pack=true
;;
-verticalbuild|-vb)
build=true
vertical_build=true
restore=true
pack=true
;;
-test|-t)
test=true
;;
Expand Down Expand Up @@ -220,6 +227,7 @@ function Build {
/p:Restore=$restore \
/p:Build=$build \
/p:ArcadeBuildFromSource=$source_build \
/p:ArcadeBuildVertical=$vertical_build \
/p:Rebuild=$rebuild \
/p:Test=$test \
/p:Pack=$pack \
Expand Down
3 changes: 2 additions & 1 deletion eng/common/tools.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -827,7 +827,8 @@ function MSBuild-Core() {
}
}

$env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs"
# Be sure quote the path in case there are spaces in the dotnet installation location.
$env:ARCADE_BUILD_TOOL_COMMAND = "`"$($buildTool.Path)`" $cmdArgs"

$exitCode = Exec-Process $buildTool.Path $cmdArgs

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

<EnableDefaultNoneItems>false</EnableDefaultNoneItems>
<_GeneratedVersionFilePath>$(IntermediateOutputPath)DefaultVersions.Generated.props</_GeneratedVersionFilePath>
<NoWarn>$(NoWarn);3021;NU5105;SYSLIB0013</NoWarn>
<NoWarn>$(NoWarn);3021;NU5105;NU5111;SYSLIB0013</NoWarn>
</PropertyGroup>

<ItemGroup>
Expand Down
11 changes: 6 additions & 5 deletions src/Microsoft.DotNet.Arcade.Sdk/tools/Build.proj
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
Configuration Build configuration: "Debug", "Release", etc.
DotNetBuildFromSource Building the entire stack from source with no external dependencies.
DotNetBuildVertical Building the entire stack in "Vertical" PoC mode. This is a superset of DotNetBuildFromSource.
DotNetOutputBlobFeedDir Directory to publish Source Build assets to (packages, symbol packages, installers, etc.).
DotNetPublishUsingPipelines Publish assets to Build Asset Registry.
DotNetSymbolServerTokenMsdl Personal access token for MSDL symbol server. Available from variable group DotNet-Symbol-Server-Pats.
Expand Down Expand Up @@ -47,7 +48,7 @@

<Import Project="RepoLayout.props"/>

<Import Project="SourceBuild/SourceBuildArcadeBuild.targets" Condition="'$(ArcadeBuildFromSource)' == 'true'" />
<Import Project="SourceBuild/SourceBuildArcadeBuild.targets" Condition="'$(ArcadeBuildFromSource)' == 'true' or '$(ArcadeBuildVertical)' == 'true'" />

<!-- Allow for repo specific Build properties such as the list of Projects to build -->
<Import Project="$(RepositoryEngineeringDir)Build.props" Condition="Exists('$(RepositoryEngineeringDir)Build.props')" />
Expand Down Expand Up @@ -121,10 +122,10 @@
<_CommonProps Include="VersionsPropsPath=$(VersionsPropsPath)"/>

<!--
When building from source we suppress restore for projects that set ExcludeFromSourceBuild=true.
When building from source we suppress restore for projects that set ExcludeFromSourceBuild=true or ExcludeFromVerticalBuild.
NuGet Restore task reports a warning for such projects, which we suppress here.
-->
<_CommonProps Include="DisableWarnForInvalidRestoreProjects=true" Condition="'$(DotNetBuildFromSource)' == 'true'"/>
<_CommonProps Include="DisableWarnForInvalidRestoreProjects=true" Condition="'$(DotNetBuildFromSource)' == 'true' or '$(DotNetBuildVertical)' == 'true'"/>

<!--
C++ projects expect VCTargetsPath property to be set. MSBuild generates this property to solution
Expand Down Expand Up @@ -296,14 +297,14 @@
-->
<MSBuild Projects="SourceBuild\AfterSourceBuild.proj"
Properties="@(_CommonProps);_NETCORE_ENGINEERING_TELEMETRY=AfterSourceBuild"
Condition="'$(ArcadeBuildFromSource)' == 'true'"/>
Condition="'$(ArcadeBuildFromSource)' == 'true' or '$(ArcadeBuildVertical)' == 'true'"/>

<!--
Publish artifacts.
-->
<MSBuild Projects="Publish.proj"
Properties="@(_PublishProps);_NETCORE_ENGINEERING_TELEMETRY=Publish"
Targets="Publish"
Condition="'$(Publish)' == 'true' and '$(DotNetBuildFromSource)' != 'true'"/>
Condition="'$(Publish)' == 'true' and '$(DotNetBuildFromSource)' != 'true' and '$(DotNetBuildVertical)' != 'true'"/>
</Target>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
<!-- This file sets properties that enable skipping of a project build if desired.
Cases where we might skip include:
- Source-build mode and "ExcludeFromSourceBuild" is set.
- Vertical-build mode and "ExcludeFromVerticalBuild" is set.
- Non source-build mode and "ExcludeFromBuild" is set
- Target filtering is used and the target filters set "ExcludeFromBuild".
Expand All @@ -26,8 +27,8 @@
In that case, Sdk.targets imports it explicitly. -->

<!--
If a project specifies ExcludeFromSourceBuild=true during a source build suppress all targets and emulate a no-op
(empty common targets like Restore, Build, Pack, etc.).
If a project specifies ExcludeFromSourceBuild=true or ExcludeFromVerticalBuild=true during a source build/vertical
build suppress all targets and emulate a no-op (empty common targets like Restore, Build, Pack, etc.).
It's also possible to set ExcludeFromBuild prior to importing the Sdk.targets
to skip building as desired in non-source build scenarios. This might be done to
Expand All @@ -36,6 +37,7 @@
<PropertyGroup>
<_SuppressAllTargets>false</_SuppressAllTargets>
<_SuppressAllTargets Condition="'$(DotNetBuildFromSource)' == 'true' and '$(ExcludeFromSourceBuild)' == 'true'">true</_SuppressAllTargets>
<_SuppressAllTargets Condition="'$(DotNetBuildVertical)' == 'true' and '$(ExcludeFromVerticalBuild)' == 'true'">true</_SuppressAllTargets>
<_SuppressAllTargets Condition="'$(ExcludeFromBuild)' == 'true'">true</_SuppressAllTargets>

<!-- If excluding, then disable a restore warning, which will fire on newer SDKs, as well as set the NuGetRestoreTargets property to empty,
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.DotNet.Arcade.Sdk/tools/Publish.proj
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,6 @@
<FileSignInfo Update="@(FileSignInfo->WithMetadataValue('CertificateName','Microsoft400'))" CertificateName="$(DotNetCertificateName)" />
</ItemGroup>

<Import Project="SourceBuild/SourceBuildArcadePublish.targets" Condition="'$(ArcadeBuildFromSource)' == 'true'" />
<Import Project="SourceBuild/SourceBuildArcadePublish.targets" Condition="'$(ArcadeBuildFromSource)' == 'true' or '$(ArcadeBuildVertical)' == 'true'" />

</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,16 @@
<ContinueOnPrebuiltBaselineError Condition="'$(FailOnPrebuiltBaselineError)' != 'true'">true</ContinueOnPrebuiltBaselineError>
</PropertyGroup>

<!-- Usage of prebuiltss against the baseline is not reported in vertical build modes,
as it is expected to be extensive. -->
<ValidateUsageAgainstBaseline
DataFile="$(PackageReportDataFile)"
BaselineDataFile="$(PrebuiltBaselineDataFile)"
BaselineDataUpdateHintFile="$(PrebuiltBaselineDataFileDefault)"
OutputBaselineFile="$(SourceBuildSelfPrebuiltReportDir)generated-new-baseline.xml"
OutputReportFile="$(SourceBuildSelfPrebuiltReportDir)baseline-comparison.xml"
ContinueOnError="$(ContinueOnPrebuiltBaselineError)"
Condition="'$(DotNetBuildFromSourceFlavor)' != 'Product'" />
Condition="'$(DotNetBuildFromSourceFlavor)' != 'Product' and '$(ArcadeBuildVertical)' != 'true'" />
</Target>

<!--
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
RunInnerSourceBuildCommand;
PackSourceBuildTarball"
Condition="
'$(ArcadeBuildFromSource)' == 'true' and
('$(ArcadeBuildFromSource)' == 'true' or '$(ArcadeBuildVertical)' == 'true') and
'$(ArcadeInnerBuildFromSource)' != 'true'"
BeforeTargets="Execute" />

Expand All @@ -40,7 +40,7 @@
-->
<Target Name="HookExecuteInnerSourceBuild"
Condition="
'$(ArcadeBuildFromSource)' == 'true' and
('$(ArcadeBuildFromSource)' == 'true' or '$(ArcadeBuildVertical)' == 'true') and
'$(ArcadeInnerBuildFromSource)' == 'true'"
DependsOnTargets="ExecuteInnerSourceBuild"
BeforeTargets="Execute"/>
Expand Down Expand Up @@ -72,8 +72,9 @@
<PropertyGroup>
<!-- Track that this is the inner build to prevent infinite recursion. -->
<InnerBuildArgs>$(InnerBuildArgs) /p:ArcadeInnerBuildFromSource=true</InnerBuildArgs>
<!-- Set DotNetBuildFromSource to avoid publishing. -->
<InnerBuildArgs>$(InnerBuildArgs) /p:DotNetBuildFromSource=true</InnerBuildArgs>
<!-- Turn on DotNetBuildFromSource or DotNetBuildVertical in the inner build -->
<InnerBuildArgs Condition="'$(ArcadeBuildFromSource)' == 'true'">$(InnerBuildArgs) /p:DotNetBuildFromSource=true</InnerBuildArgs>
<InnerBuildArgs Condition="'$(ArcadeBuildVertical)' == 'true'">$(InnerBuildArgs) /p:DotNetBuildVertical=true</InnerBuildArgs>
<!-- Use a fresh clone of the repo so that source-build modifications are isolated. -->
<InnerBuildArgs>$(InnerBuildArgs) /p:RepoRoot=$(InnerSourceBuildRepoRoot)</InnerBuildArgs>
<!-- Override the artifacts dir to cleanly separate the inner build from outer build. -->
Expand All @@ -87,7 +88,7 @@
<InnerBuildArgs>$(InnerBuildArgs) /p:SourceBuildOutputDir=$(SourceBuildOutputDir)</InnerBuildArgs>
<InnerBuildArgs>$(InnerBuildArgs) /p:SourceBuiltBlobFeedDir=$(SourceBuiltBlobFeedDir)</InnerBuildArgs>

<InnerBuildArgs>$(InnerBuildArgs) /p:DotNetBuildOffline=true</InnerBuildArgs>
<InnerBuildArgs Condition="'$(ArcadeBuildVertical)' != 'true'">$(InnerBuildArgs) /p:DotNetBuildOffline=true</InnerBuildArgs>
<InnerBuildArgs Condition=" '$(DotNetPackageVersionPropsPath)' != '' ">$(InnerBuildArgs) /p:DotNetPackageVersionPropsPath=$(DotNetPackageVersionPropsPath)</InnerBuildArgs>

<InnerBuildArgs>$(InnerBuildArgs) /p:FullAssemblySigningSupported=$(FullAssemblySigningSupported)</InnerBuildArgs>
Expand All @@ -96,7 +97,7 @@
<ItemGroup>
<!-- Override package cache to separate source-built packages from upstream. -->
<InnerBuildEnv Include="NUGET_PACKAGES=$(CurrentRepoSourceBuildPackageCache)" />
<InnerBuildEnv Include="DotNetTargetFrameworkFilter=$(SourceBuildTargetFrameworkFilter)" />
<InnerBuildEnv Include="DotNetTargetFrameworkFilter=$(SourceBuildTargetFrameworkFilter)" Condition="'$(ArcadeBuildVertical)' != 'true'" />
</ItemGroup>
</Target>

Expand Down Expand Up @@ -140,15 +141,27 @@
<CopyWipIntoInnerSourceBuildRepo Condition="'$(CopyWipIntoInnerSourceBuildRepo)' == ''">true</CopyWipIntoInnerSourceBuildRepo>

<_GitCloneToDirArgs />
</PropertyGroup>

<PropertyGroup Condition="$([MSBuild]::IsOSPlatform(Windows))">
<_GitCloneToDirArgs>$(_GitCloneToDirArgs) -Source '$(RepoRoot)'</_GitCloneToDirArgs>
<_GitCloneToDirArgs>$(_GitCloneToDirArgs) -Dest '$(InnerSourceBuildRepoRoot)'</_GitCloneToDirArgs>
<_GitCloneToDirArgs Condition="'$(CopyWipIntoInnerSourceBuildRepo)' == 'true'">$(_GitCloneToDirArgs) -CopyWip</_GitCloneToDirArgs>
<_GitCloneToDirArgs Condition="'$(CleanInnerSourceBuildRepoRoot)' == 'true'">$(_GitCloneToDirArgs) -Clean</_GitCloneToDirArgs>

<_GitCloneToDirScriptCommand>powershell -ExecutionPolicy Unrestricted -NoProfile -command "&amp; $(MSBuildThisFileDirectory)git-clone-to-dir.ps1 $(_GitCloneToDirArgs)"</_GitCloneToDirScriptCommand>
</PropertyGroup>

<PropertyGroup Condition="!$([MSBuild]::IsOSPlatform(Windows))">
<_GitCloneToDirArgs>$(_GitCloneToDirArgs) --source &quot;$(RepoRoot)&quot;</_GitCloneToDirArgs>
<_GitCloneToDirArgs>$(_GitCloneToDirArgs) --dest &quot;$(InnerSourceBuildRepoRoot)&quot;</_GitCloneToDirArgs>
<_GitCloneToDirArgs Condition="'$(CopyWipIntoInnerSourceBuildRepo)' == 'true'">$(_GitCloneToDirArgs) --copy-wip</_GitCloneToDirArgs>
<_GitCloneToDirArgs Condition="'$(CleanInnerSourceBuildRepoRoot)' == 'true'">$(_GitCloneToDirArgs) --clean</_GitCloneToDirArgs>

<_GitCloneToDirScriptFile>$(MSBuildThisFileDirectory)git-clone-to-dir.sh</_GitCloneToDirScriptFile>
<_GitCloneToDirScriptCommand>$(MSBuildThisFileDirectory)git-clone-to-dir.sh $(_GitCloneToDirArgs)</_GitCloneToDirScriptCommand>
</PropertyGroup>

<Exec Command="$(_GitCloneToDirScriptFile) $(_GitCloneToDirArgs)" />
<Exec Command="$(_GitCloneToDirScriptCommand)" />

<!--
If the repo has submodules, use 'git clone' on each submodule to put it in the inner repo. We
Expand Down
111 changes: 111 additions & 0 deletions src/Microsoft.DotNet.Arcade.Sdk/tools/SourceBuild/git-clone-to-dir.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
<#
.SYNOPSIS
Creates a git clone of <source> into <dest> directory, optionally saving WIP changes.
.PARAMETER Source
The source Git directory
.PARAMETER Dest
The destination Git directory. Created if doesn't exist.
.PARAMETER Clean
If dest directory already exists, delete it.
.PARAMETER CopyWip
Transfer most types of uncommitted change into the destination directory. Useful for dev workflows.
#>

param (
[Parameter(Mandatory=$true)]
[string]$Source,
[Parameter(Mandatory=$true)]
[string]$Dest,
[switch]$Clean,
[switch]$CopyWip
)

if (-not (Test-Path $Source -PathType Container)) {
Write-Host "-Source not a directory: $Source"
exit 1
}

Write-Host "Cloning repository at: $Source -> $Dest ..."

if (Test-Path $Dest) {
Write-Host "Destination already exists!"
if (Test-Path $Dest -PathType Leaf) {
Write-Host "Existing destination is a regular file, not a directory. This is unusual: aborting."
exit 1
}
elseif ($Clean) {
Write-Host "Clean is enabled: removing $dest and continuing..."
Remove-Item -Path $Dest -Recurse -Force
}
else {
Write-Host "Clean is not enabled: aborting."
exit 1
}
}

Push-Location -Path $Source

if (-not (Test-Path $Dest)) {
New-Item -Path $Dest -ItemType Directory | Out-Null

if ($CopyWip) {
# Copy over changes that haven't been committed, for dev inner loop.
# This gets changes (whether staged or not) but misses untracked files.
$stashCommit = (git stash create)

if ($stashCommit) {
Write-Host "WIP changes detected: created temporary stash $stashCommit to transfer to inner repository..."
}
else {
Write-Host "No WIP changes detected..."
}
}

Write-Host "Creating empty clone at: $Dest"

# Combine the results of git rev-parse --git-dir with the "shallow" subdirectory
# to get the full path to the shallow file.

$shallowFile = Join-Path (git rev-parse --git-dir) "shallow"

if (Test-Path $shallowFile) {
Write-Host "Source repository is shallow..."
if ($stashCommit) {
Write-Host "WIP stash is not supported in a shallow repository: aborting."
exit 1
}

# If source repo is shallow, old versions of Git refuse to clone it to another directory. First,
# remove the 'shallow' file to trick Git into allowing the clone.
$shallowContent = Get-Content -Path $shallowFile
Remove-Item -Path $shallowFile

# Then, run the clone:
# * 'depth=1' avoids encountering the leaf commit in the shallow repo that points to a parent
# that doesn't exist. (The commit marked "grafted".) Git would fail here, otherwise.
# * '--no-local' allows a shallow clone from a git dir on the same filesystem. This means the
# clone will not use hard links and takes up more space. However, since we're doing a shallow
# clone anyway, the difference is probably not significant. (This has not been measured.)
# * '-c protocol.file.allow=alwyas allows cloning from the local filesystem even on newer versions
# of git which have disabled this for security reasons.
git clone -c protocol.file.allow=always --depth=1 --no-local --no-checkout "$Source" "$Dest"

# Restore the 'shallow' file
Set-Content -Path $shallowFile -Value $shallowContent
} else {
git clone -c protocol.file.allow=always --no-checkout "$Source" "$Dest"
}

echo "Checking out sources..."
# If no changes were stashed, stashCommit is empty string, and this is a simple checkout.
git -C $Dest checkout $stashCommit
echo "Clone complete: $Source -> $Dest"

}

Pop-Location
2 changes: 2 additions & 0 deletions src/Microsoft.DotNet.Arcade.Sdk/tools/Tests.props
Original file line number Diff line number Diff line change
Expand Up @@ -39,5 +39,7 @@

<!-- exclude test projects from source-build by default -->
<ExcludeFromSourceBuild Condition="'$(ExcludeFromSourceBuild)' == ''">true</ExcludeFromSourceBuild>
<!-- Vertical Build PoC. Exclude test projects. -->
<ExcludeFromVerticalBuild Condition="'$(ExcludeFromVerticalBuild)' == ''">true</ExcludeFromVerticalBuild>
</PropertyGroup>
</Project>
2 changes: 1 addition & 1 deletion src/Microsoft.DotNet.Arcade.Sdk/tools/Tools.proj
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
<Exec Command='"$(DotNetTool)" tool restore' WorkingDirectory="$(RepoRoot)" />
</Target>

<Import Project="SourceBuild/SourceBuildArcadeTools.targets" Condition="'$(ArcadeBuildFromSource)' == 'true' or '$(SetUpSourceBuildIntermediateNupkgCache)' == 'true'" />
<Import Project="SourceBuild/SourceBuildArcadeTools.targets" Condition="'$(ArcadeBuildFromSource)' == 'true' or '$(ArcadeBuildVertical)' == 'true' or '$(SetUpSourceBuildIntermediateNupkgCache)' == 'true'" />

<!-- Repository extensibility point -->
<Import Project="$(RepositoryEngineeringDir)Tools.props" Condition="Exists('$(RepositoryEngineeringDir)Tools.props')" />
Expand Down
2 changes: 2 additions & 0 deletions src/Microsoft.DotNet.AsmDiff/Microsoft.DotNet.AsmDiff.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
<PackAsTool>true</PackAsTool>
<ToolCommandName>dotnet-asmdiff</ToolCommandName>
<ExcludeFromSourceBuild>true</ExcludeFromSourceBuild>
<!-- Vertical PoC - in-tree use of the tool causes some race? -->
<ExcludeFromVerticalBuild>true</ExcludeFromVerticalBuild>
<StrongNameKeyId>Open</StrongNameKeyId>
<UsingToolXliff>true</UsingToolXliff>
<!-- Use xliff localization. -->
Expand Down

0 comments on commit 0a0217f

Please sign in to comment.