Enable arcade support (#113)

* Copy eng\common from Arcade repo.

* Enable Spark to build using dotnet/arcade

* Update coverlet version to work around bug in older version.

* Fix up test directory lookup for new artifacts location.

* Don't use dummy versions for local dev builds.

* Fix up official build for new output artifacts location

* Rearrange official build

- Set official build id parameter
- Build nuget package on Build leg
- Publish the worker using MSBuild instead of cmd

* Ensure only our .nupkg gets signed and copied, and the .nupkg is placed directly in the Packages folder.

Clean up official build definition.

.gitignore

@@ -53,6 +53,7 @@ BenchmarkDotNet.Artifacts/
 project.lock.json
 project.fragment.lock.json
 artifacts/
+.dotnet/
 
 # StyleCop
 StyleCopReport.xml

NuGet.config

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />
+  </packageSources>
+</configuration>

azure-pipelines.yml

@@ -7,9 +7,7 @@ trigger:
     - master
 
 variables:
-  solution: '**/*.sln'
   buildConfiguration: 'Release'
-  dotnetCoreVersion: '2.2.105'
   _SignType: real
   _TeamName: DotNetSpark
 
@@ -23,31 +21,11 @@ jobs:
   displayName: Build and Test Sources
   pool: Hosted VS2017
 
-  steps:
-  - task: DotNetCoreInstaller@0
-    inputs:
-      version: '$(dotnetCoreVersion)'
-
-  - task: DotNetCoreCLI@2
-    displayName: '.NET build'
-    inputs:
-      command: build
-      projects: '$(solution)'
-      arguments: '--configuration $(buildConfiguration)'
-
-  - task: BatchScript@1
-    displayName: Publish Microsoft.Spark.Worker
-    inputs:
-      filename: script\publish-workers.cmd
-      arguments: $(Build.SourcesDirectory) $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker $(buildConfiguration)
-
-  - task: DotNetCoreCLI@2
-    displayName: '.NET unit tests'
-    inputs:
-      command: test
-      projects: '**/*UnitTest/*.csproj'
-      arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+  variables:
+    ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
 
+  steps:
   - task: Maven@3
     displayName: 'Maven build src'
     inputs:
@@ -64,6 +42,21 @@ jobs:
       filename: script\download-spark-distros.cmd
       arguments: $(Build.BinariesDirectory)
 
+  - script: build.cmd -pack
+              -c $(buildConfiguration)
+              -ci
+              $(_OfficialBuildIdArgs)
+              /p:PublishSparkWorker=true
+              /p:SparkWorkerPublishDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Worker
+    displayName: '.NET build'
+
+  - task: DotNetCoreCLI@2
+    displayName: '.NET unit tests'
+    inputs:
+      command: test
+      projects: '**/*UnitTest/*.csproj'
+      arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
+
   - task: DotNetCoreCLI@2
     displayName: 'E2E tests for Spark 2.3.0'
     inputs:
@@ -134,19 +127,12 @@ jobs:
     - task: CopyFiles@2
       displayName: Stage .NET artifacts
       inputs:
-        sourceFolder: $(Build.SourcesDirectory)/src/csharp/Microsoft.Spark/bin/$(buildConfiguration)
+        sourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(buildConfiguration)/Shipping
         contents: |
-          **/*.dll
-          **/*.pdb
-        targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/src/csharp/Microsoft.Spark/bin/$(buildConfiguration)
+          **/*.nupkg
+          **/*.snupkg
+        targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/artifacts/packages/$(buildConfiguration)/Shipping
 
-    - task: CopyFiles@2
-      displayName: Stage scala artifacts
-      inputs:
-        sourceFolder: $(Build.SourcesDirectory)/src/scala
-        contents: '**/*.jar'
-        targetFolder: $(Build.ArtifactStagingDirectory)/BuildArtifacts/src/scala
-
     - task: PublishBuildArtifacts@1
       inputs:
         pathtoPublish: '$(Build.ArtifactStagingDirectory)'
@@ -162,32 +148,12 @@ jobs:
       queue: buildpool.windows.10.amd64.vs2017
 
     steps:
-    - task: DotNetCoreInstaller@0
-      inputs:
-        version: '$(dotnetCoreVersion)'
-
     - task: DownloadBuildArtifacts@0
       displayName: Download Build Artifacts
       inputs:
         artifactName: Microsoft.Spark.Binaries
         downloadPath: $(Build.ArtifactStagingDirectory)
 
-    - task: CopyFiles@2
-      displayName: Copy .NET artifacts
-      inputs:
-        sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/BuildArtifacts/src/csharp/Microsoft.Spark/bin/$(buildConfiguration)
-        contents: |
-          **/*.dll
-          **/*.pdb
-        targetFolder: $(Build.SourcesDirectory)/src/csharp/Microsoft.Spark/bin/$(buildConfiguration)
-
-    - task: CopyFiles@2
-      displayName: Copy scala artifacts
-      inputs:
-        sourceFolder: $(Build.ArtifactStagingDirectory)/Microsoft.Spark.Binaries/BuildArtifacts/src/scala
-        contents: '**/*.jar'
-        targetFolder: $(Build.SourcesDirectory)/src/scala
-
     - task: MicroBuildSigningPlugin@2
       displayName: Install MicroBuild plugin
       inputs:
@@ -205,26 +171,6 @@ jobs:
         msbuildArguments: /t:Restore
         msbuildVersion: 15.0
 
-    - task: MSBuild@1
-      displayName: 'Sign .NET binaries'
-      inputs:
-        solution: eng/Sign.proj
-        msbuildArguments: /t:SignBinaries
-                          /p:SignSparkBinaries=true
-                          /p:SignAssetsDir=$(Build.SourcesDirectory)\src\csharp\Microsoft.Spark\bin\$(buildConfiguration)\
-                          /p:SignType=$(_SignType)
-        msbuildVersion: 15.0
-
-    - task: MSBuild@1
-      displayName: 'Sign .jar binaries'
-      inputs:
-        solution: eng/Sign.proj
-        msbuildArguments: /t:SignBinaries
-                          /p:SignJarBinaries=true
-                          /p:SignAssetsDir=$(Build.SourcesDirectory)\src\scala\
-                          /p:SignType=$(_SignType)
-        msbuildVersion: 15.0
-
     - task: MSBuild@1
       displayName: 'Sign worker binaries'
       inputs:
@@ -235,36 +181,25 @@ jobs:
                           /p:SignType=$(_SignType)
         msbuildVersion: 15.0
 
-    - task: DotNetCoreCLI@2
-      displayName: 'Create NuGet packages'
-      inputs:
-        command: pack
-        projects: '$(solution)'
-        arguments: '--no-build --configuration $(buildConfiguration) --output $(Build.ArtifactStagingDirectory)'
-
     - task: MSBuild@1
       displayName: 'Sign nuget/snupkg packages'
       inputs:
         solution: eng/Sign.proj
         msbuildArguments: /t:SignBinaries 
-                          /p:SignAssetsDir=$(Build.ArtifactStagingDirectory)\
+                          /p:SignAssetsDir=$(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries\
                           /p:SignNugetPackages=true
                           /p:SignType=$(_SignType)
         msbuildVersion: 15.0
 
     - task: CopyFiles@2
       displayName: Copy nupkg to publish
       inputs:
-        sourceFolder: $(Build.ArtifactStagingDirectory)
-        contents: 'Microsoft.Spark*.nupkg'
+        sourceFolder: $(Build.ArtifactStagingDirectory)\Microsoft.Spark.Binaries
+        contents: |
+          **/*.nupkg
+          **/*.snupkg
         targetFolder: $(Build.ArtifactStagingDirectory)/Packages
-
-    - task: CopyFiles@2
-      displayName: Copy snupkg to publish
-      inputs:
-        sourceFolder: $(Build.ArtifactStagingDirectory)
-        contents: 'Microsoft.Spark*.snupkg'
-        targetFolder: $(Build.ArtifactStagingDirectory)/SymbolPackages
+        flattenFolders: true
 
     - task: PowerShell@2
       displayName: Package Microsoft.Spark.Worker
@@ -283,8 +218,3 @@ jobs:
       inputs:
         pathtoPublish: '$(Build.ArtifactStagingDirectory)/Packages'
         artifactName:  Microsoft.Spark.Binaries
-
-    - task: PublishBuildArtifacts@1
-      inputs:
-        pathtoPublish: '$(Build.ArtifactStagingDirectory)/SymbolPackages'
-        artifactName:  Microsoft.Spark.Binaries

benchmark/csharp/Directory.Build.props

@@ -1,4 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" TreatAsLocalProperty="ExcludeRestorePackageImports">
   <Import Project="..\..\src\csharp\Directory.Build.props" />
+
+  <PropertyGroup>
+    <IsShipping>false</IsShipping>
+  </PropertyGroup>
 </Project>

benchmark/csharp/Directory.Build.targets

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project>
+  <Import Project="..\..\src\csharp\Directory.Build.targets" />
+</Project>

build.cmd

@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build %*"
+exit /b %ErrorLevel%

build.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+"$scriptroot/eng/common/build.sh" --build --restore $@

docs/building/ubuntu-instructions.md

@@ -190,14 +190,14 @@ You should see JARs created for the supported Spark versions:
      </details>
   3. Manually copy Worker binaries into the Samples output location. 
      ```
-     cp ~/dotnet.spark/src/csharp/Microsoft.Spark.Worker/bin/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish/* ~/dotnet.spark/examples/Microsoft.Spark.CSharp.Examples/bin/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish/
+     cp ~/dotnet.spark/artifacts/bin/Microsoft.Spark.Worker/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish/* ~/dotnet.spark/artifacts/bin/Microsoft.Spark.CSharp.Examples/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish/
      ```
 
 # Run Samples
 
 Once you build the samples, you can use `spark-submit` to submit your .NET Core apps. Make sure you have followed the [pre-requisites](#pre-requisites) section and installed Apache Spark.
 
-  1. Open a terminal and go to the directory where your app binary has been generated (e.g., `~/dotnet.spark/examples/Microsoft.Spark.CSharp.Examples/bin/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish`)
+  1. Open a terminal and go to the directory where your app binary has been generated (e.g., `~/dotnet.spark/artifacts/bin/Microsoft.Spark.CSharp.Examples/Debug/netcoreapp2.1/ubuntu.18.04-x64/publish`)
   2. Running your app follows the basic structure:
      ```bash
      spark-submit \

docs/building/windows-instructions.md

@@ -210,14 +210,14 @@ You should see JARs created for the supported Spark versions:
      </details>
   3. Manually copy Worker binaries into the Samples output location. 
      ```
-     cp c:\github\dotnet-spark\src\csharp\Microsoft.Spark.Worker\bin\Debug\netcoreapp2.1\win10-x64\publish\* C:\github\dotnet-spark\examples\Microsoft.Spark.CSharp.Examples\bin\Debug\netcoreapp2.1\win10-x64\publish\
+     cp c:\github\dotnet-spark\artifacts\bin\Microsoft.Spark.Worker\Debug\netcoreapp2.1\win10-x64\publish\* C:\github\dotnet-spark\artifacts\bin\Microsoft.Spark.CSharp.Examples\Debug\netcoreapp2.1\win10-x64\publish\
      ```
 
 # Run Samples
 
 Once you build the samples, running them will be through `spark-submit` regardless of whether you are targeting .NET Framework or .NET Core apps. Make sure you have followed the [pre-requisites](#pre-requisites) section and installed Apache Spark.
 
-  1. Open Powershell and go to the directory where your app binary has been generated (e.g., `c:\github\dotnet\spark\examples\Microsoft.Spark.CSharp.Examples\bin\Debug\net461` for .NET Framework, `c:\github\dotnet-spark\examples\Microsoft.Spark.CSharp.Examples\bin\Debug\netcoreapp2.1\win10-x64\publish` for .NET Core)
+  1. Open Powershell and go to the directory where your app binary has been generated (e.g., `c:\github\dotnet\spark\artifacts\bin\Microsoft.Spark.CSharp.Examples\Debug\net461` for .NET Framework, `c:\github\dotnet-spark\artifacts\bin\Microsoft.Spark.CSharp.Examples\Debug\netcoreapp2.1\win10-x64\publish` for .NET Core)
   2. Running your app follows the basic structure:
      ```powershell
      spark-submit.cmd `

eng/AfterSolutionBuild.targets

@@ -0,0 +1,28 @@
+<Project>
+
+  <ItemGroup>
+    <_PublishProject Include="$(RepoRoot)src\csharp\Microsoft.Spark.Worker\Microsoft.Spark.Worker.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <_PublishTarget Include="FullFramework" Framework="net461" RuntimeId="win-x64" />
+    <_PublishTarget Include="WindowsCore" Framework="netcoreapp2.1" RuntimeId="win-x64" />
+    <_PublishTarget Include="LinuxCore" Framework="netcoreapp2.1" RuntimeId="linux-x64" />
+  </ItemGroup>
+
+  <Target Name="PublishSparkWorker"
+          AfterTargets="Build"
+          Condition="'$(PublishSparkWorker)' == 'true'">
+
+    <Error Condition="'$(SparkWorkerPublishDir)' == ''"
+           Text="SparkWorkerPublishDir variable is not set." />
+
+    <Exec Command="dotnet publish &quot;@(_PublishProject)&quot; ^
+                   -c $(Configuration) ^
+                   -f %(_PublishTarget.Framework) ^
+                   -r %(_PublishTarget.RuntimeId) ^
+                   -o &quot;$(SparkWorkerPublishDir)\%(_PublishTarget.Framework)\%(_PublishTarget.RuntimeId)&quot; ^
+                   /p:OfficialBuildId=$(OfficialBuildId)" />
+
+  </Target>
+</Project>

eng/Build.props

@@ -0,0 +1,7 @@
+<Project>
+  <ItemGroup>
+    <ProjectToBuild Include="$(RepoRoot)src\csharp\**\*.csproj"/>
+    <ProjectToBuild Include="$(RepoRoot)examples\**\*.csproj;$(RepoRoot)examples\**\*.fsproj" />
+    <ProjectToBuild Include="$(RepoRoot)benchmark\csharp\**\*.csproj" />
+  </ItemGroup>
+</Project>

eng/Sign.proj

@@ -36,23 +36,11 @@
     </ItemGroup>
 
     <ItemGroup Condition="'$(SignNugetPackages)' == 'true'">
-      <FilesToSign Include="$(OutDir)*.nupkg;$(OutDir)*.snupkg">
+      <FilesToSign Include="$(OutDir)**/*.nupkg;$(OutDir)**/*.snupkg">
         <Authenticode>NuGet</Authenticode>
       </FilesToSign>
     </ItemGroup>
 
-    <ItemGroup Condition="'$(SignSparkBinaries)' == 'true'">
-      <FilesToSign Include="$(OutDir)**/Microsoft.Spark.dll">
-        <Authenticode>Microsoft</Authenticode>
-      </FilesToSign>
-    </ItemGroup>
-
-    <ItemGroup Condition="'$(SignJarBinaries)' == 'true'">
-      <FilesToSign Include="$(OutDir)**/microsoft-spark-*.jar">
-        <Authenticode>MicrosoftJARSHA2</Authenticode>
-      </FilesToSign>
-    </ItemGroup>
-
     <Error Condition="'@(FilesToSign)' == ''" Text="There are no files to sign. FilesToSign group is empty."/>
   </Target>
-</Project>
+</Project>

eng/Version.Details.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Dependencies>
+  <ProductDependencies>
+  </ProductDependencies>
+  <ToolsetDependencies>
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19257.7">
+      <Uri>https://github.com/dotnet/arcade</Uri>
+      <Sha>bda52d7619f9420de46f2c39ffc972864bbcab63</Sha>
+    </Dependency>
+  </ToolsetDependencies>
+</Dependencies>

eng/Versions.props

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <VersionPrefix>0.2.0</VersionPrefix>
+    <PreReleaseVersionLabel>prerelease</PreReleaseVersionLabel>
+    <RestoreSources>
+      $(RestoreSources);
+      https://api.nuget.org/v3/index.json;
+    </RestoreSources> 
+  </PropertyGroup>
+</Project>

eng/common/CIBuild.cmd

@@ -0,0 +1,2 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"