Skip to content

Commit

Permalink
Feature/eng common patch 2 (#1052)
Browse files Browse the repository at this point in the history
* Update Tools repo Sync to make use of patch files rather than replacing all files in the sync directory

* Update location of patch files

* Add some changes in eng/common to test things

* Update patch application logic

* Switch to pushing to upstream

* Change workflow to use upstream branches

* Add step tp delete braches after merging
  • Loading branch information
chidozieononiwu authored and benbp committed Dec 1, 2020
1 parent 6a2634b commit 42e0e2b
Show file tree
Hide file tree
Showing 8 changed files with 330 additions and 67 deletions.
14 changes: 10 additions & 4 deletions eng/common/pipelines/templates/steps/create-pull-request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ parameters:
GHTeamReviewersVariable: ''
# Multiple labels seperated by comma, e.g. "bug, APIView"
PRLabels: ''
SkipCheckingForChanges: false

steps:

Expand All @@ -35,15 +36,19 @@ steps:
echo "##vso[task.setvariable variable=HasChanges]$false"
echo "No changes so skipping code push"
}
displayName: Check for changes
condition: and(succeeded(), eq(${{ parameters.SkipCheckingForChanges }}, false))
workingDirectory: ${{ parameters.WorkingDirectory }}
ignoreLASTEXITCODE: true

- pwsh: |
# Remove the repo owner from the front of the repo name if it exists there
$repoName = "${{ parameters.RepoName }}" -replace "^${{ parameters.RepoOwner }}/", ""
echo "##vso[task.setvariable variable=RepoNameWithoutOwner]$repoName"
echo "RepoName = $repName"
displayName: Check for changes
echo "RepoName = $repoName"
displayName: Remove Repo Owner from Repo Name
condition: succeeded()
workingDirectory: ${{ parameters.WorkingDirectory }}
ignoreLASTEXITCODE: true

- task: PowerShell@2
displayName: Push changes
Expand All @@ -57,6 +62,7 @@ steps:
-CommitMsg "${{ parameters.CommitMsg }}"
-GitUrl "https://$(azuresdk-github-pat)@github.com/${{ parameters.PROwner }}/$(RepoNameWithoutOwner).git"
-PushArgs "${{ parameters.PushArgs }}"
-SkipCommit $${{parameters.SkipCheckingForChanges}}
- task: PowerShell@2
displayName: Create pull request
Expand Down
53 changes: 53 additions & 0 deletions eng/common/scripts/Add-Issue-Comment.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
[CmdletBinding(SupportsShouldProcess = $true)]
param(
[Parameter(Mandatory = $true)]
[string]$RepoOwner,

[Parameter(Mandatory = $true)]
[string]$RepoName,

[Parameter(Mandatory = $true)]
[string]$IssueNumber,

[Parameter(Mandatory = $false)]
[string]$CommentPrefix,

[Parameter(Mandatory = $true)]
[string]$Comment,

[Parameter(Mandatory = $false)]
[string]$CommentPostFix,

[Parameter(Mandatory = $true)]
[string]$AuthToken
)

. "${PSScriptRoot}\logging.ps1"

$headers = @{
Authorization = "bearer $AuthToken"
}

$apiUrl = "https://api.github.com/repos/$RepoOwner/$RepoName/issues/$IssueNumber/comments"

$commentPrefixValue = [System.Environment]::GetEnvironmentVariable($CommentPrefix)
$commentValue = [System.Environment]::GetEnvironmentVariable($Comment)
$commentPostFixValue = [System.Environment]::GetEnvironmentVariable($CommentPostFix)

if (!$commentPrefixValue) { $commentPrefixValue = $CommentPrefix }
if (!$commentValue) { $commentValue = $Comment }
if (!$commentPostFixValue) { $commentPostFixValue = $CommentPostFix }

$PRComment = "$commentPrefixValue $commentValue $commentPostFixValue"

$data = @{
body = $PRComment
}

try {
$resp = Invoke-RestMethod -Method POST -Headers $headers -Uri $apiUrl -Body ($data | ConvertTo-Json)
}
catch {
LogError "Invoke-RestMethod [ $apiUrl ] failed with exception:`n$_"
exit 1
}
56 changes: 56 additions & 0 deletions eng/common/scripts/Queue-Pipeline.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
[CmdletBinding(SupportsShouldProcess = $true)]
param(
[Parameter(Mandatory = $true)]
[string]$Organization,

[Parameter(Mandatory = $true)]
[string]$Project,

[Parameter(Mandatory = $true)]
[string]$SourceBranch,

[Parameter(Mandatory = $true)]
[int]$DefinitionId,

[Parameter(Mandatory = $false)]
[string]$VsoQueuedPipelines,

[Parameter(Mandatory = $true)]
[string]$AuthToken
)

. "${PSScriptRoot}\logging.ps1"

$headers = @{
Authorization = "Basic $AuthToken"
}

$apiUrl = "https://dev.azure.com/$Organization/$Project/_apis/build/builds?api-version=6.0"

$body = @{
sourceBranch = $SourceBranch
definition = @{ id = $DefinitionId }
}

Write-Verbose ($body | ConvertTo-Json)

try {
$resp = Invoke-RestMethod -Method POST -Headers $headers $apiUrl -Body ($body | ConvertTo-Json) -ContentType application/json
}
catch {
LogError "Invoke-RestMethod [ $apiUrl ] failed with exception:`n$_"
exit 1
}

LogDebug "Pipeline [ $($resp.definition.name) ] queued at [ $($resp._links.web.href) ]"

if ($VsoQueuedPipelines) {
$enVarValue = [System.Environment]::GetEnvironmentVariable($VsoQueuedPipelines)
$QueuedPipelineLinks = if ($enVarValue) {
"$enVarValue<br>[$($resp.definition.name)]($($resp._links.web.href))"
}else {
"[$($resp.definition.name)]($($resp._links.web.href))"
}
$QueuedPipelineLinks
Write-Host "##vso[task.setvariable variable=$VsoQueuedPipelines]$QueuedPipelineLinks"
}
22 changes: 15 additions & 7 deletions eng/common/scripts/git-branch-push.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,10 @@ param(
[string] $GitUrl,

[Parameter(Mandatory = $false)]
[string] $PushArgs = ""
[string] $PushArgs = "",

[Parameter(Mandatory = $false)]
[boolean] $SkipCommit = $false
)

# This is necessay because of the janky git command output writing to stderr.
Expand Down Expand Up @@ -57,12 +60,17 @@ if ($LASTEXITCODE -ne 0)
exit $LASTEXITCODE
}

Write-Host "git -c user.name=`"azure-sdk`" -c user.email=`"azuresdk@microsoft.com`" commit -am `"$($CommitMsg)`""
git -c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com" commit -am "$($CommitMsg)"
if ($LASTEXITCODE -ne 0)
{
Write-Error "Unable to add files and create commit LASTEXITCODE=$($LASTEXITCODE), see command output above."
exit $LASTEXITCODE
if (!$SkipCommit) {
Write-Host "git -c user.name=`"azure-sdk`" -c user.email=`"azuresdk@microsoft.com`" commit -am `"$($CommitMsg)`""
git -c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com" commit -am "$($CommitMsg)"
if ($LASTEXITCODE -ne 0)
{
Write-Error "Unable to add files and create commit LASTEXITCODE=$($LASTEXITCODE), see command output above."
exit $LASTEXITCODE
}
}
else {
Write-Host "Skipped applying commit"
}

# The number of retries can be increased if necessary. In theory, the number of retries
Expand Down
1 change: 0 additions & 1 deletion eng/common/scripts/modules/ChangeLog-Operations.psm1
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
# Common Changelog Operations

$RELEASE_TITLE_REGEX = "(?<releaseNoteTitle>^\#+.*(?<version>\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?)(\s(?<releaseStatus>\(Unreleased\)|\(\d{4}-\d{2}-\d{2}\)))?)"

# Returns a Collection of changeLogEntry object containing changelog info for all version present in the gived CHANGELOG
Expand Down
1 change: 1 addition & 0 deletions eng/common/scripts/modules/Package-Properties.psm1
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# This Files has been retired
# Helper functions for retireving useful information from azure-sdk-for-* repo
# Example Use : Import-Module .\eng\common\scripts\modules
class PackageProps
Expand Down
141 changes: 118 additions & 23 deletions eng/pipelines/eng-common-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@ parameters:
- name: DirectoryToSync
type: string
default: eng/common
- name: Repos
type: object
default:
- azure-sdk-for-android
- azure-sdk-for-c
- azure-sdk-for-cpp
- azure-sdk-for-go
- azure-sdk-for-ios
- azure-sdk-for-java
- azure-sdk-for-js
- azure-sdk-for-net
- azure-sdk-for-python

trigger: none

Expand All @@ -30,16 +42,36 @@ stages:
displayName: Sync ${{ parameters.DirectoryToSync }} Directory

steps:
- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
- pwsh: |
Set-PsDebug -Trace 1
$patchDir = "$(Build.ArtifactStagingDirectory)/patchfiles"
$patchfiles = git format-patch --output-directory $patchDir HEAD...origin/$(system.pullRequest.targetBranch) -- "${{ parameters.DirectoryToSync }}"
if ($patchfiles -and ($LASTEXITCODE -eq 0)) {
echo "##vso[task.setvariable variable=PatchFilesLocation]$patchDir"
echo "Setting PatchFilesLocation"
}
else {
Write-Host "Failed to Create PatchFiles from Pull Request [https://github.com/Azure/azure-sdk-tools/pull/$(System.PullRequest.PullRequestNumber)]"
exit 1
}
displayName: Create Patch Files from Changes in PR
workingDirectory: $(System.DefaultWorkingDirectory)
- task: PublishPipelineArtifact@1
condition: and(succeeded(), ne(variables['PatchFilesLocation'],''))
inputs:
artifactName: patchfiles
path: "$(PatchFilesLocation)"

- template: ./templates/steps/sync-directory.yml
parameters:
${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
CommitMessage: "Sync ${{ parameters.DirectoryToSync }} directory with azure-sdk-tools repository for Tools PR $(System.PullRequest.PullRequestNumber)"
DirectoryToSync: ${{ parameters.DirectoryToSync }}
PRBranchName: "sync-${{ parameters.DirectoryToSync }}-$(System.PullRequest.SourceBranch)-$(System.PullRequest.PullRequestNumber)"
PRTitle: "Sync ${{ parameters.DirectoryToSync }} directory with azure-sdk-tools for PR $(System.PullRequest.PullRequestNumber)"
PRBody: >
Sync ${{ parameters.DirectoryToSync }} directory with azure-sdk-tools for PR https://github.com/Azure/azure-sdk-tools/pull/$(System.PullRequest.PullRequestNumber)<br>
See [eng/common workflow](https://github.com/Azure/azure-sdk-tools/blob/master/eng/common/README.md#workflow)
BaseBranchName: $(system.pullRequest.targetBranch)
SkipCheckingForChanges: true
${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
CommitMessage: Sync ${{ parameters.DirectoryToSync }} directory with azure-sdk-tools repository
DirectoryToSync: ${{ parameters.DirectoryToSync }}
Expand All @@ -48,29 +80,78 @@ stages:
PRBody: >
Sync ${{ parameters.DirectoryToSync }} directory with azure-sdk-tools repository.<br>
See [eng/common workflow](https://github.com/Azure/azure-sdk-tools/blob/master/eng/common/README.md#workflow)
PRDataArtifactPath: $(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }}
Repos:
- azure-sdk-for-android
- azure-sdk-for-c
- azure-sdk-for-cpp
- azure-sdk-for-go
- azure-sdk-for-ios
- azure-sdk-for-java
- azure-sdk-for-js
- azure-sdk-for-net
- azure-sdk-for-python

- task: PublishPipelineArtifact@1
condition: succeeded()
inputs:
artifactName: ${{ parameters.ArtifactName }}
path: $(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }}
Repos: ${{ parameters.Repos }}

- stage: CreateSyncPRs
jobs:
- deployment: CreateSyncPullRequests
displayName: Create Sync Pull Requests
environment: githubmerges

pool:
vmImage: windows-2019

strategy:
runOnce:
deploy:
steps:
- checkout: self

- pwsh: |
New-Item -Path $(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }} -ItemType File
displayName: Create PRData Artifact
- ${{ each repo in parameters.Repos }}:
- task: PowerShell@2
displayName: Create pull request
condition: succeeded()
inputs:
pwsh: true
workingDirectory: ${{ parameters.WorkingDirectory }}
filePath: $(Build.SourcesDirectory)/eng/common/scripts/Submit-PullRequest.ps1
${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
arguments: >
-RepoOwner "Azure"
-RepoName "${{ repo }}"
-BaseBranch $(system.pullRequest.targetBranch)
-PROwner "Azure"
-PRBranch "sync-${{ parameters.DirectoryToSync }}-$(System.PullRequest.SourceBranch)-$(System.PullRequest.PullRequestNumber)"
-AuthToken "$(azuresdk-github-pat)"
-PRTitle "Sync ${{ parameters.DirectoryToSync }} directory with azure-sdk-tools for PR $(System.PullRequest.PullRequestNumber)"
-PRLabels "Central-EngSys, EngSys"
-PRBody "Sync ${{ parameters.DirectoryToSync }} directory with azure-sdk-tools for PR https://github.com/Azure/azure-sdk-tools/pull/$(System.PullRequest.PullRequestNumber)`
See [eng/common workflow](https://github.com/Azure/azure-sdk-tools/blob/master/eng/common/README.md#workflow)"
${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
arguments: >
-RepoOwner "Azure"
-RepoName "${{ repo }}"
-BaseBranch $(system.pullRequest.targetBranch)
-PROwner "Azure"
-PRBranch "sync-${{ parameters.DirectoryToSync }}"
-AuthToken "$(azuresdk-github-pat)"
-PRTitle "Sync ${{ parameters.DirectoryToSync }} directory with azure-sdk-tools for PR $(System.PullRequest.PullRequestNumber)"
-PRLabels "Central-EngSys, EngSys"
-PRBody "Sync ${{ parameters.DirectoryToSync }} directory with azure-sdk-tools for PR https://github.com/Azure/azure-sdk-tools/pull/$(System.PullRequest.PullRequestNumber)`
See [eng/common workflow](https://github.com/Azure/azure-sdk-tools/blob/master/eng/common/README.md#workflow)"
- pwsh: |
$PRData = "Azure;${{ repo }};$(Submitted.PullRequest.Number)"
Add-Content -Path "$(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }}" -Value $PRData
displayName: Write Sync PR Data to Artifact File
condition: succeeded()
- task: PublishPipelineArtifact@1
condition: succeeded()
displayName: Publish ${{ parameters.PRDataFileName }}
inputs:
artifactName: ${{ parameters.ArtifactName }}
path: $(Build.ArtifactStagingDirectory)/${{ parameters.PRDataFileName }}

- stage: VerifyAndMerge
jobs:
- deployment: VerifyandMergeSyncPrs
displayName: Verify and Merge Sync PRs
environment: github
environment: githubmerges

pool:
vmImage: windows-2019
Expand All @@ -94,4 +175,18 @@ stages:
-PRDataArtifactPath "$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{ parameters.PRDataFileName }}"
-AuthToken "$(azuresdk-github-pat)"
-devOpsLogging
pwsh: true
pwsh: true

- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
- ${{ each repo in parameters.Repos }}:
- pwsh: |
git clone https://github.com/azure/${{ repo }} --depth 1
pushd $(System.DefaultWorkingDirectory)/${{ repo }}
git push origin --delete "sync-${{ parameters.DirectoryToSync }}-$(System.PullRequest.SourceBranch)-$(System.PullRequest.PullRequestNumber)"
if ($lastExitCode -ne 0) {
Write-Host "Failed to delete [sync-${{ parameters.DirectoryToSync }}-$(System.PullRequest.SourceBranch)-$(System.PullRequest.PullRequestNumber)] branch in ${{ repo }}"
exit 1
}
displayName: Write Sync PR Data to Artifact File
workingDirectory: $(System.DefaultWorkingDirectory)
condition: succeeded()
Loading

0 comments on commit 42e0e2b

Please sign in to comment.