From 764d08da684b90c8952ecc5baf2dbd4eef47e7ba Mon Sep 17 00:00:00 2001 From: Pallavi Taneja Date: Wed, 23 Feb 2022 16:59:52 -0800 Subject: [PATCH] Add support for sync scripts and mass patch releases. (#26981) --- eng/bomgenerator/generateAzureSDKBOM.ps1 | 68 +++-- eng/scripts/bomhelpers.ps1 | 311 +++++++++++++++++++++++ eng/scripts/generatepatch.ps1 | 78 ++++++ eng/scripts/patchreleases.ps1 | 294 +++++++++++++++++++++ eng/scripts/syncversionclient.ps1 | 94 +++++++ 5 files changed, 829 insertions(+), 16 deletions(-) create mode 100644 eng/scripts/bomhelpers.ps1 create mode 100644 eng/scripts/generatepatch.ps1 create mode 100644 eng/scripts/patchreleases.ps1 create mode 100644 eng/scripts/syncversionclient.ps1 diff --git a/eng/bomgenerator/generateAzureSDKBOM.ps1 b/eng/bomgenerator/generateAzureSDKBOM.ps1 index 8806842a30d41..eab99f13cc06d 100644 --- a/eng/bomgenerator/generateAzureSDKBOM.ps1 +++ b/eng/bomgenerator/generateAzureSDKBOM.ps1 @@ -1,24 +1,60 @@ -$repoRoot = Resolve-Path "${PSScriptRoot}..\..\.." -$inputDir = Join-Path ${PSScriptRoot} "inputDir" -$outputDir = Join-Path ${PSScriptRoot} "outputDir" -$versionClientFileName = "version_client.txt" -$pomFileName = "pom.xml" -$defaultVersionClientFilePath = Join-Path $inputDir $versionClientFileName -$defaultPomFilePath = Join-Path $inputDir $pomFileName -$versionClientFilePath = Join-Path $repoRoot "eng" "versioning" $versionClientFileName -$bomPomFilePath = Join-Path $repoRoot "sdk" "boms" "azure-sdk-bom" $pomFileName - -if(! (Test-Path $inputDir)) { +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +$RepoRoot = Resolve-Path "${PSScriptRoot}..\..\.." +$VersionClientFileName = "version_client.txt" +$PomFileName = "pom.xml" +$InputDir = Join-Path ${PSScriptRoot} "inputdir" +$OutputDir = Join-Path ${PSScriptRoot} "outputdir" +$DefaultVersionClientFilePath = Join-Path $InputDir $VersionClientFileName +$DefaultPomFilePath = Join-Path $InputDir $PomFileName +$EngDir = Join-Path $RepoRoot "eng" +$VersionClientFilePath = Join-Path $EngDir "versioning" $VersionClientFileName +$BomPomFilePath = Join-Path $RepoRoot "sdk" "boms" "azure-sdk-bom" $PomFileName +$EngScriptDir = Join-Path $EngDir "scripts" +$BomGeneratorPomFilePath = Join-Path ${PSScriptRoot} $PomFileName +$NewBomFilePath = Join-Path $OutputDir $PomFileName + +. (Join-Path $EngScriptDir syncversionclient.ps1) + + +function UpdateBomProjectElement($OldPomFilePath, $NewPomFilePath) { + $oldFileContent = [xml](Get-Content -Path $oldPomFilePath) + $newFileContent = [xml](Get-Content -Path $NewPomFilePath) + + $oldXmlns = $oldFileContent.Project.xmlns + $oldxsi = $oldFileContent.Project.xsi + $oldschemaLocation = $oldFileContent.Project.SchemaLocation + + $newFileContent.Project.xmlns = $oldXmlns + $newFileContent.Project.xsi = $oldxsi + $newFileContent.Project.SchemaLocation = $oldschemaLocation + + $newFileContent.Save($NewPomFilePath) +} + +Write-Output "InputDir:$($InputDir)" +Write-Output "OutputDir:$($OutputDir)" +Write-Output "Updating version_client.txt file by looking at the packages released to maven." +SyncVersionClientFile -GroupId "com.azure" +Write-Output "Updated version_client.txt file." + +if(! (Test-Path $InputDir)) { New-Item -Path $PSScriptRoot -Name "inputDir" -ItemType "directory" } if(! (Test-Path $defaultVersionClientFilePath)) { - Copy-Item $versionClientFilePath -Destination $inputDir + Copy-Item $VersionClientFilePath -Destination $InputDir } -if(! (Test-Path $defaultPomFilePath)) { - Copy-Item $bomPomFilePath -Destination $inputDir +if(! (Test-Path $DefaultPomFilePath)) { + Copy-Item $BomPomFilePath -Destination $InputDir } -echo "Run the following to generate the Pom file and dependency closure report. Both files will be generated in the outputDir." -echo "mvn exec:java -Dexec.args=`"-inputDir=$inputDir -outputDir=$outputDir -mode=generate`"" \ No newline at end of file +#$args = "-Dexec.args=`"-InputDir=$($InputDir) -OutputDir=$($OutputDir) -mode=generate`"" +$cmdoutput = mvn clean install -f $BomGeneratorPomFilePath +UpdateBomProjectElement -OldPomFilePath $BomPomFilePath -NewPomFilePath $NewBomFilePath + +Write-Output "Updating azure-sdk-bom file." +Copy-Item $NewBomFilePath -Destination $BomPomFilePath -Force + diff --git a/eng/scripts/bomhelpers.ps1 b/eng/scripts/bomhelpers.ps1 new file mode 100644 index 0000000000000..70db6177f1ab7 --- /dev/null +++ b/eng/scripts/bomhelpers.ps1 @@ -0,0 +1,311 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +class MavenArtifactInfo { + [String] $GroupId + [String] $ArtifactId + [String] $LatestGAOrPatchVersion + [String] $LatestRealeasedVersion + + MavenArtifactInfo($ArtifactId, $LatestGAOrPatchVersion, $LatestRealeasedVersion) { + $this.ArtifactId = $ArtifactId + $this.LatestGAOrPatchVersion = $LatestGAOrPatchVersion + $this.LatestRealeasedVersion = $LatestRealeasedVersion + $this.GroupId = 'com.azure' + } +} + +. Join-Path ${PSScriptRoot} "common.ps1" + +function SetDependencyVersion($GroupId = "com.azure", $ArtifactId, $Version) { + $repoRoot = Resolve-Path "${PSScriptRoot}..\..\.." + $setVersionFilePath = Join-Path $repoRoot "eng" "versioning" "set_versions.py" + $cmdOutput = python $setVersionFilePath --bt client --new-version $Version --ar $ArtifactId --gi $GroupId + $cmdOutput = python $setVersionFilePath --bt client --ar $ArtifactId --gi $GroupId --increment-version +} + +function SetCurrentVersion($GroupId, $ArtifactId, $Version) { + $repoRoot = Resolve-Path "${PSScriptRoot}..\..\.." + $setVersionFilePath = Join-Path $repoRoot "eng" "versioning" "set_versions.py" + $cmdOutput = python $setVersionFilePath --bt client --new-version $Version --ar $ArtifactId --gi $GroupId +} + +function UpdateDependencyOfClientSDK() { + $repoRoot = Resolve-Path "${PSScriptRoot}..\..\.." + $updateVersionFilePath = Join-Path $repoRoot "eng" "versioning" "update_versions.py" + $cmdOutput = python $updateVersionFilePath --ut all --bt client --sr +} + +function GetAllAzComClientArtifactsFromMaven() { + $webResponseObj = Invoke-WebRequest -Uri "https://repo1.maven.org/maven2/com/azure" + $azureComArtifactIds = $webResponseObj.Links.HRef | Where-Object { ($_ -like 'azure-*') -and ($IgnoreList -notcontains $_) } | ForEach-Object { $_.substring(0, $_.length - 1) } + return $azureComArtifactIds | Where-Object { ($_ -like "azure-*") -and !($_ -like "azure-spring") } +} + +function GetVersionInfoForAnArtifactId([String]$ArtifactId) { + $mavenMetadataUrl = "https://repo1.maven.org/maven2/com/azure/$($ArtifactId)/maven-metadata.xml" + $webResponseObj = Invoke-WebRequest -Uri $mavenMetadataUrl + $versions = ([xml]$webResponseObj.Content).metadata.versioning.versions.version + $semVersions = $versions | ForEach-Object { [AzureEngSemanticVersion]::ParseVersionString($_) } + $sortedVersions = [AzureEngSemanticVersion]::SortVersions($semVersions) + $latestReleasedVersion = $sortedVersions[0].RawVersion + $latestPatchOrGAVersion = $sortedVersions | Where-Object { !($_.IsPrerelease) } | ForEach-Object { $_.RawVersion } | Select-Object -First 1 + + $mavenArtifactInfo = [MavenArtifactInfo]::new($ArtifactId, $latestPatchOrGAVersion, $latestReleasedVersion) + + return $mavenArtifactInfo +} + +function GetPatchVersion([String]$ReleaseVersion) { + $ParsedSemver = [AzureEngSemanticVersion]::new($ReleaseVersion) + if (!$ParsedSemver) { + LogError "Unexpected release version:$($ReleaseVersion).Exiting..." + exit 1 + } + + return "$($ParsedSemver.Major).$($ParsedSemver.Minor).$($ParsedSemver.Patch + 1)" +} + +function GetRemoteName() { + $mainRemoteUrl = 'https://github.com/Azure/azure-sdk-for-java.git' + foreach ($rem in git remote show) { + $remoteUrl = git remote get-url $rem + if ($remoteUrl -eq $mainRemoteUrl) { + return $rem + } + } + LogError "Could not compute the remote name." + return $null +} + +function GetPipelineName([string]$ArtifactId, [string]$ArtifactDirPath) { + $ciYmlFilePath = Join-Path ArtifactDirPath "ci.yml" + if (Test-Path $ciYmlFilePath) { + return "java - " + $ArtifactId + } + else { + $ciDirPath = Split-Path -Path $arInfo.ArtifactDirPath -Parent + $ciYmlFilePath = Join-Path $ciDirPath "ci.yml" + if (Test-Path $ciYmlFilePath) { + return "java - " + $arInfo.ServiceDirectoryName + } + } +} + +function TriggerPipeline($PatchInfos) { + $distinctPipelineNames = $PatchInfos | Select-Object {$_.PipelineName} | Get-Unique -AsString + $distinctPipelineNames | ForEach-Object { + Write-Output "Triggering pipeline {$_}" + $cmdoutput = az pipeline --name $_} +} + +function GetBranchName($ArtifactId) { + $artifactNameToLower = $ArtifactId.ToLower() + $guid = [guid]::NewGuid().Guid + return "release/$($artifactNameToLower)_$guid" +} + +class ArtifactPatchInfo { + [string]$ArtifactId + [string]$ServiceDirectoryName + [string]$ArtifactDirPath + [string]$LatestGAOrPatchVersion + [string]$CurrentPomFileVersion + [string]$ChangeLogPath + [string]$ReadMePath + [string]$PipelineName + } + + function GetDependencyToVersion($PomFilePath) { + $dependencyNameToVersion = @{} + $pomFileContent = [xml](Get-Content -Path $PomFilePath) + foreach ($dependency in $pomFileContent.project.dependencies.dependency) { + $scope = $dependency.scope + if ($scope -ne 'test') { + $dependencyNameToVersion[$dependency.artifactId] = $dependency.version + } + } + + return $dependencyNameToVersion + } + function GetChangeLogContent($NewDependencyNameToVersion, $OldDependencyNameToVersion) { + $content = @() + $content += "" + $content += "### Other Changes" + $content += "" + $content += "#### Dependency Updates" + $content += "" + + foreach ($key in $OldDependencyNameToVersion.Keys) { + $oldVersion = $($OldDependencyNameToVersion[$key]).Trim() + $newVersion = $($NewDependencyNameToVersion[$key]).Trim() + if ($oldVersion -ne $newVersion) { + $content += "- Upgraded ``$key`` from ``$oldVersion`` to version ``$newVersion``." + } + } + + $content += "" + + return $content + } + + function GitCommit($Message) { + $cmdOutput = git commit -a -m $Message + if ($LASTEXITCODE -ne 0) { + LogError "Could not commit the changes locally.Exiting..." + exit 1 + } + } + + function GeneratePatches($ArtifactPatchInfos, [string]$BranchName, [string]$RemoteName, [string]$GroupId = "com.azure") { + foreach ($patchInfo in $ArtifactPatchInfos) { + GeneratePatch -PatchInfo $patchInfo -BranchName $BranchName -RemoteName $RemoteName -GroupId $GroupId + } + + TriggerPipeline -PatchInfos $ArtifactPatchInfos + } + + function GeneratePatch($PatchInfo, [string]$BranchName, [string]$RemoteName, [string]$GroupId = "com.azure") { + $artifactId = $PatchInfo.ArtifactId + $releaseVersion = $PatchInfo.LatestGAOrPatchVersion + $serviceDirectoryName = $PatchInfo.ServiceDirectoryName + $currentPomFileVersion = $PatchInfo.CurrentPomFileVersion + $artifactDirPath = $PatchInfo.ArtifactDirPath + $changelogPath = $PatchInfo.ChangeLogPath + + if (!$artifactId) { + Write-Output "artifactId can't be null". + exit 1 + } + + if (!$BranchName) { + $BranchName = GetBranchName -ArtifactId $artifactId + } + + if(!$BranchName) { + Write-Output "BranchName can't be null". + exit 1 + } + + if (!$RemoteName) { + Write-Output "RemoteName can't be null". + exit 1 + } + + $cmdOutput = git checkout -b $BranchName $RemoteName/main + if ($LASTEXITCODE -ne 0) { + LogError "Could not checkout branch $BranchName), please check if it already exists and delete as necessary. Exiting..." + exit 1 + } + + if (!$releaseVersion) { + Write-Output "Computing the latest release version for each of the relevant artifacts from maven central." + $mavenArtifactInfo = [MavenArtifactInfo](GetVersionInfoForAnArtifactId -ArtifactId $artifactId) + + if ($null -eq $mavenArtifactInfo) { + LogError "Could not find $artifactId on maven central." + exit 1 + } + + $mavenLatestGAOrPatchVersion = $mavenArtifactInfo.LatestGAOrPatchVersion + if ([String]::IsNullOrWhiteSpace($mavenLatestGAOrPatchVersion)) { + LogError "Could not compute the latest GA\release version for $artifactId from maven central. Exiting." + exit 1 + } + + $releaseVersion = $mavenArtifactInfo.LatestGAOrPatchVersion + Write-Output "Found the latest GA/Patch version $releaseVersion. Using this to prepare the patch." + } + + $patchVersion = GetPatchVersion -ReleaseVersion $releaseVersion + Write-Output "PatchVersion is: $patchVersion" + + $releaseTag = "$($artifactId)_$($releaseVersion)" + if (!$currentPomFileVersion -or !$artifactDirPath -or !$changelogPath) { + $pkgProperties = [PackageProps](Get-PkgProperties -PackageName $artifactId -ServiceDirectory $serviceDirectoryName) + $artifactDirPath = $pkgProperties.DirectoryPath + $currentPomFileVersion = $pkgProperties.Version + $changelogPath = $pkgProperties.ChangeLogPath + } + + if (!$artifactDirPath) { + LogError "ArtifactDirPath could not be found. Exiting." + exit 1 + } + + if ($currentPomFileVersion -ne $releaseVersion) { + Write-Output "Hard reseting the sources for $artifactId to version $releaseVersion using release tag: $releaseTag." + Write-Information "Fetching all the tags from $RemoteName" + $cmdOutput = git fetch $RemoteName $releaseTag + + if ($LASTEXITCODE -ne 0) { + LogError "Could not restore the tags for release tag $releaseTag" + exit 1 + } + + $cmdOutput = git restore --source $releaseTag -W -S $artifactDirPath + if ($LASTEXITCODE -ne 0) { + LogError "Could not reset sources for $artifactId) to the release version $releaseVersion" + exit 1 + } + + ## Commit these changes. + GitCommit -Message "Reset sources for $artifactId to the release version $releaseVersion." + } + + $pomFilePath = Join-Path $artifactDirPath "pom.xml" + $oldDependencyNameToVersion = GetDependencyToVersion -PomFilePath $pomFilePath + $cmdOutput = SetCurrentVersion -GroupId $GroupId -ArtifactId $artifactId -$Version $patchVersion --gi $GroupId + if ($LASTEXITCODE -ne 0) { + LogError "Could not set the dependencies for $artifactId" + exit 1 + } + + $cmdOutput = UpdateDependencyOfClientSDK + if ($LASTEXITCODE -ne 0) { + LogError LogError "Could not update all references for for $artifactId" + exit 1 + } + + $newDependenciesToVersion = GetDependencyToVersion -PomFilePath $pomFilePath + $releaseStatus = "$(Get-Date -Format $CHANGELOG_DATE_FORMAT)" + $releaseStatus = "($releaseStatus)" + $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $changelogPath + + $Content = GetChangeLogContent -NewDependencyNameToVersion $newDependenciesToVersion -OldDependencyNameToVersion $oldDependencyNameToVersion + $newChangeLogEntry = New-ChangeLogEntry -Version $patchVersion -Status $releaseStatus -Content $Content + if ($newChangeLogEntry) { + $changeLogEntries.Insert(0, $patchVersion, $newChangeLogEntry) + } + else { + LogError "Failed to create new changelog entry for $artifactId" + exit 1 + } + + $cmdOutput = Set-ChangeLogContent -ChangeLogLocation $changelogPath -ChangeLogEntries $changeLogEntries + if ($LASTEXITCODE -ne 0) { + LogError "Could not update the changelog at $changelogPath). Exiting..." + exit 1 + } + + GitCommit -Message "Prepare $artifactId for $patchVersion patch release." + if ($PushToRemote) { + $cmdOutput = git push $RemoteName $BranchName + if ($LASTEXITCODE -ne 0) { + LogError "Could not push the changes to $RemoteName\$BranchName. Exiting..." + exit 1 + } + Write-Output "Pushed the changes to remote:$RemoteName, Branch:$BranchName" + } + + if(!$PatchInfo.PipelineName) { + $PatchInfo.PipelineName = GetPipelineName -ArtifactId $artifactId -ArtifactDirPath $artifactDirPath + } + + if(!$PatchInfo.PipelineName) { + LogError "Could not calculate the pipeline Name. Will not trigger a run." + } + + Write-Output "Patching done for $artifactId." + } + diff --git a/eng/scripts/generatepatch.ps1 b/eng/scripts/generatepatch.ps1 new file mode 100644 index 0000000000000..cadc2ec5c82df --- /dev/null +++ b/eng/scripts/generatepatch.ps1 @@ -0,0 +1,78 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +#Requires -Version 6.0 + +<# +.SYNOPSIS +This script will generate a patch release for a given artifact or service directory. + +.DESCRIPTION +This script will do a number of things when ran: + +- It will find the latest GA\patch version from the artifactId and will have you confirm if that is the release version you want to pick for the patch release. +- It will reset the sources to the release version picked above. +- It will update the compile time dependencies used by the given artifact. +- It will update the changelog and readme to point to the new version. + +.PARAMETER ArtifactIds +The artifact id. The script currently assumes groupId is com.azure + +.PARAMETER ServiceDirectoryName +Optional: The service directory that contains all the artifacts. If this is not provided the service directory is calculated from the first artifact. +Please not if all the artifacts are not in the same service directory the script won't work. + +.PARAMETER BranchName +Optional: The name of the remote branch where the patch changes will be pushed. This is not a required parameter. In case the argument is not provided +the branch name is release/{ArtifactId}_{ReleaseVersion}. The script pushes the branch to remote URL https://github.com/Azure/azure-sdk-for-java.git + +.PARAMETER PushToRemote +Optional: Whether the commited changes should be pushed to the remote branch or not.The default value is false. + +.EXAMPLE +PS> ./eng/scripts/Generate-Patch.ps1 -ArtifactId azure-mixedreality-remoterendering +This creates a remote branch "release/azure-mixedreality-remoterendering" with all the necessary changes. + +The most common usage is to call the script passing the package name. Once the script is finished then you will have modified project and change log files. +You should make any additional changes to the change log to capture the changes and then submit the PR for the final changes before you do a release. +#> + +param( + [string[]]$ArtifactIds, + [string]$ServiceDirectoryName, + [string]$BranchName, + [boolean]$PushToRemote = $false +) + +$RepoRoot = Resolve-Path "${PSScriptRoot}..\..\.." +$BomHelpersFilePath = Join-Path $PSScriptRoot "bomhelpers.ps1" +. $BomHelpersFilePath + +function TestPathThrow($Path, $PathName) { + if (!(Test-Path $Path)) { + LogError "$PathName): $Path) not found. Exiting ..." + exit 1 + } +} + +if (!$ArtifactIds -or $ArtifactIds.Length -eq 0) { + LogError "ArtifactIds can't be null or empty. Please provide at least one ArtifactId to patch." + exit 1 +} + +$RemoteName = GetRemoteName +if (!$RemoteName) { + LogError "Could not compute the remote name." + exit 1 +} +Write-Output "RemoteName is: $RemoteName" + +foreach ($artifactId in $ArtifactIds) { + $patchInfo = [ArtifactPatchInfo]::new() + $patchInfo.ArtifactId = $artifactId + $patchInfo.ServiceDirectoryName = $ServiceDirectoryName + GeneratePatch -PatchInfo $patchInfo -BranchName $BranchName -RemoteName $RemoteName -GroupId "com.azure" + TriggerPipeline -PatchInfos $patchInfo +} + +Write-Output "Patch generation completed successfully." \ No newline at end of file diff --git a/eng/scripts/patchreleases.ps1 b/eng/scripts/patchreleases.ps1 new file mode 100644 index 0000000000000..8faf3a6150662 --- /dev/null +++ b/eng/scripts/patchreleases.ps1 @@ -0,0 +1,294 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +param( + [string]$GroupId = "com.azure" +) + +Write-Information "PS Script Root is: $PSScriptRoot" +$RepoRoot = Resolve-Path "${PSScriptRoot}..\..\.." +$CommonScriptFilePath = Join-Path $RepoRoot "eng" "common" "scripts" "common.ps1" +$BomHelpersFilePath = Join-Path $PSScriptRoot "bomhelpers.ps1" +. $CommonScriptFilePath +. $BomHelpersFilePath + + +class ArtifactInfo { + [string]$GroupId = "com.azure" + [string]$ArtifactId + [string]$ServiceDirectoryName + [string]$ArtifactDirPath + [string]$LatestGAOrPatchVersion + [string]$FutureReleasedPatchVersion + [string]$CurrentPomFileVersion + [string]$ChangeLogPath + [string]$ReadMePath + [string]$PipelineName + [hashtable]$Dependencies + + ArtifactInfo([string]$ArtifactId, [string]$LatestGAOrPatchVersion) { + $this.ArtifactId = $ArtifactId + $this.LatestGAOrPatchVersion = $LatestGAOrPatchVersion + } +} + +function ConvertToPatchInfo([ArtifactInfo]$ArInfo) { + $patchInfo = [ArtifactPatchInfo]::new() + $patchInfo.ArtifactId = $ArInfo.ArtifactId + $patchInfo.ServiceDirectoryName = $ArInfo.ServiceDirectoryName + $patchInfo.ArtifactDirPath = $ArInfo.ArtifactDirPath + $patchInfo.LatestGAOrPatchVersion = $ArInfo.LatestGAOrPatchVersion + $patchInfo.CurrentPomFileVersion = $ArInfo.CurrentPomFileVersion + $patchInfo.ChangeLogPath = $ArInfo.ChangeLogPath + $patchInfo.ReadMePath = $ArInfo.ReadMePath + $patchInfo.PipelineName = $ArInfo.PipelineName + + return $patchInfo +} + +function GetVersionInfoForAllMavenArtifacts([string]$GroupId = "com.azure") { + $artifactInfos = @{} + $azComArtifactIds = GetAllAzComClientArtifactsFromMaven -GroupId $GroupId + + foreach ($artifactId in $azComArtifactIds) { + $info = GetVersionInfoForAnArtifactId -ArtifactId $artifactId + + $artifactId = $info.ArtifactId + $latestGAOrPatchVersion = $info.LatestGAOrPatchVersion + $artifactInfos[$artifactId] = [ArtifactInfo]::new($artifactId, $latestGAOrPatchVersion) + } + + return $artifactInfos +} + +function UpdateDependencies($ArtifactInfos) { + foreach ($artifactId in $ArtifactInfos.Keys) { + $deps = @{} + $sdkVersion = $ArtifactInfos[$artifactId].LatestGAOrPatchVersion + $pomFileUri = "https://repo1.maven.org/maven2/com/azure/$artifactId/$sdkVersion/$artifactId-$sdkVersion.pom" + $webResponseObj = Invoke-WebRequest -Uri $pomFileUri + $dependencies = ([xml]$webResponseObj.Content).project.dependencies.dependency | Where-Object { (([String]::IsNullOrWhiteSpace($_.scope)) -or ($_.scope -eq 'compile')) } + $dependencies | Where-Object { $_.groupId -eq $GroupId } | ForEach-Object { $deps[$_.artifactId] = $_.version } + $ArtifactInfos[$artifactId].Dependencies = $deps + } + + return +} + +function ParseCIYamlFile([string]$FileName) { + $artifactIdToPipelineName = @{} + + $templateRegex = "\s*template:(.*)"; + $artifactsRegex = "\s+Artifacts:\s*" + $artifactsRegex = "\s+Artifacts:\s*" + $artifactIdRegex = ".*name:(.*)" + $safeNameRegex = ".*safeName:(.*)" + $fileContent = Get-Content -Path $FileName + $index = 0 + + while ($index -lt $fileContent.Length -and ($fileContent[$index] -notmatch $templateRegex)) { + $index += 1 + } + + if ($index -eq $fileContent.Length) { + return + } + + do { + + while ($index -lt $fileContent.Length -and $fileContent[$index] -notmatch $artifactsRegex ) { + $index += 1 + } + + while ($index -lt $fileContent.Length -and $fileContent[$index] -notmatch $artifactIdRegex) { + $index += 1 + } + + if ($index -eq $fileContent.Length) { + return $artifactIdToPipelineName + } + + $artifactId = $Matches[1] + + while ($index -lt $fileContent.Length -and $fileContent[$index] -notmatch $safeNameRegex) { + $index += 1 + } + + if ($index -eq $fileContent.Length) { + return $artifactIdToPipelineName + } + + $artifactIdToPipelineName[$artifactId] = $Matches[1] + } while ($index -lt $fileContent.Length) + + return $artifactIdToPipelineName +} + +function UpdateCIInformation($ArtifactsToPatch, $ArtifactInfos) { + foreach ($artifactId in $ArtifactsToPatch) { + $arInfo = [ArtifactInfo]$ArtifactInfos[$artifactId] + $serviceDirectory = $arInfo.ServiceDirectoryName + + if (!$serviceDirectory) { + $pkgProperties = [PackageProps](Get-PkgProperties -PackageName $artifactId -ServiceDirectory $serviceDirectory) + $arInfo.ServiceDirectoryName = $pkgProperties.ServiceDirectory + $arInfo.ArtifactDirPath = $pkgProperties.DirectoryPath + $arInfo.CurrentPomFileVersion = $pkgProperties.Version + $arInfo.ChangeLogPath = $pkgProperties.ChangeLogPath + $arInfo.ReadMePath = $pkgProperties.ReadMePath + } + + $arInfo.PipelineName = GetPipelineName -ArtifactId $arInfo.ArtifactId -ArtifactDirPath $arInfo.ArtifactDirPath + } +} + +function FindAllArtifactsToBePatched([String]$DependencyId, [String]$PatchVersion, [hashtable]$ArtifactInfos) { + $artifactsToPatch = @{} + + foreach ($id in $ArtifactInfos.Keys) { + $arInfo = $ArtifactInfos[$id] + $futureReleasedPatchVersion = $arInfo.FutureReleasedPatchVersion + + if($futureReleasedPatchVersion) { + # This library is already being patched and hence analyzed so we don't need to analyze it again. + if ($id -ne 'azure-core' -or $id -ne 'azure-core-http-netty') { + continue; + } + } + + $depVersion = $arInfo.Dependencies[$DependencyId] + if ($depVersion -and $depVersion -ne $PatchVersion) { + $currentGAOrPatchVersion = $arInfo.LatestGAOrPatchVersion + $newPatchVersion = GetPatchVersion -ReleaseVersion $currentGAOrPatchVersion + $arInfo.FutureReleasedPatchVersion = $newPatchVersion + $artifactsToPatch[$id] = $id + $depArtifactsToPatch = FindAllArtifactsToBePatched -DependencyId $id -PatchVersion $newPatchVersion -ArtifactInfos $ArtifactInfos + foreach ($recArtifacts in $depArtifactsToPatch.Keys) { + $artifactsToPatch[$recArtifacts] = $recArtifacts + } + } + } + + return $artifactsToPatch +} + +function GetPatchSets($artifactsToPatch, [hashtable]$ArtifactInfos) { + $patchSets = @() + + foreach($artifactToPatch in $artifactsToPatch.Keys) { + $patchDependencies = @{} + $dependencies = $artifactInfos[$artifactToPatch].Dependencies + $dependencies.Keys | Where-Object { $null -ne $artifactsToPatch[$_]} | ForEach-Object {$patchDependencies[$_] = $_} + $patchDependencies[$artifactToPatch] = $artifactToPatch + + $unionSet = @{} + $patchDependencies.Keys | ForEach-Object { $unionSet[$_] = $_ } + + $reducedPatchSets = @() + # Add this set to the exiting sets and reduce duplicates. + foreach($patchSet in $patchSets) { + $matches = $patchDependencies.Keys | Where-Object {$patchSet[$_]} | Select-Object $_ -First 1 + + if($matches) { + $patchSet.Keys | ForEach-Object {$unionSet[$_] = $_ } + } else { + $reducedPatchSets += $patchSet + } + } + + $patchSets = $reducedPatchSets + $patchSets += $unionSet + } + + return $patchSets +} +function UpdateDependenciesInVersionClient([string]$ArtifactId, [hashtable]$ArtifactInfos, [string]$GroupId = "com.azure") { + ## We need to update the version_client.txt to have the correct versions in place. + $arInfo = $ArtifactInfos[$ArtifactId] + $dependencies = $arInfo.Dependencies + foreach ($depId in $dependencies.Keys) { + $depArtifactInfo = $ArtifactInfos[$depId] + $newDependencyVersion = $depArtifactInfo.FutureReleasedPatchVersion + + if (!$newDependencyVersion) { + $newDependencyVersion = $depArtifactInfo.LatestGAOrPatchVersion + } + + if ($newDependencyVersion) { + $cmdOutput = SetDependencyVersion -GroupId $GroupId -ArtifactId $depId -Version $newDependencyVersion + } + } +} +function UndoVersionClientFile() { + $repoRoot = Resolve-Path "${PSScriptRoot}..\..\.." + $versionClientFile = Join-Path $repoRoot "eng" "versioning" "version_client.txt" + $cmdOutput = git checkout $versionClientFile +} + + +$ArtifactInfos = GetVersionInfoForAllMavenArtifacts -GroupId $GroupId +$IgnoreList = @( + 'azure-client-sdk-parent', + 'azure-core-parent', + 'azure-core-test', + 'azure-sdk-all', + 'azure-sdk-bom', + 'azure-sdk-parent', + 'azure-sdk-template', + 'azure-sdk-template-bom', + 'azure-data-sdk-parent', + 'azure-spring-data-cosmos', + 'azure-core-management' +) + +$inEligibleKeys = $ArtifactInfos.Keys | Where-Object { !$ArtifactInfos[$_].LatestGAOrPatchVersion -or $IgnoreList -contains $_ } +$inEligibleKeys | ForEach-Object { $ArtifactInfos.Remove($_) } + +UpdateDependencies -ArtifactInfos $ArtifactInfos +$AzCoreArtifactId = "azure-core" +$AzCoreVersion = $ArtifactInfos[$AzCoreArtifactId].LatestGAOrPatchVersion + +# For testing only. +# $AzCoreVersion = "1.26.0" +# $ArtifactInfos[$AzCoreArtifactId].FutureReleasedPatchVersion = $AzCoreVersion +# $AzCoreNettyArtifactId = "azure-core-http-netty" +# $ArtifactInfos[$AzCoreNettyArtifactId].Dependencies[$AzCoreArtifactId] = $AzCoreVersion + +$ArtifactsToPatch = FindAllArtifactsToBePatched -DependencyId $AzCoreArtifactId -PatchVersion $AzCoreVersion -ArtifactInfos $ArtifactInfos +$ReleaseSets = GetPatchSets -ArtifactsToPatch $ArtifactsToPatch -ArtifactInfos $ArtifactInfos +$RemoteName = GetRemoteName +$CurrentBranchName = git rev-parse --abbrev-ref HEAD +if ($LASTEXITCODE -ne 0) { + LogError "Could not correctly get the current branch name." + exit 1 +} +UpdateCIInformation -ArtifactsToPatch $ArtifactsToPatch.Keys -ArtifactInfos $ArtifactInfos + +$fileContent = [System.Text.StringBuilder]::new() +$fileContent.AppendLine("BranchName;ArtifactId"); +Write-Output "Preparing patch releases for BOM updates." +## We now can run the generate_patch script for all those dependencies. +foreach ($patchSet in $ReleaseSets) { + try { + $patchInfos = [ArtifactPatchInfo[]]@() + foreach ($artifactId in $patchSet.Keys) { + $arInfo = $ArtifactInfos[$artifactId] + $patchInfo = [ArtifactPatchInfo]::new() + $patchInfo = ConvertToPatchInfo -ArInfo $arInfo + $patchInfos += $patchInfo + UpdateDependenciesInVersionClient -ArtifactId $artifactId -ArtifactInfos $ArtifactInfos + } + + $remoteBranchName = GetBranchName -ArtifactId "PatchSet" + GeneratePatches -ArtifactPatchInfos $patchInfos -BranchName $remoteBranchName -RemoteName $RemoteName -GroupId $GroupId + $fileContent.AppendLine("$remoteBranchName;$($artifactIds);"); + } + finally { + $cmdOutput = git checkout $CurrentBranchName + } +} + +New-Item -Path . -Name "ReleasePatchInfo.csv" -ItemType "file" -Value $fileContent.ToString() -Force + + + diff --git a/eng/scripts/syncversionclient.ps1 b/eng/scripts/syncversionclient.ps1 new file mode 100644 index 0000000000000..2c02fa7f45758 --- /dev/null +++ b/eng/scripts/syncversionclient.ps1 @@ -0,0 +1,94 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +param( + [Parameter(Mandatory = $false)][string]$GroupId = "com.azure" +) + +Write-Information "PS Script Root is: $PSScriptRoot" + +$RepoRoot = Resolve-Path "${PSScriptRoot}..\..\.." +$EngDir = Join-Path $RepoRoot "eng" +$EngVersioningDir = Join-Path $EngDir "versioning" +$EngCommonScriptsDir = Join-Path $EngDir "common" "scripts" + +. (Join-Path $EngCommonScriptsDir common.ps1) +. (Join-Path $PSScriptRoot bomhelpers.ps1) + +function UpdateDependencyVersion($ArtifactInfo, $EngSysVersionInfo) { + $version = $ArtifactInfo.LatestGAOrPatchVersion + $sdkName = $ArtifactInfo.ArtifactId + $groupId = $ArtifactInfo.GroupId + $engsysCurrentVersion = $EngSysVersionInfo.CurrentVersion + + if([String]::IsNullOrWhiteSpace($version)) { + return + } + + SetDependencyVersion -GroupId $groupId -ArtifactId $sdkName -Version $version + SetCurrentVersion -GroupId $groupId -ArtifactId $sdkName -Version $engsysCurrentVersion +} + +class EngSysVersionInfo{ + [String] $GroupId + [String] $Name + [String] $DependencyVersion + [String] $CurrentVersion + + EngSysVersionInfo($Name, $DependencyVersion, $CurrentVersion) { + $this.Name = $Name + $this.DependencyVersion = $DependencyVersion + $this.CurrentVersion = $CurrentVersion + $this.GroupId = 'com.azure' + } +} + +function ParseVersionClientFile($GroudpId) { + $versionClientInfo = @{} + $versionClientFilePath = Join-Path $EngVersioningDir "version_client.txt" + $regexPattern = "$($GroupId):(.*);(.*);(.*)" + + foreach($line in Get-Content $versionClientFilePath) { + if($line -match $regexPattern) { + $artifactId = $Matches.1 + $dependencyVersion = $Matches.2 + $currentVersion = $Matches.3 + + $engSysVersionInfo = [EngSysVersionInfo]::new($artifactId, $dependencyVersion, $currentVersion) + $versionClientInfo[$artifactId] = $engSysVersionInfo + } + } + + return $versionClientInfo + +} + +function SyncVersionClientFile([String]$GroupId) { + $artifactIds = GetAllAzComClientArtifactsFromMaven -GroupId $GroupId + $versionClientInfo = ParseVersionClientFile -GroudpId $GroupId + + foreach($artifactId in $artifactIds) { + $artifactInfo = GetVersionInfoForAnArtifactId -ArtifactId $artifactId + $latestPatchOrGaVersion = $ArtifactInfo.LatestGAOrPatchVersion + + if([String]::IsNullOrWhiteSpace($latestPatchOrGaVersion)) { + # This library does not have a released version so we are likely good here. + continue + } + + $engSysArtifactInfo = $versionClientInfo[$artifactId] + if($null -eq $engSysArtifactInfo) { + continue + } + + $dependencyVersion = $engSysArtifactInfo.DependencyVersion; + if($dependencyVersion -eq $latestPatchOrGaVersion) { + continue + } + + UpdateDependencyVersion -ArtifactInfo $artifactInfo -EngSysVersionInfo $engSysArtifactInfo + UpdateDependencyOfClientSDK + } +} + +