Add support for sync scripts and mass patch releases. (#26981)

eng/bomgenerator/generateAzureSDKBOM.ps1

@@ -1,24 +1,60 @@
-$repoRoot = Resolve-Path "${PSScriptRoot}..\..\.."
-$inputDir = Join-Path ${PSScriptRoot} "inputDir"
-$outputDir = Join-Path ${PSScriptRoot} "outputDir"
-$versionClientFileName = "version_client.txt"
-$pomFileName = "pom.xml"
-$defaultVersionClientFilePath = Join-Path $inputDir $versionClientFileName
-$defaultPomFilePath = Join-Path $inputDir $pomFileName
-$versionClientFilePath = Join-Path $repoRoot "eng" "versioning" $versionClientFileName
-$bomPomFilePath = Join-Path $repoRoot "sdk" "boms" "azure-sdk-bom" $pomFileName
-
-if(! (Test-Path $inputDir)) { 
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+$RepoRoot = Resolve-Path "${PSScriptRoot}..\..\.."
+$VersionClientFileName = "version_client.txt"
+$PomFileName = "pom.xml"
+$InputDir = Join-Path ${PSScriptRoot} "inputdir"
+$OutputDir = Join-Path ${PSScriptRoot} "outputdir"
+$DefaultVersionClientFilePath = Join-Path $InputDir $VersionClientFileName
+$DefaultPomFilePath = Join-Path $InputDir $PomFileName
+$EngDir = Join-Path $RepoRoot "eng"
+$VersionClientFilePath = Join-Path $EngDir "versioning" $VersionClientFileName
+$BomPomFilePath = Join-Path $RepoRoot "sdk" "boms" "azure-sdk-bom" $PomFileName
+$EngScriptDir = Join-Path  $EngDir "scripts"
+$BomGeneratorPomFilePath = Join-Path ${PSScriptRoot} $PomFileName
+$NewBomFilePath = Join-Path $OutputDir $PomFileName
+
+. (Join-Path $EngScriptDir syncversionclient.ps1)
+
+
+function UpdateBomProjectElement($OldPomFilePath, $NewPomFilePath) {
+  $oldFileContent = [xml](Get-Content -Path $oldPomFilePath)
+  $newFileContent = [xml](Get-Content -Path $NewPomFilePath)
+
+  $oldXmlns = $oldFileContent.Project.xmlns
+  $oldxsi = $oldFileContent.Project.xsi
+  $oldschemaLocation = $oldFileContent.Project.SchemaLocation
+
+  $newFileContent.Project.xmlns = $oldXmlns
+  $newFileContent.Project.xsi = $oldxsi
+  $newFileContent.Project.SchemaLocation = $oldschemaLocation
+
+  $newFileContent.Save($NewPomFilePath)
+}
+
+Write-Output "InputDir:$($InputDir)"
+Write-Output "OutputDir:$($OutputDir)"
+Write-Output "Updating version_client.txt file by looking at the packages released to maven."
+SyncVersionClientFile -GroupId "com.azure"
+Write-Output "Updated version_client.txt file."
+
+if(! (Test-Path $InputDir)) { 
   New-Item -Path $PSScriptRoot -Name "inputDir" -ItemType "directory"
 }
 
 if(! (Test-Path $defaultVersionClientFilePath)) {
- Copy-Item $versionClientFilePath -Destination $inputDir
+ Copy-Item $VersionClientFilePath -Destination $InputDir
 }
 
-if(! (Test-Path $defaultPomFilePath)) {
- Copy-Item $bomPomFilePath -Destination $inputDir
+if(! (Test-Path $DefaultPomFilePath)) {
+ Copy-Item $BomPomFilePath -Destination $InputDir
 }
 
-echo "Run the following to generate the Pom file and dependency closure report. Both files will be generated in the outputDir."
-echo "mvn exec:java -Dexec.args=`"-inputDir=$inputDir -outputDir=$outputDir -mode=generate`""
+#$args = "-Dexec.args=`"-InputDir=$($InputDir) -OutputDir=$($OutputDir) -mode=generate`""
+$cmdoutput = mvn clean install -f $BomGeneratorPomFilePath
+UpdateBomProjectElement -OldPomFilePath $BomPomFilePath -NewPomFilePath $NewBomFilePath
+
+Write-Output "Updating azure-sdk-bom file."
+Copy-Item $NewBomFilePath -Destination $BomPomFilePath -Force
+

eng/scripts/bomhelpers.ps1

@@ -0,0 +1,311 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+class MavenArtifactInfo {
+    [String] $GroupId
+    [String] $ArtifactId
+    [String] $LatestGAOrPatchVersion
+    [String] $LatestRealeasedVersion
+
+    MavenArtifactInfo($ArtifactId, $LatestGAOrPatchVersion, $LatestRealeasedVersion) {
+        $this.ArtifactId = $ArtifactId
+        $this.LatestGAOrPatchVersion = $LatestGAOrPatchVersion
+        $this.LatestRealeasedVersion = $LatestRealeasedVersion
+        $this.GroupId = 'com.azure'
+    }
+}
+
+. Join-Path ${PSScriptRoot} "common.ps1"
+
+function SetDependencyVersion($GroupId = "com.azure", $ArtifactId, $Version) {
+    $repoRoot = Resolve-Path "${PSScriptRoot}..\..\.."
+    $setVersionFilePath = Join-Path $repoRoot "eng" "versioning" "set_versions.py"
+    $cmdOutput = python $setVersionFilePath --bt client --new-version $Version --ar $ArtifactId --gi $GroupId
+    $cmdOutput = python $setVersionFilePath --bt client --ar $ArtifactId --gi $GroupId --increment-version
+}
+
+function SetCurrentVersion($GroupId, $ArtifactId, $Version) {
+    $repoRoot = Resolve-Path "${PSScriptRoot}..\..\.."
+    $setVersionFilePath = Join-Path $repoRoot "eng" "versioning" "set_versions.py"
+    $cmdOutput = python $setVersionFilePath --bt client --new-version $Version --ar $ArtifactId --gi $GroupId
+}
+
+function UpdateDependencyOfClientSDK() {
+    $repoRoot = Resolve-Path "${PSScriptRoot}..\..\.."
+    $updateVersionFilePath = Join-Path $repoRoot "eng" "versioning" "update_versions.py"
+    $cmdOutput = python $updateVersionFilePath --ut all --bt client --sr
+}
+
+function GetAllAzComClientArtifactsFromMaven() {
+    $webResponseObj = Invoke-WebRequest -Uri "https://repo1.maven.org/maven2/com/azure"
+    $azureComArtifactIds = $webResponseObj.Links.HRef | Where-Object { ($_ -like 'azure-*') -and ($IgnoreList -notcontains $_) } |  ForEach-Object { $_.substring(0, $_.length - 1) }
+    return $azureComArtifactIds | Where-Object { ($_ -like "azure-*") -and !($_ -like "azure-spring") }
+}
+
+function GetVersionInfoForAnArtifactId([String]$ArtifactId) {
+    $mavenMetadataUrl = "https://repo1.maven.org/maven2/com/azure/$($ArtifactId)/maven-metadata.xml"
+    $webResponseObj = Invoke-WebRequest -Uri $mavenMetadataUrl
+    $versions = ([xml]$webResponseObj.Content).metadata.versioning.versions.version
+    $semVersions = $versions | ForEach-Object { [AzureEngSemanticVersion]::ParseVersionString($_) }
+    $sortedVersions = [AzureEngSemanticVersion]::SortVersions($semVersions)
+    $latestReleasedVersion = $sortedVersions[0].RawVersion
+    $latestPatchOrGAVersion = $sortedVersions | Where-Object { !($_.IsPrerelease) } | ForEach-Object { $_.RawVersion } | Select-Object -First 1
+
+    $mavenArtifactInfo = [MavenArtifactInfo]::new($ArtifactId, $latestPatchOrGAVersion, $latestReleasedVersion)
+
+    return $mavenArtifactInfo
+}
+
+function GetPatchVersion([String]$ReleaseVersion) {
+    $ParsedSemver = [AzureEngSemanticVersion]::new($ReleaseVersion)
+    if (!$ParsedSemver) {
+        LogError "Unexpected release version:$($ReleaseVersion).Exiting..."
+        exit 1
+    }
+
+    return "$($ParsedSemver.Major).$($ParsedSemver.Minor).$($ParsedSemver.Patch + 1)"
+}
+
+function GetRemoteName() {
+    $mainRemoteUrl = 'https://github.com/Azure/azure-sdk-for-java.git'
+    foreach ($rem in git remote show) {
+        $remoteUrl = git remote get-url $rem
+        if ($remoteUrl -eq $mainRemoteUrl) {
+            return $rem
+        }
+    }
+    LogError "Could not compute the remote name."
+    return $null
+}
+
+function GetPipelineName([string]$ArtifactId, [string]$ArtifactDirPath) {
+  $ciYmlFilePath = Join-Path ArtifactDirPath "ci.yml"
+  if (Test-Path $ciYmlFilePath) {
+    return  "java - " + $ArtifactId 
+  }
+  else {
+    $ciDirPath = Split-Path -Path $arInfo.ArtifactDirPath -Parent
+    $ciYmlFilePath = Join-Path $ciDirPath "ci.yml"
+    if (Test-Path $ciYmlFilePath) {
+      return  "java - " + $arInfo.ServiceDirectoryName
+    }
+  }
+}
+
+function TriggerPipeline($PatchInfos) {
+  $distinctPipelineNames = $PatchInfos | Select-Object {$_.PipelineName} | Get-Unique -AsString
+  $distinctPipelineNames | ForEach-Object { 
+    Write-Output "Triggering pipeline {$_}"
+    $cmdoutput = az pipeline --name $_}
+}
+
+function GetBranchName($ArtifactId) {
+    $artifactNameToLower = $ArtifactId.ToLower()
+    $guid = [guid]::NewGuid().Guid
+    return "release/$($artifactNameToLower)_$guid"
+}
+
+class ArtifactPatchInfo {
+    [string]$ArtifactId
+    [string]$ServiceDirectoryName
+    [string]$ArtifactDirPath
+    [string]$LatestGAOrPatchVersion
+    [string]$CurrentPomFileVersion
+    [string]$ChangeLogPath
+    [string]$ReadMePath
+    [string]$PipelineName
+  }
+
+  function GetDependencyToVersion($PomFilePath) {
+    $dependencyNameToVersion = @{}
+    $pomFileContent = [xml](Get-Content -Path $PomFilePath)
+    foreach ($dependency in $pomFileContent.project.dependencies.dependency) {
+      $scope = $dependency.scope
+      if ($scope -ne 'test') {
+        $dependencyNameToVersion[$dependency.artifactId] = $dependency.version
+      }
+    }
+
+    return $dependencyNameToVersion
+  }
+  function GetChangeLogContent($NewDependencyNameToVersion, $OldDependencyNameToVersion) {
+    $content = @()
+    $content += ""
+    $content += "### Other Changes"
+    $content += ""
+    $content += "#### Dependency Updates"
+    $content += ""
+
+    foreach ($key in $OldDependencyNameToVersion.Keys) {
+      $oldVersion = $($OldDependencyNameToVersion[$key]).Trim()
+      $newVersion = $($NewDependencyNameToVersion[$key]).Trim()
+      if ($oldVersion -ne $newVersion) {
+        $content += "- Upgraded ``$key`` from ``$oldVersion`` to version ``$newVersion``."
+      }
+    }
+
+    $content += ""
+
+    return $content
+  }
+
+  function GitCommit($Message) {
+    $cmdOutput = git commit -a -m $Message
+    if ($LASTEXITCODE -ne 0) {
+      LogError "Could not commit the changes locally.Exiting..."
+      exit 1
+    }
+  }
+
+  function GeneratePatches($ArtifactPatchInfos, [string]$BranchName, [string]$RemoteName, [string]$GroupId = "com.azure") {
+    foreach ($patchInfo in $ArtifactPatchInfos) {
+      GeneratePatch -PatchInfo $patchInfo -BranchName $BranchName -RemoteName $RemoteName -GroupId $GroupId
+    }
+
+    TriggerPipeline  -PatchInfos $ArtifactPatchInfos
+  }
+
+  function GeneratePatch($PatchInfo, [string]$BranchName, [string]$RemoteName, [string]$GroupId = "com.azure") {
+    $artifactId = $PatchInfo.ArtifactId
+    $releaseVersion = $PatchInfo.LatestGAOrPatchVersion
+    $serviceDirectoryName = $PatchInfo.ServiceDirectoryName
+    $currentPomFileVersion = $PatchInfo.CurrentPomFileVersion
+    $artifactDirPath = $PatchInfo.ArtifactDirPath
+    $changelogPath = $PatchInfo.ChangeLogPath
+
+    if (!$artifactId) {
+      Write-Output "artifactId can't be null".
+      exit 1
+    }
+
+    if (!$BranchName) {
+      $BranchName = GetBranchName -ArtifactId $artifactId
+    }
+
+    if(!$BranchName) {
+        Write-Output "BranchName can't be null".
+        exit 1
+    }
+
+    if (!$RemoteName) {
+      Write-Output "RemoteName can't be null".
+      exit 1
+    }
+
+    $cmdOutput = git checkout -b $BranchName $RemoteName/main 
+    if ($LASTEXITCODE -ne 0) {
+      LogError "Could not checkout branch $BranchName), please check if it already exists and delete as necessary. Exiting..."
+      exit 1
+    }
+
+    if (!$releaseVersion) {
+      Write-Output "Computing the latest release version for each of the relevant artifacts from maven central."
+      $mavenArtifactInfo = [MavenArtifactInfo](GetVersionInfoForAnArtifactId -ArtifactId $artifactId)
+
+      if ($null -eq $mavenArtifactInfo) {
+        LogError "Could not find $artifactId on maven central."
+        exit 1
+      }
+
+      $mavenLatestGAOrPatchVersion = $mavenArtifactInfo.LatestGAOrPatchVersion
+      if ([String]::IsNullOrWhiteSpace($mavenLatestGAOrPatchVersion)) {
+        LogError "Could not compute the latest GA\release version for $artifactId from maven central. Exiting."
+        exit 1
+      }
+
+      $releaseVersion = $mavenArtifactInfo.LatestGAOrPatchVersion
+      Write-Output "Found the latest GA/Patch version $releaseVersion. Using this to prepare the patch."  
+    }
+
+    $patchVersion = GetPatchVersion -ReleaseVersion $releaseVersion
+    Write-Output "PatchVersion is: $patchVersion"
+
+    $releaseTag = "$($artifactId)_$($releaseVersion)"
+    if (!$currentPomFileVersion -or !$artifactDirPath -or !$changelogPath) {
+      $pkgProperties = [PackageProps](Get-PkgProperties -PackageName $artifactId -ServiceDirectory $serviceDirectoryName)
+      $artifactDirPath = $pkgProperties.DirectoryPath  
+      $currentPomFileVersion = $pkgProperties.Version
+      $changelogPath = $pkgProperties.ChangeLogPath
+    }
+
+    if (!$artifactDirPath) {
+      LogError "ArtifactDirPath could not be found. Exiting."
+      exit 1
+    }
+
+    if ($currentPomFileVersion -ne $releaseVersion) {
+      Write-Output "Hard reseting the sources for $artifactId to version $releaseVersion using release tag: $releaseTag." 
+      Write-Information "Fetching all the tags from $RemoteName"
+      $cmdOutput = git fetch $RemoteName $releaseTag
+
+      if ($LASTEXITCODE -ne 0) {
+        LogError "Could not restore the tags for release tag $releaseTag"
+        exit 1
+      }
+
+      $cmdOutput = git restore --source $releaseTag -W -S $artifactDirPath
+      if ($LASTEXITCODE -ne 0) {
+        LogError "Could not reset sources for $artifactId) to the release version $releaseVersion"
+        exit 1
+      }
+
+      ## Commit these changes.
+      GitCommit -Message "Reset sources for $artifactId to the release version $releaseVersion."
+    }
+
+    $pomFilePath = Join-Path $artifactDirPath "pom.xml"
+    $oldDependencyNameToVersion = GetDependencyToVersion -PomFilePath $pomFilePath
+    $cmdOutput = SetCurrentVersion -GroupId $GroupId -ArtifactId $artifactId -$Version $patchVersion --gi $GroupId
+    if ($LASTEXITCODE -ne 0) {
+      LogError "Could not set the dependencies for $artifactId"
+      exit 1
+    }
+
+    $cmdOutput = UpdateDependencyOfClientSDK
+    if ($LASTEXITCODE -ne 0) {
+      LogError  LogError "Could not update all references for for $artifactId"
+      exit 1
+    }
+
+    $newDependenciesToVersion = GetDependencyToVersion -PomFilePath $pomFilePath
+    $releaseStatus = "$(Get-Date -Format $CHANGELOG_DATE_FORMAT)"
+    $releaseStatus = "($releaseStatus)"
+    $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $changelogPath
+
+    $Content = GetChangeLogContent -NewDependencyNameToVersion $newDependenciesToVersion -OldDependencyNameToVersion $oldDependencyNameToVersion
+    $newChangeLogEntry = New-ChangeLogEntry -Version $patchVersion -Status $releaseStatus -Content $Content
+    if ($newChangeLogEntry) {
+      $changeLogEntries.Insert(0, $patchVersion, $newChangeLogEntry)
+    }
+    else {
+      LogError "Failed to create new changelog entry for $artifactId"
+      exit 1
+    }
+
+    $cmdOutput = Set-ChangeLogContent -ChangeLogLocation $changelogPath -ChangeLogEntries $changeLogEntries
+    if ($LASTEXITCODE -ne 0) {
+      LogError "Could not update the changelog at $changelogPath). Exiting..."
+      exit 1
+    }
+
+    GitCommit -Message "Prepare $artifactId for $patchVersion patch release."
+    if ($PushToRemote) {
+      $cmdOutput = git push $RemoteName $BranchName
+      if ($LASTEXITCODE -ne 0) {
+        LogError "Could not push the changes to $RemoteName\$BranchName. Exiting..."
+        exit 1
+      }
+      Write-Output "Pushed the changes to remote:$RemoteName, Branch:$BranchName"
+    }
+
+    if(!$PatchInfo.PipelineName) {
+      $PatchInfo.PipelineName = GetPipelineName -ArtifactId $artifactId -ArtifactDirPath $artifactDirPath
+    }
+
+    if(!$PatchInfo.PipelineName) {
+      LogError "Could not calculate the pipeline Name. Will not trigger a run."
+    }
+
+    Write-Output "Patching done for $artifactId."
+  }
+