From b648c16f351af07bf3a303be2c06506f79ca51b5 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 30 Jan 2025 11:09:47 -0800 Subject: [PATCH 1/4] [TESTING ONLY, WILL NOT BE CHECKED IN] Change core file to ensure version test matrix --- sdk/core/azure-core/README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-core/README.md b/sdk/core/azure-core/README.md index 62709d75b043..ef7b7f29705e 100644 --- a/sdk/core/azure-core/README.md +++ b/sdk/core/azure-core/README.md @@ -2,6 +2,8 @@ [![Build Documentation](https://img.shields.io/badge/documentation-published-blue.svg)](https://azure.github.io/azure-sdk-for-java) +JRS-CHANGE FOR TESTING PURPOSES AND WILL NOT BE CHECKED IN. + Azure Core provides shared primitives, abstractions, and helpers for modern Java Azure SDK client libraries. These libraries follow the [Azure SDK Design Guidelines for Java](https://azure.github.io/azure-sdk/java_introduction.html) @@ -116,22 +118,22 @@ polling is cancelled or reaches a terminal state. ### Configuring Builders -Builders are used to create service clients and some `TokenCredential` implementations. They can be configured with a -variety of options, including `HttpPipeline` and `HttpClient` for HTTP-based clients and more general options such as -`Configuration` and`endpoint`. To allow for simpler integration into frameworks such as Spring and to allow generic +Builders are used to create service clients and some `TokenCredential` implementations. They can be configured with a +variety of options, including `HttpPipeline` and `HttpClient` for HTTP-based clients and more general options such as +`Configuration` and`endpoint`. To allow for simpler integration into frameworks such as Spring and to allow generic methods to be used for all builders `azure-core` provides a set of interfaces that can be implemented to provide the necessary functionality. #### HttpTrait `HttpTrait` contains methods for setting key configurations for HTTP-based clients. This interface will allow you to -configure the `HttpClient`, `HttpPipeline`, `HttpPipelinePolicy`s, `RetryOptions`, `HttpLogOptions`, and `ClientOptions` +configure the `HttpClient`, `HttpPipeline`, `HttpPipelinePolicy`s, `RetryOptions`, `HttpLogOptions`, and `ClientOptions` (preferably `HttpClientOptions` as it is more specific for HTTP-based service clients). -For builders that expose `HttpTrait`, if an `HttpPipeline` or `HttpClient` isn't set a default instance will be -created based on classpath configurations and the `ClientOptions` based to the builder. This can cause confusion if -you're expecting specific behavior for your client, such as using a proxy that wasn't loaded from the environment. To -avoid this, it is recommended to always set the `HttpPipeline` or `HttpClient` in all clients if you're building if your +For builders that expose `HttpTrait`, if an `HttpPipeline` or `HttpClient` isn't set a default instance will be +created based on classpath configurations and the `ClientOptions` based to the builder. This can cause confusion if +you're expecting specific behavior for your client, such as using a proxy that wasn't loaded from the environment. To +avoid this, it is recommended to always set the `HttpPipeline` or `HttpClient` in all clients if you're building if your configurations aren't based on the environment running the application. #### Credential Traits @@ -185,7 +187,7 @@ and generally should be retried by the `RetryPolicy`. #### HttpPipeline Timeouts HttpPipeline timeouts are the next level of timeout handling the Azure SDKs provide. These timeouts are configured using -an `HttpPipelinePolicy` and configuring a timeout using either `Mono.timeout` for asynchronous requests or an +an `HttpPipelinePolicy` and configuring a timeout using either `Mono.timeout` for asynchronous requests or an `ExecutorService` with a timed `get(long, TimeUnit)` for synchronous requests. Depending on the location within the `HttpPipeline`, these timeouts may be captured by the `RetryPolicy` and retried. From 5981d1eb1db5819ef4a2e68d3222a576356c6f5d Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 30 Jan 2025 11:45:02 -0800 Subject: [PATCH 2/4] Set EnablePRGeneration to true when generating job matrix for a PR --- eng/pipelines/templates/jobs/ci.yml | 2 + eng/scripts/Language-Settings.ps1 | 129 ++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 8a6b8b76820f..261041317fd6 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -451,6 +451,8 @@ jobs: - ${{ config }} - ${{ each config in parameters.AdditionalMatrixConfigs }}: - ${{ config }} + ${{ if eq(parameters.ServiceDirectory, 'auto') }}: + EnablePRGeneration: true MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} CloudConfig: diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index c3c5a4c4eb88..5d64dd181098 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -193,6 +193,135 @@ function Get-AllPackageInfoFromRepo([string]$serviceDirectory = $null) { return $allPackageProps } +# Get-java-AdditionalValidationPackagesFromPackageSet is the implementation of the +# $AdditionalValidationPackagesFromPackageSetFn which is used +function Get-java-AdditionalValidationPackagesFromPackageSet { + param( + [Parameter(Mandatory=$true)] + $LocatedPackages, + [Parameter(Mandatory=$true)] + $diffObj, + [Parameter(Mandatory=$true)] + $AllPkgProps + ) + $additionalValidationPackages = @() + $uniqueResultSet = @() + + # For these directories, just run template + function isOther($fileName) { + $startsWithPrefixes = @(".config", ".devcontainer", ".github", ".vscode", "common", "conda", "doc", "eng", "scripts") + + $startsWith = $false + foreach ($prefix in $startsWithPrefixes) { + if ($fileName.StartsWith($prefix)) { + $startsWith = $true + } + } + + return $startsWith + } + + # this section will identify the list of packages that we should treat as + # "directly" changed for a given service level change. While that doesn't + # directly change a package within the service, I do believe we should directly include all + # packages WITHIN that service. This is because the service level file changes are likely to + # have an impact on the packages within that service. + + # JRS - After chatting with Alan, erroring on the side of caution. The thing is, + # not all ServiceDirectories will do this, only the ones with ci.yml files which + # aren't SdkType=data. What to do with changes in a ServiceDirectory that only has + # pipelines at the library level? $changedServices = @() + if ($diffObj.ChangedFiles) { + foreach($file in $diffObj.ChangedFiles) { + $pathComponents = $file -split "/" + # handle changes only in sdk/// + if ($pathComponents.Length -eq 3 -and $pathComponents[0] -eq "sdk") { + $changedServices += $pathComponents[1] + } + + # handle any changes under sdk/. + if ($pathComponents.Length -eq 2 -and $pathComponents[0] -eq "sdk") { + $changedServices += "template" + } + } + foreach ($changedService in $changedServices) { + $additionalPackages = $AllPkgProps | Where-Object { $_.ServiceDirectory -eq $changedService } + + foreach ($pkg in $additionalPackages) { + if ($uniqueResultSet -notcontains $pkg -and $LocatedPackages -notcontains $pkg) { + # notice the lack of setting IncludedForValidation to true. This is because these "changed services" + # are specifically where a file within the service, but not an individual package within that service has changed. + # we want this package to be fully validated + $uniqueResultSet += $pkg + } + } + } + } + + $othersChanged = @() + $engChanged = @() + + if ($diffObj.ChangedFiles) { + $engChanged = $diffObj.ChangedFiles | Where-Object { $_.StartsWith("eng")} + # JRS - + $engChanged = $diffObj.ChangedFiles | Where-Object { $_.StartsWith("sdk/parents")} + $othersChanged = $diffObj.ChangedFiles | Where-Object { isOther($_) } + } + + $changedServices = $changedServices | Get-Unique + + # if ($toolChanged) { + # $additionalPackages = @( + # "azure-storage-blob", + # "azure-servicebus", + # "azure-eventhub", + # "azure-data-table", + # "azure-appconfig", + # "azure-keyvault-keys", + # "azure-identity", + # "azure-mgmt-core", + # "azure-core-experimental", + # "azure-core-tracing-opentelemetry", + # "azure-core-tracing-opencensus", + # # "azure-cosmos", leave removed until we resolve what to do with the emulator tests + # "azure-ai-documentintelligence", + # "azure-ai-ml", + # "azure-ai-inference", + # "azure-ai-textanalytics", + # "azure-ai-doctranslation", + # "azure-mgmt-compute", + # "azure-communication-chat", + # "azure-communication-identity" + # ) | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 } + + # $additionalValidationPackages += $additionalPackages + # } + + if ($engChanged -or $othersChanged) { + $additionalPackages = @( + "azure-template", + "azure-core" + ) | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 } + + $additionalValidationPackages += $additionalPackages + } + + + foreach ($pkg in $additionalValidationPackages) { + if ($uniqueResultSet -notcontains $pkg -and $LocatedPackages -notcontains $pkg) { + $pkg.IncludedForValidation = $true + $uniqueResultSet += $pkg + } + } + + Write-Host "Returning additional packages for validation: $($uniqueResultSet.Count)" + foreach ($pkg in $uniqueResultSet) { + Write-Host " - $($pkg.Name)" + } + + return $uniqueResultSet +} + # Returns the maven (really sonatype) publish status of a package id and version. function IsMavenPackageVersionPublished($pkgId, $pkgVersion, $groupId) { From b4573c5f4b49ed8d23a027bcf266e85ea3630340 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 30 Jan 2025 12:06:00 -0800 Subject: [PATCH 3/4] Update PreGenerationSteps for PR matrix gen, remove language-settings changes --- eng/pipelines/templates/jobs/ci.yml | 9 ++ eng/scripts/Language-Settings.ps1 | 129 ---------------------------- 2 files changed, 9 insertions(+), 129 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 261041317fd6..479278ede5ee 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -453,6 +453,15 @@ jobs: - ${{ config }} ${{ if eq(parameters.ServiceDirectory, 'auto') }}: EnablePRGeneration: true + PreGenerationSteps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + - template: /eng/common/pipelines/templates/steps/save-package-properties.yml + parameters: + ServiceDirectory: ${{parameters.ServiceDirectory}} + ExcludePaths: ${{parameters.ExcludePaths}} MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} CloudConfig: diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 5d64dd181098..c3c5a4c4eb88 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -193,135 +193,6 @@ function Get-AllPackageInfoFromRepo([string]$serviceDirectory = $null) { return $allPackageProps } -# Get-java-AdditionalValidationPackagesFromPackageSet is the implementation of the -# $AdditionalValidationPackagesFromPackageSetFn which is used -function Get-java-AdditionalValidationPackagesFromPackageSet { - param( - [Parameter(Mandatory=$true)] - $LocatedPackages, - [Parameter(Mandatory=$true)] - $diffObj, - [Parameter(Mandatory=$true)] - $AllPkgProps - ) - $additionalValidationPackages = @() - $uniqueResultSet = @() - - # For these directories, just run template - function isOther($fileName) { - $startsWithPrefixes = @(".config", ".devcontainer", ".github", ".vscode", "common", "conda", "doc", "eng", "scripts") - - $startsWith = $false - foreach ($prefix in $startsWithPrefixes) { - if ($fileName.StartsWith($prefix)) { - $startsWith = $true - } - } - - return $startsWith - } - - # this section will identify the list of packages that we should treat as - # "directly" changed for a given service level change. While that doesn't - # directly change a package within the service, I do believe we should directly include all - # packages WITHIN that service. This is because the service level file changes are likely to - # have an impact on the packages within that service. - - # JRS - After chatting with Alan, erroring on the side of caution. The thing is, - # not all ServiceDirectories will do this, only the ones with ci.yml files which - # aren't SdkType=data. What to do with changes in a ServiceDirectory that only has - # pipelines at the library level? $changedServices = @() - if ($diffObj.ChangedFiles) { - foreach($file in $diffObj.ChangedFiles) { - $pathComponents = $file -split "/" - # handle changes only in sdk/// - if ($pathComponents.Length -eq 3 -and $pathComponents[0] -eq "sdk") { - $changedServices += $pathComponents[1] - } - - # handle any changes under sdk/. - if ($pathComponents.Length -eq 2 -and $pathComponents[0] -eq "sdk") { - $changedServices += "template" - } - } - foreach ($changedService in $changedServices) { - $additionalPackages = $AllPkgProps | Where-Object { $_.ServiceDirectory -eq $changedService } - - foreach ($pkg in $additionalPackages) { - if ($uniqueResultSet -notcontains $pkg -and $LocatedPackages -notcontains $pkg) { - # notice the lack of setting IncludedForValidation to true. This is because these "changed services" - # are specifically where a file within the service, but not an individual package within that service has changed. - # we want this package to be fully validated - $uniqueResultSet += $pkg - } - } - } - } - - $othersChanged = @() - $engChanged = @() - - if ($diffObj.ChangedFiles) { - $engChanged = $diffObj.ChangedFiles | Where-Object { $_.StartsWith("eng")} - # JRS - - $engChanged = $diffObj.ChangedFiles | Where-Object { $_.StartsWith("sdk/parents")} - $othersChanged = $diffObj.ChangedFiles | Where-Object { isOther($_) } - } - - $changedServices = $changedServices | Get-Unique - - # if ($toolChanged) { - # $additionalPackages = @( - # "azure-storage-blob", - # "azure-servicebus", - # "azure-eventhub", - # "azure-data-table", - # "azure-appconfig", - # "azure-keyvault-keys", - # "azure-identity", - # "azure-mgmt-core", - # "azure-core-experimental", - # "azure-core-tracing-opentelemetry", - # "azure-core-tracing-opencensus", - # # "azure-cosmos", leave removed until we resolve what to do with the emulator tests - # "azure-ai-documentintelligence", - # "azure-ai-ml", - # "azure-ai-inference", - # "azure-ai-textanalytics", - # "azure-ai-doctranslation", - # "azure-mgmt-compute", - # "azure-communication-chat", - # "azure-communication-identity" - # ) | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 } - - # $additionalValidationPackages += $additionalPackages - # } - - if ($engChanged -or $othersChanged) { - $additionalPackages = @( - "azure-template", - "azure-core" - ) | ForEach-Object { $me=$_; $AllPkgProps | Where-Object { $_.Name -eq $me } | Select-Object -First 1 } - - $additionalValidationPackages += $additionalPackages - } - - - foreach ($pkg in $additionalValidationPackages) { - if ($uniqueResultSet -notcontains $pkg -and $LocatedPackages -notcontains $pkg) { - $pkg.IncludedForValidation = $true - $uniqueResultSet += $pkg - } - } - - Write-Host "Returning additional packages for validation: $($uniqueResultSet.Count)" - foreach ($pkg in $uniqueResultSet) { - Write-Host " - $($pkg.Name)" - } - - return $uniqueResultSet -} - # Returns the maven (really sonatype) publish status of a package id and version. function IsMavenPackageVersionPublished($pkgId, $pkgVersion, $groupId) { From 3a403942cfc49b1916092a1fce0ace5b3da8c531 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 30 Jan 2025 12:12:48 -0800 Subject: [PATCH 4/4] Update sparse checkout paths for PR matrix gen --- eng/pipelines/templates/jobs/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 479278ede5ee..d22ed528430d 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -453,6 +453,10 @@ jobs: - ${{ config }} ${{ if eq(parameters.ServiceDirectory, 'auto') }}: EnablePRGeneration: true + SparseCheckoutPaths: + - '**/*.xml' + - '**/*.md' + - '**/ci*.yml' # necessary for save-package-properties.yml PreGenerationSteps: - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)'