Skip to content

Commit

Permalink
[Fix] Allowed filtering for graviton DBRs in `ClusterAPI.SelectSparkV…
Browse files Browse the repository at this point in the history
…ersion` (#1004)

## Changes
- We need to filter out Graviton DBRs, otherwise `SelectSparkVersion`
will fail with `spark versions query returned multiple results`

## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` passing
- [x] `make fmt` applied
- [ ] relevant integration tests applied
  • Loading branch information
nkvuong authored Aug 5, 2024
1 parent 325f174 commit fd65dce
Showing 1 changed file with 16 additions and 2 deletions.
18 changes: 16 additions & 2 deletions service/compute/ext_spark_version.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ type SparkVersionRequest struct {
Scala string `json:"scala,omitempty" tf:"optional,default:2.12"`
SparkVersion string `json:"spark_version,omitempty" tf:"optional,default:"`
Photon bool `json:"photon,omitempty" tf:"optional,default:false"`
Graviton bool `json:"graviton,omitempty" tf:"optional,default:false"`
}

type sparkVersionsType []string
Expand All @@ -47,7 +48,6 @@ func (s sparkVersionsType) Less(i, j int) bool {
return semver.Compare("v"+extractDbrVersions(s[i]), "v"+extractDbrVersions(s[j])) > 0
}

// LatestSparkVersion returns latest version matching the request parameters
func (sv GetSparkVersionsResponse) Select(req SparkVersionRequest) (string, error) {
var versions []string
for _, version := range sv.Versions {
Expand All @@ -57,6 +57,7 @@ func (sv GetSparkVersionsResponse) Select(req SparkVersionRequest) (string, erro
(strings.Contains(version.Key, "-hls-") == req.Genomics) &&
(strings.Contains(version.Key, "-gpu-") == req.GPU) &&
(strings.Contains(version.Key, "-photon-") == req.Photon) &&
(strings.Contains(version.Key, "-aarch64-") == req.Graviton) &&
(strings.Contains(version.Name, "Beta") == req.Beta))
if matches && req.LongTermSupport {
matches = (matches && (strings.Contains(version.Name, "LTS") || strings.Contains(version.Key, "-esr-")))
Expand All @@ -75,12 +76,25 @@ func (sv GetSparkVersionsResponse) Select(req SparkVersionRequest) (string, erro
if req.Latest {
sort.Sort(sparkVersionsType(versions))
} else {
return "", fmt.Errorf("spark versions query returned multiple results. Please change your search criteria and try again")
return "", fmt.Errorf("spark versions query returned multiple results %#v. Please change your search criteria and try again", versions)
}
}
return versions[0], nil
}

// SelectSparkVersion returns latest DBR version matching the request parameters.
// If there are multiple versions matching the request, it will error (if latest = false)
// or return the latest version.
// Possible parameters are:
// - LongTermSupport: LTS versions only
// - Beta: Beta versions only
// - ML: ML versions only
// - Genomics: Genomics versions only
// - GPU: GPU versions only
// - Scala: Scala version
// - SparkVersion: Apache Spark version
// - Photon: Photon versions only (deprecated)
// - Graviton: Graviton versions only (deprecated)
func (a *ClustersAPI) SelectSparkVersion(ctx context.Context, r SparkVersionRequest) (string, error) {
sv, err := a.SparkVersions(ctx)
if err != nil {
Expand Down

0 comments on commit fd65dce

Please sign in to comment.