Skip to content

Commit

Permalink
ci: reshard macOS builds (#14681)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dbolduc
dbolduc authored Sep 5, 2024
1 parent ca97e68 commit 718f218
Show file tree
Hide file tree
Showing 3 changed files with 214 additions and 49 deletions.
105 changes: 79 additions & 26 deletions .github/workflows/macos-bazel.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ jobs:
- Spanner
- Storage
- Compute
- Larger Libraries
- Shard1
- Shard2
- Other
include:
- shard: Core
Expand All @@ -64,7 +65,7 @@ jobs:
- shard: Compute
targets:
- //google/cloud/compute/...
- shard: Larger Libraries
- shard: Shard1
targets:
# Run this:
#
Expand All @@ -74,24 +75,52 @@ jobs:
# awk '{ s += $1; print s, $0}'
#
# and pick the approximate midpoint
- //google/cloud/servicecontrol/...
#
# Also, it is a good idea to keep `sql` and `bigquerycontrol`
# together, as they both require our internal REST libraries.
- //google/cloud/bigquerycontrol/...
- +//google/cloud/appengine/...
- +//google/cloud/dataproc/...
- +//google/cloud/monitoring/...
- +//google/cloud/retail/...
- +//google/cloud/discoveryengine/...
- +//google/cloud/sql/...
- +//google/cloud/dialogflow_cx/...
- +//google/cloud/dialogflow_es/...
- +//google/cloud/aiplatform/...
- shard: Shard2
targets:
- //google/cloud/networkconnectivity/...
- +//google/cloud/networkservices/...
- +//google/cloud/policytroubleshooter/...
- +//google/cloud/profiler/...
- +//google/cloud/redis/...
- +//google/cloud/securitycenter/...
- +//google/cloud/servicedirectory/...
- +//google/cloud/tpu/...
- +//google/cloud/trace/...
- +//google/cloud/vision/...
- +//google/cloud/workflows/...
- +//google/cloud/beyondcorp/...
- +//google/cloud/billing/...
- +//google/cloud/binaryauthorization/...
- +//google/cloud/gkemulticloud/...
- +//google/cloud/logging/...
- +//google/cloud/notebooks/...
- +//google/cloud/osconfig/...
- +//google/cloud/servicecontrol/...
- +//google/cloud/speech/...
- +//google/cloud/support/...
- +//google/cloud/video/...
- +//google/cloud/datacatalog/...
- +//google/cloud/iam/...
- +//google/cloud/kms/...
- +//google/cloud/beyondcorp/...
- +//google/cloud/contentwarehouse/...
- +//google/cloud/dataproc/...
- +//google/cloud/run/...
- +//google/cloud/talent/...
- +//google/cloud/retail/...
- +//google/cloud/contentwarehouse/...
- +//google/cloud/dataplex/...
- +//google/cloud/kms/...
- +//google/cloud/bigquery/...
- +//google/cloud/appengine/...
- +//google/cloud/monitoring/...
- +//google/cloud/sql/...
- +//google/cloud/dialogflow_cx/...
- +//google/cloud/aiplatform/...
- +//google/cloud/dialogflow_es/...
- +//google/cloud/resourcemanager/...
- shard: Other
targets:
- //...
Expand All @@ -113,25 +142,49 @@ jobs:
- -//google/cloud/storage/...
# From Compute
- -//google/cloud/compute/...
# From Large Libraries
# From Shard1
- -//google/cloud/appengine/...
- -//google/cloud/dataproc/...
- -//google/cloud/monitoring/...
- -//google/cloud/retail/...
- -//google/cloud/discoveryengine/...
- -//google/cloud/sql/...
- -//google/cloud/dialogflow_cx/...
- -//google/cloud/dialogflow_es/...
- -//google/cloud/aiplatform/...
# From Shard2
- -//google/cloud/networkconnectivity/...
- -//google/cloud/networkservices/...
- -//google/cloud/policytroubleshooter/...
- -//google/cloud/profiler/...
- -//google/cloud/redis/...
- -//google/cloud/securitycenter/...
- -//google/cloud/servicedirectory/...
- -//google/cloud/tpu/...
- -//google/cloud/trace/...
- -//google/cloud/vision/...
- -//google/cloud/workflows/...
- -//google/cloud/beyondcorp/...
- -//google/cloud/billing/...
- -//google/cloud/binaryauthorization/...
- -//google/cloud/gkemulticloud/...
- -//google/cloud/logging/...
- -//google/cloud/notebooks/...
- -//google/cloud/osconfig/...
- -//google/cloud/servicecontrol/...
- -//google/cloud/speech/...
- -//google/cloud/support/...
- -//google/cloud/video/...
- -//google/cloud/datacatalog/...
- -//google/cloud/iam/...
- -//google/cloud/kms/...
- -//google/cloud/beyondcorp/...
- -//google/cloud/contentwarehouse/...
- -//google/cloud/dataproc/...
- -//google/cloud/run/...
- -//google/cloud/talent/...
- -//google/cloud/retail/...
- -//google/cloud/contentwarehouse/...
- -//google/cloud/dataplex/...
- -//google/cloud/kms/...
- -//google/cloud/bigquery/...
- -//google/cloud/appengine/...
- -//google/cloud/monitoring/...
- -//google/cloud/sql/...
- -//google/cloud/dialogflow_cx/...
- -//google/cloud/aiplatform/...
- -//google/cloud/dialogflow_es/...
- -//google/cloud/bigquerycontrol/...
- -//google/cloud/resourcemanager/...
steps:
- uses: actions/checkout@v4
with:
Expand Down
151 changes: 133 additions & 18 deletions .github/workflows/macos-cmake.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,19 @@ jobs:
matrix:
exclude-from-full-trick: [ true ]
os: [ macos-14 ]
shard: [ Core, Compute, Shard1, Other ]
shard: [ Core1, Core2, Compute, Shard1, Shard2, Shard3, Shard4, Other ]
exclude:
# Only full builds include shards with generated code.
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
shard: Compute
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
shard: Shard1
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
shard: Shard2
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
shard: Shard3
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
shard: Shard4
- exclude-from-full-trick: ${{ ! inputs.full-matrix }}
shard: Other
steps:
Expand All @@ -64,55 +70,164 @@ jobs:
shell: bash
run: |
echo "vcpkg-version=$(cat ci/etc/vcpkg-version.txt)" >> "${GITHUB_OUTPUT}"
core_features=(
core1_features=(
bigtable
spanner
)
core2_features=(
pubsub
pubsublite
spanner
storage
)
# These are the libraries with the most "clients". To build the list
# run something like this and find the midpoint:
#
# git grep -l 'class.*Client' 'google/cloud/**_client.h' |
# egrep -v "(bigtable/|internal/|pubsub/|spanner/|storage/)" |
# egrep -v "(bigtable/|internal/|pubsub/|pubsublite/|spanner/|storage/|testing_util/)" |
# cut -f -3 -d/| sort | uniq -c | sort -n |
# awk '{ s += $1; print s, $0}'
#
shard1_features=(
bigquerycontrol
dataproc
monitoring
retail
discoveryengine
sql
dialogflow_cx
dialogflow_es
aiplatform
)
shard2_features=(
servicecontrol
speech
support
video
datacatalog
iam
kms
beyondcorp
contentwarehouse
dataproc
run
talent
retail
contentwarehouse
dataplex
kms
bigquery
resourcemanager
appengine
monitoring
sql
dialogflow_cx
aiplatform
dialogflow_es
)
if [[ "${{ matrix.shard }}" == "Core" ]]; then
features="$(printf ",%s" "${core_features[@]}")"
shard3_features=(
channel
cloudbuild
cloudcontrolspartner
composer
containeranalysis
datastore
eventarc
functions
iap
language
metastore
networkconnectivity
networkservices
policytroubleshooter
profiler
redis
securitycenter
servicedirectory
tpu
trace
vision
workflows
beyondcorp
billing
binaryauthorization
gkemulticloud
logging
notebooks
osconfig
)
shard4_features=(
domains
edgecontainer
edgenetwork
essentialcontacts
filestore
gkebackup
gkeconnect
gkehub
ids
managedidentities
managedkafka
memcache
migrationcenter
netapp
networkmanagement
networksecurity
optimization
orgpolicy
oslogin
policysimulator
privateca
privilegedaccessmanager
publicca
rapidmigrationassessment
recaptchaenterprise
recommender
resourcesettings
scheduler
secretmanager
securesourcemanager
securitycentermanagement
servicehealth
servicemanagement
serviceusage
shell
storagecontrol
storageinsights
storagetransfer
tasks
telcoautomation
texttospeech
timeseriesinsights
translate
videointelligence
vmmigration
vmwareengine
vpcaccess
webrisk
websecurityscanner
workstations
automl
)
if [[ "${{ matrix.shard }}" == "Core1" ]]; then
features="$(printf ",%s" "${core1_features[@]}")"
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
elif [[ "${{ matrix.shard }}" == "Core2" ]]; then
features="$(printf ",%s" "${core2_features[@]}")"
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
elif [[ "${{matrix.shard}}" == "Compute" ]]; then
echo "features=compute" >> "${GITHUB_OUTPUT}"
elif [[ "${{ matrix.shard }}" == "Shard1" ]]; then
features="$(printf ",%s" "${shard1_features[@]}")"
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
elif [[ "${{ matrix.shard }}" == "Shard2" ]]; then
features="$(printf ",%s" "${shard2_features[@]}")"
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
elif [[ "${{ matrix.shard }}" == "Shard3" ]]; then
features="$(printf ",%s" "${shard3_features[@]}")"
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
elif [[ "${{ matrix.shard }}" == "Shard4" ]]; then
features="$(printf ",%s" "${shard4_features[@]}")"
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
else
skipped_features=("${core_features[@]}")
skipped_features=("${core1_features[@]}")
skipped_features+=("${core2_features[@]}")
skipped_features+=(compute)
skipped_features+=("${shard1_features[@]}")
skipped_features+=("${shard2_features[@]}")
skipped_features+=("${shard3_features[@]}")
skipped_features+=("${shard4_features[@]}")
skipped="$(printf ",-%s" "${skipped_features[@]}")"
echo "features=__ga_libraries__,__experimental_libraries__,${skipped:2}" >> "${GITHUB_OUTPUT}"
echo "features=__ga_libraries__,__experimental_libraries__,${skipped:1}" >> "${GITHUB_OUTPUT}"
fi
- name: Install bash >= 5.x and Ninja
run: brew install bash ninja
Expand Down
7 changes: 2 additions & 5 deletions .github/workflows/windows-cmake.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,10 +210,7 @@ jobs:
features="$(printf ",%s" "${shard3_features[@]}")"
echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
else
# The "-" before the first element gets consumed somewhere, so pad
# it out.
skipped_features=(null)
skipped_features+=("${core1_features[@]}")
skipped_features=("${core1_features[@]}")
skipped_features+=("${core2_features[@]}")
skipped_features+=(compute)
skipped_features+=("${shard1_features[@]}")
Expand All @@ -224,7 +221,7 @@ jobs:
# Windows.
skipped_features+=(asset channel)
skipped="$(printf ",-%s" "${skipped_features[@]}")"
echo "features=__ga_libraries__,__experimental_libraries__,${skipped:2}" >> "${GITHUB_OUTPUT}"
echo "features=__ga_libraries__,__experimental_libraries__,${skipped:1}" >> "${GITHUB_OUTPUT}"
fi
- name: Download and Install CMake
shell: bash
Expand Down

0 comments on commit 718f218

Please sign in to comment.