From ce169aaa3fdf183e7e620fd6b80bf599ba8b4c53 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 1 Nov 2023 12:17:02 -0700 Subject: [PATCH 1/6] Migrate tests to windows 2022 by default. For the windows 2019 test, we need to upgrade python for the latest gcloud to work. PiperOrigin-RevId: 578600130 --- .github/workflows/test_cpp.yml | 44 ++++++++++++++++++++-------------- .github/workflows/test_upb.yml | 2 +- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 511f9ec189e3c..560411f13c00d 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -324,8 +324,8 @@ jobs: # for Apple Silicon to detect issues there. bazel: build --cpu=darwin_arm64 //src/... - name: Windows - os: windows-2019 - cache_key: windows-2019 + os: windows-2022 + cache_key: windows-2022 bazel: test //src/... @com_google_protobuf_examples//... --test_tag_filters=-conformance --build_tag_filters=-conformance name: ${{ matrix.name }} Bazel runs-on: ${{ matrix.os }} @@ -351,14 +351,6 @@ jobs: flags: -DCMAKE_CXX_STANDARD=14 cache-prefix: macos-cmake - name: Windows CMake - os: windows-2019 - flags: >- - -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF - -Dprotobuf_BUILD_SHARED_LIBS=OFF - -Dprotobuf_BUILD_EXAMPLES=ON - vsversion: '2019' - cache-prefix: windows-2019-cmake - - name: Windows CMake 2022 os: windows-2022 flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF @@ -366,29 +358,38 @@ jobs: -Dprotobuf_BUILD_EXAMPLES=ON vsversion: '2022' cache-prefix: windows-2022-cmake - - name: Windows CMake 32-bit + - name: Windows CMake 2019 os: windows-2019 flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF + -Dprotobuf_BUILD_SHARED_LIBS=OFF + -Dprotobuf_BUILD_EXAMPLES=ON vsversion: '2019' + cache-prefix: windows-2019-cmake + python-version: '3.8' + - name: Windows CMake 32-bit + os: windows-2022 + flags: >- + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF + vsversion: '2022' windows-arch: 'win32' - cache-prefix: windows-2019-win32-cmake + cache-prefix: windows-2022-win32-cmake - name: Windows CMake Shared - os: windows-2019 + os: windows-2022 flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=ON - vsversion: '2019' - cache-prefix: windows-2019-cmake + vsversion: '2022' + cache-prefix: windows-2022-cmake - name: Windows CMake Install - os: windows-2019 + os: windows-2022 install-flags: -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_TESTS=OFF flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF - vsversion: '2019' - cache-prefix: windows-2019-cmake + vsversion: '2022' + cache-prefix: windows-2022-cmake name: ${{ matrix.name }} runs-on: ${{ matrix.os }} steps: @@ -398,6 +399,13 @@ jobs: ref: ${{ inputs.safe-checkout }} submodules: recursive + - name: Install Python + if: ${{ matrix.python-version }} + uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Setup MSVC if: ${{ runner.os == 'Windows' }} uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml index 21ec30887658e..3aed7a77761f5 100644 --- a/.github/workflows/test_upb.yml +++ b/.github/workflows/test_upb.yml @@ -64,7 +64,7 @@ jobs: strategy: fail-fast: false # Don't cancel all jobs if one fails. name: Windows - runs-on: windows-2019 + runs-on: windows-2022 steps: - name: Checkout pending changes uses: protocolbuffers/protobuf-ci/checkout@v2 From 5415d957a6a48f598da41413f2842fc45983f6b9 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 1 Nov 2023 16:18:47 -0700 Subject: [PATCH 2/6] Fix windows 2019 build. setup-gcloud doesn't honor the environment variables set by setup-python, so we need to manually set CLOUDSDK_PYTHON before running it. PiperOrigin-RevId: 578674246 --- .github/workflows/test_cpp.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 560411f13c00d..cc2004bc3e1b1 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -366,6 +366,7 @@ jobs: -Dprotobuf_BUILD_EXAMPLES=ON vsversion: '2019' cache-prefix: windows-2019-cmake + # windows-2019 has python3.7 installed, which is incompatible with the latest gcloud python-version: '3.8' - name: Windows CMake 32-bit os: windows-2022 @@ -399,13 +400,6 @@ jobs: ref: ${{ inputs.safe-checkout }} submodules: recursive - - name: Install Python - if: ${{ matrix.python-version }} - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ matrix.python-version }} - cache: pip - - name: Setup MSVC if: ${{ runner.os == 'Windows' }} uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 @@ -413,6 +407,17 @@ jobs: arch: ${{ matrix.windows-arch || 'x64' }} vsversion: ${{ matrix.vsversion }} + # Workaround for incompatibility between gcloud and windows-2019 runners. + - name: Install Python + if: ${{ matrix.python-version }} + uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: ${{ matrix.python-version }} + - name: Use custom python for gcloud + if: ${{ matrix.python-version }} + run: echo "CLOUDSDK_PYTHON=${Python3_ROOT_DIR}\\python3" >> $GITHUB_ENV + shell: bash + - name: Setup sccache uses: protocolbuffers/protobuf-ci/sccache@v2 with: From 70042e26075b7535ff2fafbfc4bb21f37804b4ab Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 1 Nov 2023 22:17:29 -0700 Subject: [PATCH 3/6] Enable caching of pip dependencies PiperOrigin-RevId: 578740011 --- .github/workflows/test_python.yml | 1 + .github/workflows/test_upb.yml | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index e207ee3a92f18..0bec9a06c399f 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -76,6 +76,7 @@ jobs: with: python-version: ${{ matrix.version }} cache: pip + cache-dependency-path: 'python/requirements.txt' - name: Validate version run: python3 --version | grep ${{ matrix.version }} || (echo "Invalid Python version - $(python3 --version)" && exit 1) diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml index 3aed7a77761f5..9eb236f051b95 100644 --- a/.github/workflows/test_upb.yml +++ b/.github/workflows/test_upb.yml @@ -70,6 +70,10 @@ jobs: uses: protocolbuffers/protobuf-ci/checkout@v2 with: ref: ${{ inputs.safe-checkout }} + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + cache: pip + cache-dependency-path: 'python/requirements.txt' - name: Run tests uses: protocolbuffers/protobuf-ci/bazel@v2 with: @@ -92,6 +96,10 @@ jobs: uses: protocolbuffers/protobuf-ci/checkout@v2 with: ref: ${{ inputs.safe-checkout }} + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + cache: pip + cache-dependency-path: 'python/requirements.txt' - name: Run tests uses: protocolbuffers/protobuf-ci/bazel@v2 with: @@ -195,7 +203,7 @@ jobs: with: name: requirements path: requirements - - uses: actions/setup-python@v2 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.architecture }} @@ -250,7 +258,7 @@ jobs: path: wheels - name: Delete Binary Wheels run: find wheels -type f | grep -v none-any | xargs rm - - uses: actions/setup-python@v2 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: python-version: ${{ matrix.python-version }} - name: Setup Python venv From 3105b8d57eebb74867daadf1ca32dfa6a55040df Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Nov 2023 11:12:33 -0700 Subject: [PATCH 4/6] Pin python at 3.11. Github changed their mac runners to use 3.12 by default, which removed setuptools and is causing breakages in some of our tests. PiperOrigin-RevId: 578917538 --- .github/workflows/test_upb.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml index 9eb236f051b95..ebe59aa7e11ac 100644 --- a/.github/workflows/test_upb.yml +++ b/.github/workflows/test_upb.yml @@ -100,6 +100,7 @@ jobs: with: cache: pip cache-dependency-path: 'python/requirements.txt' + python-version: '3.11' # 3.12 doesn't have setuptools - name: Run tests uses: protocolbuffers/protobuf-ci/bazel@v2 with: From 1155c804efe5457071b813ca9cc17c7535e7ba09 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Sat, 21 Oct 2023 09:37:33 -0700 Subject: [PATCH 5/6] Update `cc_file_list_aspect` to handle targets with missing `hdrs/textual_hdrs` Our stale file regeneration logic broke with Bazel 6.4.0, and I suspect it was caused by this change: https://github.com/bazelbuild/bazel/pull/19534 Our logic assumed that any target with a `CcInfo` provider must have `hdrs` and `textual_hdrs` attributes, but it seems that this is no longer true for `cc_proto_library` starting with Bazel 6.4.0. The fix is just to use `getattr` and treat the item as an empty list if it's missing. PiperOrigin-RevId: 575473886 --- pkg/cc_dist_library.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cc_dist_library.bzl b/pkg/cc_dist_library.bzl index 42553ca3f8281..9fc2c414e6e36 100644 --- a/pkg/cc_dist_library.bzl +++ b/pkg/cc_dist_library.bzl @@ -170,12 +170,12 @@ def _cc_file_list_aspect_impl(target, ctx): return [CcFileList( hdrs = _get_transitive_sources( - _flatten_target_files(rule_attr.hdrs).to_list(), + _flatten_target_files(getattr(rule_attr, "hdrs", [])).to_list(), "hdrs", rule_attr.deps, ), textual_hdrs = _get_transitive_sources( - _flatten_target_files(rule_attr.textual_hdrs).to_list(), + _flatten_target_files(getattr(rule_attr, "textual_hdrs", [])).to_list(), "textual_hdrs", rule_attr.deps, ), From 1577c305cf0928b4b29433812a4dfcb26d37249d Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Nov 2023 13:06:40 -0700 Subject: [PATCH 6/6] Error on staleness failure --- .github/workflows/staleness_check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml index 273a4b01176f2..01077caa5a2b2 100644 --- a/.github/workflows/staleness_check.yml +++ b/.github/workflows/staleness_check.yml @@ -50,6 +50,7 @@ jobs: # tests along with user changes. Any stale files will be automatically fixed in a follow-up # commit. run: | + set -ex if [[ -z $COMMIT_TRIGGERED_RUN || -z $MAIN_RUN ]]; then bazel query 'attr(tags, "staleness_test", //...)' | xargs bazel test $BAZEL_FLAGS || \ echo "Please run ./regenerate_stale_files.sh to regenerate stale files"