From 69b2a07e3190a8cf89025c45635ac2b4fcf9c520 Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Thu, 4 Jul 2024 19:13:57 +0200 Subject: [PATCH 1/8] Upgrade to opentelemetry-proto 1.3.2 --- MODULE.bazel | 2 +- bazel/repository.bzl | 6 +- cmake/opentelemetry-proto.cmake | 2 +- docs/maintaining-dependencies.md | 260 +++++++++++++++++++++++++++++++ third_party/opentelemetry-proto | 2 +- third_party_release | 2 +- 6 files changed, 267 insertions(+), 7 deletions(-) create mode 100644 docs/maintaining-dependencies.md diff --git a/MODULE.bazel b/MODULE.bazel index 7b84c2b719..3161ffb1d3 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,7 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "curl", version = "8.4.0") bazel_dep(name = "grpc", version = "1.62.1", repo_name = "com_github_grpc_grpc") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "github_nlohmann_json") -bazel_dep(name = "opentelemetry-proto", version = "1.3.1", repo_name = "com_github_opentelemetry_proto") +bazel_dep(name = "opentelemetry-proto", version = "1.3.2", repo_name = "com_github_opentelemetry_proto") bazel_dep(name = "opentracing-cpp", version = "1.6.0", repo_name = "com_github_opentracing") bazel_dep(name = "platforms", version = "0.0.8") bazel_dep(name = "prometheus-cpp", version = "1.2.4", repo_name = "com_github_jupp0r_prometheus_cpp") diff --git a/bazel/repository.bzl b/bazel/repository.bzl index bac1e45b34..508b95a39a 100644 --- a/bazel/repository.bzl +++ b/bazel/repository.bzl @@ -88,10 +88,10 @@ def opentelemetry_cpp_deps(): http_archive, name = "com_github_opentelemetry_proto", build_file = "@io_opentelemetry_cpp//bazel:opentelemetry_proto.BUILD", - sha256 = "bed250ceec8e4a83aa5604d7d5595a61945059dc662edd058a9da082283f7a00", - strip_prefix = "opentelemetry-proto-1.3.1", + sha256 = "c069c0d96137cf005d34411fa67dd3b6f1f8c64af1e7fb2fe0089a41c425acd7", + strip_prefix = "opentelemetry-proto-1.3.2", urls = [ - "https://github.com/open-telemetry/opentelemetry-proto/archive/v1.3.1.tar.gz", + "https://github.com/open-telemetry/opentelemetry-proto/archive/v1.3.2.tar.gz", ], ) diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 19516c3b71..dd6213c1c6 100644 --- a/cmake/opentelemetry-proto.cmake +++ b/cmake/opentelemetry-proto.cmake @@ -49,7 +49,7 @@ else() "opentelemetry-proto=[ \\t]*([A-Za-z0-9_\\.\\-]+)") set(opentelemetry-proto "${CMAKE_MATCH_1}") else() - set(opentelemetry-proto "v1.3.1") + set(opentelemetry-proto "v1.3.2") endif() unset(OTELCPP_THIRD_PARTY_RELEASE_CONTENT) endif() diff --git a/docs/maintaining-dependencies.md b/docs/maintaining-dependencies.md new file mode 100644 index 0000000000..2566bf90c5 --- /dev/null +++ b/docs/maintaining-dependencies.md @@ -0,0 +1,260 @@ + +# Maintaining dependencies + +## opentelemetry-proto + +### Origin + +The repository for opentelemetry-proto is: + +* [repository](https://github.com/open-telemetry/opentelemetry-proto) + +Check release notes at: + +* [release-notes](https://github.com/open-telemetry/opentelemetry-proto/releases) + +### Upgrade + +When upgrading opentelemetry-proto to a newer release, +a few places in the code need adjustment. + +In this example, we upgrade from 1.3.1 to 1.3.2 + +#### directory third_party/opentelemetry-proto + +This is a `git submodule`, it needs to point to the new tag. + +```shell +cd third_party/opentelemetry-proto +git log -1 +``` + +The current submodule show something like: + +``` +commit b3060d2104df364136d75a35779e6bd48bac449a (HEAD, tag: v1.3.1) +Author: Damien Mathieu <42@dmathieu.com> +Date: Thu Apr 25 17:55:35 2024 +0200 + + generate profiles proto for CI (#552) +``` + +Pull new tags: + +```shell +git pull --tag origin +``` + +Upgrade to a new tag: + +```shell +git pull origin v1.3.2 +``` + +Check the new state: + +```shell +git log -1 +``` + +``` +commit 40b3c1b746767cbc13c2e39da3eaf1a23e54ffdd (HEAD, tag: v1.3.2) +Author: jack-berg <34418638+jack-berg@users.noreply.github.com> +Date: Fri Jun 28 08:19:11 2024 -0500 + + Prepare changelog for 1.3.2 release (#563) + + Co-authored-by: Armin Ruech <7052238+arminru@users.noreply.github.com> +``` + +Go back to the root of opentelemetry-cpp + +```shell +cd ../.. +git status +``` + +``` +On branch upgrade_proto_1.3.2 +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: third_party/opentelemetry-proto (new commits) +``` + +Add the upgraded submodule: + +```shell +git add third_party/opentelemetry-proto +``` + +#### file third_party_release + +Update the line pointing to the opentelemetry-proto tag. + +``` +opentelemetry-proto=v1.3.2 +``` + +Typical change: + +```shell +[malff@malff-desktop opentelemetry-cpp]$ git diff third_party_release +diff --git a/third_party_release b/third_party_release +index 0bbf67f3..7362473f 100644 +--- a/third_party_release ++++ b/third_party_release +@@ -19,7 +19,7 @@ benchmark=v1.8.3 + googletest=1.14.0 + ms-gsl=v3.1.0-67-g6f45293 + nlohmann-json=v3.11.3 +-opentelemetry-proto=v1.3.1 ++opentelemetry-proto=v1.3.2 + opentracing-cpp=v1.6.0 + prometheus-cpp=v1.2.4 + vcpkg=2024.02.14 + +``` + +#### file bazel/repository.bzl + +Locate the entry for opentelemetry-proto: + +``` + # OTLP Protocol definition + maybe( + http_archive, + name = "com_github_opentelemetry_proto", + build_file = "@io_opentelemetry_cpp//bazel:opentelemetry_proto.BUILD", + sha256 = "bed250ceec8e4a83aa5604d7d5595a61945059dc662edd058a9da082283f7a00", + strip_prefix = "opentelemetry-proto-1.3.1", + urls = [ + "https://github.com/open-telemetry/opentelemetry-proto/archive/v1.3.1.tar.gz", + ], + ) +``` + +Update the URL to the new tag: + +``` + urls = [ + "https://github.com/open-telemetry/opentelemetry-proto/archive/v1.3.2.tar.gz", + ], +``` + +Update strip_prefix to the new tag: + +``` + strip_prefix = "opentelemetry-proto-1.3.2", +``` + +Download the new URL: + +```shell +wget https://github.com/open-telemetry/opentelemetry-proto/archive/v1.3.2.tar.gz +``` + +Run a checksum on the new file: + +```shell +sha256sum v1.3.2.tar.gz +``` + +``` +c069c0d96137cf005d34411fa67dd3b6f1f8c64af1e7fb2fe0089a41c425acd7 v1.3.2.tar.gz +``` + +Update the checksum in file bazel/repository.bzl: + +``` + sha256 = "c069c0d96137cf005d34411fa67dd3b6f1f8c64af1e7fb2fe0089a41c425acd7", +``` + +Typical change: + +```shell +[malff@malff-desktop opentelemetry-cpp]$ git diff bazel/repository.bzl +diff --git a/bazel/repository.bzl b/bazel/repository.bzl +index bac1e45b..508b95a3 100644 +--- a/bazel/repository.bzl ++++ b/bazel/repository.bzl +@@ -88,10 +88,10 @@ def opentelemetry_cpp_deps(): + http_archive, + name = "com_github_opentelemetry_proto", + build_file = "@io_opentelemetry_cpp//bazel:opentelemetry_proto.BUILD", +- sha256 = "bed250ceec8e4a83aa5604d7d5595a61945059dc662edd058a9da082283f7a00", +- strip_prefix = "opentelemetry-proto-1.3.1", ++ sha256 = "c069c0d96137cf005d34411fa67dd3b6f1f8c64af1e7fb2fe0089a41c425acd7", ++ strip_prefix = "opentelemetry-proto-1.3.2", + urls = [ +- "https://github.com/open-telemetry/opentelemetry-proto/archive/v1.3.1.tar.gz", ++ "https://github.com/open-telemetry/opentelemetry-proto/archive/v1.3.2.tar.gz", + ], + ) +``` + +#### file cmake/opentelemetry-proto.cmake + +Update the tag in the CMake logic: + +``` + set(opentelemetry-proto "v1.3.2") +``` + +Typical change: + +```shell +[malff@malff-desktop opentelemetry-cpp]$ git diff cmake/opentelemetry-proto.cmake +diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake +index 19516c3b..dd6213c1 100644 +--- a/cmake/opentelemetry-proto.cmake ++++ b/cmake/opentelemetry-proto.cmake +@@ -49,7 +49,7 @@ else() + "opentelemetry-proto=[ \\t]*([A-Za-z0-9_\\.\\-]+)") + set(opentelemetry-proto "${CMAKE_MATCH_1}") + else() +- set(opentelemetry-proto "v1.3.1") ++ set(opentelemetry-proto "v1.3.2") + endif() + unset(OTELCPP_THIRD_PARTY_RELEASE_CONTENT) + endif() +``` + +If opentelemetry-proto contains new files, +the makefile needs to be adjusted to build the new code. + +#### file MODULE.bazel + +Make sure the new tag is available in bazel central: + +* [bazel-central](https://registry.bazel.build/modules/opentelemetry-proto) + +If missing, file a PR to add it, or contact the maintainer: + +* [repository](https://github.com/bazelbuild/bazel-central-registry/tree/main/modules/opentelemetry-proto) + +Update the opentelemetry-proto version to the new tag: + +``` +bazel_dep(name = "opentelemetry-proto", version = "1.3.2", repo_name = "com_github_opentelemetry_proto") +``` + +Typical change: + +```shell +[malff@malff-desktop opentelemetry-cpp]$ git diff MODULE.bazel +diff --git a/MODULE.bazel b/MODULE.bazel +index 7b84c2b7..3161ffb1 100644 +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -13,7 +13,7 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") + bazel_dep(name = "curl", version = "8.4.0") + bazel_dep(name = "grpc", version = "1.62.1", repo_name = "com_github_grpc_grpc") + bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "github_nlohmann_json") +-bazel_dep(name = "opentelemetry-proto", version = "1.3.1", repo_name = "com_github_opentelemetry_proto") ++bazel_dep(name = "opentelemetry-proto", version = "1.3.2", repo_name = "com_github_opentelemetry_proto") + bazel_dep(name = "opentracing-cpp", version = "1.6.0", repo_name = "com_github_opentracing") + bazel_dep(name = "platforms", version = "0.0.8") + bazel_dep(name = "prometheus-cpp", version = "1.2.4", repo_name = "com_github_jupp0r_prometheus_cpp") +``` + diff --git a/third_party/opentelemetry-proto b/third_party/opentelemetry-proto index b3060d2104..40b3c1b746 160000 --- a/third_party/opentelemetry-proto +++ b/third_party/opentelemetry-proto @@ -1 +1 @@ -Subproject commit b3060d2104df364136d75a35779e6bd48bac449a +Subproject commit 40b3c1b746767cbc13c2e39da3eaf1a23e54ffdd diff --git a/third_party_release b/third_party_release index 0bbf67f3af..7362473f60 100644 --- a/third_party_release +++ b/third_party_release @@ -19,7 +19,7 @@ benchmark=v1.8.3 googletest=1.14.0 ms-gsl=v3.1.0-67-g6f45293 nlohmann-json=v3.11.3 -opentelemetry-proto=v1.3.1 +opentelemetry-proto=v1.3.2 opentracing-cpp=v1.6.0 prometheus-cpp=v1.2.4 vcpkg=2024.02.14 From dbd38cc63ba0e037b5fe22e061120d6cfa8e0858 Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Thu, 4 Jul 2024 19:20:26 +0200 Subject: [PATCH 2/8] format --- docs/maintaining-dependencies.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/maintaining-dependencies.md b/docs/maintaining-dependencies.md index 2566bf90c5..49df90a383 100644 --- a/docs/maintaining-dependencies.md +++ b/docs/maintaining-dependencies.md @@ -63,7 +63,7 @@ Author: jack-berg <34418638+jack-berg@users.noreply.github.com> Date: Fri Jun 28 08:19:11 2024 -0500 Prepare changelog for 1.3.2 release (#563) - + Co-authored-by: Armin Ruech <7052238+arminru@users.noreply.github.com> ``` @@ -204,7 +204,7 @@ Update the tag in the CMake logic: Typical change: ```shell -[malff@malff-desktop opentelemetry-cpp]$ git diff cmake/opentelemetry-proto.cmake +[malff@malff-desktop opentelemetry-cpp]$ git diff cmake/opentelemetry-proto.cmake diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake index 19516c3b..dd6213c1 100644 --- a/cmake/opentelemetry-proto.cmake From 681ea9487aa6c7d618558a40193c543d29c150be Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Thu, 4 Jul 2024 19:26:48 +0200 Subject: [PATCH 3/8] format --- docs/maintaining-dependencies.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/maintaining-dependencies.md b/docs/maintaining-dependencies.md index 49df90a383..acf39da6b9 100644 --- a/docs/maintaining-dependencies.md +++ b/docs/maintaining-dependencies.md @@ -31,7 +31,7 @@ git log -1 The current submodule show something like: -``` +```shell commit b3060d2104df364136d75a35779e6bd48bac449a (HEAD, tag: v1.3.1) Author: Damien Mathieu <42@dmathieu.com> Date: Thu Apr 25 17:55:35 2024 +0200 @@ -57,7 +57,7 @@ Check the new state: git log -1 ``` -``` +```shell commit 40b3c1b746767cbc13c2e39da3eaf1a23e54ffdd (HEAD, tag: v1.3.2) Author: jack-berg <34418638+jack-berg@users.noreply.github.com> Date: Fri Jun 28 08:19:11 2024 -0500 @@ -74,7 +74,7 @@ cd ../.. git status ``` -``` +```shell On branch upgrade_proto_1.3.2 Changes not staged for commit: (use "git add ..." to update what will be committed) @@ -92,7 +92,7 @@ git add third_party/opentelemetry-proto Update the line pointing to the opentelemetry-proto tag. -``` +```text opentelemetry-proto=v1.3.2 ``` @@ -113,14 +113,13 @@ index 0bbf67f3..7362473f 100644 opentracing-cpp=v1.6.0 prometheus-cpp=v1.2.4 vcpkg=2024.02.14 - ``` #### file bazel/repository.bzl Locate the entry for opentelemetry-proto: -``` +```text # OTLP Protocol definition maybe( http_archive, @@ -136,7 +135,7 @@ Locate the entry for opentelemetry-proto: Update the URL to the new tag: -``` +```text urls = [ "https://github.com/open-telemetry/opentelemetry-proto/archive/v1.3.2.tar.gz", ], @@ -144,7 +143,7 @@ Update the URL to the new tag: Update strip_prefix to the new tag: -``` +```text strip_prefix = "opentelemetry-proto-1.3.2", ``` @@ -160,13 +159,13 @@ Run a checksum on the new file: sha256sum v1.3.2.tar.gz ``` -``` +```shell c069c0d96137cf005d34411fa67dd3b6f1f8c64af1e7fb2fe0089a41c425acd7 v1.3.2.tar.gz ``` Update the checksum in file bazel/repository.bzl: -``` +```text sha256 = "c069c0d96137cf005d34411fa67dd3b6f1f8c64af1e7fb2fe0089a41c425acd7", ``` @@ -197,7 +196,7 @@ index bac1e45b..508b95a3 100644 Update the tag in the CMake logic: -``` +```cmake set(opentelemetry-proto "v1.3.2") ``` @@ -235,7 +234,7 @@ If missing, file a PR to add it, or contact the maintainer: Update the opentelemetry-proto version to the new tag: -``` +```text bazel_dep(name = "opentelemetry-proto", version = "1.3.2", repo_name = "com_github_opentelemetry_proto") ``` @@ -257,4 +256,3 @@ index 7b84c2b7..3161ffb1 100644 bazel_dep(name = "platforms", version = "0.0.8") bazel_dep(name = "prometheus-cpp", version = "1.2.4", repo_name = "com_github_jupp0r_prometheus_cpp") ``` - From 688120e2e3989970b8987c898eff684ac23cfdbd Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Sat, 6 Jul 2024 17:59:17 +0200 Subject: [PATCH 4/8] Added comments. --- docs/maintaining-dependencies.md | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/maintaining-dependencies.md b/docs/maintaining-dependencies.md index acf39da6b9..0b62316bce 100644 --- a/docs/maintaining-dependencies.md +++ b/docs/maintaining-dependencies.md @@ -1,8 +1,29 @@ # Maintaining dependencies +In general, several places in the code base need to be adjusted when +upgrading a dependency to a new version. + +This documentation contains notes about which place to fix, +to make maintenance easier and less error prone. + +This doc is only useful when up to date, +so make sure to add details about missing parts if any. + +Also, another good place to start when upgrading something to N+1 +is to find the commit that upgraded to version N (use `git blame`), +and inspect the commit for the last upgrade. + ## opentelemetry-proto +### Comments + +Unlike other opentelemetry SIGs, opentelemetry-cpp generates code +from opentelemetry-proto as part of the opentelemetry-cpp build. + +Only the source code of opentelemetry-proto is required, +which is why this repository is used as a git submodule under third_party. + ### Origin The repository for opentelemetry-proto is: @@ -238,6 +259,13 @@ Update the opentelemetry-proto version to the new tag: bazel_dep(name = "opentelemetry-proto", version = "1.3.2", repo_name = "com_github_opentelemetry_proto") ``` +File `MODULE.bazel` is not used in the github CI for repository +opentelemetry-cpp, so using a tag that does not exist (yet) in bazel central +will not break the CI build. + +See the known issues section. + + Typical change: ```shell @@ -256,3 +284,18 @@ index 7b84c2b7..3161ffb1 100644 bazel_dep(name = "platforms", version = "0.0.8") bazel_dep(name = "prometheus-cpp", version = "1.2.4", repo_name = "com_github_jupp0r_prometheus_cpp") ``` + +### Known issues + +For bazel, two different methods to build exists. + +First, the code can build using file `bazel/repository.bzl`. +This option does not depend on bazel central, and is used in CI today. + +Secondly, there is also a build using modules, with file `MODULE.bazel`. + +Using modules does introduce a dependency on bazel central. +Users building locally with modules will need the opentelemetry-proto build +with the new tag to exist in bazel central. + +This work with modules is still experimental, and subject to change. From 2df5c779efc1361a468c199929026a411dd0fd5f Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Sat, 6 Jul 2024 18:03:02 +0200 Subject: [PATCH 5/8] format --- docs/maintaining-dependencies.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/maintaining-dependencies.md b/docs/maintaining-dependencies.md index 0b62316bce..4cda572f5f 100644 --- a/docs/maintaining-dependencies.md +++ b/docs/maintaining-dependencies.md @@ -265,7 +265,6 @@ will not break the CI build. See the known issues section. - Typical change: ```shell From 5ae0bab44400945fac6ade1796b0079d2016bbf3 Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Sat, 6 Jul 2024 18:24:36 +0200 Subject: [PATCH 6/8] Added comments. --- docs/maintaining-dependencies.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/maintaining-dependencies.md b/docs/maintaining-dependencies.md index 4cda572f5f..4a62fe7ce4 100644 --- a/docs/maintaining-dependencies.md +++ b/docs/maintaining-dependencies.md @@ -24,6 +24,12 @@ from opentelemetry-proto as part of the opentelemetry-cpp build. Only the source code of opentelemetry-proto is required, which is why this repository is used as a git submodule under third_party. +Code is generated by the protobuf compiler during the build, +so that generated code is never checked in. + +This allows more flexibility, should the compiler (protobuf) be +upgraded even when the source code (opentelemetry-proto) is unchanged. + ### Origin The repository for opentelemetry-proto is: From a9431728cb4205ccccd27e3b5923ef8513aed961 Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Sat, 6 Jul 2024 19:04:09 +0200 Subject: [PATCH 7/8] Added comments. --- docs/maintaining-dependencies.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/maintaining-dependencies.md b/docs/maintaining-dependencies.md index 4a62fe7ce4..779fc1ea65 100644 --- a/docs/maintaining-dependencies.md +++ b/docs/maintaining-dependencies.md @@ -249,6 +249,26 @@ index 19516c3b..dd6213c1 100644 If opentelemetry-proto contains new files, the makefile needs to be adjusted to build the new code. +Depending on the actual changes in the opentelemetry-proto files, +the C++ code in opentelemetry-cpp may need adjustments. + +Typically, when opentelemetry-proto: + +* defines a new message (for example, for profiling) +* adds new optional fields to an existing message (for example, trace flags) + +the existing C++ code should work as is with the new opentelemetry-proto +format. + +In this case, it is better to: + +* upgrade the opentelemetry-proto version independently with one PR. +* upgrade the C++ code later (to use the new message, of provide new fields) + in a different PR. + +When the C++ code requires a newer minimum version of opentelemetry-proto, +make sure to document this, including in the release notes. + #### file MODULE.bazel Make sure the new tag is available in bazel central: From 4456dc8e5aaf2c7e35ee62f5a10184bdf2e08ade Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Wed, 17 Jul 2024 23:32:15 +0200 Subject: [PATCH 8/8] Fix doc, now that moduels are used in CI. --- docs/maintaining-dependencies.md | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/docs/maintaining-dependencies.md b/docs/maintaining-dependencies.md index 779fc1ea65..a1969785d8 100644 --- a/docs/maintaining-dependencies.md +++ b/docs/maintaining-dependencies.md @@ -285,9 +285,9 @@ Update the opentelemetry-proto version to the new tag: bazel_dep(name = "opentelemetry-proto", version = "1.3.2", repo_name = "com_github_opentelemetry_proto") ``` -File `MODULE.bazel` is not used in the github CI for repository +File `MODULE.bazel` is used in the github CI for repository opentelemetry-cpp, so using a tag that does not exist (yet) in bazel central -will not break the CI build. +will break the CI build. See the known issues section. @@ -315,12 +315,7 @@ index 7b84c2b7..3161ffb1 100644 For bazel, two different methods to build exists. First, the code can build using file `bazel/repository.bzl`. -This option does not depend on bazel central, and is used in CI today. +This option does not depend on bazel central. Secondly, there is also a build using modules, with file `MODULE.bazel`. - -Using modules does introduce a dependency on bazel central. -Users building locally with modules will need the opentelemetry-proto build -with the new tag to exist in bazel central. - -This work with modules is still experimental, and subject to change. +This option does depend on bazel central, and CI depends on it.