Merge branch 'main' into mixin-pubsub

.github/workflows/macos-bazel.yml

@@ -38,6 +38,7 @@ jobs:
         - Spanner
         - Storage
         - Compute
+        - AIPlatform
         - Shard1
         - Shard2
         - Other
@@ -65,6 +66,9 @@ jobs:
         - shard: Compute
           targets:
           - //google/cloud/compute/...
+        - shard: AIPlatform
+          targets:
+          - //google/cloud/aiplatform/...
         - shard: Shard1
           targets:
           # Run this:
@@ -87,7 +91,6 @@ jobs:
           - +//google/cloud/sql/...
           - +//google/cloud/dialogflow_cx/...
           - +//google/cloud/dialogflow_es/...
-          - +//google/cloud/aiplatform/...
         - shard: Shard2
           targets:
           - //google/cloud/networkconnectivity/...
@@ -142,6 +145,8 @@ jobs:
           - -//google/cloud/storage/...
           # From Compute
           - -//google/cloud/compute/...
+          # From AIPlatform
+          - -//google/cloud/aiplatform/...
           # From Shard1
           - -//google/cloud/appengine/...
           - -//google/cloud/dataproc/...
@@ -151,7 +156,6 @@ jobs:
           - -//google/cloud/sql/...
           - -//google/cloud/dialogflow_cx/...
           - -//google/cloud/dialogflow_es/...
-          - -//google/cloud/aiplatform/...
           # From Shard2
           - -//google/cloud/networkconnectivity/...
           - -//google/cloud/networkservices/...

.github/workflows/macos-cmake.yml

@@ -40,11 +40,13 @@ jobs:
       matrix:
         exclude-from-full-trick: [ true ]
         os: [ macos-14 ]
-        shard: [ Core1, Core2, Compute, Shard1, Shard2, Shard3, Shard4, Shard5, Other ]
+        shard: [ Core1, Core2, Compute, AIPlatform, Shard1, Shard2, Shard3, Shard4, Shard5, Other ]
         exclude:
         # Only full builds include shards with generated code.
         - exclude-from-full-trick: ${{ ! inputs.full-matrix }}
           shard: Compute
+        - exclude-from-full-trick: ${{ ! inputs.full-matrix }}
+          shard: AIPlatform
         - exclude-from-full-trick: ${{ ! inputs.full-matrix }}
           shard: Shard1
         - exclude-from-full-trick: ${{ ! inputs.full-matrix }}
@@ -80,6 +82,7 @@ jobs:
           pubsub
           pubsublite
           storage
+          storage_grpc
         )
         # These are the libraries with the most "clients". To build the list
         # run something like this and find the midpoint:
@@ -94,7 +97,6 @@ jobs:
           monitoring
           retail
           discoveryengine
-          aiplatform
         )
         shard2_features=(
           servicecontrol
@@ -210,6 +212,8 @@ jobs:
           echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
         elif [[ "${{matrix.shard}}" == "Compute" ]]; then
           echo "features=compute" >> "${GITHUB_OUTPUT}"
+        elif [[ "${{matrix.shard}}" == "AIPlatform" ]]; then
+          echo "features=aiplatform" >> "${GITHUB_OUTPUT}"
         elif [[ "${{ matrix.shard }}" == "Shard1" ]]; then
           features="$(printf ",%s" "${shard1_features[@]}")"
           echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
@@ -229,6 +233,7 @@ jobs:
           skipped_features=("${core1_features[@]}")
           skipped_features+=("${core2_features[@]}")
           skipped_features+=(compute)
+          skipped_features+=(aiplatform)
           skipped_features+=("${shard1_features[@]}")
           skipped_features+=("${shard2_features[@]}")
           skipped_features+=("${shard3_features[@]}")

.github/workflows/windows-bazel.yml

@@ -34,7 +34,7 @@ jobs:
         # - dbg creates very large debugging files and GHA has limited storage.
         # - fastbuild also takes too much storage.
         compilation_mode: [ opt ]
-        shard: [ Core, Compute, Other ]
+        shard: [ Core, Compute, AIPlatform, Other ]
         include:
         - shard: Core
           targets:
@@ -49,6 +49,9 @@ jobs:
         - shard: Compute
           targets:
           - //google/cloud/compute/...
+        - shard: AIPlatform
+          targets:
+          - //google/cloud/aiplatform/...
         - shard: Other
           targets:
           - //...
@@ -65,6 +68,8 @@ jobs:
           - -//google/cloud/storage/...
           # From Compute
           - -//google/cloud/compute/...
+          # From AIPlatform
+          - -//google/cloud/aiplatform/...
     steps:
     - uses: actions/checkout@v4
       with:
@@ -81,6 +86,9 @@ jobs:
     # Note that in other runners the publisher is GitHub. If we trust GitHub
     # to run the VM, we should trust their runners.
     - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # @v1.13.0
+    - name: Pre Build Disk Space
+      shell: bash
+      run: df -m
     - name: Build google-cloud-cpp
       shell: bash
       run: |
@@ -93,6 +101,9 @@ jobs:
         export BAZEL_REMOTE_CACHE_RW_MODE=${{ inputs.bazel-cache-mode }}
         export EXECUTE_INTEGRATION_TESTS=${{ inputs.execute-integration-tests }}
         ci/gha/builds/windows-bazel.sh ${{ matrix.compilation_mode }} ${{ join(matrix.targets, ' ') }}
+    - name: Post Build Disk Space
+      shell: bash
+      run: df -m
     env:
       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/cloud-cpp-community-gha-cache/bazel-cache/${{ matrix.msvc }}/${{ matrix.compilation_mode }}
       GHA_TEST_BUCKET: "gcs-grpc-team-cloud-cpp-testing-bucket"

.github/workflows/windows-cmake.yml

@@ -42,11 +42,13 @@ jobs:
         msvc: [ msvc-2022 ]
         build_type: [ Debug, Release ]
         arch: [ x64, x86 ]
-        shard: [Core1, Core2, Compute, Shard1, Shard2, Shard3, Other]
+        shard: [Core1, Core2, Core3, Compute, AIPlatform, Shard1, Shard2, Shard3, Other]
         exclude:
         # Also skip shards (Compute and Other) that contain only generated code.
         - exclude-from-full-trick: ${{ ! inputs.full-matrix }}
           shard: Compute
+        - exclude-from-full-trick: ${{ ! inputs.full-matrix }}
+          shard: AIPlatform
         - exclude-from-full-trick: ${{ ! inputs.full-matrix }}
           shard: Shard1
         - exclude-from-full-trick: ${{ ! inputs.full-matrix }}
@@ -80,7 +82,8 @@ jobs:
       run: |
         echo "vcpkg-version=$(cat ci/etc/vcpkg-version.txt)" >> "${GITHUB_OUTPUT}"
         core1_features=(bigtable pubsub pubsublite)
-        core2_features=(spanner storage)
+        core2_features=(spanner)
+        core3_features=(storage storage_grpc)
         # These are the libraries with the most "clients". To build the list
         # run something like this and create shards:
         #
@@ -98,7 +101,6 @@ jobs:
           sql
           dialogflow_cx
           dialogflow_es
-          aiplatform
         )
         shard2_features=(
           beyondcorp
@@ -174,8 +176,13 @@ jobs:
         elif [[ "${{ matrix.shard }}" == "Core2" ]]; then
           features="$(printf ",%s" "${core2_features[@]}")"
           echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
+        elif [[ "${{ matrix.shard }}" == "Core3" ]]; then
+          features="$(printf ",%s" "${core3_features[@]}")"
+          echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
         elif [[ "${{matrix.shard}}" == "Compute" ]]; then
           echo "features=compute" >> "${GITHUB_OUTPUT}"
+        elif [[ "${{matrix.shard}}" == "AIPlatform" ]]; then
+          echo "features=aiplatform" >> "${GITHUB_OUTPUT}"
         elif [[ "${{matrix.shard}}" == "Shard1" ]]; then
           features="$(printf ",%s" "${shard1_features[@]}")"
           echo "features=${features:1}" >> "${GITHUB_OUTPUT}"
@@ -188,7 +195,9 @@ jobs:
         else
           skipped_features=("${core1_features[@]}")
           skipped_features+=("${core2_features[@]}")
+          skipped_features+=("${core3_features[@]}")
           skipped_features+=(compute)
+          skipped_features+=(aiplatform)
           skipped_features+=("${shard1_features[@]}")
           skipped_features+=("${shard2_features[@]}")
           skipped_features+=("${shard3_features[@]}")
@@ -199,6 +208,9 @@ jobs:
           skipped="$(printf ",-%s" "${skipped_features[@]}")"
           echo "features=__ga_libraries__,__experimental_libraries__,${skipped:1}" >> "${GITHUB_OUTPUT}"
         fi
+    - name: Pre Build Disk Space
+      shell: bash
+      run: df -m
     - name: Download and Install CMake
       shell: bash
       run: |
@@ -241,6 +253,9 @@ jobs:
         export CMAKE_OUT='c:\b'
         export EXECUTE_INTEGRATION_TESTS=${{ inputs.execute-integration-tests }}
         ci/gha/builds/windows-cmake.sh ${{ matrix.build_type }} ${{ steps.dynamic.outputs.features }}
+    - name: Post Build Disk Space
+      shell: bash
+      run: df -m
     env:
       SCCACHE_GCS_BUCKET: cloud-cpp-community-gha-cache
       SCCACHE_GCS_KEY_PREFIX: sccache/${{ matrix.msvc }}/${{ matrix.arch}}/${{ matrix.build_type }}

ci/cloudbuild/builds/lib/universe_domain.sh

@@ -20,6 +20,8 @@ if ((CI_CLOUDBUILD_BUILDS_LIB_UNIVERSE_DOMAIN_SH__++ != 0)); then
   return 0
 fi # include guard
 
+source module ci/cloudbuild/builds/lib/bazel.sh
+
 # Only create the SA key file if the secret is available.
 if [[ -n "${UD_SERVICE_ACCOUNT}" ]]; then
   ORIG_UMASK=$(umask)
@@ -33,3 +35,12 @@ function ud::bazel_run() {
   io::log "Executing bazel run $1 with obscured arguments:"
   bazel run --ui_event_filters=-info -- "$@"
 }
+
+function ud::bazel_test() {
+  mapfile -t args < <(bazel::common_args)
+  io::log "Executing bazel test $1 with obscured arguments:"
+  bazel test "${args[@]}" --sandbox_add_mount_pair=/tmp \
+    --test_env=UD_SA_KEY_FILE="${UD_SA_KEY_FILE}" \
+    --test_env=UD_REGION="${UD_REGION}" \
+    --test_env=UD_PROJECT="${UD_PROJECT}" -- "$@"
+}

ci/cloudbuild/builds/universe-domain-demo.sh

@@ -28,6 +28,7 @@ export CXX=clang++
 if [[ -n "${UD_SA_KEY_FILE}" ]]; then
   ud::bazel_run //google/cloud/universe_domain/demo:kms_demo \
     "${UD_PROJECT}" "${UD_REGION}" "${UD_SA_KEY_FILE}"
+  ud::bazel_test //google/cloud/storage/tests:universe_domain_integration_test
 else
   source module ci/etc/integration-tests-config.sh
   bazel run -- //google/cloud/universe_domain/demo:kms_demo \

examples/api_key.cc

@@ -46,6 +46,7 @@ google::api::apikeys::v2::Key CreateApiKey(
   google::api::apikeys::v2::CreateKeyRequest request;
   request.set_parent(location.FullName());
   request.mutable_key()->set_display_name(std::move(display_name));
+  // As an example, restrict the API key's scope to the Natural Language API.
   request.mutable_key()->mutable_restrictions()->add_api_targets()->set_service(
       "language.googleapis.com");
 

google/cloud/pubsub/samples/topic_admin_samples.cc

@@ -23,6 +23,8 @@
 #include "google/cloud/project.h"
 #include "google/cloud/testing_util/example_driver.h"
 #include "absl/strings/match.h"
+#include <google/protobuf/timestamp.pb.h>
+#include <google/protobuf/util/time_util.h>
 #include <sstream>
 
 using google::cloud::pubsub::examples::Cleanup;
@@ -210,6 +212,65 @@ void CreateTopicWithKinesisIngestion(
    argv.at(4), argv.at(5));
 }
 
+void CreateTopicWithCloudStorageIngestion(
+    google::cloud::pubsub_admin::TopicAdminClient client,
+    std::vector<std::string> const& argv) {
+  //! [pubsub_create_topic_with_cloud_storage_ingestion]
+  namespace pubsub = ::google::cloud::pubsub;
+  namespace pubsub_admin = ::google::cloud::pubsub_admin;
+  [](pubsub_admin::TopicAdminClient client, std::string project_id,
+     std::string topic_id, std::string bucket, std::string input_format,
+     std::string text_delimiter, std::string match_glob,
+     std::string minimum_object_create_time) {
+    google::pubsub::v1::Topic request;
+    request.set_name(
+        pubsub::Topic(std::move(project_id), std::move(topic_id)).FullName());
+    auto* cloud_storage = request.mutable_ingestion_data_source_settings()
+                              ->mutable_cloud_storage();
+    cloud_storage->set_bucket(bucket);
+    if (input_format == "text") {
+      cloud_storage->mutable_text_format()->set_delimiter(text_delimiter);
+    } else if (input_format == "avro") {
+      cloud_storage->mutable_avro_format();
+    } else if (input_format == "pubsub_avro") {
+      cloud_storage->mutable_pubsub_avro_format();
+    } else {
+      std::cout << "input_format must be in ('text', 'avro', 'pubsub_avro'); "
+                   "got value: "
+                << input_format << std::endl;
+      return;
+    }
+
+    if (!match_glob.empty()) {
+      cloud_storage->set_match_glob(match_glob);
+    }
+
+    if (!minimum_object_create_time.empty()) {
+      google::protobuf::Timestamp timestamp;
+      if (!google::protobuf::util::TimeUtil::FromString(
+              minimum_object_create_time,
+              cloud_storage->mutable_minimum_object_create_time())) {
+        std::cout << "Invalid minimum object create time: "
+                  << minimum_object_create_time << std::endl;
+      }
+    }
+
+    auto topic = client.CreateTopic(request);
+    // Note that kAlreadyExists is a possible error when the library retries.
+    if (topic.status().code() == google::cloud::StatusCode::kAlreadyExists) {
+      std::cout << "The topic already exists\n";
+      return;
+    }
+    if (!topic) throw std::move(topic).status();
+
+    std::cout << "The topic was successfully created: " << topic->DebugString()
+              << "\n";
+  }
+  //! [pubsub_create_topic_with_cloud_storage_ingestion]
+  (std::move(client), argv.at(0), argv.at(1), argv.at(2), argv.at(3),
+   argv.at(4), argv.at(5), argv.at(6));
+}
+
 void GetTopic(google::cloud::pubsub_admin::TopicAdminClient client,
               std::vector<std::string> const& argv) {
   namespace pubsub = ::google::cloud::pubsub;
@@ -564,6 +625,9 @@ void AutoRun(std::vector<std::string> const& argv) {
       "fake-service-account@fake-gcp-project.iam.gserviceaccount.com";
   auto const* const kinesis_updated_gcp_service_account =
       "fake-update-service-account@fake-gcp-project.iam.gserviceaccount.com";
+  auto const cloud_storage_topic_id =
+      "cloud-storage-" + RandomTopicId(generator) + "_ingestion_topic";
+  auto const cloud_storage_bucket = project_id + "-pubsub-bucket";
 
   using ::google::cloud::StatusCode;
   auto ignore_emulator_failures =
@@ -613,7 +677,26 @@ void AutoRun(std::vector<std::string> const& argv) {
     DeleteTopic(topic_admin_client, {project_id, kinesis_topic_id});
   });
 
+  std::cout << "\nRunning CreateTopicWithCloudStorage() sample" << std::endl;
+
+  ignore_emulator_failures(
+      [&] {
+        CreateTopicWithCloudStorageIngestion(
+            topic_admin_client,
+            {project_id, cloud_storage_topic_id, "mikeprieto-bucket", "text",
+             "\n", "**.txt", "2024-09-26T00:00:00Z"});
+        std::cout << "\nAfter Running CreateTopicWithCloudStorage() sample"
+                  << std::endl;
+        cleanup.Defer([topic_admin_client, project_id,
+                       cloud_storage_topic_id]() mutable {
+          std::cout << "\nRunning DeleteTopic() sample" << std::endl;
+          DeleteTopic(topic_admin_client, {project_id, cloud_storage_topic_id});
+        });
+      },
+      StatusCode::kInvalidArgument);
+
   std::cout << "\nRunning UpdateTopicType() sample" << std::endl;
+
   UpdateTopicType(
       topic_admin_client,
       {project_id, kinesis_topic_id, kinesis_stream_arn, kinesis_consumer_arn,
@@ -679,6 +762,11 @@ int main(int argc, char* argv[]) {  // NOLINT(bugprone-exception-escape)
           {"project-id", "topic-id", "stream-arn", "consumer-arn",
            "aws-role-arn", "gcp-service-account"},
           CreateTopicWithKinesisIngestion),
+      CreateTopicAdminCommand(
+          "create-topic-with-cloud-storage-ingestion",
+          {"project-id", "topic-id", "bucket", "input-format", "text-delimiter",
+           "match-glob", "minimum-object-create-time"},
+          CreateTopicWithCloudStorageIngestion),
       CreateTopicAdminCommand(
           "create-topic-with-schema",
           {"project-id", "topic-id", "schema-id", "encoding"},

google/cloud/spanner/admin/integration_tests/backup_extra_integration_test.cc

@@ -288,8 +288,8 @@ TEST_F(BackupExtraIntegrationTest, CreateBackupWithVersionTime) {
 
 /// @test Verify creating a backup with an expired version_time fails.
 TEST_F(BackupExtraIntegrationTest, CreateBackupWithExpiredVersionTime) {
-  auto instance_id =
-      spanner_testing::PickRandomInstance(generator_, ProjectId());
+  auto instance_id = spanner_testing::PickRandomInstance(
+      generator_, ProjectId(), "NOT labels.edition:enterprise");
   ASSERT_THAT(instance_id, IsOk());
   Instance in(ProjectId(), *instance_id);
   Database db(in, spanner_testing::RandomDatabaseName(generator_));
@@ -333,8 +333,8 @@ TEST_F(BackupExtraIntegrationTest, CreateBackupWithExpiredVersionTime) {
 
 /// @test Verify creating a backup with a future version_time fails.
 TEST_F(BackupExtraIntegrationTest, CreateBackupWithFutureVersionTime) {
-  auto instance_id =
-      spanner_testing::PickRandomInstance(generator_, ProjectId());
+  auto instance_id = spanner_testing::PickRandomInstance(
+      generator_, ProjectId(), "NOT labels.edition:enterprise");
   ASSERT_THAT(instance_id, IsOk());
   Instance in(ProjectId(), *instance_id);
   Database db(in, spanner_testing::RandomDatabaseName(generator_));