feat(common): new *Option to configure HTTP proxy (#12766)

Applications can use this option to configure the HTTP proxy settings.
Some applications need to set the username and password and this cannot
be done through environment variables (for `libcurl`).

google/cloud/common_options.h

@@ -19,6 +19,7 @@
 #include "google/cloud/version.h"
 #include <set>
 #include <string>
+#include <utility>
 #include <vector>
 
 namespace google {
@@ -106,6 +107,108 @@ struct AuthorityOption {
   using Type = std::string;
 };
 
+/**
+ * The configuration for a HTTP Proxy.
+ *
+ * This configuration can be used for both REST-based and gRPC-based clients.
+ * The client library sets the underlying configuration parameters based on
+ * the values in this struct.
+ *
+ * The full URI is constructed as:
+ *
+ * {scheme}://{username}:{password}@{hostname}:{port}
+ *
+ * Any empty values are omitted, except for the `scheme` which defaults to
+ * `https`. If the `hostname` value is empty, no HTTP proxy is configured.
+ */
+class ProxyConfig {
+ public:
+  ProxyConfig() = default;
+
+  /// The HTTP proxy host.
+  std::string const& hostname() const { return hostname_; }
+
+  /// The HTTP proxy port.
+  std::string const& port() const { return port_; }
+
+  /// The HTTP proxy username.
+  std::string const& username() const { return username_; }
+
+  /// The HTTP proxy password.
+  std::string const& password() const { return password_; }
+
+  /// The HTTP proxy scheme (http or https).
+  std::string const& scheme() const { return scheme_; }
+
+  ///@{
+  ///@ name Modifiers.
+  ProxyConfig& set_hostname(std::string v) & {
+    hostname_ = std::move(v);
+    return *this;
+  }
+  ProxyConfig&& set_hostname(std::string v) && {
+    return std::move(set_hostname(std::move(v)));
+  }
+
+  ProxyConfig& set_port(std::string v) & {
+    port_ = std::move(v);
+    return *this;
+  }
+  ProxyConfig&& set_port(std::string v) && {
+    return std::move(set_port(std::move(v)));
+  }
+
+  ProxyConfig& set_username(std::string v) & {
+    username_ = std::move(v);
+    return *this;
+  }
+  ProxyConfig&& set_username(std::string v) && {
+    return std::move(set_username(std::move(v)));
+  }
+
+  ProxyConfig& set_password(std::string v) & {
+    password_ = std::move(v);
+    return *this;
+  }
+  ProxyConfig&& set_password(std::string v) && {
+    return std::move(set_password(std::move(v)));
+  }
+
+  ProxyConfig& set_scheme(std::string v) & {
+    scheme_ = std::move(v);
+    return *this;
+  }
+  ProxyConfig&& set_scheme(std::string v) && {
+    return std::move(set_scheme(std::move(v)));
+  }
+  ///@}
+
+ private:
+  std::string hostname_;
+  std::string port_;
+  std::string username_;
+  std::string password_;
+  std::string scheme_ = "https";
+};
+
+/**
+ * Configure the HTTP Proxy.
+ *
+ * Both HTTP and gRPC-based clients can be configured to use an HTTP proxy for
+ * requests. Setting the `ProxyOption` will configure the client to use a
+ * proxy as described by the `ProxyConfig` value.
+ *
+ * @see https://github.com/grpc/grpc/blob/master/doc/core/default_http_proxy_mapper.md
+ * @see https://curl.se/libcurl/c/CURLOPT_PROXYUSERNAME.html
+ * @see https://curl.se/libcurl/c/CURLOPT_PROXYPASSWORD.html
+ * @see https://curl.se/libcurl/c/CURLOPT_PROXY.html
+ *
+ * @ingroup options
+ */
+struct ProxyOption {
+  using Type = ProxyConfig;
+};
+
 /**
  * A list of all the common options.
  */

google/cloud/grpc_options.cc

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 #include "google/cloud/grpc_options.h"
-#include "google/cloud/common_options.h"
+#include "google/cloud/internal/absl_str_cat_quiet.h"
 #include "google/cloud/internal/absl_str_join_quiet.h"
 #include "google/cloud/internal/background_threads_impl.h"
 
@@ -34,6 +34,23 @@ void ConfigurePollContext(grpc::ClientContext& context, Options const& opts) {
   }
 }
 
+std::string MakeGrpcHttpProxy(ProxyConfig const& config) {
+  if (config.hostname().empty()) return {};
+  auto result = absl::StrCat(config.scheme(), "://");
+  char const* sep = "";
+  if (!config.username().empty()) {
+    sep = "@";
+    absl::StrAppend(&result, config.username());
+  }
+  if (!config.password().empty()) {
+    sep = "@";
+    absl::StrAppend(&result, ":", config.password());
+  }
+  absl::StrAppend(&result, sep, config.hostname());
+  if (!config.port().empty()) absl::StrAppend(&result, ":", config.port());
+  return result;
+}
+
 grpc::ChannelArguments MakeChannelArguments(Options const& opts) {
   auto channel_arguments = opts.get<GrpcChannelArgumentsNativeOption>();
   for (auto const& p : opts.get<GrpcChannelArgumentsOption>()) {
@@ -62,6 +79,9 @@ grpc::ChannelArguments MakeChannelArguments(Options const& opts) {
                              static_cast<int>(kKeepaliveTimeout.count()));
   }
 
+  auto const proxy = MakeGrpcHttpProxy(opts.get<ProxyOption>());
+  if (!proxy.empty()) channel_arguments.SetString(GRPC_ARG_HTTP_PROXY, proxy);
+
   return channel_arguments;
 }
 

google/cloud/grpc_options.h

@@ -16,6 +16,7 @@
 #define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_GRPC_OPTIONS_H
 
 #include "google/cloud/background_threads.h"
+#include "google/cloud/common_options.h"
 #include "google/cloud/completion_queue.h"
 #include "google/cloud/options.h"
 #include "google/cloud/tracing_options.h"
@@ -217,6 +218,9 @@ void ConfigureContext(grpc::ClientContext& context, Options const& opts);
 /// Configure the ClientContext for polling operations using options.
 void ConfigurePollContext(grpc::ClientContext& context, Options const& opts);
 
+/// Creates the value for GRPC_ARG_HTTP_PROXY based on @p config.
+std::string MakeGrpcHttpProxy(ProxyConfig const& config);
+
 /// Creates a new `grpc::ChannelArguments` configured with @p opts.
 grpc::ChannelArguments MakeChannelArguments(Options const& opts);
 

google/cloud/grpc_options_test.cc

@@ -92,6 +92,42 @@ TEST(GrpcOptionList, RegularOptions) {
   TestGrpcOption<GrpcTracingOptionsOption>(TracingOptions{});
 }
 
+TEST(GrpcChannelArguments, MakeGrpcHttpProxy) {
+  EXPECT_EQ(internal::MakeGrpcHttpProxy(ProxyConfig()), "");
+  EXPECT_EQ(internal::MakeGrpcHttpProxy(ProxyConfig().set_port("port")), "");
+  EXPECT_EQ(internal::MakeGrpcHttpProxy(ProxyConfig().set_hostname("hostname")),
+            "https://hostname");
+  EXPECT_EQ(internal::MakeGrpcHttpProxy(
+                ProxyConfig().set_hostname("hostname").set_port("port")),
+            "https://hostname:port");
+  EXPECT_EQ(internal::MakeGrpcHttpProxy(ProxyConfig()
+                                            .set_hostname("hostname")
+                                            .set_port("port")
+                                            .set_username("username")),
+            "https://username@hostname:port");
+  EXPECT_EQ(internal::MakeGrpcHttpProxy(ProxyConfig()
+                                            .set_hostname("hostname")
+                                            .set_port("port")
+                                            .set_password("password")),
+            "https://:password@hostname:port");
+  EXPECT_EQ(internal::MakeGrpcHttpProxy(ProxyConfig()
+                                            .set_hostname("hostname")
+                                            .set_port("port")
+                                            .set_username("username")
+                                            .set_password("password")),
+            "https://username:password@hostname:port");
+  EXPECT_EQ(internal::MakeGrpcHttpProxy(ProxyConfig()
+                                            .set_hostname("hostname")
+                                            .set_port("port")
+                                            .set_username("username")
+                                            .set_password("password")
+                                            .set_scheme("http")),
+            // Split http to avoid tidy
+            "ht"
+            "tp"
+            "://username:password@hostname:port");
+}
+
 TEST(GrpcChannelArguments, MakeChannelArguments) {
   // This test will just set all 3 options related to channel arguments and
   // ensure that `MakeChannelArguments` combines them in the correct order.
@@ -101,7 +137,8 @@ TEST(GrpcChannelArguments, MakeChannelArguments) {
   auto opts = Options{}
                   .set<GrpcChannelArgumentsOption>({{"baz", "quux"}})
                   .set<UserAgentProductsOption>({"user_agent"})
-                  .set<GrpcChannelArgumentsNativeOption>(native);
+                  .set<GrpcChannelArgumentsNativeOption>(native)
+                  .set<ProxyOption>(ProxyConfig().set_hostname("hostname"));
 
   grpc::ChannelArguments expected;
   expected.SetString("foo", "bar");
@@ -111,6 +148,7 @@ TEST(GrpcChannelArguments, MakeChannelArguments) {
                   static_cast<int>(ms(std::chrono::hours(24)).count()));
   expected.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS,
                   static_cast<int>(ms(std::chrono::seconds(60)).count()));
+  expected.SetString(GRPC_ARG_HTTP_PROXY, "https://hostname");
 
   CheckGrpcChannelArguments(expected, internal::MakeChannelArguments(opts));
 }

google/cloud/internal/curl_impl.cc

@@ -193,6 +193,10 @@ CurlImpl::CurlImpl(CurlHandle handle,
   transfer_stall_minimum_rate_ = options.get<TransferStallMinimumRateOption>();
   download_stall_timeout_ = options.get<DownloadStallTimeoutOption>();
   download_stall_minimum_rate_ = options.get<DownloadStallMinimumRateOption>();
+
+  proxy_ = CurlOptProxy(options);
+  proxy_username_ = CurlOptProxyUsername(options);
+  proxy_password_ = CurlOptProxyPassword(options);
 }
 
 CurlImpl::~CurlImpl() {
@@ -304,6 +308,18 @@ Status CurlImpl::MakeRequest(HttpMethod method, RestContext& context,
   status =
       handle_.SetOption(CURLOPT_FOLLOWLOCATION, follow_location_ ? 1L : 0L);
   if (!status.ok()) return OnTransferError(context, std::move(status));
+  if (proxy_) {
+    status = handle_.SetOption(CURLOPT_PROXY, proxy_->c_str());
+    if (!status.ok()) return OnTransferError(context, std::move(status));
+  }
+  if (proxy_username_) {
+    status = handle_.SetOption(CURLOPT_PROXYUSERNAME, proxy_username_->c_str());
+    if (!status.ok()) return OnTransferError(context, std::move(status));
+  }
+  if (proxy_password_) {
+    status = handle_.SetOption(CURLOPT_PROXYPASSWORD, proxy_password_->c_str());
+    if (!status.ok()) return OnTransferError(context, std::move(status));
+  }
 
   if (method == HttpMethod::kGet) {
     status = handle_.SetOption(CURLOPT_NOPROGRESS, 1L);
@@ -749,6 +765,28 @@ void CurlImpl::OnTransferDone() {
   factory_->CleanupMultiHandle(std::move(multi_), HandleDisposition::kKeep);
 }
 
+absl::optional<std::string> CurlOptProxy(Options const& options) {
+  if (!options.has<ProxyOption>()) return absl::nullopt;
+  auto const& cfg = options.get<ProxyOption>();
+  if (cfg.hostname().empty()) return absl::nullopt;
+  if (cfg.port().empty()) {
+    return absl::StrCat(cfg.scheme(), "://", cfg.hostname());
+  }
+  return absl::StrCat(cfg.scheme(), "://", cfg.hostname(), ":", cfg.port());
+}
+
+absl::optional<std::string> CurlOptProxyUsername(Options const& options) {
+  auto const& cfg = options.get<ProxyOption>();
+  if (cfg.username().empty()) return absl::nullopt;
+  return cfg.username();
+}
+
+absl::optional<std::string> CurlOptProxyPassword(Options const& options) {
+  auto const& cfg = options.get<ProxyOption>();
+  if (cfg.password().empty()) return absl::nullopt;
+  return cfg.password();
+}
+
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace rest_internal
 }  // namespace cloud

google/cloud/internal/curl_impl.h

@@ -24,6 +24,7 @@
 #include "google/cloud/options.h"
 #include "google/cloud/status_or.h"
 #include "google/cloud/version.h"
+#include "absl/types/optional.h"
 #include "absl/types/span.h"
 #include <array>
 #include <chrono>
@@ -136,6 +137,10 @@ class CurlImpl {
   std::chrono::seconds download_stall_timeout_;
   std::uint32_t download_stall_minimum_rate_;
 
+  absl::optional<std::string> proxy_;
+  absl::optional<std::string> proxy_username_;
+  absl::optional<std::string> proxy_password_;
+
   CurlReceivedHeaders received_headers_;
   std::string url_;
   HttpStatusCode http_code_;
@@ -172,6 +177,15 @@ class CurlImpl {
   SpillBuffer spill_;
 };
 
+/// Compute the CURLOPT_PROXY setting from @p options.
+absl::optional<std::string> CurlOptProxy(Options const& options);
+
+/// Compute the CURLOPT_PROXYUSERNAME setting from @p options.
+absl::optional<std::string> CurlOptProxyUsername(Options const& options);
+
+/// Compute the CURLOPT_PROXYPASSWORD setting from @p options.
+absl::optional<std::string> CurlOptProxyPassword(Options const& options);
+
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace rest_internal
 }  // namespace cloud

google/cloud/internal/curl_impl_test.cc

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "google/cloud/internal/curl_impl.h"
+#include "google/cloud/common_options.h"
 #include "google/cloud/testing_util/status_matchers.h"
 #include <gmock/gmock.h>
 #include <vector>
@@ -146,6 +147,41 @@ TEST_F(CurlImplTest, SetUrlPathContainsHttp) {
   EXPECT_THAT(impl.url(), Eq("HTTP://endpoint.googleapis.com/resource/verb"));
 }
 
+TEST_F(CurlImplTest, CurlOptProxy) {
+  EXPECT_EQ(CurlOptProxy(Options{}), absl::nullopt);
+  EXPECT_EQ(CurlOptProxy(Options{}.set<ProxyOption>(
+                ProxyConfig().set_hostname("hostname"))),
+            absl::make_optional(std::string("https://hostname")));
+  EXPECT_EQ(
+      CurlOptProxy(Options{}.set<ProxyOption>(ProxyConfig()
+                                                  .set_hostname("hostname")
+                                                  .set_port("1080")
+                                                  .set_scheme("http"))),
+      absl::make_optional(std::string("htt" /*silence*/ "p://hostname:1080")));
+}
+
+TEST_F(CurlImplTest, CurlOptProxyUsername) {
+  EXPECT_EQ(CurlOptProxyUsername(Options{}), absl::nullopt);
+  EXPECT_EQ(CurlOptProxyUsername(Options{}.set<ProxyOption>(
+                ProxyConfig().set_hostname("hostname"))),
+            absl::nullopt);
+  EXPECT_EQ(
+      CurlOptProxyUsername(Options{}.set<ProxyOption>(
+          ProxyConfig().set_hostname("hostname").set_username("username"))),
+      absl::make_optional(std::string("username")));
+}
+
+TEST_F(CurlImplTest, CurlOptProxyPassword) {
+  EXPECT_EQ(CurlOptProxyPassword(Options{}), absl::nullopt);
+  EXPECT_EQ(CurlOptProxyPassword(Options{}.set<ProxyOption>(
+                ProxyConfig().set_hostname("hostname"))),
+            absl::nullopt);
+  EXPECT_EQ(
+      CurlOptProxyPassword(Options{}.set<ProxyOption>(
+          ProxyConfig().set_hostname("hostname").set_password("password"))),
+      absl::make_optional(std::string("password")));
+}
+
 }  // namespace
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace rest_internal