WIP GCP XML support - dedicated proto type for upgrade path to JSON A…

…PI later

cpp/arcticdb/entity/protobufs.hpp

@@ -10,6 +10,7 @@
 #include <storage.pb.h>
 #include <encoding.pb.h>
 #include <s3_storage.pb.h>
+#include <gcp_storage.pb.h>
 #include <lmdb_storage.pb.h>
 #include <mongo_storage.pb.h>
 #include <in_memory_storage.pb.h>
@@ -24,6 +25,7 @@ namespace arcticdb::proto {
     namespace encoding = arcticc::pb2::encoding_pb2;
     namespace storage = arcticc::pb2::storage_pb2;
     namespace s3_storage = arcticc::pb2::s3_storage_pb2;
+    namespace gcp_storage = arcticc::pb2::gcp_storage_pb2;
     namespace lmdb_storage = arcticc::pb2::lmdb_storage_pb2;
     namespace mapped_file_storage = arcticc::pb2::mapped_file_storage_pb2;
     namespace mongo_storage = arcticc::pb2::mongo_storage_pb2;

cpp/arcticdb/storage/config_cache.hpp

@@ -90,10 +90,10 @@ class ConfigCache {
                     storages.emplace_back(create_storage(path, mode, s3::S3Settings(settings).update(s3_storage)));
                 },
                 [&storage_conf, &storages, &path, mode] (const s3::GCPXMLSettings& settings) {
-                    util::check(storage_conf.config().Is<arcticdb::proto::s3_storage::Config>(), "Only support S3 native settings");
-                    arcticdb::proto::s3_storage::Config s3_storage;
-                    storage_conf.config().UnpackTo(&s3_storage);
-                    storages.emplace_back(create_storage(path, mode, s3::GCPXMLSettings(settings).update(s3_storage)));
+                    util::check(storage_conf.config().Is<arcticdb::proto::gcp_storage::Config>(), "Only support S3 native settings");
+                    arcticdb::proto::gcp_storage::Config gcp_storage;
+                    storage_conf.config().UnpackTo(&gcp_storage);
+                    storages.emplace_back(create_storage(path, mode, s3::GCPXMLSettings(settings).update(gcp_storage)));
                 },
                 [&storage_conf, &storages, &path, mode](const auto &) {
                     storages.emplace_back(create_storage(path, mode, storage_conf));

cpp/arcticdb/storage/python_bindings.cpp

@@ -131,7 +131,27 @@ void register_bindings(py::module& storage, py::exception<arcticdb::ArcticExcept
         });
 
     py::class_<s3::GCPXMLSettings>(storage, "GCPXMLSettings")
-        .def(py::init<s3::S3Settings>());
+        .def(py::init<>())
+        .def(py::pickle(
+            [](const s3::GCPXMLSettings &settings) {
+                return py::make_tuple(settings.aws_auth(), settings.aws_profile());
+            },
+            [](py::tuple t) {
+                util::check(t.size() == 2, "Invalid GCPSMLSettings pickle objects");
+                s3::GCPXMLSettings settings(t[static_cast<uint32_t>(S3SettingsPickleOrder::AWS_AUTH)].cast<s3::AWSAuthMethod>(),
+                    t[static_cast<uint32_t>(S3SettingsPickleOrder::AWS_PROFILE)].cast<std::string>()
+                );
+                return settings;
+            }
+        ))
+        .def_property("bucket", &s3::GCPXMLSettings::bucket, &s3::GCPXMLSettings::set_bucket)
+        .def_property("endpoint", &s3::GCPXMLSettings::endpoint, &s3::GCPXMLSettings::set_endpoint)
+        .def_property("access", &s3::GCPXMLSettings::access, &s3::GCPXMLSettings::set_access)
+        .def_property("secret", &s3::GCPXMLSettings::secret, &s3::GCPXMLSettings::set_secret)
+        .def_property("prefix", &s3::GCPXMLSettings::prefix, &s3::GCPXMLSettings::set_prefix)
+        .def_property("aws_auth", &s3::GCPXMLSettings::aws_auth, &s3::GCPXMLSettings::set_aws_auth)
+        .def_property("aws_profile", &s3::GCPXMLSettings::aws_profile, &s3::GCPXMLSettings::set_aws_profile)
+        ;
 
     py::class_<NativeVariantStorage>(storage, "NativeVariantStorage")
         .def(py::init<>())
@@ -211,8 +231,7 @@ void register_bindings(py::module& storage, py::exception<arcticdb::ArcticExcept
         .def_property("ssl", &S3Override::ssl, &S3Override::set_ssl);
 
     py::class_<GCPXMLOverride>(storage, "GCPXMLOverride")
-        .def(py::init<S3Override>())
-        .def_property("s3_override", &GCPXMLOverride::s3_override, &GCPXMLOverride::set_s3_override);
+        .def(py::init<>());
 
     py::class_<AzureOverride>(storage, "AzureOverride")
         .def(py::init<>())

cpp/arcticdb/storage/s3/s3_settings.hpp

@@ -18,6 +18,84 @@ enum class AWSAuthMethod : uint32_t {
     STS_PROFILE_CREDENTIALS_PROVIDER = 2
 };
 
+class GCPXMLSettings {
+    std::string bucket_;
+    std::string endpoint_;
+    std::string access_;
+    std::string secret_;
+    std::string prefix_;
+    AWSAuthMethod aws_auth_;
+    std::string aws_profile_;
+
+public:
+    GCPXMLSettings() { }
+
+    explicit GCPXMLSettings(AWSAuthMethod aws_auth, const std::string& aws_profile) : aws_auth_(aws_auth), aws_profile_(aws_profile) {
+
+    }
+
+    GCPXMLSettings update(const arcticc::pb2::gcp_storage_pb2::Config& config){
+        prefix_ = config.prefix();
+        return *this;
+    }
+
+    [[nodiscard]] std::string endpoint() const {
+        return endpoint_;
+    }
+
+    void set_endpoint(std::string_view endpoint) {
+        endpoint_ = endpoint;
+    }
+
+    [[nodiscard]] std::string access() const {
+        return access_;
+    }
+
+    void set_access(const std::string_view access) {
+        access_ = access;
+    }
+
+    [[nodiscard]] std::string secret() const {
+        return secret_;
+    }
+
+    void set_secret(const std::string_view secret) {
+        secret_ = secret;
+    }
+
+    [[nodiscard]] AWSAuthMethod aws_auth() const {
+        return aws_auth_;
+    }
+
+    void set_aws_auth(const AWSAuthMethod aws_auth) {
+        aws_auth_ = aws_auth;
+    }
+
+    [[nodiscard]] std::string aws_profile() const {
+        return aws_profile_;
+    }
+
+    void set_aws_profile(const std::string_view aws_profile) {
+        aws_profile_ = aws_profile;
+    }
+
+    [[nodiscard]] std::string bucket() const {
+        return bucket_;
+    }
+
+    void set_bucket(const std::string_view bucket) {
+        bucket_ = bucket;
+    };
+
+    void set_prefix(const std::string_view prefix) {
+        prefix_ = prefix;
+    }
+
+    [[nodiscard]] std::string prefix() const {
+        return prefix_;
+    }
+};
+
 class S3Settings {
 private:
     std::string bucket_name_;
@@ -75,6 +153,26 @@ class S3Settings {
         return *this;
     }
 
+    S3Settings update(const GCPXMLSettings& gcp_xml_settings) {
+        bucket_name_ = gcp_xml_settings.bucket();
+        credential_name_ = gcp_xml_settings.access();
+        credential_key_ = gcp_xml_settings.secret();
+        endpoint_ = gcp_xml_settings.endpoint();
+        prefix_ = gcp_xml_settings.prefix();
+        aws_auth_ = gcp_xml_settings.aws_auth();
+        aws_profile_ = gcp_xml_settings.aws_profile();
+        use_virtual_addressing_ = false;
+        use_mock_storage_for_testing_ = false;
+        use_internal_client_wrapper_for_testing_ = false;
+        use_raw_prefix_ = false;
+        max_connections_ = 0;
+        connect_timeout_ = 0;
+        request_timeout_ = 0;
+        ssl_ = false;
+        https_ = false; // TODO aseaton sort this out
+        return *this;
+    }
+
     std::string bucket_name() const {
         return bucket_name_;
     }
@@ -152,22 +250,4 @@ class S3Settings {
     }
 };
 
-class GCPXMLSettings {
-public:
-    explicit GCPXMLSettings(S3Settings s3_settings) : s3_settings_(std::move(s3_settings)) {
-
-    }
-
-    GCPXMLSettings update(const arcticc::pb2::s3_storage_pb2::Config& config){
-        s3_settings_.update(config);
-        return *this;
-    }
-
-    S3Settings s3_settings() const {
-        return s3_settings_;
-    }
-
-private:
-    S3Settings s3_settings_;
-};
 }

cpp/arcticdb/storage/s3/s3_storage.cpp

@@ -179,7 +179,8 @@ S3Storage::S3Storage(const LibraryPath &library_path, OpenMode mode, const S3Set
 
 GCPXMLStorage::GCPXMLStorage(const arcticdb::storage::LibraryPath& lib,
                              arcticdb::storage::OpenMode mode,
-                             const arcticdb::storage::s3::GCPXMLSettings& conf) : S3Storage(lib, mode, conf.s3_settings()) {
+                             const arcticdb::storage::s3::GCPXMLSettings& conf) :
+                             S3Storage(lib, mode, S3Settings{AWSAuthMethod::DISABLED, "", false}.update(conf)) {
 
 }
 

cpp/arcticdb/storage/storage_override.hpp

@@ -128,24 +128,10 @@ class S3Override {
 };
 
 class GCPXMLOverride {
-    S3Override s3_override_;
 public:
-    explicit GCPXMLOverride(S3Override s3_override) : s3_override_(std::move(s3_override)) {
+    void modify_storage_config(arcticdb::proto::storage::VariantStorage& storage, bool) const {
 
     }
-
-    [[nodiscard]] S3Override s3_override() const {
-        return s3_override_;
-    }
-
-    void set_s3_override(S3Override s3_override) {
-        s3_override_ = s3_override;
-    }
-
-    void modify_storage_config(arcticdb::proto::storage::VariantStorage& storage,
-                               bool override_https) const {
-        return s3_override_.modify_storage_config(storage, override_https);
-    };
 };
 
 class AzureOverride {

cpp/arcticdb/storage/storages.hpp

@@ -276,13 +276,13 @@ inline std::shared_ptr<Storages> create_storages(const LibraryPath& library_path
                                                                   s3::S3Settings(settings).update(s3_storage)));
                             },
                             [&storage_config, &storages, &library_path, mode](const s3::GCPXMLSettings& settings) {
-                                util::check(storage_config.config().Is<arcticdb::proto::s3_storage::Config>(),
-                                            "Only support S3 native settings");
-                                arcticdb::proto::s3_storage::Config s3_storage;
-                                storage_config.config().UnpackTo(&s3_storage);
+                                util::check(storage_config.config().Is<arcticdb::proto::gcp_storage::Config>(),
+                                            "Only support GCP native settings");
+                                arcticdb::proto::gcp_storage::Config gcp_storage;
+                                storage_config.config().UnpackTo(&gcp_storage);
                                 storages.push_back(create_storage(library_path,
                                                                   mode,
-                                                                  s3::GCPXMLSettings(settings).update(s3_storage)));
+                                                                  s3::GCPXMLSettings(settings).update(gcp_storage)));
                             },
                             [&storage_config, &storages, &library_path, mode](const auto&) {
                                 storages.push_back(create_storage(library_path, mode, storage_config));

cpp/proto/arcticc/pb2/gcp_storage.proto

@@ -0,0 +1,14 @@
+/*
+Copyright 2025 Man Group Operations Limited
+
+Use of this software is governed by the Business Source License 1.1 included in the file licenses/BSL.txt.
+
+As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+*/
+syntax = "proto3";
+
+package arcticc.pb2.gcp_storage_pb2;
+
+message Config {
+    string prefix = 1;
+}

cpp/proto/arcticc/pb2/proto/CMakeLists.txt

@@ -11,6 +11,7 @@ SET(PROTO_IN_FILES
         mongo_storage.proto
         in_memory_storage.proto
         s3_storage.proto
+        gcp_storage.proto
         azure_storage.proto
         nfs_backed_storage.proto
         mapped_file_storage.proto