chore: Update vendored sources to duckdb/duckdb@56619fa (#281)

* chore: Update vendored sources to duckdb/duckdb@56619fa

Merge pull request duckdb/duckdb#13368 from samansmink/add-http-proxy

* ext fix

---------

Co-authored-by: krlmlr <krlmlr@users.noreply.github.com>
Co-authored-by: Kirill Müller <kirill@cynkra.com>

patch/0007-ext-fix.patch

@@ -0,0 +1,25 @@
+From ac8318dda4c948a4c1179f303854d91331bf9c2a Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Kirill=20M=C3=BCller?= <kirill@cynkra.com>
+Date: Sun, 8 Sep 2024 16:22:50 +0200
+Subject: [PATCH] ext fix
+
+---
+ src/duckdb/src/main/extension/extension_helper.cpp | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/duckdb/src/main/extension/extension_helper.cpp b/src/duckdb/src/main/extension/extension_helper.cpp
+index 0db77193..c821caed 100644
+--- a/src/duckdb/src/main/extension/extension_helper.cpp
++++ b/src/duckdb/src/main/extension/extension_helper.cpp
+@@ -232,7 +232,7 @@ bool ExtensionHelper::TryAutoLoadExtension(DatabaseInstance &instance, const str
+ 		if (dbconfig.options.autoinstall_known_extensions) {
+ 			auto autoinstall_repo =
+ 			    ExtensionRepository::GetRepositoryByUrl(dbconfig.options.autoinstall_extension_repo);
+-			ExtensionHelper::InstallExtension(dbconfig, fs, extension_name, false, autoinstall_repo, false);
++			ExtensionHelper::InstallExtension(instance, fs, extension_name, false, autoinstall_repo, false);
+ 		}
+ 		ExtensionHelper::LoadExternalExtension(instance, fs, extension_name);
+ 		return true;
+-- 
+2.43.0
+

src/duckdb/src/common/http_util.cpp

@@ -0,0 +1,25 @@
+#include "duckdb/common/http_util.hpp"
+
+#include "duckdb/common/operator/cast_operators.hpp"
+#include "duckdb/common/string_util.hpp"
+
+namespace duckdb {
+
+void HTTPUtil::ParseHTTPProxyHost(string &proxy_value, string &hostname_out, idx_t &port_out, idx_t default_port) {
+	auto proxy_split = StringUtil::Split(proxy_value, ":");
+	if (proxy_split.size() == 1) {
+		hostname_out = proxy_split[0];
+		port_out = default_port;
+	} else if (proxy_split.size() == 2) {
+		idx_t port;
+		if (!TryCast::Operation<string_t, idx_t>(proxy_split[1], port, false)) {
+			throw InvalidInputException("Failed to parse port from http_proxy '%s'", proxy_value);
+		}
+		hostname_out = proxy_split[0];
+		port_out = port;
+	} else {
+		throw InvalidInputException("Failed to parse http_proxy '%s' into a host and port", proxy_value);
+	}
+}
+
+} // namespace duckdb

src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "1-dev4728"
+#define DUCKDB_PATCH_VERSION "1-dev4748"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 0
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.0.1-dev4728"
+#define DUCKDB_VERSION "v1.0.1-dev4748"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "8ecc90c8d6"
+#define DUCKDB_SOURCE_ID "56619faf05"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

src/duckdb/src/include/duckdb/common/file_opener.hpp

@@ -43,6 +43,24 @@ class FileOpener {
 	                                                           Value &result);
 	DUCKDB_API static SettingLookupResult TryGetCurrentSetting(optional_ptr<FileOpener> opener, const string &key,
 	                                                           Value &result, FileOpenerInfo &info);
+
+	template <class TYPE>
+	static SettingLookupResult TryGetCurrentSetting(optional_ptr<FileOpener> opener, const string &key, TYPE &result,
+	                                                optional_ptr<FileOpenerInfo> info) {
+		Value output;
+		SettingLookupResult lookup_result;
+
+		if (info) {
+			lookup_result = TryGetCurrentSetting(opener, key, output, *info);
+		} else {
+			lookup_result = TryGetCurrentSetting(opener, key, output, *info);
+		}
+
+		if (lookup_result) {
+			result = output.GetValue<TYPE>();
+		}
+		return lookup_result;
+	}
 };
 
 } // namespace duckdb

src/duckdb/src/include/duckdb/common/http_util.hpp

@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/http_util.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/types.hpp"
+
+namespace duckdb {
+
+class HTTPUtil {
+public:
+	static void ParseHTTPProxyHost(string &proxy_value, string &hostname_out, idx_t &port_out, idx_t default_port = 80);
+};
+} // namespace duckdb

src/duckdb/src/include/duckdb/main/config.hpp

@@ -176,6 +176,12 @@ struct DBConfigOptions {
 	bool object_cache_enable = false;
 	//! Whether or not the global http metadata cache is used
 	bool http_metadata_cache_enable = false;
+	//! HTTP Proxy config as 'hostname:port'
+	string http_proxy;
+	//! HTTP Proxy username for basic auth
+	string http_proxy_username;
+	//! HTTP Proxy password for basic auth
+	string http_proxy_password;
 	//! Force checkpoint when CHECKPOINT is called or on shutdown, even if no changes have been made
 	bool force_checkpoint = false;
 	//! Run a checkpoint on successful shutdown and delete the WAL, to leave only a single database file behind

src/duckdb/src/include/duckdb/main/extension_helper.hpp

@@ -88,8 +88,8 @@ class ExtensionHelper {
 	                                                         optional_ptr<ExtensionRepository> repository = nullptr,
 	                                                         bool throw_on_origin_mismatch = false,
 	                                                         const string &version = "");
-	static unique_ptr<ExtensionInstallInfo> InstallExtension(DBConfig &config, FileSystem &fs, const string &extension,
-	                                                         bool force_install,
+	static unique_ptr<ExtensionInstallInfo> InstallExtension(DatabaseInstance &db, FileSystem &fs,
+	                                                         const string &extension, bool force_install,
 	                                                         optional_ptr<ExtensionRepository> repository = nullptr,
 	                                                         bool throw_on_origin_mismatch = false,
 	                                                         const string &version = "");
@@ -112,15 +112,15 @@ class ExtensionHelper {
 
 	//! Get the extension directory base on the current config
 	static string ExtensionDirectory(ClientContext &context);
-	static string ExtensionDirectory(DBConfig &config, FileSystem &fs);
+	static string ExtensionDirectory(DatabaseInstance &db, FileSystem &fs);
 
 	static bool CheckExtensionSignature(FileHandle &handle, ParsedExtensionMetaData &parsed_metadata,
 	                                    const bool allow_community_extensions);
 	static ParsedExtensionMetaData ParseExtensionMetaData(const char *metadata);
 	static ParsedExtensionMetaData ParseExtensionMetaData(FileHandle &handle);
 
 	//! Get the extension url template, containing placeholders for version, platform and extension name
-	static string ExtensionUrlTemplate(optional_ptr<const DBConfig> config, const ExtensionRepository &repository,
+	static string ExtensionUrlTemplate(optional_ptr<const DatabaseInstance> db, const ExtensionRepository &repository,
 	                                   const string &version);
 	//! Return the extension url template with the variables replaced
 	static string ExtensionFinalizeUrlTemplate(const string &url, const string &name);
@@ -202,7 +202,7 @@ class ExtensionHelper {
 	                                            const string &extension_name);
 	static string AddExtensionInstallHintToErrorMsg(ClientContext &context, const string &base_error,
 	                                                const string &extension_name);
-	static string AddExtensionInstallHintToErrorMsg(DBConfig &config, const string &base_error,
+	static string AddExtensionInstallHintToErrorMsg(DatabaseInstance &db, const string &base_error,
 	                                                const string &extension_name);
 
 	//! For tagged releases we use the tag, else we use the git commit hash
@@ -213,15 +213,15 @@ class ExtensionHelper {
 
 private:
 	static unique_ptr<ExtensionInstallInfo> InstallExtensionInternal(
-	    DBConfig &config, FileSystem &fs, const string &local_path, const string &extension, bool force_install,
+	    DatabaseInstance &db, FileSystem &fs, const string &local_path, const string &extension, bool force_install,
 	    bool throw_on_origin_mismatch, const string &version, optional_ptr<ExtensionRepository> repository,
 	    optional_ptr<HTTPLogger> http_logger = nullptr, optional_ptr<ClientContext> context = nullptr);
 	static const vector<string> PathComponents();
 	static string DefaultExtensionFolder(FileSystem &fs);
 	static bool AllowAutoInstall(const string &extension);
-	static ExtensionInitResult InitialLoad(DBConfig &config, FileSystem &fs, const string &extension);
-	static bool TryInitialLoad(DBConfig &config, FileSystem &fs, const string &extension, ExtensionInitResult &result,
-	                           string &error);
+	static ExtensionInitResult InitialLoad(DatabaseInstance &db, FileSystem &fs, const string &extension);
+	static bool TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const string &extension,
+	                           ExtensionInitResult &result, string &error);
 	//! Version tags occur with and without 'v', tag in extension path is always with 'v'
 	static const string NormalizeVersionTag(const string &version_tag);
 

src/duckdb/src/include/duckdb/main/secret/default_secrets.hpp

@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/secret/default_secrets.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+
+namespace duckdb {
+class DatabaseInstance;
+class ClientContext;
+class BaseSecret;
+struct CreateSecretInput;
+class SecretManager;
+struct SecretType;
+class CreateSecretFunction;
+
+struct CreateHTTPSecretFunctions {
+public:
+	//! Get the default secret types
+	static vector<SecretType> GetDefaultSecretTypes();
+	//! Get the default secret functions
+	static vector<CreateSecretFunction> GetDefaultSecretFunctions();
+
+protected:
+	//! HTTP secret CONFIG provider
+	static unique_ptr<BaseSecret> CreateHTTPSecretFromConfig(ClientContext &context, CreateSecretInput &input);
+	//! HTTP secret ENV provider
+	static unique_ptr<BaseSecret> CreateHTTPSecretFromEnv(ClientContext &context, CreateSecretInput &input);
+};
+
+} // namespace duckdb

src/duckdb/src/include/duckdb/main/secret/secret.hpp

@@ -213,6 +213,15 @@ class KeyValueSecret : public BaseSecret {
 		return true;
 	}
 
+	bool TrySetValue(const string &key, const CreateSecretInput &input) {
+		auto lookup = input.options.find(key);
+		if (lookup != input.options.end()) {
+			secret_map[key] = lookup->second;
+			return true;
+		}
+		return false;
+	}
+
 	//! the map of key -> values that make up the secret
 	case_insensitive_tree_t<Value> secret_map;
 	//! keys that are sensitive and should be redacted
@@ -228,11 +237,20 @@ class KeyValueSecret : public BaseSecret {
 class KeyValueSecretReader {
 public:
 	//! Manually pass in a secret reference
-	KeyValueSecretReader(const KeyValueSecret &secret_p, FileOpener &opener_p) : secret(secret_p), opener(opener_p) {};
+	KeyValueSecretReader(const KeyValueSecret &secret_p, FileOpener &opener_p) : secret(secret_p) {};
 
 	//! Initializes the KeyValueSecretReader by fetching the secret automatically
-	KeyValueSecretReader(FileOpener &opener_p, FileOpenerInfo &info, const char **secret_types, idx_t secret_types_len);
-	KeyValueSecretReader(FileOpener &opener_p, FileOpenerInfo &info, const char *secret_type);
+	KeyValueSecretReader(FileOpener &opener_p, optional_ptr<FileOpenerInfo> info, const char **secret_types,
+	                     idx_t secret_types_len);
+	KeyValueSecretReader(FileOpener &opener_p, optional_ptr<FileOpenerInfo> info, const char *secret_type);
+
+	//! Initialize KeyValueSecretReader from a db instance
+	KeyValueSecretReader(DatabaseInstance &db, const char **secret_types, idx_t secret_types_len, string path);
+	KeyValueSecretReader(DatabaseInstance &db, const char *secret_type, string path);
+
+	// Initialize KeyValueSecretReader from a client context
+	KeyValueSecretReader(ClientContext &context, const char **secret_types, idx_t secret_types_len, string path);
+	KeyValueSecretReader(ClientContext &context, const char *secret_type, string path);
 
 	~KeyValueSecretReader();
 
@@ -279,6 +297,8 @@ class KeyValueSecretReader {
 	}
 
 protected:
+	void Initialize(const char **secret_types, idx_t secret_types_len);
+
 	[[noreturn]] void ThrowNotFoundError(const string &secret_key);
 	[[noreturn]] void ThrowNotFoundError(const string &secret_key, const string &setting_name);
 
@@ -287,9 +307,12 @@ class KeyValueSecretReader {
 	//! Optionally an owning pointer to the secret entry
 	shared_ptr<SecretEntry> secret_entry;
 
-	//! Fetching the settings
-	optional_ptr<FileOpener> opener;
-	optional_ptr<FileOpenerInfo> opener_info;
+	//! Secrets/settings will be fetched either through a context (local + global settings) or a databaseinstance
+	//! (global only)
+	optional_ptr<DatabaseInstance> db;
+	optional_ptr<ClientContext> context;
+
+	string path;
 };
 
 } // namespace duckdb

src/duckdb/src/include/duckdb/main/secret/secret_manager.hpp

@@ -153,8 +153,12 @@ class SecretManager {
 	                                 const string &storage = "");
 
 private:
+	//! Register a secret type
+	void RegisterSecretTypeInternal(SecretType &type);
 	//! Lookup a SecretType
 	SecretType LookupTypeInternal(const string &type);
+	//! Register a secret provider
+	void RegisterSecretFunctionInternal(CreateSecretFunction function, OnCreateConflict on_conflict);
 	//! Lookup a CreateSecretFunction
 	optional_ptr<CreateSecretFunction> LookupFunctionInternal(const string &type, const string &provider);
 	//! Register a new Secret

src/duckdb/src/include/duckdb/main/settings.hpp

@@ -487,6 +487,33 @@ struct HomeDirectorySetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct HTTPProxy {
+	static constexpr const char *Name = "http_proxy";
+	static constexpr const char *Description = "HTTP proxy host";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct HTTPProxyUsername {
+	static constexpr const char *Name = "http_proxy_username";
+	static constexpr const char *Description = "Username for HTTP proxy";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct HTTPProxyPassword {
+	static constexpr const char *Name = "http_proxy";
+	static constexpr const char *Description = "Password for HTTP proxy";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct IntegerDivisionSetting {
 	static constexpr const char *Name = "integer_division";
 	static constexpr const char *Description =

src/duckdb/src/main/config.cpp

@@ -97,6 +97,9 @@ static const ConfigurationOption internal_options[] = {
     DUCKDB_GLOBAL(ForceCompressionSetting),
     DUCKDB_GLOBAL(ForceBitpackingModeSetting),
     DUCKDB_LOCAL(HomeDirectorySetting),
+    DUCKDB_GLOBAL(HTTPProxy),
+    DUCKDB_GLOBAL(HTTPProxyUsername),
+    DUCKDB_GLOBAL(HTTPProxyPassword),
     DUCKDB_LOCAL(LogQueryPathSetting),
     DUCKDB_GLOBAL(EnableMacrosDependencies),
     DUCKDB_GLOBAL(EnableViewDependencies),