diff --git a/src/duckdb/src/common/enum_util.cpp b/src/duckdb/src/common/enum_util.cpp index 5938a7e8..9a0db08f 100644 --- a/src/duckdb/src/common/enum_util.cpp +++ b/src/duckdb/src/common/enum_util.cpp @@ -6467,6 +6467,29 @@ SecretPersistType EnumUtil::FromString(const char *value) { throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented in FromString", value)); } +template<> +const char* EnumUtil::ToChars(SecretSerializationType value) { + switch(value) { + case SecretSerializationType::CUSTOM: + return "CUSTOM"; + case SecretSerializationType::KEY_VALUE_SECRET: + return "KEY_VALUE_SECRET"; + default: + throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented in ToChars", value)); + } +} + +template<> +SecretSerializationType EnumUtil::FromString(const char *value) { + if (StringUtil::Equals(value, "CUSTOM")) { + return SecretSerializationType::CUSTOM; + } + if (StringUtil::Equals(value, "KEY_VALUE_SECRET")) { + return SecretSerializationType::KEY_VALUE_SECRET; + } + throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented in FromString", value)); +} + template<> const char* EnumUtil::ToChars(SequenceInfo value) { switch(value) { diff --git a/src/duckdb/src/core_functions/aggregate/distributive/bitstring_agg.cpp b/src/duckdb/src/core_functions/aggregate/distributive/bitstring_agg.cpp index 36920a47..f01cc50a 100644 --- a/src/duckdb/src/core_functions/aggregate/distributive/bitstring_agg.cpp +++ b/src/duckdb/src/core_functions/aggregate/distributive/bitstring_agg.cpp @@ -8,6 +8,8 @@ #include "duckdb/execution/expression_executor.hpp" #include "duckdb/common/types/cast_helpers.hpp" #include "duckdb/common/operator/subtract.hpp" +#include "duckdb/common/serializer/deserializer.hpp" +#include "duckdb/common/serializer/serializer.hpp" namespace duckdb { @@ -43,6 +45,21 @@ struct BitstringAggBindData : public FunctionData { } return false; } + + static void Serialize(Serializer &serializer, const optional_ptr bind_data_p, + const AggregateFunction &) { + auto &bind_data = bind_data_p->Cast(); + serializer.WriteProperty(100, "min", bind_data.min); + serializer.WriteProperty(101, "max", bind_data.max); + } + + static unique_ptr Deserialize(Deserializer &deserializer, AggregateFunction &) { + Value min; + Value max; + deserializer.ReadProperty(100, "min", min); + deserializer.ReadProperty(101, "max", max); + return make_uniq(min, max); + } }; struct BitStringAggOperation { @@ -247,7 +264,9 @@ static void BindBitString(AggregateFunctionSet &bitstring_agg, const LogicalType auto function = AggregateFunction::UnaryAggregateDestructor, TYPE, string_t, BitStringAggOperation>( type, LogicalType::BIT); - function.bind = BindBitstringAgg; // create new a 'BitstringAggBindData' + function.bind = BindBitstringAgg; // create new a 'BitstringAggBindData' + function.serialize = BitstringAggBindData::Serialize; + function.deserialize = BitstringAggBindData::Deserialize; function.statistics = BitstringPropagateStats; // stores min and max from column stats in BitstringAggBindData bitstring_agg.AddFunction(function); // uses the BitstringAggBindData to access statistics for creating bitstring function.arguments = {type, type, type}; diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp index 4c49147f..b43f28f2 100644 --- a/src/duckdb/src/function/table/version/pragma_version.cpp +++ b/src/duckdb/src/function/table/version/pragma_version.cpp @@ -1,5 +1,5 @@ #ifndef DUCKDB_PATCH_VERSION -#define DUCKDB_PATCH_VERSION "3-dev142" +#define DUCKDB_PATCH_VERSION "3-dev156" #endif #ifndef DUCKDB_MINOR_VERSION #define DUCKDB_MINOR_VERSION 1 @@ -8,10 +8,10 @@ #define DUCKDB_MAJOR_VERSION 1 #endif #ifndef DUCKDB_VERSION -#define DUCKDB_VERSION "v1.1.3-dev142" +#define DUCKDB_VERSION "v1.1.3-dev156" #endif #ifndef DUCKDB_SOURCE_ID -#define DUCKDB_SOURCE_ID "7f34190f3f" +#define DUCKDB_SOURCE_ID "9cba6a2a03" #endif #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database.hpp" diff --git a/src/duckdb/src/include/duckdb/common/enum_util.hpp b/src/duckdb/src/include/duckdb/common/enum_util.hpp index 7cae3772..3e6a235a 100644 --- a/src/duckdb/src/include/duckdb/common/enum_util.hpp +++ b/src/duckdb/src/include/duckdb/common/enum_util.hpp @@ -278,6 +278,8 @@ enum class SecretDisplayType : uint8_t; enum class SecretPersistType : uint8_t; +enum class SecretSerializationType : uint8_t; + enum class SequenceInfo : uint8_t; enum class SetOperationType : uint8_t; @@ -728,6 +730,9 @@ const char* EnumUtil::ToChars(SecretDisplayType value); template<> const char* EnumUtil::ToChars(SecretPersistType value); +template<> +const char* EnumUtil::ToChars(SecretSerializationType value); + template<> const char* EnumUtil::ToChars(SequenceInfo value); @@ -1218,6 +1223,9 @@ SecretDisplayType EnumUtil::FromString(const char *value); template<> SecretPersistType EnumUtil::FromString(const char *value); +template<> +SecretSerializationType EnumUtil::FromString(const char *value); + template<> SequenceInfo EnumUtil::FromString(const char *value); diff --git a/src/duckdb/src/include/duckdb/core_functions/aggregate/minmax_n_helpers.hpp b/src/duckdb/src/include/duckdb/core_functions/aggregate/minmax_n_helpers.hpp index 6dbdb8bc..55a7a150 100644 --- a/src/duckdb/src/include/duckdb/core_functions/aggregate/minmax_n_helpers.hpp +++ b/src/duckdb/src/include/duckdb/core_functions/aggregate/minmax_n_helpers.hpp @@ -40,6 +40,8 @@ struct HeapEntry { HeapEntry(HeapEntry &&other) noexcept { if (other.value.IsInlined()) { value = other.value; + capacity = 0; + allocated_data = nullptr; } else { capacity = other.capacity; allocated_data = other.allocated_data; diff --git a/src/duckdb/src/include/duckdb/main/secret/secret.hpp b/src/duckdb/src/include/duckdb/main/secret/secret.hpp index f3257f00..ee721d7f 100644 --- a/src/duckdb/src/include/duckdb/main/secret/secret.hpp +++ b/src/duckdb/src/include/duckdb/main/secret/secret.hpp @@ -37,8 +37,7 @@ struct CreateSecretInput { case_insensitive_map_t options; }; -typedef unique_ptr (*secret_deserializer_t)(Deserializer &deserializer, BaseSecret base_secret, - const named_parameter_type_map_t &options); +typedef unique_ptr (*secret_deserializer_t)(Deserializer &deserializer, BaseSecret base_secret); typedef unique_ptr (*create_secret_function_t)(ClientContext &context, CreateSecretInput &input); //! A CreateSecretFunction is a function adds a provider for a secret type. @@ -81,6 +80,13 @@ struct SecretType { string default_provider; }; +enum class SecretSerializationType : uint8_t { + //! The secret is serialized with a custom serialization function + CUSTOM = 0, + //! The secret has been serialized as a KeyValueSecret + KEY_VALUE_SECRET = 1 +}; + //! Base class from which BaseSecret classes can be made. class BaseSecret { friend class SecretManager; @@ -181,30 +187,14 @@ class KeyValueSecret : public BaseSecret { // FIXME: use serialization scripts template - static unique_ptr Deserialize(Deserializer &deserializer, BaseSecret base_secret, - const named_parameter_type_map_t &options) { + static unique_ptr Deserialize(Deserializer &deserializer, BaseSecret base_secret) { auto result = make_uniq(base_secret); Value secret_map_value; deserializer.ReadProperty(201, "secret_map", secret_map_value); for (const auto &entry : ListValue::GetChildren(secret_map_value)) { auto kv_struct = StructValue::GetChildren(entry); - auto key = kv_struct[0].ToString(); - auto raw_value = kv_struct[1].ToString(); - - auto it = options.find(key); - if (it == options.end()) { - throw IOException("Failed to deserialize secret '%s', it contains an unexpected key: '%s'", - base_secret.GetName(), key); - } - auto &logical_type = it->second; - Value value; - if (logical_type.id() == LogicalTypeId::VARCHAR) { - value = Value(raw_value); - } else { - value = Value(raw_value).DefaultCastAs(logical_type); - } - result->secret_map[key] = value; + result->secret_map[kv_struct[0].ToString()] = kv_struct[1]; } Value redact_set_value; diff --git a/src/duckdb/src/main/connection.cpp b/src/duckdb/src/main/connection.cpp index 65f5f255..4dd7ad11 100644 --- a/src/duckdb/src/main/connection.cpp +++ b/src/duckdb/src/main/connection.cpp @@ -19,7 +19,7 @@ namespace duckdb { Connection::Connection(DatabaseInstance &database) - : context(make_shared_ptr(database.shared_from_this())) { + : context(make_shared_ptr(database.shared_from_this())), warning_cb(nullptr) { ConnectionManager::Get(database).AddConnection(*context); #ifdef DEBUG EnableProfiling(); diff --git a/src/duckdb/src/main/secret/secret.cpp b/src/duckdb/src/main/secret/secret.cpp index 8535ef63..9a39b717 100644 --- a/src/duckdb/src/main/secret/secret.cpp +++ b/src/duckdb/src/main/secret/secret.cpp @@ -87,7 +87,8 @@ void KeyValueSecret::Serialize(Serializer &serializer) const { map_values.push_back(Value::STRUCT(map_struct)); } - auto map_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR); + // Warning: the secret map is serialized into a single MAP value with type ANY + auto map_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::ANY); auto map = Value::MAP(ListType::GetChildType(map_type), map_values); serializer.WriteProperty(201, "secret_map", map); diff --git a/src/duckdb/src/main/secret/secret_manager.cpp b/src/duckdb/src/main/secret/secret_manager.cpp index f783a0c3..045d1efe 100644 --- a/src/duckdb/src/main/secret/secret_manager.cpp +++ b/src/duckdb/src/main/secret/secret_manager.cpp @@ -97,6 +97,20 @@ unique_ptr SecretManager::DeserializeSecret(Deserializer &deserializ vector scope; deserializer.ReadList(103, "scope", [&](Deserializer::List &list, idx_t i) { scope.push_back(list.ReadElement()); }); + auto serialization_type = + deserializer.ReadPropertyWithExplicitDefault(104, "serialization_type", SecretSerializationType::CUSTOM); + + switch (serialization_type) { + // This allows us to skip looking up the secret type for deserialization altogether + case SecretSerializationType::KEY_VALUE_SECRET: + return KeyValueSecret::Deserialize(deserializer, {scope, type, provider, name}); + // Continues below: we need to do a type lookup to find the secret deserialize method + case SecretSerializationType::CUSTOM: + break; + default: + throw IOException("Unrecognized secret serialization type found in secret '%s': %s", secret_path, + EnumUtil::ToString(serialization_type)); + } SecretType deserialized_type; if (!TryLookupTypeInternal(type, deserialized_type)) { @@ -108,15 +122,7 @@ unique_ptr SecretManager::DeserializeSecret(Deserializer &deserializ "Attempted to deserialize secret type '%s' which does not have a deserialization method", type); } - auto function_entry = LookupFunctionInternal(type, provider); - if (!function_entry) { - throw IOException("Attempted to deserialize secret (type: '%s', provider: '%s', path: '%s') which does not " - "have any functions registered", - type, provider, secret_path); - } - - return deserialized_type.deserializer(deserializer, {scope, type, provider, name}, - function_entry->named_parameters); + return deserialized_type.deserializer(deserializer, {scope, type, provider, name}); } void SecretManager::RegisterSecretType(SecretType &type) { diff --git a/src/duckdb/src/planner/binder/query_node/plan_setop.cpp b/src/duckdb/src/planner/binder/query_node/plan_setop.cpp index ce0b52d4..1afca342 100644 --- a/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +++ b/src/duckdb/src/planner/binder/query_node/plan_setop.cpp @@ -132,7 +132,7 @@ unique_ptr Binder::CreatePlan(BoundSetOperationNode &node) { node.right_binder->has_unplanned_dependent_joins; // create actual logical ops for setops - LogicalOperatorType logical_type; + LogicalOperatorType logical_type = LogicalOperatorType::LOGICAL_INVALID; switch (node.setop_type) { case SetOperationType::UNION: case SetOperationType::UNION_BY_NAME: diff --git a/src/duckdb/third_party/brotli/enc/memory.cpp b/src/duckdb/third_party/brotli/enc/memory.cpp index a4c8b8db..8a5c6712 100644 --- a/src/duckdb/third_party/brotli/enc/memory.cpp +++ b/src/duckdb/third_party/brotli/enc/memory.cpp @@ -64,7 +64,7 @@ void duckdb_brotli::BrotliWipeOutMemoryManager(MemoryManager* m) { #else /* BROTLI_ENCODER_EXIT_ON_OOM */ -static void SortPointers(void** items, const size_t n) { +void SortPointers(void** items, const size_t n) { /* Shell sort. */ /* TODO(eustas): fine-tune for "many slots" case */ static const size_t gaps[] = {23, 10, 4, 1}; @@ -135,7 +135,7 @@ static void CollectGarbagePointers(MemoryManager* m) { } } -void* BrotliAllocate(MemoryManager* m, size_t n) { +void* duckdb_brotli::BrotliAllocate(MemoryManager* m, size_t n) { void* result = m->alloc_func(m->opaque, n); if (!result) { m->is_oom = BROTLI_TRUE; @@ -146,14 +146,14 @@ void* BrotliAllocate(MemoryManager* m, size_t n) { return result; } -void BrotliFree(MemoryManager* m, void* p) { +void duckdb_brotli::BrotliFree(MemoryManager* m, void* p) { if (!p) return; m->free_func(m->opaque, p); if (m->new_freed == MAX_NEW_FREED) CollectGarbagePointers(m); m->pointers[NEW_FREED_OFFSET + (m->new_freed++)] = p; } -void BrotliWipeOutMemoryManager(MemoryManager* m) { +void duckdb_brotli::BrotliWipeOutMemoryManager(MemoryManager* m) { size_t i; CollectGarbagePointers(m); /* Now all unfreed pointers are in perm-allocated list. */