[Experimental] Tracking duckdb main branch

extension_config_wasm.cmake

@@ -2,11 +2,11 @@
 # DuckDB-Wasm extension base config
 ################################################################################
 #
+
 duckdb_extension_load(json DONT_LINK)
 duckdb_extension_load(parquet DONT_LINK)
 duckdb_extension_load(autocomplete DONT_LINK)
 
-duckdb_extension_load(fts DONT_LINK)
 duckdb_extension_load(icu DONT_LINK)
 duckdb_extension_load(tpcds DONT_LINK)
 duckdb_extension_load(tpch DONT_LINK)

lib/CMakeLists.txt

@@ -243,10 +243,6 @@ if (DUCKDB_WASM_LOADABLE_EXTENSIONS)
 
   target_link_libraries(duckdb_web duckdb arrow rapidjson ${THREAD_LIBS})
 else()
-  add_library(
-    duckdb_web_fts
-    ${CMAKE_SOURCE_DIR}/src/extensions/fts_extension.cc)
-
   add_library(
     duckdb_web_parquet
     ${CMAKE_SOURCE_DIR}/src/extensions/parquet_extension.cc)
@@ -264,8 +260,7 @@ else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDUCKDB_JSON_EXTENSION")
   endif()
 
-  target_link_libraries(duckdb_web duckdb duckdb_web_fts duckdb_web_parquet ${DUCKDB_WEB_JSON} arrow rapidjson ${THREAD_LIBS})
-  target_link_libraries(duckdb_web_fts duckdb duckdb_fts)
+  target_link_libraries(duckdb_web duckdb duckdb_web_parquet ${DUCKDB_WEB_JSON} arrow rapidjson ${THREAD_LIBS})
   target_link_libraries(duckdb_web_parquet duckdb duckdb_parquet)
   target_link_libraries(duckdb_web_json duckdb duckdb_json)
 endif()
@@ -275,6 +270,8 @@ endif()
 
 # We need "-s WARN_ON_UNDEFINED_SYMBOLS=0" to instantiate the module with our
 # own imports.
+     # --pre-js "/Users/carlo/databricks/duckdb_delta/file_system_packager.js" \
+#
 if(EMSCRIPTEN)
   add_executable(duckdb_wasm ${CMAKE_SOURCE_DIR}/src/wasm_main.cc)
   target_link_libraries(duckdb_wasm duckdb_web ${THREAD_LIBS})
@@ -371,7 +368,7 @@ if(NOT EMSCRIPTEN)
       ${CMAKE_SOURCE_DIR}/test/webdb_test.cc
       ${CMAKE_SOURCE_DIR}/test/tester.cc)
 
-  set(TEST_LIBS duckdb_web duckdb_web_fts duckdb_web_parquet ${DUCKDB_WEB_JSON} gtest gmock gflags ${THREAD_LIBS})
+  set(TEST_LIBS duckdb_web duckdb_web_parquet ${DUCKDB_WEB_JSON} gtest gmock gflags ${THREAD_LIBS})
 
   add_executable(tester ${TEST_CC})
   target_link_libraries(tester ${TEST_LIBS})

lib/cmake/duckdb.cmake

@@ -15,7 +15,7 @@ endif()
 set(DUCKDB_CXX_FLAGS "${DUCKDB_CXX_FLAGS} -Wno-unqualified-std-cast-call -DDUCKDB_DEBUG_NO_SAFETY -DDUCKDB_FROM_DUCKDB_WASM")
 message("DUCKDB_CXX_FLAGS=${DUCKDB_CXX_FLAGS}")
 
-set(DUCKDB_EXTENSIONS "fts;json")
+set(DUCKDB_EXTENSIONS "json;core_functions")
 # Escape semicolons in DUCKDB_EXTENSIONS before passing to ExternalProject_Add
 string(REPLACE ";" "$<SEMICOLON>" DUCKDB_EXTENSIONS_PACKED "${DUCKDB_EXTENSIONS}")
 
@@ -46,6 +46,7 @@ ExternalProject_Add(
              -DDISABLE_BUILTIN_EXTENSIONS=TRUE
              -DUSE_WASM_THREADS=${USE_WASM_THREADS}
              -DDUCKDB_EXPLICIT_PLATFORM=${DUCKDB_EXPLICIT_PLATFORM}
+             -DSMALLER_BINARY=1
   BUILD_BYPRODUCTS
     <INSTALL_DIR>/lib/libduckdb_re2.a
     <INSTALL_DIR>/lib/libduckdb_static.a
@@ -59,7 +60,7 @@ ExternalProject_Add(
     <INSTALL_DIR>/lib/libduckdb_utf8proc.a
     <INSTALL_DIR>/lib/libduckdb_fastpforlib.a
     <INSTALL_DIR>/lib/libparquet_extension.a
-    <INSTALL_DIR>/lib/libfts_extension.a
+    <INSTALL_DIR>/lib/libcore_functions_extension.a
     <INSTALL_DIR>/lib/libjson_extension.a)
 
 ExternalProject_Get_Property(duckdb_ep install_dir)
@@ -90,6 +91,7 @@ target_link_libraries(
   INTERFACE ${install_dir}/lib/libduckdb_pg_query.a
   INTERFACE ${install_dir}/lib/libduckdb_utf8proc.a
   INTERFACE ${install_dir}/lib/libduckdb_fastpforlib.a
+  INTERFACE ${install_dir}/lib/libcore_functions_extension.a
   INTERFACE dl)
 
 target_include_directories(
@@ -105,10 +107,6 @@ target_include_directories(
   INTERFACE ${DUCKDB_SOURCE_DIR}/third_party/thrift
   INTERFACE ${DUCKDB_SOURCE_DIR}/third_party/zstd)
 
-add_library(duckdb_fts STATIC IMPORTED)
-set_property(TARGET duckdb_fts PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libfts_extension.a)
-target_include_directories(duckdb_fts INTERFACE ${DUCKDB_SOURCE_DIR}/extension/fts/include)
-
 add_library(duckdb_parquet STATIC IMPORTED)
 set_property(TARGET duckdb_parquet PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libparquet_extension.a)
 target_include_directories(duckdb_parquet INTERFACE ${DUCKDB_SOURCE_DIR}/extension/parquet/include)
@@ -117,7 +115,10 @@ add_library(duckdb_json STATIC IMPORTED)
 set_property(TARGET duckdb_json PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libjson_extension.a)
 target_include_directories(duckdb_json INTERFACE ${DUCKDB_SOURCE_DIR}/extension/json/include)
 
+add_library(duckdb_core_functions STATIC IMPORTED)
+set_property(TARGET duckdb_core_functions PROPERTY IMPORTED_LOCATION ${install_dir}/lib/libcore_functions_extension.a)
+target_include_directories(duckdb_core_functions INTERFACE ${DUCKDB_SOURCE_DIR}/extension/json/include)
+
 add_dependencies(duckdb duckdb_ep)
-add_dependencies(duckdb_fts duckdb_ep)
 add_dependencies(duckdb_parquet duckdb_ep)
 add_dependencies(duckdb_json duckdb_ep)

lib/include/duckdb/web/io/web_filesystem.h

@@ -132,7 +132,7 @@ class WebFileSystem : public duckdb::FileSystem {
        public:
         /// Constructor
         WebFileHandle(std::shared_ptr<WebFile> file)
-            : duckdb::FileHandle(file->GetFileSystem(), file->GetFileName()),
+            : duckdb::FileHandle(file->GetFileSystem(), file->GetFileName(), FileOpenFlags::FILE_FLAGS_READ),
               file_(file),
               readahead_(nullptr),
               position_(0) {

lib/src/io/buffered_filesystem.cc

@@ -42,7 +42,7 @@ void BufferedFileHandle::Close() { file_ref_->Release(); }
 /// Constructor
 BufferedFileHandle::BufferedFileHandle(duckdb::FileSystem &file_system,
                                        std::shared_ptr<FilePageBuffer::FileRef> file_buffers)
-    : duckdb::FileHandle(file_system, std::string{file_buffers->GetPath()}),
+    : duckdb::FileHandle(file_system, std::string{file_buffers->GetPath()}, FileOpenFlags::FILE_FLAGS_READ),
       file_ref_(std::move(file_buffers)),
       file_position_(0) {}
 

lib/src/io/memory_filesystem.cc

@@ -17,7 +17,8 @@ MemoryFileSystem::FileBuffer::FileBuffer(size_t id, std::string path, std::vecto
     : file_id(id), file_path(std::move(path)), buffer(std::move(buffer)), handles() {}
 /// Constructor
 MemoryFileSystem::FileHandle::FileHandle(MemoryFileSystem &file_system, FileBuffer &buffer, size_t position)
-    : duckdb::FileHandle(file_system, buffer.file_path),
+    : duckdb::FileHandle(file_system, buffer.file_path,
+                         FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE),
       file_system_(file_system),
       buffer_(buffer),
       position_(position) {}

lib/src/webdb.cc

@@ -45,7 +45,6 @@
 #include "duckdb/web/config.h"
 #include "duckdb/web/csv_insert_options.h"
 #include "duckdb/web/environment.h"
-#include "duckdb/web/extensions/fts_extension.h"
 #include "duckdb/web/extensions/json_extension.h"
 #include "duckdb/web/extensions/parquet_extension.h"
 #include "duckdb/web/functions/table_function_relation.h"
@@ -831,7 +830,6 @@ arrow::Status WebDB::Open(std::string_view args_json) {
         auto db = make_shared_ptr<duckdb::DuckDB>(config_->path, &db_config);
 #ifndef WASM_LOADABLE_EXTENSIONS
         duckdb_web_parquet_init(db.get());
-        duckdb_web_fts_init(db.get());
 #if defined(DUCKDB_JSON_EXTENSION)
         duckdb_web_json_init(db.get());
 #endif

patches/duckdb/duckdb_smaller_binary_no_select.patch

@@ -0,0 +1,37 @@
+diff --git a/src/execution/expression_executor.cpp b/src/execution/expression_executor.cpp
+index 63a24f479e..eadf1dea0d 100644
+--- a/src/execution/expression_executor.cpp
++++ b/src/execution/expression_executor.cpp
+@@ -236,7 +236,7 @@ idx_t ExpressionExecutor::Select(const Expression &expr, ExpressionState *state,
+ 	D_ASSERT(true_sel || false_sel);
+ 	D_ASSERT(expr.return_type.id() == LogicalTypeId::BOOLEAN);
+ 	switch (expr.expression_class) {
+-#ifndef DUCKDB_SMALLER_BINARY
++#ifndef DUCKDB_SMALLER_BINARY_NO_SELECT
+ 	case ExpressionClass::BOUND_BETWEEN:
+ 		return Select(expr.Cast<BoundBetweenExpression>(), state, sel, count, true_sel, false_sel);
+ #endif
+diff --git a/src/execution/expression_executor/execute_between.cpp b/src/execution/expression_executor/execute_between.cpp
+index 3418351362..52cf454530 100644
+--- a/src/execution/expression_executor/execute_between.cpp
++++ b/src/execution/expression_executor/execute_between.cpp
+@@ -7,7 +7,7 @@
+
+ namespace duckdb {
+
+-#ifndef DUCKDB_SMALLER_BINARY
++#ifndef DUCKDB_SMALLER_BINARY_NO_SELECT
+ struct BothInclusiveBetweenOperator {
+ 	template <class T>
+ 	static inline bool Operation(T input, T lower, T upper) {
+@@ -133,8 +133,8 @@ void ExpressionExecutor::Execute(const BoundBetweenExpression &expr, ExpressionS
+
+ idx_t ExpressionExecutor::Select(const BoundBetweenExpression &expr, ExpressionState *state, const SelectionVector *sel,
+                                  idx_t count, SelectionVector *true_sel, SelectionVector *false_sel) {
+-#ifdef DUCKDB_SMALLER_BINARY
+-	throw InternalException("ExpressionExecutor::Select not available with DUCKDB_SMALLER_BINARY");
++#ifdef DUCKDB_SMALLER_BINARY_NO_SELECT
++	throw InternalException("ExpressionExecutor::Select not available with DUCKDB_SMALLER_BINARY_NO_SELECT");
+ #else
+ 	// resolve the children
+ 	Vector input(state->intermediate_chunk.data[0]);

patches/duckdb/extension_install_rework.patch

@@ -5,7 +5,7 @@ index 2a6fffa994..f4ef687ddd 100644
 @@ -96,6 +96,10 @@ private:
  	shared_ptr<DatabaseCacheEntry> db_cache_entry;
 
- 	duckdb_ext_api_v0 (*create_api_v0)();
+ 	duckdb_ext_api_v1 (*create_api_v1)();
 +public:
 +	static void SetPreferredRepository(const string& extension, const string &repository);
 +	static string GetPreferredRepository(const string& extension);

patches/duckdb/hardcode_abi_type.patch

@@ -0,0 +1,13 @@
+diff --git a/src/main/extension/extension_load.cpp b/src/main/extension/extension_load.cpp
+index a7a7c62fef..d560a3b3b3 100644
+--- a/src/main/extension/extension_load.cpp
++++ b/src/main/extension/extension_load.cpp
+@@ -451,7 +451,7 @@ bool ExtensionHelper::TryInitialLoad(DatabaseInstance &db, FileSystem &fs, const
+ 	result.filebase = lowercase_extension_name;
+ 	result.filename = filename;
+ 	result.lib_hdl = lib_hdl;
+-	result.abi_type = parsed_metadata.abi_type;
++	result.abi_type = ExtensionABIType::CPP;
+
+ 	if (!direct_load) {
+ 		auto info_file_name = filename + ".info";