Merge pull request #101 from samansmink/minor-scipt-additions

Add new Makefiles for experimental C API based extensions

makefiles/duckdb_extension.Makefile

@@ -1,4 +1,4 @@
-# Reusable makefile for building out-of-tree extension with the DuckDB extension template
+# Reusable makefile for building out-of-tree extension with the DuckDB C++ based extension template
 #
 # Inputs
 #   EXT_NAME          : Upper case string describing the name of the out-of-tree extension

makefiles/duckdb_extension_c_api.Makefile

@@ -0,0 +1,211 @@
+# Reusable makefile for C API based extensions
+#
+# Inputs
+#   EXTENSION_NAME         : name of the extension (lower case)
+#   MINIMUM_DUCKDB_VERSION : the minimum version of DuckDB that the extension supports
+#   EXTENSION_VERSION      : the version of the extension, if left blank it will be autodetected
+#   DUCKDB_PLATFORM        : the platform of the extension, if left blank it will be autodetected
+#   DUCKDB_TEST_VERSION    : the version of DuckDB to test with, if left blank will default to latest stable on PyPi
+#   DUCKDB_GIT_VERSION     : set by CI currently, should probably be removed at some point
+#   LINUX_CI_IN_DOCKER     : indicates that the build is being run in/out of Docker in the linux CI
+#   SKIP_TESTS             : makes the test targets turn into NOPs
+
+.PHONY: platform extension_version test_extension_release test_extension_debug test_extension_release_internal test_extension_debug_internal tests_skipped clean_build clean_configure nop set_duckdb_tag set_duckdb_version output_distribution_matrix venv configure_ci check_configure
+
+#############################################
+### Platform dependent config
+#############################################
+PYTHON_BIN=python3
+
+ifeq ($(OS),Windows_NT)
+	EXTENSION_LIB_FILENAME=$(EXTENSION_NAME).dll
+	PYTHON_VENV_BIN=./configure/venv/Scripts/python.exe
+else
+	PYTHON_VENV_BIN=./configure/venv/bin/python3
+    UNAME_S := $(shell uname -s)
+    ifeq ($(UNAME_S),Linux)
+        EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).so
+    endif
+    ifeq ($(UNAME_S),Darwin)
+        EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).dylib
+    endif
+endif
+
+#############################################
+### Main extension parameters
+#############################################
+
+# The minimum DuckDB version that this extension supports
+ifeq ($(MINIMUM_DUCKDB_VERSION),)
+	MINIMUM_DUCKDB_VERSION = v0.0.1
+endif
+
+EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension
+
+#############################################
+### Platform Detection
+#############################################
+
+# Write the platform we are building for
+platform: configure/platform.txt
+
+# Either autodetect or use the provided value
+PLATFORM_COMMAND?=
+ifeq ($(DUCKDB_PLATFORM),)
+    PLATFORM_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py --duckdb-platform
+else
+	# Sets the platform using DUCKDB_PLATFORM variable
+	PLATFORM_COMMAND=echo $(DUCKDB_PLATFORM) > configure/platform.txt
+endif
+
+configure/platform.txt:
+	@ $(PLATFORM_COMMAND)
+
+#############################################
+### Extension Version Detection
+#############################################
+
+# Either autodetect or use the provided value
+VERSION_COMMAND?=
+ifeq ($(EXTENSION_VERSION),)
+    VERSION_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py --extension-version
+else
+	# Sets the platform using DUCKDB_PLATFORM variable
+	VERSION_COMMAND=echo "$(EXTENSION_VERSION)" > configure/extension_version.txt
+endif
+
+extension_version: configure/extension_version.txt
+
+configure/extension_version.txt:
+	@ $(VERSION_COMMAND)
+
+#############################################
+### Testing
+#############################################
+
+# Note: to override the default test runner, create a symlink to a different venv
+TEST_RUNNER=$(PYTHON_VENV_BIN) -m duckdb_sqllogictest
+
+TEST_RUNNER_BASE=$(TEST_RUNNER) --test-dir test/sql $(EXTRA_EXTENSIONS_PARAM)
+TEST_RUNNER_DEBUG=$(TEST_RUNNER_BASE) --external-extension build/debug/rusty_quack.duckdb_extension
+TEST_RUNNER_RELEASE=$(TEST_RUNNER_BASE) --external-extension build/release/rusty_quack.duckdb_extension
+
+# By default latest duckdb is installed, set DUCKDB_TEST_VERSION to switch to a different version
+DUCKDB_INSTALL_VERSION?=
+ifneq ($(DUCKDB_TEST_VERSION),)
+	DUCKDB_INSTALL_VERSION===$(DUCKDB_TEST_VERSION)
+endif
+
+ifneq ($(DUCKDB_GIT_VERSION),)
+	DUCKDB_INSTALL_VERSION===$(DUCKDB_GIT_VERSION)
+endif
+
+TEST_RELEASE_TARGET=test_extension_release_internal
+TEST_DEBUG_TARGET=test_extension_debug_internal
+
+# Disable testing outside docker: the unittester is currently dynamically linked by default
+ifeq ($(LINUX_CI_IN_DOCKER),1)
+	SKIP_TESTS=1
+endif
+
+# TODO: for some weird reason the Ubuntu 22.04 Runners on Github Actions don't actually grab the glibc 2.24 wheels but the
+#       gilbc 2.17 ones. What this means is that we can't run the tests on linux_amd64 because we are installing the duckdb
+#	    linux_amd64_gcc4 test runner
+ifeq ($(DUCKDB_PLATFORM),linux_amd64)
+	SKIP_TESTS=1
+endif
+
+ifeq ($(SKIP_TESTS),1)
+	TEST_RELEASE_TARGET=tests_skipped
+	TEST_DEBUG_TARGET=tests_skipped
+endif
+
+test_extension_release: $(TEST_RELEASE_TARGET)
+test_extension_debug: $(TEST_DEBUG_TARGET)
+
+test_extension_release_internal: check_configure
+	@echo "Running RELEASE tests.."
+	@$(TEST_RUNNER_RELEASE)
+
+test_extension_debug_internal: check_configure
+	@echo "Running DEBUG tests.."
+	@$(TEST_RUNNER_DEBUG)
+
+tests_skipped:
+	@echo "Skipping tests.."
+
+
+#############################################
+### Misc
+#############################################
+
+clean_build:
+	rm -rf build
+	rm -rf duckdb_unittest_tempdir
+
+clean_configure:
+	rm -rf configure
+
+nop:
+	@echo "NOP"
+
+set_duckdb_tag: nop
+
+set_duckdb_version: nop
+
+output_distribution_matrix:
+	cat extension-ci-tools/config/distribution_matrix.json
+
+#############################################
+### Building
+#############################################
+build_extension_with_metadata_debug: check_configure
+	$(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \
+			-l build/debug/$(EXTENSION_LIB_FILENAME) \
+			-o build/debug/$(EXTENSION_FILENAME) \
+			-n $(EXTENSION_NAME) \
+			-dv $(MINIMUM_DUCKDB_VERSION) \
+			-evf configure/extension_version.txt \
+			-pf configure/platform.txt
+	$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/debug/$(EXTENSION_FILENAME)', 'build/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')"
+
+build_extension_with_metadata_release: check_configure
+	$(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \
+			-l build/release/$(EXTENSION_LIB_FILENAME) \
+			-o build/release/$(EXTENSION_FILENAME) \
+			-n $(EXTENSION_NAME) \
+			-dv $(MINIMUM_DUCKDB_VERSION) \
+			-evf configure/extension_version.txt \
+			-pf configure/platform.txt
+	$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/release/$(EXTENSION_FILENAME)', 'build/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')"
+
+#############################################
+### Python
+#############################################
+
+# Installs the test runner using the selected DuckDB version (latest stable by default)
+# TODO: switch to PyPI distribution
+venv: configure/venv
+
+configure/venv:
+	$(PYTHON_BIN) -m venv configure/venv
+	$(PYTHON_VENV_BIN) -m pip install 'duckdb$(DUCKDB_INSTALL_VERSION)'
+	$(PYTHON_VENV_BIN) -m pip install git+https://github.com/duckdb/duckdb-sqllogictest-python
+
+#############################################
+### Configure
+#############################################
+
+CONFIGURE_CI_STEP?=
+ifeq ($(LINUX_CI_IN_DOCKER),1)
+	CONFIGURE_CI_STEP=nop
+else
+	CONFIGURE_CI_STEP=configure
+endif
+
+configure_ci: $(CONFIGURE_CI_STEP)
+
+# Because the configure_ci may differ from configure, we don't automatically run configure on make build, this makes the error a bit nicer
+check_configure:
+	@$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/platform.txt'), 'The configure step appears to not be run. Please try running make configure'"
+	@$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/venv'), 'The configure step appears to not be run. Please try running make configure'"

makefiles/duckdb_extension_rs.Makefile

@@ -0,0 +1,39 @@
+# Reusable makefile for the Rust extensions targeting the C extension API
+#
+# Inputs
+#   EXTENSION_NAME          : name of the extension (lower case)
+#   EXTENSION_LIB_FILENAME  : the library name that is produced by the build
+#   LOCAL_DUCKDB_RS_PATH    : overrides the duckdb-rs path
+
+.PHONY: build_extension_library_debug build_extension_library_release
+
+#############################################
+### Development config
+#############################################
+
+# Allows overriding the duckdb-rs crates with a local version
+CARGO_OVERRIDE_DUCKDB_RS_FLAG?=
+ifneq ($(LOCAL_DUCKDB_RS_PATH),)
+	CARGO_OVERRIDE_DUCKDB_RS_FLAG=--config 'patch.crates-io.duckdb.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb"' --config 'patch.crates-io.libduckdb-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/libduckdb-sys"' --config 'patch.crates-io.duckdb-loadable-macros-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb-loadable-macros-sys"'
+endif
+
+#############################################
+### Rust Build targets
+#############################################
+
+build_extension_library_debug: check_configure
+	DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG)
+	$(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)"
+	$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/debug/$(EXTENSION_LIB_FILENAME)', 'build/debug/$(EXTENSION_LIB_FILENAME)')"
+
+build_extension_library_release: check_configure
+	DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release
+	$(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)"
+	$(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/release/$(EXTENSION_LIB_FILENAME)', 'build/release/$(EXTENSION_LIB_FILENAME)')"
+
+#############################################
+### Misc
+#############################################
+
+clean_rust:
+	cargo clean

scripts/append_extension_metadata.py

@@ -39,7 +39,9 @@ def main():
 
     arg_parser.add_argument('-dv', '--duckdb-version', type=str, help='The DuckDB version to encode, depending on the ABI type '
                                                                'this encodes the duckdb version or the C API version', required=True)
-    arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode', required=True)
+    arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode')
+    arg_parser.add_argument('-evf', '--extension-version-file', type=str, help='The file containing the Extension version to encode')
+
     arg_parser.add_argument('--abi-type', type=str, help='The ABI type to encode, set to C_STRUCT by default', default='C_STRUCT')
 
     args = arg_parser.parse_args()
@@ -70,6 +72,23 @@ def main():
     else:
         raise Exception(f"Neither --duckdb-platform nor --duckdb-platform-file found, please specify the platform using either")
 
+    EXTENSION_VERSION = ""
+    if args.extension_version:
+        EXTENSION_VERSION = args.extension_version
+    elif args.extension_version_file:
+        try:
+            with open(args.extension_version_file, 'r') as file:
+                EXTENSION_VERSION = file.read().strip()
+        except Exception as e:
+            print(f"Failed to read extension version from file {args.extension_version_file}")
+            raise
+        if not EXTENSION_VERSION:
+            raise Exception(f"Extension version file passed to script is empty: {args.extension_version_file}")
+    else:
+        raise Exception(f"Neither --extension-version nor --extension-version-file found, please specify the extension version using either")
+
+
+
     # Then append the metadata to the tmp file
     print(f" - Metadata:")
     with open(OUTPUT_FILE_TMP, 'ab') as file:
@@ -82,8 +101,8 @@ def main():
         file.write(padded_byte_string(""))
         print(f"   - FIELD5 (abi_type)          = {args.abi_type}")
         file.write(padded_byte_string(args.abi_type))
-        print(f"   - FIELD4 (extension_version) = {args.extension_version}")
-        file.write(padded_byte_string(args.extension_version))
+        print(f"   - FIELD4 (extension_version) = {EXTENSION_VERSION}")
+        file.write(padded_byte_string(EXTENSION_VERSION))
         print(f"   - FIELD3 (duckdb_version)    = {args.duckdb_version}")
         file.write(padded_byte_string(args.duckdb_version))
         print(f"   - FIELD2 (duckdb_platform)   = {PLATFORM}")

scripts/configure_helper.py

@@ -0,0 +1,45 @@
+import subprocess
+import argparse
+from pathlib import Path
+import os
+
+def main():
+    arg_parser = argparse.ArgumentParser(description='Script to aid in running the configure step of the extension build process')
+
+
+    arg_parser.add_argument('-o', '--output-directory', type=str, help='Specify the output directory', default='configure')
+
+    arg_parser.add_argument('-ev', '--extension-version', help='Write the autodetected extension version', action='store_true')
+    arg_parser.add_argument('-p', '--duckdb-platform', help='Write the auto-detected duckdb platform', action='store_true')
+
+    args = arg_parser.parse_args()
+
+    OUTPUT_DIR = args.output_directory
+
+    Path(OUTPUT_DIR).mkdir(parents=True, exist_ok=True)
+
+    # Write version
+    if args.extension_version:
+        git_tag = subprocess.getoutput("git tag --points-at HEAD")
+        if git_tag:
+            EXTENSION_VERSION = git_tag
+        else:
+            EXTENSION_VERSION = subprocess.getoutput("git --no-pager log -1 --format=%h")
+
+        version_file = Path(os.path.join(OUTPUT_DIR, "extension_version.txt"))
+        with open(version_file, 'w') as f:
+            print(f"Writing version {EXTENSION_VERSION} to {version_file}")
+            f.write(EXTENSION_VERSION)
+
+    # Write duck
+    if args.duckdb_platform:
+        import duckdb
+        platform_file = Path(os.path.join(OUTPUT_DIR, "platform.txt"))
+        duckdb_platform = duckdb.execute('pragma platform').fetchone()[0]
+        with open(platform_file, 'w') as f:
+            print(f"Writing platform {duckdb_platform} to {platform_file}")
+            f.write(duckdb_platform)
+
+
+if __name__ == '__main__':
+    main()