diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index c248f7c..47f32b4 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -1,4 +1,4 @@ -# Reusable makefile for building out-of-tree extension with the DuckDB extension template +# Reusable makefile for building out-of-tree extension with the DuckDB C++ based extension template # # Inputs # EXT_NAME : Upper case string describing the name of the out-of-tree extension diff --git a/makefiles/duckdb_extension_c_api.Makefile b/makefiles/duckdb_extension_c_api.Makefile new file mode 100644 index 0000000..6ae8635 --- /dev/null +++ b/makefiles/duckdb_extension_c_api.Makefile @@ -0,0 +1,211 @@ +# Reusable makefile for C API based extensions +# +# Inputs +# EXTENSION_NAME : name of the extension (lower case) +# MINIMUM_DUCKDB_VERSION : the minimum version of DuckDB that the extension supports +# EXTENSION_VERSION : the version of the extension, if left blank it will be autodetected +# DUCKDB_PLATFORM : the platform of the extension, if left blank it will be autodetected +# DUCKDB_TEST_VERSION : the version of DuckDB to test with, if left blank will default to latest stable on PyPi +# DUCKDB_GIT_VERSION : set by CI currently, should probably be removed at some point +# LINUX_CI_IN_DOCKER : indicates that the build is being run in/out of Docker in the linux CI +# SKIP_TESTS : makes the test targets turn into NOPs + +.PHONY: platform extension_version test_extension_release test_extension_debug test_extension_release_internal test_extension_debug_internal tests_skipped clean_build clean_configure nop set_duckdb_tag set_duckdb_version output_distribution_matrix venv configure_ci check_configure + +############################################# +### Platform dependent config +############################################# +PYTHON_BIN=python3 + +ifeq ($(OS),Windows_NT) + EXTENSION_LIB_FILENAME=$(EXTENSION_NAME).dll + PYTHON_VENV_BIN=./configure/venv/Scripts/python.exe +else + PYTHON_VENV_BIN=./configure/venv/bin/python3 + UNAME_S := $(shell uname -s) + ifeq ($(UNAME_S),Linux) + EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).so + endif + ifeq ($(UNAME_S),Darwin) + EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).dylib + endif +endif + +############################################# +### Main extension parameters +############################################# + +# The minimum DuckDB version that this extension supports +ifeq ($(MINIMUM_DUCKDB_VERSION),) + MINIMUM_DUCKDB_VERSION = v0.0.1 +endif + +EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension + +############################################# +### Platform Detection +############################################# + +# Write the platform we are building for +platform: configure/platform.txt + +# Either autodetect or use the provided value +PLATFORM_COMMAND?= +ifeq ($(DUCKDB_PLATFORM),) + PLATFORM_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py --duckdb-platform +else + # Sets the platform using DUCKDB_PLATFORM variable + PLATFORM_COMMAND=echo $(DUCKDB_PLATFORM) > configure/platform.txt +endif + +configure/platform.txt: + @ $(PLATFORM_COMMAND) + +############################################# +### Extension Version Detection +############################################# + +# Either autodetect or use the provided value +VERSION_COMMAND?= +ifeq ($(EXTENSION_VERSION),) + VERSION_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py --extension-version +else + # Sets the platform using DUCKDB_PLATFORM variable + VERSION_COMMAND=echo "$(EXTENSION_VERSION)" > configure/extension_version.txt +endif + +extension_version: configure/extension_version.txt + +configure/extension_version.txt: + @ $(VERSION_COMMAND) + +############################################# +### Testing +############################################# + +# Note: to override the default test runner, create a symlink to a different venv +TEST_RUNNER=$(PYTHON_VENV_BIN) -m duckdb_sqllogictest + +TEST_RUNNER_BASE=$(TEST_RUNNER) --test-dir test/sql $(EXTRA_EXTENSIONS_PARAM) +TEST_RUNNER_DEBUG=$(TEST_RUNNER_BASE) --external-extension build/debug/rusty_quack.duckdb_extension +TEST_RUNNER_RELEASE=$(TEST_RUNNER_BASE) --external-extension build/release/rusty_quack.duckdb_extension + +# By default latest duckdb is installed, set DUCKDB_TEST_VERSION to switch to a different version +DUCKDB_INSTALL_VERSION?= +ifneq ($(DUCKDB_TEST_VERSION),) + DUCKDB_INSTALL_VERSION===$(DUCKDB_TEST_VERSION) +endif + +ifneq ($(DUCKDB_GIT_VERSION),) + DUCKDB_INSTALL_VERSION===$(DUCKDB_GIT_VERSION) +endif + +TEST_RELEASE_TARGET=test_extension_release_internal +TEST_DEBUG_TARGET=test_extension_debug_internal + +# Disable testing outside docker: the unittester is currently dynamically linked by default +ifeq ($(LINUX_CI_IN_DOCKER),1) + SKIP_TESTS=1 +endif + +# TODO: for some weird reason the Ubuntu 22.04 Runners on Github Actions don't actually grab the glibc 2.24 wheels but the +# gilbc 2.17 ones. What this means is that we can't run the tests on linux_amd64 because we are installing the duckdb +# linux_amd64_gcc4 test runner +ifeq ($(DUCKDB_PLATFORM),linux_amd64) + SKIP_TESTS=1 +endif + +ifeq ($(SKIP_TESTS),1) + TEST_RELEASE_TARGET=tests_skipped + TEST_DEBUG_TARGET=tests_skipped +endif + +test_extension_release: $(TEST_RELEASE_TARGET) +test_extension_debug: $(TEST_DEBUG_TARGET) + +test_extension_release_internal: check_configure + @echo "Running RELEASE tests.." + @$(TEST_RUNNER_RELEASE) + +test_extension_debug_internal: check_configure + @echo "Running DEBUG tests.." + @$(TEST_RUNNER_DEBUG) + +tests_skipped: + @echo "Skipping tests.." + + +############################################# +### Misc +############################################# + +clean_build: + rm -rf build + rm -rf duckdb_unittest_tempdir + +clean_configure: + rm -rf configure + +nop: + @echo "NOP" + +set_duckdb_tag: nop + +set_duckdb_version: nop + +output_distribution_matrix: + cat extension-ci-tools/config/distribution_matrix.json + +############################################# +### Building +############################################# +build_extension_with_metadata_debug: check_configure + $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ + -l build/debug/$(EXTENSION_LIB_FILENAME) \ + -o build/debug/$(EXTENSION_FILENAME) \ + -n $(EXTENSION_NAME) \ + -dv $(MINIMUM_DUCKDB_VERSION) \ + -evf configure/extension_version.txt \ + -pf configure/platform.txt + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/debug/$(EXTENSION_FILENAME)', 'build/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + +build_extension_with_metadata_release: check_configure + $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ + -l build/release/$(EXTENSION_LIB_FILENAME) \ + -o build/release/$(EXTENSION_FILENAME) \ + -n $(EXTENSION_NAME) \ + -dv $(MINIMUM_DUCKDB_VERSION) \ + -evf configure/extension_version.txt \ + -pf configure/platform.txt + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/release/$(EXTENSION_FILENAME)', 'build/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + +############################################# +### Python +############################################# + +# Installs the test runner using the selected DuckDB version (latest stable by default) +# TODO: switch to PyPI distribution +venv: configure/venv + +configure/venv: + $(PYTHON_BIN) -m venv configure/venv + $(PYTHON_VENV_BIN) -m pip install 'duckdb$(DUCKDB_INSTALL_VERSION)' + $(PYTHON_VENV_BIN) -m pip install git+https://github.com/duckdb/duckdb-sqllogictest-python + +############################################# +### Configure +############################################# + +CONFIGURE_CI_STEP?= +ifeq ($(LINUX_CI_IN_DOCKER),1) + CONFIGURE_CI_STEP=nop +else + CONFIGURE_CI_STEP=configure +endif + +configure_ci: $(CONFIGURE_CI_STEP) + +# Because the configure_ci may differ from configure, we don't automatically run configure on make build, this makes the error a bit nicer +check_configure: + @$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/platform.txt'), 'The configure step appears to not be run. Please try running make configure'" + @$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/venv'), 'The configure step appears to not be run. Please try running make configure'" \ No newline at end of file diff --git a/makefiles/duckdb_extension_rs.Makefile b/makefiles/duckdb_extension_rs.Makefile new file mode 100644 index 0000000..027fea5 --- /dev/null +++ b/makefiles/duckdb_extension_rs.Makefile @@ -0,0 +1,39 @@ +# Reusable makefile for the Rust extensions targeting the C extension API +# +# Inputs +# EXTENSION_NAME : name of the extension (lower case) +# EXTENSION_LIB_FILENAME : the library name that is produced by the build +# LOCAL_DUCKDB_RS_PATH : overrides the duckdb-rs path + +.PHONY: build_extension_library_debug build_extension_library_release + +############################################# +### Development config +############################################# + +# Allows overriding the duckdb-rs crates with a local version +CARGO_OVERRIDE_DUCKDB_RS_FLAG?= +ifneq ($(LOCAL_DUCKDB_RS_PATH),) + CARGO_OVERRIDE_DUCKDB_RS_FLAG=--config 'patch.crates-io.duckdb.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb"' --config 'patch.crates-io.libduckdb-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/libduckdb-sys"' --config 'patch.crates-io.duckdb-loadable-macros-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb-loadable-macros-sys"' +endif + +############################################# +### Rust Build targets +############################################# + +build_extension_library_debug: check_configure + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/debug/$(EXTENSION_LIB_FILENAME)', 'build/debug/$(EXTENSION_LIB_FILENAME)')" + +build_extension_library_release: check_configure + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/release/$(EXTENSION_LIB_FILENAME)', 'build/release/$(EXTENSION_LIB_FILENAME)')" + +############################################# +### Misc +############################################# + +clean_rust: + cargo clean \ No newline at end of file diff --git a/scripts/append_extension_metadata.py b/scripts/append_extension_metadata.py index 277d5d3..d15262d 100644 --- a/scripts/append_extension_metadata.py +++ b/scripts/append_extension_metadata.py @@ -39,7 +39,9 @@ def main(): arg_parser.add_argument('-dv', '--duckdb-version', type=str, help='The DuckDB version to encode, depending on the ABI type ' 'this encodes the duckdb version or the C API version', required=True) - arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode', required=True) + arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode') + arg_parser.add_argument('-evf', '--extension-version-file', type=str, help='The file containing the Extension version to encode') + arg_parser.add_argument('--abi-type', type=str, help='The ABI type to encode, set to C_STRUCT by default', default='C_STRUCT') args = arg_parser.parse_args() @@ -70,6 +72,23 @@ def main(): else: raise Exception(f"Neither --duckdb-platform nor --duckdb-platform-file found, please specify the platform using either") + EXTENSION_VERSION = "" + if args.extension_version: + EXTENSION_VERSION = args.extension_version + elif args.extension_version_file: + try: + with open(args.extension_version_file, 'r') as file: + EXTENSION_VERSION = file.read().strip() + except Exception as e: + print(f"Failed to read extension version from file {args.extension_version_file}") + raise + if not EXTENSION_VERSION: + raise Exception(f"Extension version file passed to script is empty: {args.extension_version_file}") + else: + raise Exception(f"Neither --extension-version nor --extension-version-file found, please specify the extension version using either") + + + # Then append the metadata to the tmp file print(f" - Metadata:") with open(OUTPUT_FILE_TMP, 'ab') as file: @@ -82,8 +101,8 @@ def main(): file.write(padded_byte_string("")) print(f" - FIELD5 (abi_type) = {args.abi_type}") file.write(padded_byte_string(args.abi_type)) - print(f" - FIELD4 (extension_version) = {args.extension_version}") - file.write(padded_byte_string(args.extension_version)) + print(f" - FIELD4 (extension_version) = {EXTENSION_VERSION}") + file.write(padded_byte_string(EXTENSION_VERSION)) print(f" - FIELD3 (duckdb_version) = {args.duckdb_version}") file.write(padded_byte_string(args.duckdb_version)) print(f" - FIELD2 (duckdb_platform) = {PLATFORM}") diff --git a/scripts/configure_helper.py b/scripts/configure_helper.py new file mode 100644 index 0000000..9685410 --- /dev/null +++ b/scripts/configure_helper.py @@ -0,0 +1,45 @@ +import subprocess +import argparse +from pathlib import Path +import os + +def main(): + arg_parser = argparse.ArgumentParser(description='Script to aid in running the configure step of the extension build process') + + + arg_parser.add_argument('-o', '--output-directory', type=str, help='Specify the output directory', default='configure') + + arg_parser.add_argument('-ev', '--extension-version', help='Write the autodetected extension version', action='store_true') + arg_parser.add_argument('-p', '--duckdb-platform', help='Write the auto-detected duckdb platform', action='store_true') + + args = arg_parser.parse_args() + + OUTPUT_DIR = args.output_directory + + Path(OUTPUT_DIR).mkdir(parents=True, exist_ok=True) + + # Write version + if args.extension_version: + git_tag = subprocess.getoutput("git tag --points-at HEAD") + if git_tag: + EXTENSION_VERSION = git_tag + else: + EXTENSION_VERSION = subprocess.getoutput("git --no-pager log -1 --format=%h") + + version_file = Path(os.path.join(OUTPUT_DIR, "extension_version.txt")) + with open(version_file, 'w') as f: + print(f"Writing version {EXTENSION_VERSION} to {version_file}") + f.write(EXTENSION_VERSION) + + # Write duck + if args.duckdb_platform: + import duckdb + platform_file = Path(os.path.join(OUTPUT_DIR, "platform.txt")) + duckdb_platform = duckdb.execute('pragma platform').fetchone()[0] + with open(platform_file, 'w') as f: + print(f"Writing platform {duckdb_platform} to {platform_file}") + f.write(duckdb_platform) + + +if __name__ == '__main__': + main() \ No newline at end of file