Merge pull request #98 from samansmink/minor-ci-tweaks

Prepare ci tools for extensions without DuckDB submodule

.github/workflows/TestCITools.yml

@@ -23,7 +23,6 @@ jobs:
       override_ci_tools_repository: ${{ github.repository }}
       ci_tools_version: ${{ github.sha }}
       extra_toolchains: 'parser_tools;fortran;omp;go;python3'
-      custom_toolchain_script: true
 
   delta-extension-main:
     name: Rust builds (using Delta extension)

.github/workflows/_extension_distribution.yml

@@ -49,7 +49,7 @@ on:
       matrix_parse_script:
         required: false
         type: string
-        default: "./duckdb/scripts/modify_distribution_matrix.py"
+        default: "./extension-ci-tools/scripts/modify_distribution_matrix.py"
       # Enable building the DuckDB Shell
       build_duckdb_shell:
         required: false
@@ -76,11 +76,6 @@ on:
         required: false
         type: string
         default: ""
-      # If set, the setup will look for a script in `./scripts/custom-toolchain-script.sh` to run
-      custom_toolchain_script:
-        required: false
-        type: boolean
-        default: false
       rust_logs:
         required: false
         type: boolean
@@ -283,6 +278,8 @@ jobs:
           echo "OPENSSL_DIR=/duckdb_build_dir/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> docker_env.txt
           echo "OPENSSL_USE_STATIC_LIBS=true" >> docker_env.txt
           echo "DUCKDB_PLATFORM=${{ matrix.duckdb_arch }}" >> docker_env.txt
+          echo "DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }}" >> docker_env.txt
+          echo "LINUX_CI_IN_DOCKER=1" >> docker_env.txt
           echo "TOOLCHAIN_FLAGS=${{ matrix.duckdb_arch == 'linux_arm64' && '-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran' || '' }}" >> docker_env.txt
 
       - name: Generate timestamp for Ccache entry
@@ -304,14 +301,35 @@ jobs:
           restore-keys: |
             ccache-extension-distribution-${{ matrix.duckdb_arch }}-
 
-      - name: Build extension
+      - name: Run configure (outside Docker)
+        shell: bash
+        env:
+          DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }}
+          LINUX_CI_IN_DOCKER: 0
+        run: |
+          make configure_ci
+
+      - name: Run configure (inside Docker)
+        shell: bash
+        run: |
+          docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make configure_ci
+
+      - name: Build extension (inside Docker)
         run: |
           docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make release
 
-      - name: Test extension
+      - name: Test extension (inside docker)
         if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }}
         run: |
-          docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test
+          docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test_release
+
+      - name: Test extension (outside docker)
+        if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }}
+        env:
+          DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }}
+          LINUX_CI_IN_DOCKER: 0
+        run: |
+          make test_release
 
       - uses: actions/upload-artifact@v3
         with:
@@ -439,24 +457,23 @@ jobs:
           echo "CPPFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV
           echo "CXXFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV
 
-      - name: Run custom toolchain script
-        if: ${{ inputs.custom_toolchain_script }}
+      - name: Run configure
         shell: bash
+        env:
+          DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }}
         run: |
-          bash scripts/setup-custom-toolchain.sh
+          make configure_ci
 
       - name: Build extension
         shell: bash
-        env:
-          DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
         run: |
           make release
 
       - name: Test Extension
         if: ${{ matrix.osx_build_arch == 'arm64' && inputs.skip_tests == false }}
         shell: bash
         run: |
-          make test
+          make test_release
 
       - uses: actions/upload-artifact@v3
         with:
@@ -583,21 +600,23 @@ jobs:
           vcpkgGitCommitId: ${{ inputs.vcpkg_commit }}
           vcpkgGitURL: ${{ inputs.vcpkg_url }}
 
-      - name: Run custom toolchain script
-        if: ${{ inputs.custom_toolchain_script }}
+      - name: Run configure
         shell: bash
+        env:
+          DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
+          DUCKDB_PLATFORM_RTOOLS: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 1 || 0 }}
+          DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }}
         run: |
-          bash scripts/setup-custom-toolchain.sh
+          make configure_ci
 
       - name: Build extension
-        if: ${{ inputs.skip_tests == true }}
         env:
           DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
           DUCKDB_PLATFORM_RTOOLS: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 1 || 0 }}
         run: |
           make release
 
-      - name: Build & test extension
+      - name: Test extension
         if: ${{ inputs.skip_tests == false }}
         env:
           DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }}
@@ -701,11 +720,12 @@ jobs:
         with:
           key: ${{ github.job }}-${{ matrix.duckdb_arch }}
 
-      - name: Run custom toolchain script
-        if: ${{ inputs.custom_toolchain_script }}
+      - name: Run configure
         shell: bash
+        env:
+          DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }}
         run: |
-          bash scripts/setup-custom-toolchain.sh
+          make configure_ci
 
       - name: Build Wasm module
         run: |

makefiles/duckdb_extension.Makefile

@@ -6,8 +6,9 @@
 #   EXT_FLAGS         : Extra CMake flags to pass to the build
 #   EXT_RELEASE_FLAGS : Extra CMake flags to pass to the release build
 #   EXT_DEBUG_FLAGS   : Extra CMake flags to pass to the debug build
+#   SKIP_TESTS        : Replaces all test targets with a NOP step
 
-.PHONY: all clean format debug release pull update wasm_mvp wasm_eh wasm_threads
+.PHONY: all clean clean-python format debug release pull update wasm_mvp wasm_eh wasm_threads test test_release test_debug test_reldebug test_release_internal test_debug_internal test_reldebug_internal set_duckdb_version set_duckdb_tag  output_distribution_matrix
 
 all: release
 
@@ -78,15 +79,35 @@ reldebug:
 # Main tests
 test: test_release
 
-test_release: release
-	./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*"
+TEST_RELEASE_TARGET=test_release_internal
+TEST_DEBUG_TARGET=test_debug_internal
+TEST_RELDEBUG_TARGET=test_reldebug_internal
 
-test_debug: debug
-	./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*"
+# Disable testing outside docker: the unittester is currently dynamically linked by default
+ifeq ($(LINUX_CI_IN_DOCKER),0)
+	SKIP_TESTS=1
+endif
+
+ifeq ($(SKIP_TESTS),1)
+	TEST_RELEASE_TARGET=tests_skipped
+	TEST_DEBUG_TARGET=tests_skipped
+	TEST_RELDEBUG_TARGET=tests_skipped
+endif
 
-test_reldebug: reldebug
+test_release: $(TEST_RELEASE_TARGET)
+test_debug: $(TEST_DEBUG_TARGET)
+test_reldebug: $(TEST_RELDEBUG_TARGET)
+
+test_release_internal:
+	./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*"
+test_debug_internal:
+	./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*"
+test_reldebug_internal:
 	./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*"
 
+tests_skipped:
+	@echo "Tests are skipped in this run..."
+
 # WASM config
 VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake
 WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 $(TOOLCHAIN_FLAGS) $(VCPKG_EMSDK_FLAGS)
@@ -138,3 +159,6 @@ set_duckdb_tag:
 
 output_distribution_matrix:
 	cat duckdb/.github/config/distribution_matrix.json
+
+configure_ci:
+	@echo "configure_ci step is skipped for this extension build..."

scripts/append_extension_metadata.py

@@ -33,7 +33,10 @@ def main():
 
     arg_parser.add_argument('-o', '--out-file', type=str, help='Explicit path for the output file', default='')
 
-    arg_parser.add_argument('-p', '--duckdb-platform', type=str, help='The DuckDB platform to encode', required=True)
+    # The platform
+    arg_parser.add_argument('-p', '--duckdb-platform', type=str, help='The DuckDB platform to encode')
+    arg_parser.add_argument('-pf', '--duckdb-platform-file', type=str, help='The file containing the DuckDB platform to encode')
+
     arg_parser.add_argument('-dv', '--duckdb-version', type=str, help='The DuckDB version to encode, depending on the ABI type '
                                                                'this encodes the duckdb version or the C API version', required=True)
     arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode', required=True)
@@ -51,6 +54,22 @@ def main():
     print(f" - Output file: {OUTPUT_FILE}")
     shutil.copyfile(args.library_file, OUTPUT_FILE_TMP)
 
+    # Handle the platform
+    PLATFORM = ""
+    if args.duckdb_platform:
+        PLATFORM = args.duckdb_platform
+    elif args.duckdb_platform_file:
+        try:
+            with open(args.duckdb_platform_file, 'r') as file:
+                PLATFORM = file.read().strip()
+        except Exception as e:
+            print(f"Failed to read platform from file {args.duckdb_platform_file}")
+            raise
+        if not PLATFORM:
+            raise Exception(f"Platform file passed to script is empty: {args.duckdb_platform_file}")
+    else:
+        raise Exception(f"Neither --duckdb-platform nor --duckdb-platform-file found, please specify the platform using either")
+
     # Then append the metadata to the tmp file
     print(f" - Metadata:")
     with open(OUTPUT_FILE_TMP, 'ab') as file:
@@ -67,8 +86,8 @@ def main():
         file.write(padded_byte_string(args.extension_version))
         print(f"   - FIELD3 (duckdb_version)    = {args.duckdb_version}")
         file.write(padded_byte_string(args.duckdb_version))
-        print(f"   - FIELD2 (duckdb_platform)   = {args.duckdb_platform}")
-        file.write(padded_byte_string(args.duckdb_platform))
+        print(f"   - FIELD2 (duckdb_platform)   = {PLATFORM}")
+        file.write(padded_byte_string(PLATFORM))
         print(f"   - FIELD1 (header signature)  = 4 (special value to identify a duckdb extension)")
         file.write(padded_byte_string("4"))