diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml deleted file mode 100644 index aa42fd5..0000000 --- a/.github/workflows/Linux.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: Linux -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - linux: - name: Linux Release - runs-on: ubuntu-latest - container: ${{ matrix.container }} - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4'] - vcpkg_version: [ '2023.04.15' ] - include: - - arch: 'linux_amd64_gcc4' - container: 'quay.io/pypa/manylinux2014_x86_64' - vcpkg_triplet: 'x64-linux' - - arch: 'linux_amd64' - container: 'ubuntu:18.04' - vcpkg_triplet: 'x64-linux' - - arch: 'linux_arm64' - container: 'ubuntu:18.04' - vcpkg_triplet: 'arm64-linux' - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - GEN: Ninja - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - - steps: - - name: Install required ubuntu packages - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - run: | - apt-get update -y -qq - apt-get install -y -qq software-properties-common - add-apt-repository ppa:git-core/ppa - apt-get update -y -qq - apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client - apt-get install -y -qq tar pkg-config - - - name: Install Git 2.18.5 - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - run: | - wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz - tar xvf v2.18.5.tar.gz - cd git-2.18.5 - make - make prefix=/usr install - git --version - - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - - name: Setup ManyLinux2014 - if: ${{ matrix.arch == 'linux_amd64_gcc4' }} - run: | - ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh openssl python_alias - - - name: Setup Ubuntu - if: ${{ matrix.arch == 'linux_amd64' || matrix.arch == 'linux_arm64' }} - uses: ./duckdb/.github/actions/ubuntu_18_setup - with: - aarch64_cross_compile: 1 - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 - - # Build extension - - name: Build extension - env: - GEN: ninja - STATIC_LIBCPP: 1 - CC: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-gcc' || '' }} - CXX: ${{ matrix.arch == 'linux_arm64' && 'aarch64-linux-gnu-g++' || '' }} - run: | - make release - - - name: Test extension - if: ${{ matrix.arch == 'linux_amd64_gcc4' || matrix.arch == 'linux_amd64' }} - run: | - make test - - - uses: actions/upload-artifact@v2 - with: - name: ${{matrix.arch}}-extensions - path: | - build/release/extension/iceberg/iceberg.duckdb_extension - - - name: Deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} - BUCKET_NAME: ${{ secrets.S3_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_KEY }} - run: | - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME false - fi \ No newline at end of file diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml deleted file mode 100644 index 4a522bb..0000000 --- a/.github/workflows/MacOS.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: MacOS -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - macos: - name: MacOS Release (${{ matrix.osx_build_arch }}) - runs-on: macos-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - vcpkg_version: [ '2023.04.15' ] - vcpkg_triplet: [ 'x64-osx', 'arm64-osx' ] - include: - - vcpkg_triplet: 'x64-osx' - osx_build_arch: 'x86_64' - duckdb_arch: 'osx_amd64' - - vcpkg_triplet: 'arm64-osx' - osx_build_arch: 'arm64' - duckdb_arch: 'osx_arm64' - - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - OSX_BUILD_ARCH: ${{ matrix.osx_build_arch }} - GEN: Ninja - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Install Ninja - run: brew install ninja - - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - with: - key: ${{ github.job }}-${{ matrix.duckdb_version }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 - - - name: Build extension - shell: bash - run: | - make release - - - name: Test Extension - if: ${{ matrix.osx_build_arch == 'x86_64'}} - shell: bash - run: | - make test_release - - - uses: actions/upload-artifact@v2 - with: - name: osx-${{ matrix.osx_build_arch }}-extension - path: | - build/release/extension/iceberg/iceberg.duckdb_extension - - - name: Deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} - BUCKET_NAME: ${{ secrets.S3_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_KEY }} - run: | - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME false - fi \ No newline at end of file diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml new file mode 100644 index 0000000..0d3dec2 --- /dev/null +++ b/.github/workflows/MainDistributionPipeline.yml @@ -0,0 +1,32 @@ +# +# This workflow calls the main distribution pipeline from DuckDB to build, test and (optionally) release the extension +# +name: Main Extension Distribution Pipeline +on: + push: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true + +jobs: + duckdb-stable-build: + name: Build extension binaries + uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@v0.10.0 + with: + extension_name: iceberg + duckdb_version: 'v0.10.0' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' + + duckdb-stable-deploy: + name: Deploy extension binaries + needs: duckdb-stable-build + uses: duckdb/duckdb/.github/workflows/_extension_deploy.yml@v0.10.0 + secrets: inherit + with: + extension_name: iceberg + duckdb_version: 'v0.10.0' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' + deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} \ No newline at end of file diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml deleted file mode 100644 index e387449..0000000 --- a/.github/workflows/Windows.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Windows -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - windows: - name: Release - runs-on: windows-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - vcpkg_version: [ '2023.04.15' ] - vcpkg_triplet: ['x64-windows-static-md'] - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - GEN: Ninja - VCPKG_ROOT: ${{ github.workspace }}\vcpkg - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Checkout DuckDB to version - # Add commits/tags to build against other DuckDB versions - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - - name: Build extension - run: | - make release - - # TODO: Windows tests currently failing due to path separator issue. -# - name: Test Extension -# shell: bash -# run: | -# build/release/test/Release/unittest.exe --test-dir . "*.test" - - - uses: actions/upload-artifact@v2 - with: - name: windows-extension - path: | - build/release/extension/iceberg/iceberg.duckdb_extension - - - name: error log - if: failure() - shell: bash - run: cat D:/a/duckdb_iceberg/duckdb_iceberg/vcpkg/buildtrees/avro-cpp/install-x64-windows-static-md-dbg-out.log - - - name: Deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} - BUCKET_NAME: ${{ secrets.S3_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_KEY }} - run: | - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - python -m pip install awscli - ./scripts/extension-upload.sh iceberg ${{ github.ref_name }} $DUCKDB_VERSION windows_amd64 $BUCKET_NAME true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - python -m pip install awscli - ./scripts/extension-upload.sh iceberg `git log -1 --format=%h` $DUCKDB_VERSION windows_amd64 $BUCKET_NAME false - fi \ No newline at end of file diff --git a/Makefile b/Makefile index b2c5296..6757fa6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all clean format debug release duckdb_debug duckdb_release pull update +.PHONY: all clean format debug release duckdb_debug duckdb_release pull update wasm_mvp wasm_eh wasm_threads all: release @@ -6,6 +6,12 @@ MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) PROJ_DIR := $(dir $(MKFILE_PATH)) DISABLE_SANITIZER_FLAG ?= +ifeq ($(OS),Windows_NT) + TEST_PATH="/test/Release/unittest.exe" +else + TEST_PATH="/test/unittest" +endif + OSX_BUILD_UNIVERSAL_FLAG= ifneq (${OSX_BUILD_ARCH}, "") OSX_BUILD_UNIVERSAL_FLAG=-DOSX_BUILD_ARCH=${OSX_BUILD_ARCH} @@ -17,7 +23,7 @@ endif ifeq (${DISABLE_SANITIZER}, 1) DISABLE_SANITIZER_FLAG=-DENABLE_SANITIZER=FALSE -DENABLE_UBSAN=0 endif - +#### VCPKG config VCPKG_TOOLCHAIN_PATH?= ifneq ("${VCPKG_TOOLCHAIN_PATH}", "") TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_MANIFEST_DIR='${PROJ_DIR}' -DVCPKG_BUILD=1 -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}' @@ -26,17 +32,28 @@ ifneq ("${VCPKG_TARGET_TRIPLET}", "") TOOLCHAIN_FLAGS:=${TOOLCHAIN_FLAGS} -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' endif +#### Enable Ninja as generator ifeq ($(GEN),ninja) GENERATOR=-G "Ninja" FORCE_COLOR=-DFORCE_COLORED_OUTPUT=1 endif -BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 -DBUILD_EXTENSIONS="httpfs" ${OSX_BUILD_UNIVERSAL_FLAG} ${STATIC_LIBCPP} ${TOOLCHAIN_FLAGS} +BUILD_FLAGS=-DEXTENSION_STATIC_BUILD=1 -DBUILD_EXTENSIONS="httpfs" ${OSX_BUILD_UNIVERSAL_FLAG} ${STATIC_LIBCPP} ${TOOLCHAIN_FLAGS} -DDUCKDB_EXPLICIT_PLATFORM='${DUCKDB_PLATFORM}' -CLIENT_FLAGS := +EXT_NAME=iceberg + +#### Configuration for this extension +EXTENSION_NAME=ICEBERG +EXTENSION_FLAGS=\ +-DDUCKDB_EXTENSION_NAMES="iceberg" \ +-DDUCKDB_EXTENSION_${EXTENSION_NAME}_PATH="$(PROJ_DIR)" \ +-DDUCKDB_EXTENSION_${EXTENSION_NAME}_LOAD_TESTS=1 \ +-DDUCKDB_EXTENSION_${EXTENSION_NAME}_SHOULD_LINK=1 \ +-DDUCKDB_EXTENSION_${EXTENSION_NAME}_INCLUDE_PATH="$(PROJ_DIR)src/include" \ +-DDUCKDB_EXTENSION_${EXTENSION_NAME}_TEST_PATH="$(PROJ_DIR)test/sql" -# These flags will make DuckDB build the extension -EXTENSION_FLAGS=-DDUCKDB_EXTENSION_NAMES="iceberg" -DDUCKDB_EXTENSION_ICEBERG_PATH="$(PROJ_DIR)" -DDUCKDB_EXTENSION_ICEBERG_SHOULD_LINK=1 -DDUCKDB_EXTENSION_ICEBERG_INCLUDE_PATH="$(PROJ_DIR)src/include" + +CLIENT_FLAGS := pull: git submodule init @@ -84,12 +101,10 @@ release_python: release # Main tests test: test_release - test_release: release - ./build/release/test/unittest --test-dir . "[sql]" - + ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" test_debug: debug - ./build/debug/test/unittest --test-dir . "[sql]" + ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" # Client tests test_js: test_debug_js @@ -121,4 +136,24 @@ data_large: data data_clean python3 scripts/test_data_generator/generate_iceberg.py 1 data/iceberg/generated_spec2_1 2 data_clean: - rm -rf data/iceberg/generated_* \ No newline at end of file + rm -rf data/iceberg/generated_* + +WASM_LINK_TIME_FLAGS= + +wasm_mvp: + mkdir -p build/wasm_mvp + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_mvp -DCMAKE_CXX_FLAGS="-DDUCKDB_CUSTOM_PLATFORM=wasm_mvp" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_mvp + cd build/wasm_mvp/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) + +wasm_eh: + mkdir -p build/wasm_eh + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_eh -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_eh" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_eh + cd build/wasm_eh/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) + +wasm_threads: + mkdir -p ./build/wasm_threads + emcmake cmake $(GENERATOR) -DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 -Bbuild/wasm_threads -DCMAKE_CXX_FLAGS="-fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1 -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -DDUCKDB_CUSTOM_PLATFORM=wasm_threads" -DSKIP_EXTENSIONS="parquet" -S duckdb $(TOOLCHAIN_FLAGS) $(EXTENSION_FLAGS) -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + emmake make -j8 -Cbuild/wasm_threads + cd build/wasm_threads/extension/${EXT_NAME} && emcc $f -sSIDE_MODULE=1 -o ../../${EXT_NAME}.duckdb_extension.wasm -O3 ${EXT_NAME}.duckdb_extension $(WASM_LINK_TIME_FLAGS) diff --git a/README.md b/README.md index 364ac68..30c7c1a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,4 @@ -> **Disclaimer:** This extension is currently in an experimental state. Feel free to try it out, but be aware that minimal testing and -benchmarking was done. - -> Warning: This extension currently builds with the main branch of DuckDB. A PR is being worked on. When the PR is merged, -this extension will be updated and usable from (nightly) DuckDB releases. +> **Disclaimer:** This extension is currently in an experimental state. Feel free to try it out, but be aware that things may not work as expected # DuckDB extension for Apache Iceberg diff --git a/duckdb b/duckdb index 275f4a7..20b1486 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 275f4a7e9564db08cc3cda211e1f63a7967308cf +Subproject commit 20b1486d1192f9fbd2328d1122b5afe5f1747fce diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh deleted file mode 100755 index 2304642..0000000 --- a/scripts/extension-upload.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Usage: ./extension-upload.sh -# : Name of the extension -# : Version (commit / version tag) of the extension -# : Version (commit / version tag) of DuckDB -# : Architecture target of the extension binary -# : S3 bucket to upload to -# : Set this as the latest version ("true" / "false", default: "false") - -set -e - -ext="build/release/extension/$1/$1.duckdb_extension" - -# (Optionally) Sign binary -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - duckdb/scripts/compute-extension-hash.sh $ext > $ext.hash - openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign - cat $ext.sign >> $ext -fi - -# compress extension binary -gzip < "${ext}" > "$ext.gz" - -# upload compressed extension binary to S3 -aws s3 cp $ext.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read - -# upload to latest if copy_to_latest is set to true -if [[ $6 = 'true' ]]; then - aws s3 cp $ext.gz s3://$5/$1/latest/$3/$4/$1.duckdb_extension.gz --acl public-read -fi - -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - rm private.pem -fi \ No newline at end of file diff --git a/scripts/start-rest-catalog.sh b/scripts/start-rest-catalog.sh index 393caca..ba8fca8 100755 --- a/scripts/start-rest-catalog.sh +++ b/scripts/start-rest-catalog.sh @@ -14,11 +14,15 @@ python3 provision.py UNPARTITIONED_TABLE_PATH=$(curl -s http://127.0.0.1:8181/v1/namespaces/default/tables/table_unpartitioned | jq -r '."metadata-location"') SQL=$(cat <<-END -SET s3_access_key_id='admin'; -SET s3_secret_access_key='password'; -SET s3_endpoint='127.0.0.1:9000'; -SET s3_url_style='path'; -SET s3_use_ssl=false; + +CREATE SECRET ( + TYPE S3, + KEY_ID 'admin', + SECRET 'password', + ENDPOINT '127.0.0.1:9000', + URL_STYLE 'path', + USE_SSL 0 +); SELECT * FROM iceberg_scan('${UNPARTITIONED_TABLE_PATH}'); END diff --git a/test/sql/iceberg_snapshots.test b/test/sql/iceberg_snapshots.test index 943f68b..6b93f12 100644 --- a/test/sql/iceberg_snapshots.test +++ b/test/sql/iceberg_snapshots.test @@ -8,6 +8,8 @@ SELECT * FROM ICEBERG_SNAPSHOTS('data/iceberg/lineitem_iceberg'); ---- Catalog Error +require notwindows + require iceberg query IIII