diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index aa42fd5..cab4e5d 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -15,9 +15,7 @@ jobs: strategy: matrix: # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4'] - vcpkg_version: [ '2023.04.15' ] + arch: ['linux_amd64', 'linux_amd64_gcc4'] include: - arch: 'linux_amd64_gcc4' container: 'quay.io/pypa/manylinux2014_x86_64' @@ -25,9 +23,6 @@ jobs: - arch: 'linux_amd64' container: 'ubuntu:18.04' vcpkg_triplet: 'x64-linux' - - arch: 'linux_arm64' - container: 'ubuntu:18.04' - vcpkg_triplet: 'arm64-linux' env: VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} GEN: Ninja @@ -59,12 +54,6 @@ jobs: fetch-depth: 0 submodules: 'true' - - name: Checkout DuckDB to version - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - name: Setup ManyLinux2014 if: ${{ matrix.arch == 'linux_amd64_gcc4' }} run: | @@ -94,34 +83,4 @@ jobs: - name: Test extension if: ${{ matrix.arch == 'linux_amd64_gcc4' || matrix.arch == 'linux_amd64' }} run: | - make test - - - uses: actions/upload-artifact@v2 - with: - name: ${{matrix.arch}}-extensions - path: | - build/release/extension/iceberg/iceberg.duckdb_extension - - - name: Deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} - BUCKET_NAME: ${{ secrets.S3_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_KEY }} - run: | - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME false - fi \ No newline at end of file + make test \ No newline at end of file diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml index 4a522bb..baa4b23 100644 --- a/.github/workflows/MacOS.yml +++ b/.github/workflows/MacOS.yml @@ -14,8 +14,6 @@ jobs: strategy: matrix: # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - vcpkg_version: [ '2023.04.15' ] vcpkg_triplet: [ 'x64-osx', 'arm64-osx' ] include: - vcpkg_triplet: 'x64-osx' @@ -64,34 +62,4 @@ jobs: if: ${{ matrix.osx_build_arch == 'x86_64'}} shell: bash run: | - make test_release - - - uses: actions/upload-artifact@v2 - with: - name: osx-${{ matrix.osx_build_arch }}-extension - path: | - build/release/extension/iceberg/iceberg.duckdb_extension - - - name: Deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} - BUCKET_NAME: ${{ secrets.S3_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_KEY }} - run: | - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - python3 -m pip install pip awscli - ./scripts/extension-upload.sh iceberg `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME false - fi \ No newline at end of file + make test_release \ No newline at end of file diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 6cf4e76..0d3dec2 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,21 +14,19 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@6812703823d1d66566bc7eaac2b6e4b273c85333 + uses: duckdb/duckdb/.github/workflows/_extension_distribution.yml@v0.10.0 with: - vcpkg_commit: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 - duckdb_version: v0.9.2 extension_name: iceberg - exclude_archs: "wasm_mvp;wasm_eh;wasm_threads" + duckdb_version: 'v0.10.0' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' duckdb-stable-deploy: name: Deploy extension binaries needs: duckdb-stable-build - uses: ./.github/workflows/_extension_deploy.yml + uses: duckdb/duckdb/.github/workflows/_extension_deploy.yml@v0.10.0 secrets: inherit with: - duckdb_version: v0.9.2 extension_name: iceberg - exclude_archs: "wasm_mvp;wasm_eh;wasm_threads" - deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} - deploy_versioned: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} + duckdb_version: 'v0.10.0' + exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools' + deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} \ No newline at end of file diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml deleted file mode 100644 index e387449..0000000 --- a/.github/workflows/Windows.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Windows -on: [push, pull_request,repository_dispatch] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true -defaults: - run: - shell: bash - -jobs: - windows: - name: Release - runs-on: windows-latest - strategy: - matrix: - # Add commits/tags to build against other DuckDB versions - duckdb_version: [ '' ] - vcpkg_version: [ '2023.04.15' ] - vcpkg_triplet: ['x64-windows-static-md'] - env: - VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} - GEN: Ninja - VCPKG_ROOT: ${{ github.workspace }}\vcpkg - VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}\vcpkg\scripts\buildsystems\vcpkg.cmake - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Setup vcpkg - uses: lukka/run-vcpkg@v11.1 - with: - vcpkgGitCommitId: a42af01b72c28a8e1d7b48107b33e4f286a55ef6 - - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Checkout DuckDB to version - # Add commits/tags to build against other DuckDB versions - if: ${{ matrix.duckdb_version != ''}} - run: | - cd duckdb - git checkout ${{ matrix.duckdb_version }} - - - name: Build extension - run: | - make release - - # TODO: Windows tests currently failing due to path separator issue. -# - name: Test Extension -# shell: bash -# run: | -# build/release/test/Release/unittest.exe --test-dir . "*.test" - - - uses: actions/upload-artifact@v2 - with: - name: windows-extension - path: | - build/release/extension/iceberg/iceberg.duckdb_extension - - - name: error log - if: failure() - shell: bash - run: cat D:/a/duckdb_iceberg/duckdb_iceberg/vcpkg/buildtrees/avro-cpp/install-x64-windows-static-md-dbg-out.log - - - name: Deploy - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} - BUCKET_NAME: ${{ secrets.S3_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.DUCKDB_EXTENSION_SIGNING_KEY }} - run: | - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then - echo 'No key set, skipping' - elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then - python -m pip install awscli - ./scripts/extension-upload.sh iceberg ${{ github.ref_name }} $DUCKDB_VERSION windows_amd64 $BUCKET_NAME true - elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then - python -m pip install awscli - ./scripts/extension-upload.sh iceberg `git log -1 --format=%h` $DUCKDB_VERSION windows_amd64 $BUCKET_NAME false - fi \ No newline at end of file diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml deleted file mode 100644 index e1caa57..0000000 --- a/.github/workflows/_extension_deploy.yml +++ /dev/null @@ -1,123 +0,0 @@ -# -# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml -# -# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the -# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as -# this workflow can be configured to use a custom deploy script. - - -name: Extension Deployment -on: - workflow_call: - inputs: - # The name of the extension - extension_name: - required: true - type: string - # DuckDB version to build against - duckdb_version: - required: true - type: string - # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' - exclude_archs: - required: false - type: string - default: "" - # Whether to upload this deployment as the latest. This may overwrite a previous deployment. - deploy_latest: - required: false - type: boolean - default: false - # Whether to upload this deployment under a versioned path. These will not be deleted automatically - deploy_versioned: - required: false - type: boolean - default: false - # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times - artifact_postfix: - required: false - type: string - default: "" - # Override the default deploy script with a custom script - deploy_script: - required: false - type: string - default: "./scripts/extension-upload-ci.sh*" - # Override the default matrix parse script with a custom script - matrix_parse_script: - required: false - type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" - -jobs: - generate_matrix: - name: Generate matrix - runs-on: ubuntu-latest - outputs: - deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - id: parse-matrices - run: | - cat ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json - grep wasm distribution_matrix.json || (head -n -1 ./duckdb/.github/config/distribution_matrix.json > distribution_matrix.json && echo ',"wasm":{"include":[{"duckdb_arch":"wasm_mvp","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_eh","vcpkg_triplet":"wasm32-emscripten"},{"duckdb_arch":"wasm_threads","vcpkg_triplet":"wasm32-emscripten"}]}}' >> distribution_matrix.json) - python3 ${{ inputs.matrix_parse_script }} --input distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - deploy_matrix="`cat deploy_matrix.json`" - echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT - echo `cat $GITHUB_OUTPUT` - - deploy: - name: Deploy - runs-on: ubuntu-latest - needs: generate_matrix - if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} - strategy: - matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - uses: actions/download-artifact@v2 - with: - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} - path: | - /tmp/extension - - - name: Deploy - shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_ORG_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_DUCKDB_ORG_REGION }} - BUCKET_NAME: ${{ secrets.S3_DUCKDB_ORG_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} - run: | - pwd - python3 -m pip install pip awscli - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - git fetch --tags - export EXT_VERSION=`git tag --points-at HEAD` - export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} - ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} diff --git a/duckdb b/duckdb index 275f4a7..20b1486 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 275f4a7e9564db08cc3cda211e1f63a7967308cf +Subproject commit 20b1486d1192f9fbd2328d1122b5afe5f1747fce diff --git a/scripts/extension-upload-ci.sh b/scripts/extension-upload-ci.sh deleted file mode 100755 index 32097f2..0000000 --- a/scripts/extension-upload-ci.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -# Extension upload script - -# Usage: ./extension-upload.sh -# : Name of the extension -# : Version (commit / version tag) of the extension -# : Version (commit / version tag) of DuckDB -# : Architecture target of the extension binary -# : S3 bucket to upload to -# : Set this as the latest version ("true" / "false", default: "false") -# : Set this as a versioned version that will prevent its deletion - -set -e - -if [[ $4 == wasm* ]]; then - ext="/tmp/extension/$1.duckdb_extension.wasm" -else - ext="/tmp/extension/$1.duckdb_extension" -fi - -echo $ext - -script_dir="$(dirname "$(readlink -f "$0")")" - -# calculate SHA256 hash of extension binary -cat $ext > $ext.append - -if [[ $4 == wasm* ]]; then - # 0 for custom section - # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256) - # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02] - echo -n -e '\x00' >> $ext.append - echo -n -e '\x93\x02' >> $ext.append - # 10 in hex = 16 in decimal, lenght of name, 1 byte - echo -n -e '\x10' >> $ext.append - echo -n -e 'duckdb_signature' >> $ext.append - # the name of the WebAssembly custom section, 16 bytes - # 100 in hex, 256 in decimal - # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)], - # for a grand total of 2 bytes - echo -n -e '\x80\x02' >> $ext.append -fi - -# (Optionally) Sign binary -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash - openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign - rm -f private.pem -fi - -# Signature is always there, potentially defaulting to 256 zeros -truncate -s 256 $ext.sign - -# append signature to extension binary -cat $ext.sign >> $ext.append - -# compress extension binary -if [[ $4 == wasm_* ]]; then - gzip < $ext.append > "$ext.compressed" -else - brotli < $ext.append > "$ext.compressed" -fi - -set -e - -# Abort if AWS key is not set -if [ -z "$AWS_ACCESS_KEY_ID" ]; then - echo "No AWS key found, skipping.." - exit 0 -fi - -# upload versioned version -if [[ $7 = 'true' ]]; then - if [[ $4 == wasm* ]]; then - aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$1/$2/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" - else - aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read - fi -fi - -# upload to latest version -if [[ $6 = 'true' ]]; then - if [[ $4 == wasm* ]]; then - aws s3 cp $ext.compressed s3://$5/duckdb-wasm/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm" - else - aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read - fi -fi diff --git a/scripts/extension-upload.sh b/scripts/extension-upload.sh deleted file mode 100755 index 2304642..0000000 --- a/scripts/extension-upload.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Usage: ./extension-upload.sh -# : Name of the extension -# : Version (commit / version tag) of the extension -# : Version (commit / version tag) of DuckDB -# : Architecture target of the extension binary -# : S3 bucket to upload to -# : Set this as the latest version ("true" / "false", default: "false") - -set -e - -ext="build/release/extension/$1/$1.duckdb_extension" - -# (Optionally) Sign binary -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem - duckdb/scripts/compute-extension-hash.sh $ext > $ext.hash - openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign - cat $ext.sign >> $ext -fi - -# compress extension binary -gzip < "${ext}" > "$ext.gz" - -# upload compressed extension binary to S3 -aws s3 cp $ext.gz s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read - -# upload to latest if copy_to_latest is set to true -if [[ $6 = 'true' ]]; then - aws s3 cp $ext.gz s3://$5/$1/latest/$3/$4/$1.duckdb_extension.gz --acl public-read -fi - -if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then - rm private.pem -fi \ No newline at end of file