From 9f573603a3a4c6479003c84b28296d668165fc81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20M=C3=BChleisen?= Date: Mon, 29 Apr 2024 09:55:42 +0200 Subject: [PATCH] moar CI --- .../actions/manylinux_2014_setup/action.yml | 111 +++++++ .github/actions/ubuntu_18_setup/action.yml | 100 ++++++ .github/workflows/Java.yml | 308 ++++++++++++++++++ 3 files changed, 519 insertions(+) create mode 100644 .github/actions/manylinux_2014_setup/action.yml create mode 100644 .github/actions/ubuntu_18_setup/action.yml create mode 100644 .github/workflows/Java.yml diff --git a/.github/actions/manylinux_2014_setup/action.yml b/.github/actions/manylinux_2014_setup/action.yml new file mode 100644 index 00000000..8f524e61 --- /dev/null +++ b/.github/actions/manylinux_2014_setup/action.yml @@ -0,0 +1,111 @@ +name: "Setup manylinux2014 image" +description: "Installs/configures additional CI dependencies" +inputs: + aws-cli: + description: 'Setup aws-cli' + default: 0 + ninja-build: + description: 'Setup ninja-build' + default: 0 + vcpkg: + description: 'Setup vcpkg (installs to $GITHUB_WORKSPACE/vcpkg)' + default: 0 + openssl: + description: 'Setup OpenSSL (requires vcpkg to also be installed)' + default: 0 + ccache: + description: 'Setup Ccache' + default: 0 + jdk: + description: 'Setup JDK' + default: 0 + odbc: + description: 'Setup ODBC' + default: 0 + ssh: + description: 'Setup SSH' + default: 0 + glibc32: + description: 'Setup 32bit glibc' + default: 0 + nodejs: + description: 'Setup NodeJS' + default: 0 + gcc_4_8: + description: 'Setup GCC 4.8 (installs to /usr/bin/g++, default will still be GCC 10)' + default: 0 + python_alias: + description: 'Create an alias for python3 to python3.9' + default: 0 + +runs: + using: "composite" + steps: + - name: Setup general dependencies + shell: bash + run: scripts/setup_manylinux2014.sh general + + - name: Install AWS CLI + if: ${{ inputs.aws-cli == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh aws-cli + + - name: Setup dependencies for ODBC + if: ${{ inputs.odbc == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh odbc + + - name: Setup dependencies for ccache + if: ${{ inputs.ccache == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh ccache + + - name: Setup JDK + if: ${{ inputs.jdk == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh jdk + + - name: Setup SSH + if: ${{ inputs.ssh == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh ssh + + - name: Setup NodeJS + if: ${{ inputs.nodejs == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh nodejs + + - name: Setup 32bit compiler + if: ${{ inputs.glibc32 == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh glibc32 + + - name: Setup python3 as python3.9 + if: ${{ inputs.python_alias == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh python_alias + + - name: Setup old (GCC 4.8) compiler + if: ${{ inputs.gcc_4_8 == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh gcc_4_8 + + # Note instead of using scripts/setup_manylinux2014.sh vcpkg, we prefer to use + # lukka/run-vcpkg@v11.1 here as it configures vcpkg to cache to GH actions. + - name: Setup vcpkg + if: ${{ inputs.vcpkg == 1 }} + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 + + - name: Install OpenSSL + if: ${{ inputs.openssl == 1 }} + shell: bash + run: scripts/setup_manylinux2014.sh openssl + + - name: Setup Ccache + if: ${{ inputs.ccache == 1 }} + uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} \ No newline at end of file diff --git a/.github/actions/ubuntu_18_setup/action.yml b/.github/actions/ubuntu_18_setup/action.yml new file mode 100644 index 00000000..d8938ab1 --- /dev/null +++ b/.github/actions/ubuntu_18_setup/action.yml @@ -0,0 +1,100 @@ +name: "Setup Ubuntu 18" +description: "Setup an Ubuntu 18 docker container with the required libraries" +inputs: + openssl: + description: 'OpenSSL' + default: 0 + python: + description: 'Python' + default: 1 + aarch64_cross_compile: + description: 'Install dependencies for aarch64 cross-compiling' + default: 0 + vcpkg: + description: 'Install vcpkg' + default: 0 + ccache: + description: 'Install ccache' + default: 0 + +runs: + using: "composite" + steps: + - name: Install + shell: bash + run: | + apt-get update -y -qq + apt-get install -y -qq software-properties-common + add-apt-repository ppa:git-core/ppa + apt-get update -y -qq + apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client pkg-config + + - name: Install + shell: bash + if: ${{ inputs.aarch64_cross_compile == 1 }} + run: | + apt-get install -y -qq gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + + - name: Install Git 2.18.5 + shell: bash + run: | + wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz + tar xvf v2.18.5.tar.gz + cd git-2.18.5 + make + make prefix=/usr install + git --version + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install CMake 3.21 + shell: bash + run: | + wget https://github.com/Kitware/CMake/releases/download/v3.21.3/cmake-3.21.3-linux-x86_64.sh + chmod +x cmake-3.21.3-linux-x86_64.sh + ./cmake-3.21.3-linux-x86_64.sh --skip-license --prefix=/usr/local + cmake --version + + - name: Install Python 3.8 + if: ${{ inputs.python }} == 1 + shell: bash + run: | + wget https://www.python.org/ftp/python/3.8.17/Python-3.8.17.tgz + tar xvf Python-3.8.17.tgz + cd Python-3.8.17 + mkdir -p pythonbin + ./configure --with-ensurepip=install + make -j + make install + python3.8 --version + python3.8 -m pip install pip + python3.8 -m pip install requests awscli + + - name: Version Check + shell: bash + run: | + ldd --version ldd + python3 --version + git --version + git log -1 --format=%h + + - name: Setup vcpkg + if: ${{ inputs.vcpkg == 1 }} + uses: lukka/run-vcpkg@v11.1 + with: + vcpkgGitCommitId: a1a1cbc975abf909a6c8985a6a2b8fe20bbd9bd6 + + - name: Setup Ccache + if: ${{ inputs.ccache == 1 }} + uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + + - name: Instal OpenSSL through vcpkg + if: ${{ inputs.openssl == 1 }} + shell: bash + run: | + cd $VCPKG_ROOT && ./vcpkg install openssl --triplet=${{ inputs.aarch64_cross_compile == 1 && 'arm64-linux' || 'x64-linux' }} \ No newline at end of file diff --git a/.github/workflows/Java.yml b/.github/workflows/Java.yml new file mode 100644 index 00000000..973515e0 --- /dev/null +++ b/.github/workflows/Java.yml @@ -0,0 +1,308 @@ +name: Java JDBC +on: + push: + pull_request: + workflow_dispatch: + repository_dispatch: + +env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }} + +jobs: + java-linux-amd64: + name: Java Linux (amd64) + runs-on: ubuntu-latest + container: + image: quay.io/pypa/manylinux2014_x86_64 + env: + GEN: ninja + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - uses: ./.github/actions/manylinux_2014_setup + with: + ninja-build: 1 + ccache: 1 + jdk: 1 + python_alias: 1 + aws-cli: 1 + + - name: Build + shell: bash + run: make build + + - name: Java Tests + shell: bash + if: ${{ inputs.skip_tests != 'true' }} + run: make test + + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + run: | + cp build/release/duckdb_jdbc.jar duckdb_jdbc-linux-amd64.jar + # ./scripts/upload-assets-to-staging.sh github_release duckdb_jdbc-linux-amd64.jar + - uses: actions/upload-artifact@v3 + with: + name: java-linux-amd64 + path: | + build/release/duckdb_jdbc.jar + + java-linux-aarch64: + name: Java Linux (aarch64) + runs-on: ubuntu-latest + container: + image: quay.io/pypa/manylinux2014_arm64 + needs: java-linux-amd64 + env: + GEN: ninja + DUCKDB_PLATFORM: linux_arm64 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - uses: ./.github/actions/ubuntu_18_setup + with: + ccache: 1 + aarch64_cross_compile: 1 + + - name: Install Stuff + shell: bash + run: > + curl -L https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u345-b01/OpenJDK8U-jdk_x64_linux_hotspot_8u345b01.tar.gz | tar xvz + + - name: Build + shell: bash + run: CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ make build + + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + run: | + cp build/release/duckdb_jdbc.jar duckdb_jdbc-linux-aarch64.jar + # ./scripts/upload-assets-to-staging.sh github_release duckdb_jdbc-linux-aarch64.jar + + - uses: actions/upload-artifact@v3 + with: + name: java-linux-aarch64 + path: | + build/release/duckdb_jdbc.jar + + + java-windows-amd64: + name: Java Windows (arm64) + runs-on: windows-latest + needs: java-linux-amd64 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + + - name: Build + shell: bash + run: > + python scripts/windows_ci.py + + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_GENERATOR_PLATFORM=x64 -DJDBC_DRIVER=1 -DBUILD_EXTENSIONS=json -DENABLE_EXTENSION_AUTOLOADING=1 -DENABLE_EXTENSION_AUTOINSTALL=1 -DBUILD_SHELL=0 -DOVERRIDE_GIT_DESCRIBE="$OVERRIDE_GIT_DESCRIBE" + + cmake --build . --config Release + - name: Java Tests + if: ${{ inputs.skip_tests != 'true' }} + shell: bash + working-directory: tools/jdbc + run: make test_release + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + run: | + cp tools/jdbc/duckdb_jdbc.jar duckdb_jdbc-windows-amd64.jar + ./scripts/upload-assets-to-staging.sh github_release duckdb_jdbc-windows-amd64.jar + - uses: actions/upload-artifact@v3 + with: + name: java-windows-amd64 + path: | + tools/jdbc/duckdb_jdbc.jar + + + java-osx-universal: + name: Java OSX (Universal) + runs-on: macos-14 + needs: java-linux-amd64 + env: + BUILD_JDBC: 1 + BUILD_JSON: 1 + OSX_BUILD_UNIVERSAL: 1 + ENABLE_EXTENSION_AUTOLOADING: 1 + ENABLE_EXTENSION_AUTOINSTALL: 1 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' }} + - name: Build + shell: bash + run: make + - name: Java Tests + if: ${{ inputs.skip_tests != 'true' }} + shell: bash + run: make test_release + - name: Java Example + shell: bash + run: | + (cd examples/jdbc; make; make maven) + - name: Deploy + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} + run: | + cp build/release/duckdb_jdbc.jar duckdb_jdbc-osx-universal.jar + ./scripts/upload-assets-to-staging.sh github_release duckdb_jdbc-osx-universal.jar + - uses: actions/upload-artifact@v3 + with: + name: java-osx-universal + path: | + build/release/duckdb_jdbc.jar + + + java-combine: + if: ${{ inputs.override_git_describe == '' }} + name: Java Combine + runs-on: ubuntu-latest + needs: + - java-linux-aarch64 + - java-linux-amd64 + - java-windows-amd64 + - java-osx-universal + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - shell: bash + run: mkdir jdbc-artifacts + + - uses: actions/download-artifact@v3 + with: + name: java-linux-aarch64 + path: jdbc-artifacts/java-linux-aarch64 + + - uses: actions/download-artifact@v3 + with: + name: java-linux-amd64 + path: jdbc-artifacts/java-linux-amd64 + + - uses: actions/download-artifact@v3 + with: + name: java-windows-amd64 + path: jdbc-artifacts/java-windows-amd64 + + - uses: actions/download-artifact@v3 + with: + name: java-osx-universal + path: jdbc-artifacts/java-osx-universal + + - name: Combine JARs + shell: bash + run: | + if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb" ]] ; then + export XML=' + + + + ossrh + hfmuehleisen + PASSWORD + + + ' + mkdir ~/.m2 + echo $XML | sed "s/PASSWORD/${{ secrets.MAVEN_PASSWORD }}/" > ~/.m2/settings.xml + echo "${{ secrets.MAVEN_PGP_PK }}" | base64 -d > maven_pgp_key + gpg --import maven_pgp_key + python scripts/jdbc_maven_deploy.py ${{ github.ref_name }} jdbc-artifacts . + fi + ls -lahR jdbc-artifacts + + - uses: actions/upload-artifact@v3 + with: + name: java-jars + path: | + jdbc-artifacts + + jdbc-compliance: + name: JDBC Compliance + runs-on: ubuntu-20.04 + if: ${{ inputs.skip_tests != 'true' }} + needs: java-linux-amd64 + container: quay.io/pypa/manylinux2014_x86_64 + env: + BUILD_JDBC: 1 + GEN: ninja + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.git_ref }} + + - uses: ./.github/actions/manylinux_2014_setup + with: + ninja-build: 1 + ccache: 1 + jdk: 1 + python_alias: 1 + aws-cli: 1 + + - name: Install + shell: bash + run: | + git clone https://github.com/cwida/jdbccts.git + + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases + with: + key: ${{ github.job }} + save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb' }} + + - name: Build + shell: bash + run: make release + + - name: Test + shell: bash + run: (cd jdbccts && make DUCKDB_JAR=../build/release/duckdb_jdbc.jar test)