From 1ff19ef859e71e7b5434ef5f0ecceb891f0345f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Mon, 22 Jul 2024 15:47:36 +0200 Subject: [PATCH 1/2] Scikit-build-core build system rework (#1988) Rework build system using `scikit-build-core` Rework all CI/CD logic to use new build system Fix CI/CD numpy compatibility tests --------- Co-authored-by: Isaiah Norton --- .github/disabled-workflows/release.yml | 26 - .github/workflows/build-wheels.yml | 146 ++++ .github/workflows/ci.yml | 103 +-- .github/workflows/daily-test-build-numpy.yml | 53 +- .github/workflows/daily-test-build.yml | 47 +- .github/workflows/daily-tests.yml | 11 +- .github/workflows/issue-if-azure-fail.yml | 33 - .gitignore | 6 - .readthedocs.yml | 6 +- CMakeLists.txt | 52 ++ MANIFEST.in | 12 - azure-pipelines.yml | 4 - cmake/DownloadTileDB.cmake | 125 +++ doc/local-build.sh | 14 +- doc/requirements_doc.txt | 4 - misc/azure-ci.yml | 98 --- misc/azure-libtiledb-darwin.yml | 97 --- misc/azure-print-logs.yml | 14 - misc/azure-release.yml | 257 ------- misc/clean | 2 - misc/requirements_ci.txt | 6 - misc/requirements_test.txt | 6 - misc/requirements_wheel.txt | 16 - pyproject.toml | 52 +- requirements.txt | 8 - requirements_dev.txt | 19 - setup.cfg | 4 - setup.py | 759 ------------------- tiledb/CMakeLists.txt | 82 ++ tiledb/__init__.py | 18 +- tiledb/cc/CMakeLists.txt | 50 ++ tiledb/tests/test_schema_evolution.py | 3 +- tiledb/tests/test_timestamp_overrides.py | 4 +- 33 files changed, 578 insertions(+), 1559 deletions(-) delete mode 100644 .github/disabled-workflows/release.yml create mode 100644 .github/workflows/build-wheels.yml delete mode 100644 .github/workflows/issue-if-azure-fail.yml create mode 100644 CMakeLists.txt delete mode 100644 MANIFEST.in delete mode 100644 azure-pipelines.yml create mode 100644 cmake/DownloadTileDB.cmake delete mode 100644 doc/requirements_doc.txt delete mode 100644 misc/azure-ci.yml delete mode 100644 misc/azure-libtiledb-darwin.yml delete mode 100644 misc/azure-print-logs.yml delete mode 100644 misc/clean delete mode 100644 misc/requirements_ci.txt delete mode 100644 misc/requirements_test.txt delete mode 100644 misc/requirements_wheel.txt delete mode 100644 requirements.txt delete mode 100644 requirements_dev.txt delete mode 100644 setup.cfg create mode 100644 tiledb/CMakeLists.txt create mode 100644 tiledb/cc/CMakeLists.txt diff --git a/.github/disabled-workflows/release.yml b/.github/disabled-workflows/release.yml deleted file mode 100644 index f579e482e9..0000000000 --- a/.github/disabled-workflows/release.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: TileDB Python Release - -on: - push: - branches: - - 'release-*' - tags: - - '**' - -jobs: - build: - name: Create Release - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token - with: - tag_name: ${{ github.ref }} - release_name: TileDB-Py ${{ github.ref }} - draft: false - prerelease: false \ No newline at end of file diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml new file mode 100644 index 0000000000..6f9ac9f6d4 --- /dev/null +++ b/.github/workflows/build-wheels.yml @@ -0,0 +1,146 @@ +name: Build and test wheels + +on: + workflow_dispatch: + push: + branches: + - release-* + - "*wheel*" # must quote since "*" is a YAML reserved character; we want a string + tags: + - "*" + pull_request: + branches: + - "*wheel*" # must quote since "*" is a YAML reserved character; we want a string + +jobs: + build_wheels: + name: Wheel ${{ matrix.buildplat[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.python }} + runs-on: ${{ matrix.buildplat[0] }} + strategy: + matrix: + buildplat: + - [ubuntu-22.04, manylinux_x86_64] + - [macos-13, macosx_x86_64] + - [macos-14, macosx_arm64] + - [windows-2022, win_amd64] + python: ["cp38", "cp39", "cp310", "cp311", "cp312"] + + steps: + - uses: actions/checkout@v4 + + - name: "Brew setup on macOS" # x-ref c8e49ba8f8b9ce + if: ${{ startsWith(matrix.os, 'macos-') == true }} + run: | + set -e pipefail + brew update + brew install automake pkg-config ninja llvm + + - name: Build wheels + uses: pypa/cibuildwheel@v2.18.1 + env: + CIBW_BUILD_VERBOSITY: 3 + CIBW_ENVIRONMENT_MACOS: > + CC=clang + CXX=clang++ + MACOSX_DEPLOYMENT_TARGET: "11.0" + CIBW_ARCHS: all + CIBW_PRERELEASE_PYTHONS: True + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }} + # __init__.py interferes with the tests and is included as local file instead of + # used from wheels. To be honest, tests should not be in the source folder at all. + CIBW_BEFORE_TEST: rm {project}/tiledb/__init__.py + with: + output-dir: wheelhouse + + - uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.buildplat[1] }}-${{ strategy.job-index }} + path: "./wheelhouse/*.whl" + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + outputs: + sdist_name: ${{ steps.get_sdist_name.outputs.sdist_name }} + steps: + - uses: actions/checkout@v4 + + - name: Build sdist + run: pipx run build --sdist + + - name: Get sdist package name + id: get_sdist_name + run: | + echo "sdist_name=$(ls dist/ | head -n 1)" >> "$GITHUB_OUTPUT" + + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + test_sdist: + name: Test source distribution package + needs: [build_sdist] + strategy: + matrix: + os: + - macos-13 + - macos-14 + - windows-2022 + - ubuntu-22.04 + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] + runs-on: ${{ matrix.os }} + steps: + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Download sdist artifact + uses: actions/download-artifact@v4 + with: + name: sdist + path: dist + + - name: Install sdist artifact + run: pip install --verbose dist/${{ needs.build_sdist.outputs.sdist_name }} + + - uses: actions/checkout@v4 + + - name: Install test dependencies + run: pip install pytest hypothesis psutil pyarrow + + - name: Run tests + shell: bash + run: | + PROJECT_CWD=$PWD + rm tiledb/__init__.py + cd .. + pytest -vv --showlocals $PROJECT_CWD + + upload_pypi: + needs: [build_wheels, test_sdist] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + outputs: + package_version: ${{ steps.get_package_version.outputs.package_version }} + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - id: get_package_version + run: | + echo "package_version=$(ls dist/ | head -n 1 | cut -d - -f 2)" >> "$GITHUB_OUTPUT" + + - name: Upload to test-pypi + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + - name: Upload to pypi + if: startsWith(github.ref, 'refs/tags/') + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66fce61bb4..1b96871ad3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,27 +14,29 @@ jobs: shell: bash strategy: matrix: - os: [ubuntu-latest, macos-12, windows-latest] + os: + - ubuntu-latest + - macos-13 + # libfaketime tests fail on macos arm. Disable tests for now. + # - macos-14 + - windows-latest python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] fail-fast: false env: - # 11.7 necessary due to: https://github.com/actions/setup-python/issues/682#issuecomment-1604261330 - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-12' && matrix.python-version == '3.8' && '11.7' || '11' }} - #MACOSX_DEPLOYMENT_TARGET: "10.11" - # On windows-2019 we are using the Visual Studio generator, which is multi-config and places the build artifacts in a subdirectory + MACOSX_DEPLOYMENT_TARGET: "11" steps: - name: Checkout TileDB-Py `dev` - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup MSVC toolset (VS 2022) uses: TheMrMilchmann/setup-msvc-dev@v3 - if: matrix.os == 'windows-latest' + if: startsWith(matrix.os, 'windows') with: arch: x64 - name: Install Ninja (VS 2022) uses: seanmiddleditch/gha-setup-ninja@v4 - if: matrix.os == 'windows-latest' + if: startsWith(matrix.os, 'windows') - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -57,79 +59,29 @@ jobs: # - https://github.com/actions/runner-images/pull/7125 - name: "Install homebrew dependencies" run: brew install pkg-config - if: matrix.os == 'macos-12' - - - name: "Install dependencies" - run: python -m pip install --upgrade -r misc/requirements_ci.txt - - - name: "Get TILEDB_VERSION" - run: echo "LIBTILEDB_VERSION=$(python setup.py get_tiledb_version | tail -n 1)" >> $GITHUB_ENV - - - name: "Get LIBTILEDB_SHA" - run: echo "LIBTILEDB_SHA=$(git ls-remote https://github.com/TileDB-Inc/TileDB $LIBTILEDB_VERSION | cut -c1-7)" >> $GITHUB_ENV - - - name: "Download TileDB From Zip And Build TileDB-Py (Windows)" - run: | - choco install wget --no-progress - - if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-windows-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.zip; then - mkdir libtiledb - unzip tiledb-windows-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.zip -d libtiledb - cp libtiledb/bin/tiledb.dll tiledb - python setup.py develop --tiledb=libtiledb - else - # Build from source as fallback - python setup.py build_ext --inplace - python setup.py develop - fi - env: - TILEDB_FORCE_ALL_DEPS: True - CMAKE_GENERATOR: "Ninja" - if: matrix.os == 'windows-latest' - - - name: "Download TileDB From Tarball And Build TileDB-Py (macOS)" - run: | - set -xeo pipefail - - if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-macos-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz; then - mkdir libtiledb - sudo tar -vzxf tiledb-macos-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz -C libtiledb - python setup.py develop --tiledb=libtiledb - else - # Build from source as fallback - python setup.py build_ext --inplace - python setup.py develop - fi - if: matrix.os == 'macos-12' - - - name: "Download TileDB From Tarball And Build TileDB-Py (Linux)" - run: | - set -xeo pipefail - - if wget https://github.com/TileDB-Inc/TileDB/releases/download/$LIBTILEDB_VERSION/tiledb-linux-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz; then - mkdir libtiledb - sudo tar -vzxf tiledb-linux-x86_64-$LIBTILEDB_VERSION-$LIBTILEDB_SHA.tar.gz -C libtiledb - python setup.py develop --tiledb=libtiledb - else - # Build from source as fallback - python setup.py build_ext --inplace - python setup.py develop - fi - if: matrix.os == 'ubuntu-latest' - - - name: "Check build directory" - run: ls -Rl + if: startsWith(matrix.os, 'macos') - name: "Install libfaketime (linux and macOS)" - if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-12' + if: ${{ ! startsWith(matrix.os, 'windows') }} run: | git clone https://github.com/wolfcw/libfaketime/ cd libfaketime sudo make install cd .. + - name: "Build and Install TileDB-Py" + # We use pipx here to produce wheel/sdist to upload as artifact in case of error + run: | + pipx run --python ${{ matrix.python-version }} build + WHEEL_NAME=$(ls dist/*.whl) + pip install --verbose ${WHEEL_NAME}[test] + - name: "Run tests" - run: pytest -vv --showlocals + run: | + PROJECT_CWD=$PWD + rm tiledb/__init__.py + cd /tmp + pytest -vv --showlocals $PROJECT_CWD - name: "Print log files (failed build only)" run: | @@ -144,3 +96,10 @@ jobs: cat $f done; if: failure() + + - name: "Upload files for debug" + if: always() + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }}-${{ matrix.python-version }} + path: "." diff --git a/.github/workflows/daily-test-build-numpy.yml b/.github/workflows/daily-test-build-numpy.yml index f2c8310a33..13cb1ed2dc 100644 --- a/.github/workflows/daily-test-build-numpy.yml +++ b/.github/workflows/daily-test-build-numpy.yml @@ -1,6 +1,12 @@ name: TileDB Python CI - With Earliest Supported Version of NumPy on: + workflow_dispatch: + inputs: + libtiledb_version: + description: TileDB Core Version + required: true + type: string workflow_call: inputs: libtiledb_version: @@ -15,7 +21,7 @@ jobs: shell: bash strategy: matrix: - os: [ubuntu-latest, macos-12, windows-latest] + os: [ubuntu-latest, macos-13, macos-14, windows-latest] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] include: # https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg @@ -37,24 +43,22 @@ jobs: numpy-version: "1.17.3" fail-fast: false env: - TILEDB_VERSION: ${{ github.event.inputs.libtiledb_version }} - # 11.7 necessary due to: https://github.com/actions/setup-python/issues/682#issuecomment-1604261330 - #MACOSX_DEPLOYMENT_TARGET: "10.15" - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-12' && matrix.python-version == '3.8' && '11.7' || '11' }} + TILEDB_VERSION: ${{ inputs.libtiledb_version }} + MACOSX_DEPLOYMENT_TARGET: "11" VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' steps: - name: Checkout TileDB-Py `dev` - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup MSVC toolset (VS 2022) uses: TheMrMilchmann/setup-msvc-dev@v3 - if: matrix.os == 'windows-latest' + if: startsWith(matrix.os, 'windows') with: arch: x64 - name: Install Ninja (VS 2022) uses: seanmiddleditch/gha-setup-ninja@v4 - if: matrix.os == 'windows-latest' + if: startsWith(matrix.os, 'windows') - name: Enable vcpkg binary caching uses: actions/github-script@v6 @@ -65,10 +69,10 @@ jobs: - name: "Set CMAKE_GENERATOR" run: export CMAKE_GENERATOR="Ninja" - if: matrix.os == 'windows-latest' + if: startsWith(matrix.os, 'windows') - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -81,26 +85,25 @@ jobs: - name: Print env run: printenv - - name: Use Oldest Version of NumPy - run: python -m pip install numpy==${{ matrix.numpy-version }} - - name: Checkout TileDB-Py `dev` uses: actions/checkout@v3 - - name: Install dependencies - run: python -m pip install --upgrade -r misc/requirements_ci.txt - - - name: Test without pandas - run: python -m pip uninstall -y pandas - - name: Build TileDB-Py + run: pip install --verbose .[test] + + - name: Install Numpy run: | - echo "CMAKE_GENERATOR=$CMAKE_GENERATOR" - python setup.py build_ext --inplace --werror - python setup.py develop + pip install numpy==${{ matrix.numpy-version }} - - name: Test TileDB-Py - run: pytest -vv + - name: Print installed Python dependencies + run: pip list + + - name: Run tests + run: | + PROJECT_CWD=$PWD + rm tiledb/__init__.py + cd /tmp + pytest -vv --showlocals $PROJECT_CWD create_issue_on_fail: permissions: @@ -115,4 +118,4 @@ jobs: with: name: nightly build with earliest supported numpy label: bug,nightly-failure - assignee: kounelisagis,nguyenv,KiterLuc,ihnorton + assignee: kounelisagis,nguyenv,KiterLuc,ihnorton \ No newline at end of file diff --git a/.github/workflows/daily-test-build.yml b/.github/workflows/daily-test-build.yml index afdce4c7d8..11daa5294c 100644 --- a/.github/workflows/daily-test-build.yml +++ b/.github/workflows/daily-test-build.yml @@ -8,24 +8,6 @@ on: type: string jobs: - test-wheels-on-azure: - runs-on: ubuntu-latest - defaults: - run: - shell: bash - steps: - - name: Get current date - id: date - run: echo "::set-output name=date::$(date +'%a-%Y-%m-%d')" - - name: Get libtiledb short SHA - run: echo "LIBTILEDB_SHA=$(git ls-remote https://github.com/TileDB-Inc/TileDB HEAD | cut -c1-7)" >> $GITHUB_ENV - - name: Create Test Branch for Azure Wheel Nightly Build - uses: peterjgrainger/action-create-branch@v2.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - branch: "azure-wheel-test-${{ steps.date.outputs.date }}-against-${{ env.LIBTILEDB_SHA }}" - test: runs-on: ${{ matrix.os }} defaults: @@ -34,11 +16,10 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-12, windows-latest] - uninstall_pandas: [true, false] fail-fast: false env: - TILEDB_VERSION: ${{ github.event.inputs.libtiledb_version }} + TILEDB_VERSION: ${{ inputs.libtiledb_version }} MACOSX_DEPLOYMENT_TARGET: "11" VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite' @@ -65,7 +46,7 @@ jobs: if: matrix.os == 'windows-latest' - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.11" @@ -85,25 +66,17 @@ jobs: if: matrix.os == 'macos-12' - name: Checkout TileDB-Py `dev` - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - name: Install dependencies - run: python -m pip install --upgrade -r misc/requirements_ci.txt + - name: Build and install TileDB-Py and dependencies + run: python -m pip install --verbose .[test] - - name: Test without pandas - run: python -m pip uninstall -y pandas - if: ${{ matrix.uninstall_pandas }} - - - name: Build TileDB-Py + - name: Run tests run: | - echo "CMAKE_GENERATOR=$CMAKE_GENERATOR" - python setup.py build_ext --inplace --werror - python setup.py develop - env: - TILEDB_FORCE_ALL_DEPS: True - - - name: Test TileDB-Py - run: pytest -vv + PROJECT_CWD=$PWD + rm tiledb/__init__.py + cd /tmp + pytest -vv --showlocals $PROJECT_CWD create_issue_on_fail: permissions: diff --git a/.github/workflows/daily-tests.yml b/.github/workflows/daily-tests.yml index d1610a6410..63ea0eba8e 100644 --- a/.github/workflows/daily-tests.yml +++ b/.github/workflows/daily-tests.yml @@ -10,19 +10,14 @@ jobs: ci1: uses: ./.github/workflows/daily-test-build.yml with: - libtiledb_version: 'dev' + libtiledb_version: '2.24.0' ci2: uses: ./.github/workflows/daily-test-build.yml with: - libtiledb_version: 'release-2.24' + libtiledb_version: '2.23.0' ci3: - uses: ./.github/workflows/daily-test-build.yml - with: - libtiledb_version: 'release-2.23' - - ci4: uses: ./.github/workflows/daily-test-build-numpy.yml with: - libtiledb_version: 'release-2.23' + libtiledb_version: '2.23.0' \ No newline at end of file diff --git a/.github/workflows/issue-if-azure-fail.yml b/.github/workflows/issue-if-azure-fail.yml deleted file mode 100644 index de37ccad55..0000000000 --- a/.github/workflows/issue-if-azure-fail.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Create Issue if Build Fails on Azure - -on: [repository_dispatch, workflow_dispatch] - -jobs: - clean-branch: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Get current date - id: date - run: echo "::set-output name=date::$(date +'%a-%Y-%m-%d')" - - name: Get libtiledb short SHA - run: echo "LIBTILEDB_SHA=$(git ls-remote https://github.com/TileDB-Inc/TileDB HEAD | cut -c1-7)" >> $GITHUB_ENV - - name: Clean Up Test Branch - uses: dawidd6/action-delete-branch@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branches: "azure-wheel-test-${{ steps.date.outputs.date }}-against-${{ env.LIBTILEDB_SHA }}" - - notify-fail: - permissions: - issues: write - if: github.event.action == 'failed' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Create Issue if Build Fails - uses: TileDB-Inc/github-actions/open-issue@main - with: - name: nightly Azure wheel - label: bug,nightly-azure-failure - assignee: KiterLuc,kounelisagis,nguyenc,ihnorton diff --git a/.gitignore b/.gitignore index bf25166948..44c68fbba0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -# project.toml: Forces build isolation and other problems; ecosystem still spotty. -pyproject.toml - # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -108,9 +105,6 @@ ENV/ # mypy .mypy_cache/ -# setuptools-scm -tiledb/_generated_version.py - # IntelliJ .idea diff --git a/.readthedocs.yml b/.readthedocs.yml index 1d12d14c2d..8ce42b247e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -16,7 +16,7 @@ sphinx: python: install: - # this order is important: we need to get cmake - - requirements: doc/requirements_doc.txt - - method: setuptools + - method: pip path: . + extra_requirements: + - doc diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..84ce2a57f2 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,52 @@ +cmake_minimum_required(VERSION 3.15...3.26) + +project(${SKBUILD_PROJECT_NAME}) + +set(PYBIND11_NEWPYTHON ON) + +find_package( + Python + COMPONENTS Interpreter Development.Module NumPy + REQUIRE +) + +find_package(pybind11 CONFIG REQUIRED) + +if (TILEDB_PATH) + # If TILEDB_PATH variable is present prepend it to the CMAKE_PREFIX_PATH + # so that find_package will be able to find TileDB core library there + set(CMAKE_PREFIX_PATH "${TILEDB_PATH};${CMAKE_PREFIX_PATH}") +endif() + + +# In the future we should use vcpkg +find_package(TileDB QUIET) +if (NOT TileDB_FOUND) + include(cmake/DownloadTileDB.cmake) + if (TILEDB_VERSION) + message(STATUS "Downloading TileDB version \"${TILEDB_VERSION}\" ...") + if (TILEDB_HASH) + fetch_prebuilt_tiledb( + VERSION ${TILEDB_VERSION} + RELLIST_HASH SHA256=${TILEDB_HASH} + ) + else() + fetch_prebuilt_tiledb( + VERSION ${TILEDB_VERSION} + ) + endif() + else() + message(STATUS "Downloading TileDB default version ...") + # Download latest release + fetch_prebuilt_tiledb( + VERSION 2.24.0 + RELLIST_HASH SHA256=e5aa7e6d747bb956730e438d0b29c90096226d4a553c26c5cb9161680f1d489c + ) + endif() + find_package(TileDB REQUIRED) + set(TILEDB_DOWNLOADED TRUE) +else() + message(STATUS "Found external TileDB core library") +endif() + +add_subdirectory(tiledb) diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 0d0512bcd9..0000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,12 +0,0 @@ -global-exclude .gitignore - -exclude MANIFEST.in -exclude .dockerignore -exclude .readthedocs.yml -exclude azure-pipelines.yml -exclude misc/* - -include misc/requirements_wheel.txt -include README.md - -prune doc diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 74ee33f3a7..0000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,4 +0,0 @@ -stages: - #- template: ./misc/azure-ci.yml - - template: ./misc/azure-release.yml - #- template: ./misc/azure-publish.yml diff --git a/cmake/DownloadTileDB.cmake b/cmake/DownloadTileDB.cmake new file mode 100644 index 0000000000..8db5015bdb --- /dev/null +++ b/cmake/DownloadTileDB.cmake @@ -0,0 +1,125 @@ +# +# FindTileDB_EP.cmake +# +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +include(FetchContent) + +function(fetch_tiledb_release_list VERSION) + # Local constants + set(UPSTREAM_URL "https://github.com/TileDB-Inc/TileDB/releases/download") + list(LENGTH ARGV COUNT) + if (${COUNT} GREATER 1) + list(GET ARGV 1 EXPECTED_HASH) + endif() + + if(NOT VERSION) + set(VERSION latest) + endif() + + if(EXPECTED_HASH) + file(DOWNLOAD + ${UPSTREAM_URL}/${VERSION}/releases.csv + ${CMAKE_CURRENT_BINARY_DIR}/releases.csv + SHOW_PROGRESS + EXPECTED_HASH ${EXPECTED_HASH} + ) + else() + message(WARNING "Downloading release list without SHA checksum!") + file(DOWNLOAD + ${UPSTREAM_URL}/${VERSION}/releases.csv + ${CMAKE_CURRENT_BINARY_DIR}/releases.csv + SHOW_PROGRESS + ) + endif() + + file(STRINGS + ${CMAKE_CURRENT_BINARY_DIR}/releases.csv + RELLIST + ) + + # Remove csv table headers + list(POP_FRONT RELLIST) + + foreach(LINE ${RELLIST}) + string(REPLACE "," ";" LINE ${LINE}) + list(LENGTH LINE LENGTH) + + list(GET LINE 0 PLATFORM) + list(GET LINE 1 URL) + list(GET LINE 2 SHA) + + set(RELEASE_VAR TILEDB_${PLATFORM}) + set(URL_${RELEASE_VAR} ${URL} PARENT_SCOPE) + set(HASH_${RELEASE_VAR} ${SHA} PARENT_SCOPE) + endforeach() +endfunction() + +function(detect_artifact_name OUT_VAR) + if (WIN32) # Windows + SET(${OUT_VAR} TILEDB_WINDOWS-X86_64 PARENT_SCOPE) + elseif(APPLE) # OSX + if (CMAKE_OSX_ARCHITECTURES) + set(ACTUAL_TARGET ${CMAKE_OSX_ARCHITECTURES}) + else() + set(ACTUAL_TARGET ${CMAKE_SYSTEM_PROCESSOR}) + endif() + + if (ACTUAL_TARGET MATCHES "(x86_64)|(AMD64|amd64)|(^i.86$)") + SET(${OUT_VAR} TILEDB_MACOS-X86_64 PARENT_SCOPE) + elseif (ACTUAL_TARGET STREQUAL arm64 OR ACTUAL_TARGET MATCHES "^aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") + SET(${OUT_VAR} TILEDB_MACOS-ARM64 PARENT_SCOPE) + endif() + else() # Linux + SET(${OUT_VAR} TILEDB_LINUX-X86_64 PARENT_SCOPE) + endif() +endfunction() + +function(fetch_prebuilt_tiledb) + # Arguments + set(oneValueArgs VERSION ARTIFACT_NAME RELLIST_HASH) + set(multiValueArgs) + cmake_parse_arguments( + FETCH_PREBUILT_TILEDB + "${options}" + "${oneValueArgs}" + "${multiValueArgs}" + ${ARGN} + ) + + fetch_tiledb_release_list(${FETCH_PREBUILT_TILEDB_VERSION} ${FETCH_PREBUILT_TILEDB_RELLIST_HASH}) + + if(NOT FETCH_PREBUILT_TILEDB_ARTIFACT_NAME) + detect_artifact_name(FETCH_PREBUILT_TILEDB_ARTIFACT_NAME) + endif() + + string(STRIP ${HASH_${FETCH_PREBUILT_TILEDB_ARTIFACT_NAME}} HASH_${FETCH_PREBUILT_TILEDB_ARTIFACT_NAME}) + FetchContent_Declare( + tiledb + URL ${URL_${FETCH_PREBUILT_TILEDB_ARTIFACT_NAME}} + URL_HASH SHA256=${HASH_${FETCH_PREBUILT_TILEDB_ARTIFACT_NAME}} + ) + FetchContent_MakeAvailable(tiledb) + set(TileDB_DIR ${tiledb_SOURCE_DIR}/lib/cmake/TileDB PARENT_SCOPE) +endfunction() \ No newline at end of file diff --git a/doc/local-build.sh b/doc/local-build.sh index 4ce2ee67ec..e82cef3aea 100755 --- a/doc/local-build.sh +++ b/doc/local-build.sh @@ -49,20 +49,9 @@ if [ ! -d "${tiledb}" ]; then die "invalid tiledb installation directory (use --tiledb)" fi -setup_venv() { - if [ ! -d "${venv_dir}" ]; then - virtualenv "${venv_dir}" || die "could not create virtualenv" - fi - source "${venv_dir}/bin/activate" || die "could not activate virtualenv" - pip install 'Sphinx==1.6.7' \ - 'breathe' \ - 'sphinx_rtd_theme' \ - -r requirements_doc.txt || die "could not install doc dependencies" -} - build_ext() { pushd "${ext_dir}" - python setup.py install --tiledb="${tiledb}" || die "could not install tiledb-py" + TILEDB_PATH=${tiledb} pip install .[doc] || die "could not install tiledb-py" popd } @@ -78,7 +67,6 @@ build_site() { } run() { - setup_venv build_ext build_site echo "Build complete. Open '${build_dir}/html/index.html' in your browser." diff --git a/doc/requirements_doc.txt b/doc/requirements_doc.txt deleted file mode 100644 index 70c063c5b6..0000000000 --- a/doc/requirements_doc.txt +++ /dev/null @@ -1,4 +0,0 @@ -jinja2==3.0.0 -sphinx-rtd-theme==2.0.0 - --r ../requirements_dev.txt diff --git a/misc/azure-ci.yml b/misc/azure-ci.yml deleted file mode 100644 index d388255b45..0000000000 --- a/misc/azure-ci.yml +++ /dev/null @@ -1,98 +0,0 @@ -stages: - - stage: CI - condition: not(or(startsWith(variables['Build.SourceBranch'], 'refs/tags'), startsWith(variables['Build.SourceBranchName'], 'release-'))) - jobs: - - job: - pool: - vmImage: $(imageName) - strategy: - matrix: - mac: - imageName: "macOS-12" - python.version: "3.10" - MACOSX_DEPLOYMENT_TARGET: 11 - windows: - imageName: "windows-latest" - python.version: "3.10" - linux_py3: - imageName: "ubuntu-latest" - python.version: "3.10" - maxParallel: 4 - timeoutInMinutes: 120 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "$(python.version)" - architecture: "x64" - - # Print python and pip version information for debugging. - # Azure pipelines windows images have been unstable or out of sync, causing - # build failures in the pip step below when the 'bash' task uses the wrong - # python or has issue that causes un-corrected cygwin-style paths to be - # passed to pip. - - bash: | - echo "==== Python information ====" - which python - which pip - python --version - echo "============================" - displayName: "Print python version in bash task" - - - script: | - printenv - displayName: "Print env" - - - script: | - python -m pip install --upgrade -r misc/requirements_ci.txt - displayName: "Install dependencies" - - - bash: | - brew install pkg-config - displayName: "Homebrew setup" - condition: eq(variables['Agent.OS'], 'Darwin') - - - script: | - set TILEDB_FORCE_ALL_DEPS=ON - # vcvarsall is necessary so that numpy uses the correct compiler - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - pip install -e -vv . - displayName: "Build TileDB and TileDB-Py extension (Windows)" - condition: eq(variables['Agent.OS'], 'Windows_NT') - - - bash: | - unset SYSTEM - set -xeo pipefail - pip install -e -vv . - displayName: "Build TileDB and TileDB-Py extension (POSIX)" - condition: ne(variables['Agent.OS'], 'Windows_NT') - - - bash: | - set -xeo pipefail - - pytest -vv - - # Test wheel build, install, and run - python setup.py bdist_wheel - #whl_file=`pwd`/dist/`ls dist/*.whl` - mkdir /tmp/wheel_test - cp dist/*.whl /tmp/wheel_test - pushd /tmp/wheel_test - ls - pip install *.whl - python -c "import tiledb ; tiledb.libtiledb.version()" - displayName: "Run tests" - - - bash: | - set -xeo pipefail - # Display log files if the build failed - echo "Dumping log files for failed build" - echo "----------------------------------" - for f in $(find $BUILD_REPOSITORY_LOCALPATH/build -name *.log); - do echo "------" - echo $f - echo "======" - cat $f - done; - condition: failed() # only run this job if the build step failed - displayName: "Print log files (failed build only)" diff --git a/misc/azure-libtiledb-darwin.yml b/misc/azure-libtiledb-darwin.yml deleted file mode 100644 index 8b1e894478..0000000000 --- a/misc/azure-libtiledb-darwin.yml +++ /dev/null @@ -1,97 +0,0 @@ -steps: - - task: Cache@2 - inputs: - key: 'libtiledb v0 | "$(Agent.OS)" | "$(imageName)" | "$(LIBTILEDB_SHA)" | "$(CMAKE_OSX_ARCHITECTURES)" | setup.py | **/azure-*.yml, !tiledb_src/**, !tiledb_build/**' - path: '$(TILEDB_INSTALL)' - cacheHitVar: LIBTILEDB_CACHE_RESTORED - - - bash: | - find $PIPELINE_WORKSPACE/.libtiledb_dist/${LIBTILEDB_SHA} - condition: eq(variables.LIBTILEDB_CACHE_RESTORED, 'true') - displayName: "Print files restored from cache" - - - bash: | - brew install pkg-config - displayName: "Homebrew setup" - condition: eq(variables['Agent.OS'], 'Darwin') - - - bash: | - set -xe pipefail - unset SYSTEM - - git clone ${LIBTILEDB_REPO} $TILEDB_SRC - git -C $(TILEDB_SRC) checkout $(LIBTILEDB_SHA) - - mkdir -p $TILEDB_BUILD - cd $TILEDB_BUILD - - # note: CMAKE_OSX_ARCHITECTURES is ignored on non-macOS platforms - cmake -B $TILEDB_BUILD -S $TILEDB_SRC/ -DTILEDB_VCPKG=ON -DTILEDB_WERROR=OFF -DTILEDB_SERIALIZATION=ON -DTILEDB_GCS=${TILEDB_GCS} -DTILEDB_S3=ON -DTILEDB_AZURE=ON -DCOMPILER_SUPPORTS_AVX2=FALSE -DTILEDB_TESTS=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$TILEDB_INSTALL -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} - - cmake --build $TILEDB_BUILD --config Release -j3 - cmake --build $TILEDB_BUILD --target install-tiledb --config Release - - if [[ "$AGENT_OS" == "Darwin" ]]; then - cp $TILEDB_BUILD/externals/install/lib/libz.1.dylib $TILEDB_INSTALL/lib || true - fi - - if [[ "$AGENT_OS" == "Darwin" ]]; then - otool -L ${TILEDB_INSTALL}/lib/libtiledb.dylib; - fi - condition: and(ne(variables['Agent.OS'], 'Windows_NT'), ne(variables.LIBTILEDB_CACHE_RESTORED, 'true')) - displayName: "Bld libtiledb (POSIX)" - - - script: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - echo ON - git clone $(LIBTILEDB_REPO) $(TILEDB_SRC) - git -C $(TILEDB_SRC) checkout $(LIBTILEDB_SHA) - - mkdir $(TILEDB_INSTALL) - cd $(TILEDB_BUILD) - :: use cmake directly because the powershell arg/quoting rules are bonkers - - cmake -A X64 -DCMAKE_BUILD_TYPE=Release -DTILEDB_VCPKG=ON -DTILEDB_WERROR=ON -DTILEDB_S3=ON -DTILEDB_SERIALIZATION=ON -DTILEDB_TOOLS=OFF -DTILEDB_SUPERBUILD=ON -DTILEDB_FORCE_ALL_DEPS=ON -DTILEDB_CPP_API=ON -DTILEDB_TESTS=OFF -DTILEDB_HDFS=OFF -DTILEDB_LOG_OUTPUT_ON_FAILURE=ON -DBUILD_SHARED_LIBS=ON -DTILEDB_VERBOSE=ON -DMSVC_MP_FLAG="/MP3" -DCMAKE_INSTALL_PREFIX=$(TILEDB_INSTALL) $(TILEDB_SRC) . - - cmake --build . --config Release -j3 - cmake --build . --target install-tiledb --config Release - condition: and(eq(variables['Agent.OS'], 'Windows_NT'), ne(variables.LIBTILEDB_CACHE_RESTORED, 'true')) - displayName: "Bld libtiledb (Windows)" - - - bash: | - set -xe - if [[ "$AGENT_OS" == "Windows_NT" ]]; then - 7z a ${TILEDB_INSTALL}/libtiledb-${LIBTILEDB_VERSION}-${LIBTILEDB_SHA}. ${TILEDB_INSTALL}/* - elif [[ "$AGENT_OS" == "Darwin" ]]; then - tar -czf ${TILEDB_INSTALL}/libtiledb-${LIBTILEDB_VERSION}-${LIBTILEDB_SHA} -C ${TILEDB_INSTALL} . - else - tar -czf ${TILEDB_INSTALL}/libtiledb-${LIBTILEDB_VERSION}-${LIBTILEDB_SHA} -C ${TILEDB_INSTALL} lib64 include - fi - displayName: "Archive build" - - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: $(TILEDB_INSTALL)/libtiledb-$(LIBTILEDB_VERSION)-$(LIBTILEDB_SHA) - artifactName: 'libtiledb-$(Agent.OS)$(CMAKE_OSX_ARCHITECTURES)' - - - bash: | - set -x - # Print cmake version - echo "CMake version:" - cmake --version - - echo "--- Listing files (BUILD_REPOSITORY_LOCALPATH): ${BUILD_REPOSITORY_LOCALPATH} ---" - ls $BUILD_REPOSITORY_LOCALPATH || true - echo "--- Finding files (TILEDB_INSTALL): '${TILEDB_INSTALL}'---" - find $TILEDB_INSTALL || true - - echo "--- Printing libtiledb git version ---" - libtiledb_sha_actual=$(git -C $TILEDB_SRC show-ref -s $LIBTILEDB_VERSION) - if [[ "$libtiledb_sha_actual" != "$(LIBTILEDB_SHA)" ]]; then - echo "variable LIBTILEDB_SHA ('$LIBTILEDB_SHA') does not match SHA of LIBTILEDB_VERSION checkout ('$libtiledb_sha_actual')"; - fi - echo "--- Printing libtiledb linkage ---" - otool -L `find $LIBTILEDB_INSTALL -name *libtiledb*` - echo "----------------------------------" - displayName: "Print debug info" - condition: always() diff --git a/misc/azure-print-logs.yml b/misc/azure-print-logs.yml deleted file mode 100644 index 5106ebd1d0..0000000000 --- a/misc/azure-print-logs.yml +++ /dev/null @@ -1,14 +0,0 @@ -steps: -- bash: | - set -e pipefail - # Display log files if the build failed - echo "Dumping log files for failed build" - echo "----------------------------------" - for f in $(find $BUILD_REPOSITORY_LOCALPATH -name *.log); - do echo "------" - echo $f - echo "======" - cat $f - done; - condition: failed() # only run this job if the build step failed - displayName: "Print log files (failed build only)" diff --git a/misc/azure-release.yml b/misc/azure-release.yml index 1d1578d855..e69de29bb2 100644 --- a/misc/azure-release.yml +++ b/misc/azure-release.yml @@ -1,257 +0,0 @@ -stages: - - stage: Release - variables: - ? ${{ if startsWith(variables['Build.SourceBranchName'], 'azure-wheel-test-') }} - : TILEDBPY_VERSION: dev - LIBTILEDB_VERSION: dev - LIBTILEDB_SHA: dev - ${{ else }}: - TILEDBPY_VERSION: 0.30.2 - # NOTE: *must* update both LIBTILEDB_VERSION and LIBTILEDB_SHA - LIBTILEDB_VERSION: "2.24.2" - # NOTE: *must* update both LIBTILEDB_VERSION and LIBTILEDB_SHA - LIBTILEDB_SHA: 76cd03c39d459b7659ccccb692864d81dd87d36c - LIBTILEDB_REPO: https://github.com/TileDB-Inc/TileDB - TILEDB_SRC: "$(Build.Repository.Localpath)/tiledb_src" - TILEDB_BUILD: "$(Build.Repository.Localpath)/tiledb_build" - TILEDB_GCS: ON # set here, override for macOS-arm64 - # UPDATE TILEDB_INSTALL: - # - IN macOS sections below too! - # - IN azure-libtiledb-darwin.yml 'path' input, if applicable - TILEDB_INSTALL: "$(Pipeline.Workspace)/.libtiledb_dist/$(LIBTILEDB_SHA)" - condition: or(startsWith(variables['Build.SourceBranch'], 'refs/tags'), startsWith(variables['Build.SourceBranchName'], 'release-'), startsWith(variables['Build.SourceBranchName'], 'azure-wheel-test-')) - - jobs: - - job: build1_libtiledb - timeoutInMinutes: 120 - strategy: - matrix: - macOS_libtiledb: - imageName: "macOS-12" - CMAKE_OSX_ARCHITECTURES: "x86_64" - MACOSX_DEPLOYMENT_TARGET: 11 - TILEDB_INSTALL: "$(Pipeline.Workspace)/.libtiledb_dist/$(LIBTILEDB_SHA)-macos-x86_64" - macOS_libtiledb_arm64: - imageName: "macOS-12" - CMAKE_OSX_ARCHITECTURES: "arm64" - MACOSX_DEPLOYMENT_TARGET: 11 - BUILD_MAGIC_MACOS_UNIVERSAL: "ON" - TILEDB_INSTALL: "$(Pipeline.Workspace)/.libtiledb_dist/$(LIBTILEDB_SHA)-macos-arm64" - windows_libtiledb: - imageName: "windows-latest" - CMAKE_OSX_ARCHITECTURES: "" - pool: - vmImage: $(imageName) - - steps: - - task: UsePythonVersion@0 - - template: azure-libtiledb-darwin.yml - - - job: build1_libtiledb_on_linux - timeoutInMinutes: 120 - pool: - vmImage: "ubuntu-latest" - container: ghcr.io/ihnorton/tiledb-manylinux2014_x86_64:2023-04-02 - variables: - CXXFLAGS: "-Wno-unused-parameter -lrt -DKJ_USE_EPOLL10 -D__BIONIC__=1" - CFLAGS: "-Wno-unused-parameter -lrt -DKJ_USE_EPOLL=0 -D__BIONIC__=1" - CMAKE_OSX_ARCHITECTURES: "" - steps: - - task: UsePythonVersion@0 - - template: azure-libtiledb-darwin.yml - - - job: build2_python - dependsOn: [build1_libtiledb, build1_libtiledb_on_linux] - condition: always() - variables: - cibw_test_requires: "pytest" - USE_CIBW_VERSION: 2.17.0 - strategy: - matrix: - linux_py: - imageName: "ubuntu-latest" - CIBW_SKIP: "cp36-* cp37-* *_i686 pp* *musllinux*" - CIBW_BUILD_VERBOSITY: 3 - CIBW_ARCHS_MACOS: "" - macOS_py: - imageName: "macOS-12" - MACOSX_DEPLOYMENT_TARGET: 11 - TILEDB_INSTALL: "$(Pipeline.Workspace)/.libtiledb_dist/$(LIBTILEDB_SHA)-macos-x86_64" - CIBW_ARCHS_MACOS: "x86_64" - CIBW_SKIP: "cp36-* cp37-* pp*" - CIBW_BUILD_VERBOSITY: 3 - macOS_arm64_py: - imageName: "macOS-12" - MACOSX_DEPLOYMENT_TARGET: 11 - TILEDB_INSTALL: "$(Pipeline.Workspace)/.libtiledb_dist/$(LIBTILEDB_SHA)-macos-arm64" - CIBW_BUILD_VERBOSITY: 3 - CIBW_ARCHS_MACOS: "arm64" - # NumPy is only available in CPython 3.8+ on macOS-arm64 - CIBW_SKIP: "cp36-* cp37-* pp*" - windows_py: - imageName: "windows-latest" - CIBW_SKIP: "cp36-* cp37-* *-win32 pp*" - CIBW_BUILD_VERBOSITY: 3 - CIBW_ARCHS_MACOS: "" - pool: - vmImage: $(imageName) - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.10" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - - - script: git tag -f $(TILEDBPY_VERSION) - - - task: DownloadPipelineArtifact@2 - displayName: "Download libtiledb artifact" - inputs: - artifactName: 'libtiledb-$(Agent.OS)$(CIBW_ARCHS_MACOS)' - path: $(TILEDB_INSTALL) - - # we have to archive the files because azp breaks newlines in a bare directory restore - - bash: | - set -x - if [[ "$AGENT_OS" == "Windows_NT" ]]; then - 7z x -o${TILEDB_INSTALL}/ ${TILEDB_INSTALL}/libtiledb-${LIBTILEDB_VERSION}-${LIBTILEDB_SHA} -y - else - tar xzf ${TILEDB_INSTALL}/libtiledb-${LIBTILEDB_VERSION}-${LIBTILEDB_SHA} -C ${TILEDB_INSTALL}; - find ${TILEDB_INSTALL} - fi - - # Copy libz (temporary work-around for delocate on macOS) - if [[ "$AGENT_OS" == "Darwin" ]]; then - cp ${TILEDB_INSTALL}/lib/libz.1.dylib $BUILD_REPOSITORY_LOCALPATH - install_name_tool -change libz.1.dylib ${BUILD_REPOSITORY_LOCALPATH}/libz.1.dylib ${TILEDB_INSTALL}/lib/libtiledb.dylib - fi - displayName: "Extract libtiledb files" - - - bash: | - # Set the CIBW_ENVIRONMENT from bash in order to get forward slashes because somewhere in the - # cmd/cibw/python chain we end up losing the slashes entirely if we use a job-level variable. - MPATH=$(python -c 'import os; print(os.environ["TILEDB_INSTALL"].replace("\\","/"))') - export CIBW_ENVIRONMENT="TILEDB_PATH=${MPATH}" - # !!! DO NOT PUT OTHER VARIABLES IN THIS SECTION - vars w/out expansions go below !!!" - echo "##vso[task.setvariable variable=CIBW_ENVIRONMENT]$CIBW_ENVIRONMENT" - displayName: "Set CIBW_ENVIRONMENT" - condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) - - - bash: | - set -xe pipefail - - mv ${TILEDB_INSTALL} .libtiledb - export TILEDB_INSTALL=.libtiledb - export CIBW_ENVIRONMENT="TILEDB_PATH=${TILEDB_INSTALL} TILEDB_WHEEL_BUILD=1" - # use the requirements_wheel.txt with numpy pins to ensure ABI compatibility - export CIBW_TEST_COMMAND="python -c 'import tiledb'" - # copy libtiledb into usr/local for auditwheel to find - export CIBW_BEFORE_BUILD="cp -R .libtiledb/* /usr/local" - ls -lR "${TILEDB_INSTALL}" - - python -c "import os; print(os.environ.get('CIBW_ENVIRONMENT', None))" - git rev-parse HEAD - python3 -m pip install --upgrade -r requirements_dev.txt - python3 -m build . --sdist --outdir wheelhouse - pip3 install cibuildwheel==${USE_CIBW_VERSION} - cibuildwheel --output-dir wheelhouse . - displayName: "Build and test wheels (Linux)" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - - - bash: | - set -xe pipefail - - export TILEDB_WHEEL_BUILD=1 - export SYSTEM_VERSION_COMPAT=0 # https://github.com/numba/numba-benchmark/issues/14#issuecomment-1167675905 - # use the requirements_wheel.txt with numpy pins to ensure ABI compatibility - if [[ "$CIBW_ARCHS" != "arm64" ]]; then - export CIBW_TEST_COMMAND="python -c 'import tiledb'" - fi - echo "${TILEDB_INSTALL}" - - python -c "import os; print(os.environ.get('CIBW_ENVIRONMENT', None))" - git rev-parse HEAD - python3 -m pip install --upgrade -r requirements_dev.txt - python3 -m build . --sdist --outdir wheelhouse - pip3 install cibuildwheel==${USE_CIBW_VERSION} - cibuildwheel --output-dir wheelhouse . - displayName: "Build and test wheels (macOS)" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - - - script: | - echo ON - set "TILEDB_WHEEL_BUILD=1" - :: # Have not managed to get this working AZP quoting breaks the command - :: set CIBW_TEST_COMMAND=python -c \"import tiledb\"" - echo "cibw env: " - echo "%CIBW_ENVIRONMENT%" - echo "tiledb_install: " - echo "%TILEDB_INSTALL%" - python -c "import os; print(os.environ['CIBW_ENVIRONMENT'])" - python -c "import platform; print('py compiler: ', platform.python_compiler())" - :: this runs under cmd on windows, which we need to use vcvarsall - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - git rev-parse HEAD - python -m pip install --upgrade -r requirements_dev.txt - python3 -m build . --sdist --outdir wheelhouse - pip install cibuildwheel==%USE_CIBW_VERSION% - cibuildwheel --output-dir wheelhouse . - :: delete the sdist because we *do not* want files with CRLF endings - :: (if windows builds finish last then the windows sdist will be - :: overwrite any others in the artifact publishing step) - del /q wheelhouse\*.tar.gz - displayName: "Build and test wheels (Windows)" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - - - template: azure-print-logs.yml - - - task: PublishBuildArtifacts@1 - inputs: { pathtoPublish: "wheelhouse" } - - - bash: | - set -x - echo "TILEDB_SRC -----------------------------------------" - find $TILEDB_SRC || true - echo "TILEDB_BUILD -----------------------------------------" - find $TILEDB_BUILD || true - echo "TILEDB_INSTALL -----------------------------------------" - find $TILEDB_INSTALL || true - displayName: "List all the files" - condition: always() - - - job: trigger_fail_build - dependsOn: [build2_python] - condition: and(failed(), startsWith(variables['Build.SourceBranchName'], 'azure-wheel-test-')) - pool: server - steps: - - task: InvokeRESTAPI@1 - inputs: - connectionType: "connectedServiceName" - connectedServiceName: "TileDB-Py-Test" - method: "POST" - body: | - { - "event_type": "failed", - "client_payload": {"build_id:": "$(Build.BuildId)"} - } - urlSuffix: "repos/$(Build.Repository.Name)/dispatches" - waitForCompletion: "false" - - - job: trigger_success_build - dependsOn: [build2_python] - condition: and(succeeded(), startsWith(variables['Build.SourceBranchName'], 'azure-wheel-test-')) - pool: server - steps: - - task: InvokeRESTAPI@1 - condition: succeeded() - inputs: - connectionType: "connectedServiceName" - connectedServiceName: "TileDB-Py-Test" - method: "POST" - body: | - { - "event_type": "succeeded", - "client_payload": {"build_id:": "$(Build.BuildId)"} - } - urlSuffix: "repos/$(Build.Repository.Name)/dispatches" - waitForCompletion: "false" diff --git a/misc/clean b/misc/clean deleted file mode 100644 index d1edeb01fb..0000000000 --- a/misc/clean +++ /dev/null @@ -1,2 +0,0 @@ -clean: - rm -f tiledb/*.cpp tiledb/*.so tiledb/*.dll tiledb/*.pyc tiledb/native/libtiledb.* diff --git a/misc/requirements_ci.txt b/misc/requirements_ci.txt deleted file mode 100644 index 6d1dfbb160..0000000000 --- a/misc/requirements_ci.txt +++ /dev/null @@ -1,6 +0,0 @@ -black -clang-format == 16.0.3 -dask -distributed --r ../requirements_dev.txt --r requirements_test.txt diff --git a/misc/requirements_test.txt b/misc/requirements_test.txt deleted file mode 100644 index d6625b22bf..0000000000 --- a/misc/requirements_test.txt +++ /dev/null @@ -1,6 +0,0 @@ -pandas ; python_version > '3.5' -psutil -pyarrow -pytest -hypothesis -hypothesis[numpy] diff --git a/misc/requirements_wheel.txt b/misc/requirements_wheel.txt deleted file mode 100644 index b7b04f8f64..0000000000 --- a/misc/requirements_wheel.txt +++ /dev/null @@ -1,16 +0,0 @@ -numpy==1.17.* ; python_version == '3.8' and platform_machine not in 'arm64|aarch64' -numpy==1.19.* ; python_version == '3.8' and platform_machine == 'aarch64' -numpy==1.21.* ; python_version == '3.8' and platform_machine == 'arm64' -numpy>=2.0.0rc2 ; python_version >= '3.9' - -#------------------------------- - -cmake >= 3.23 -cython -pybind11 -setuptools >= 64 -setuptools_scm >= 8 -wheel >= 0.30 - -contextvars ;python_version<"3.7" -dataclasses ;python_version<"3.7" diff --git a/pyproject.toml b/pyproject.toml index 11632bb46d..edcce9a4ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,28 +1,27 @@ [build-system] requires = [ - "setuptools>=64", - "wheel", + "scikit-build-core", "pybind11", "Cython", "numpy==1.17.* ; python_version == '3.8' and platform_machine not in 'arm64|aarch64'", "numpy==1.19.* ; python_version == '3.8' and platform_machine == 'aarch64'", "numpy==1.21.* ; python_version == '3.8' and platform_machine == 'arm64'", - "numpy>=2.0.0rc2 ; python_version >= '3.9'", + "numpy>=2.0.0rc2 ; python_version >= '3.9'" ] -build-backend = "setuptools.build_meta" +build-backend = "scikit_build_core.build" [project] requires-python = ">=3.8" name = "tiledb" description = "Pythonic interface to the TileDB array storage manager" readme = "README.md" +license = {text = "MIT"} authors = [ {name = "TileDB, Inc.", email = "help@tiledb.io"} ] maintainers = [ {name = "TileDB, Inc.", email = "help@tiledb.io"} ] -license = {text = "MIT"} classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers", @@ -49,22 +48,46 @@ dependencies = [ ] dynamic = ["version"] -[tool.setuptools.packages.find] -exclude = ["*.pyc", ".pytest_cache/*", ".hypothesis/*"] +[project.optional-dependencies] +doc = [ + "jinja2==3.0.0", + "sphinx-rtd-theme==2.0.0", + "Sphinx", + "breathe", +] +test = [ + "pytest", + "hypothesis", + "psutil", + "pyarrow", +] [project.urls] homepage = "https://github.com/TileDB-Inc/TileDB-Py" [tool.setuptools_scm] -version_scheme = "guess-next-dev" -local_scheme = "no-local-version" version_file = "tiledb/_generated_version.py" +[tool.scikit-build] +wheel.expand-macos-universal-tags = true +metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" +wheel.packages = ["tiledb", "examples", "external"] +wheel.license-files = ["LICENSE", "external/LICENSE-*.txt"] + +[tool.scikit-build.cmake.define] +TILEDB_PATH = {env="TILEDB_PATH"} +TILEDB_VERSION = {env="TILEDB_VERSION"} +TILEDB_HASH = {env="TILEDB_HASH"} + [tool.pytest.ini_options] python_classes = "*Test*" python_files = "test_*.py" testpaths = ["tiledb/tests"] -addopts = "--ignore=tiledb/tests/perf --ignore=tiledb/tests/__pycache__" +addopts = [ + "--import-mode=importlib", + "--ignore=tiledb/tests/perf", + "--ignore=tiledb/tests/__pycache__", +] filterwarnings = [ "error", "default::pytest.PytestWarning", @@ -85,3 +108,12 @@ select = ["NPY201"] [tool.ruff.per-file-ignores] "tiledb/__init__.py" = ["F401"] + +[tool.cibuildwheel] +test-requires = [ + "pytest", + "hypothesis", + "psutil", + "pyarrow", +] +test-command = "pytest {project}" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3f7906469b..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -numpy>=1.17 ; python_version == '3.8' and platform_machine not in 'arm64|aarch64' -numpy>=1.19 ; python_version == '3.8' and platform_machine == 'aarch64' -numpy>=1.21 ; python_version == '3.8' and platform_machine == 'arm64' -numpy>=1.25 ; python_version >= '3.9' -packaging - -contextvars ;python_version<"3.7" -dataclasses ;python_version<"3.7" diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 2ee24f2956..0000000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,19 +0,0 @@ -numpy>=1.17 ; python_version == '3.8' and platform_machine not in 'arm64|aarch64' -numpy>=1.19 ; python_version == '3.8' and platform_machine == 'aarch64' -numpy>=1.21 ; python_version == '3.8' and platform_machine == 'arm64' -numpy>=1.25 ; python_version >= '3.9' - -# ------------------------------------------------ -# ** MUST sync with misc/requirements_wheel.txt ** -# ------------------------------------------------ -build -cmake >= 3.23 -cython -pybind11 -setuptools >= 64 -setuptools_scm >= 8 -wheel >= 0.30 -contextvars ;python_version<"3.7" -dataclasses ;python_version<"3.7" -pytest --r misc/requirements_test.txt diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 83a232199d..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[metadata] -license_files = - LICENSE - external/LICENSE-*.txt diff --git a/setup.py b/setup.py index 65c057c612..e69de29bb2 100644 --- a/setup.py +++ b/setup.py @@ -1,759 +0,0 @@ -import glob -import multiprocessing -import os -import shutil -import subprocess -import sys -from ctypes import CDLL, POINTER, Structure, byref, c_char_p, c_int, c_void_p - -import numpy as np -from pybind11.setup_helpers import Pybind11Extension -from setuptools import Extension, find_packages, setup - -### DO NOT USE ON CI -## Optional multithreaded build -# from pybind11.setup_helpers import ParallelCompile -# ParallelCompile('10').install() -### DO NOT USE ON CI - -# Target branch. Note: -# - this is for builds-from-source -# - release builds are controlled by `misc/azure-release.yml` -# - this should be set to the current core release, not `dev` -TILEDB_VERSION = "2.24.2" - -# allow overriding w/ environment variable -TILEDB_VERSION = ( - os.environ.get("TILEDB_VERSION") - or os.environ.get("LIBTILEDB_VERSION") # For CI override - or TILEDB_VERSION -) - -# Use `setup.py [] --debug` for a debug build of libtiledb -TILEDB_DEBUG_BUILD = False -# Use `setup.py [] --release-symbols` for a release build with symbols libtiledb -TILEDB_SYMBOLS_BUILD = False - -# Allow to override TILEDB_FORCE_ALL_DEPS with environment variable -TILEDB_FORCE_ALL_DEPS = "TILEDB_FORCE_ALL_DEPS" in os.environ -TILEDB_DISABLE_SERIALIZATION = "TILEDB_DISABLE_SERIALIZATION" in os.environ -CMAKE_ARCHITECTURE = os.environ.get("CMAKE_ARCHITECTURE", None) -CMAKE_GENERATOR = os.environ.get("CMAKE_GENERATOR", None) - -# Directory containing this file -CONTAINING_DIR = os.path.abspath(os.path.dirname(__file__)) - -# Build directory path -BUILD_DIR = os.path.join(CONTAINING_DIR, "build") - -# TileDB package source directory -TILEDB_PKG_DIR = os.path.join(CONTAINING_DIR, "tiledb") - -# Set deployment target for mac -# -# TO OVERRIDE: -# set MACOSX_DEPLOYMENT_TARGET before calling setup.py -if sys.platform == "darwin": - if "MACOSX_DEPLOYMENT_TARGET" not in os.environ: - os.environ["MACOSX_DEPLOYMENT_TARGET"] = "11" - -# Is this process building a wheel? -WHEEL_BUILD = ("bdist_wheel" in sys.argv) or ("TILEDB_WHEEL_BUILD" in os.environ) - -# Is this being built under conda-forge? -CONDA_FORGE_BUILD = os.environ.get("TILEDB_CONDA_BUILD") is not None - - -def is_windows(): - return os.name == "nt" - - -def _libtiledb_exists(library_dirs): - """ - Checks the given list of paths and returns true if any contain the TileDB library. - :return: The path to the TileDB library, or None. - """ - - print("libtiledb_exists checking 'library_dirs': {}".format(library_dirs)) - - names = libtiledb_library_names() - if len(library_dirs) > 0: - paths = [os.path.join(d, n) for d in library_dirs for n in names] - for p in paths: - if os.path.exists(p): - return p - raise RuntimeError( - "Could not find given --tiledb library path(s):\n{}".format( - "\n".join(paths) - ) - ) - # If no explicit path is given check to see if TileDB is globally installed. - lib_name = names[0] - - try: - # note: this is a relative path on linux - # https://bugs.python.org/issue21042 - - lib = CDLL(lib_name) - - if os.name == "posix": - # Workaround to retrieving full path of shared library based - # on https://stackoverflow.com/a/35683698 - - dlinfo = lib.dlinfo - dlinfo.argtypes = c_void_p, c_int, c_void_p - dlinfo.restype = c_int - - class LINKMAP(Structure): - _fields_ = [("l_addr", c_void_p), ("l_name", c_char_p)] - - lmptr = POINTER(LINKMAP)() - dlinfo(lib._handle, 2, byref(lmptr)) - lib_name = lmptr.contents.l_name.decode() - - return lib_name - except: - pass - - return None - - -def libtiledb_exists(library_dirs): - lib = _libtiledb_exists(library_dirs) - print("libtiledb_exists found: '{}'".format(lib)) - return lib - - -def libtiledb_library_names(): - """ - :return: List of TileDB shared library names. - """ - if os.name == "posix": - if sys.platform == "darwin": - return ["libtiledb.dylib"] - else: - return ["libtiledb.so"] - elif os.name == "nt": - return ["tiledb.dll"] - else: - raise RuntimeError("Unsupported OS name " + os.name) - - -def download_libtiledb(): - """ - Downloads the native TileDB source. - :return: Path to extracted source directory. - """ - dest_name = "TileDB-{}".format(TILEDB_VERSION) - os.path.join(BUILD_DIR, dest_name) - os.path.join(BUILD_DIR, f"TileDB-{TILEDB_VERSION[:8]}") - - os.path.join(BUILD_DIR, dest_name) - build_dir = os.path.join(BUILD_DIR, f"TileDB-{TILEDB_VERSION[:8]}") - # note that this will only run once locally - if not os.path.exists(build_dir): - - # ---- - # restore for 2.19: https://github.com/TileDB-Inc/TileDB/pull/4484 - # url = "https://github.com/TileDB-Inc/TileDB/archive/{}.zip".format( - # TILEDB_VERSION - # ) - # print("Downloading TileDB package from {}...".format(TILEDB_VERSION)) - # try: - # with zipfile.ZipFile(io.BytesIO(urlopen(url).read())) as z: - # z.extractall(BUILD_DIR) - # except URLError: - # # try falling back to wget, maybe SSL is broken - # subprocess.check_call(["wget", url], shell=True) - # with zipfile.ZipFile("{}.zip".format(TILEDB_VERSION)) as z: - # z.extractall(BUILD_DIR) - # shutil.move(dest, build_dir) - # ---- - - # NOTE: build_dir is where we want the source - args = [ - "git", - "clone", - "--depth=1", - "--branch", - TILEDB_VERSION, - "https://github.com/TileDB-Inc/TileDB", - build_dir, - ] - subprocess.check_output(args) - - return build_dir - - -def build_libtiledb(src_dir): - """ - Builds and installs the native TileDB library. - :param src_dir: Path to libtiledb source directory. - :return: Path to the directory where the library was installed. - """ - libtiledb_build_dir = os.path.join(src_dir, "build") - libtiledb_install_dir = os.path.join(src_dir, "dist") - if not os.path.exists(libtiledb_build_dir): - os.makedirs(libtiledb_build_dir) - - print("Building libtiledb in directory {}...".format(libtiledb_build_dir)) - werror = os.environ.get("TILEDB_WERROR", "OFF") - cmake = os.environ.get("CMAKE", "cmake") - cmake_cmd = [ - cmake, - "-DCMAKE_INSTALL_PREFIX={}".format(libtiledb_install_dir), - "-DTILEDB_TESTS=OFF", - "-DTILEDB_S3=ON", - "-DTILEDB_WERROR={}".format(werror), - "-DTILEDB_HDFS={}".format("ON" if os.name == "posix" else "OFF"), - "-DTILEDB_INSTALL_LIBDIR=lib", - "-DTILEDB_CPP_API=ON", - "-DTILEDB_LOG_OUTPUT_ON_FAILURE=ON", - "-DTILEDB_FORCE_ALL_DEPS:BOOL={}".format( - "ON" if TILEDB_FORCE_ALL_DEPS else "OFF" - ), - "-DTILEDB_SERIALIZATION:BOOL={}".format( - "OFF" if TILEDB_DISABLE_SERIALIZATION else "ON" - ), - ] - - deployment_target = os.environ.get("MACOSX_DEPLOYMENT_TARGET", None) - if deployment_target: - cmake_cmd.append(f"-DCMAKE_OSX_DEPLOYMENT_TARGET={deployment_target}") - - extra_cmake_args = os.environ.get("CMAKE_ARGS", []) - if extra_cmake_args: - cmake_cmd.extend(extra_cmake_args.split()) - - if TILEDB_DEBUG_BUILD: - build_type = "Debug" - elif TILEDB_SYMBOLS_BUILD: - build_type = "RelWithDebInfo" - else: - build_type = "Release" - - cmake_cmd.append("-DCMAKE_BUILD_TYPE={}".format(build_type)) - - if os.name == "nt": - cmake_cmd.extend(["-DMSVC_MP_FLAG=/MP4"]) - - if CMAKE_ARCHITECTURE: - cmake_cmd.extend(["-A", CMAKE_ARCHITECTURE]) - - if CMAKE_GENERATOR: - cmake_cmd.extend(["-G", CMAKE_GENERATOR]) - - # cmake target directory -- important - cmake_cmd.append(src_dir) - - print("CMake configure command: {}".format(cmake_cmd)) - - have_make = True - try: - subprocess.check_call(["make", "-v"]) - except: - have_make = False - - if have_make and not os.name == "nt": - njobs = multiprocessing.cpu_count() or 2 - build_cmd = ["make", "-j{:d}".format(njobs)] - install_cmd = ["make", "install-tiledb"] - else: - build_cmd = ["cmake", "--build", ".", "--config", build_type] - install_cmd = [ - "cmake", - "--build", - ".", - "--config", - build_type, - "--target", - "install-tiledb", - ] - - # Build and install libtiledb - # - run cmake - # - run build via 'cmake --build' - # - run install-tiledb - subprocess.check_call(cmake_cmd, cwd=libtiledb_build_dir) - subprocess.check_call(build_cmd, cwd=libtiledb_build_dir) - subprocess.check_call(install_cmd, cwd=libtiledb_build_dir) - - return libtiledb_install_dir - - -def find_or_install_libtiledb(setuptools_cmd): - """ - Find the TileDB library required for building the Cython extension. If not found, - download, build and install TileDB, copying the resulting shared libraries - into a path where they will be found by package_data or the build process. - - :param setuptools_cmd: The setuptools command instance. - """ - tiledb_ext = None - main_ext = None - tiledb_cc_ext = None - print("ext_modules: ", setuptools_cmd.distribution.ext_modules) - for ext in setuptools_cmd.distribution.ext_modules: - if ext.name == "tiledb.libtiledb": - tiledb_ext = ext - elif ext.name == "tiledb.main": - main_ext = ext - elif ext.name == "tiledb.cc": - tiledb_cc_ext = ext - - print("tiledb_ext: ", tiledb_ext) - print("main_ext: ", main_ext) - print("tiledb_cc_ext: ", tiledb_cc_ext) - print("tiledb_ext.library_dirs: ", tiledb_ext.library_dirs) - wheel_build = getattr(tiledb_ext, "tiledb_wheel_build", False) - from_source = getattr(tiledb_ext, "tiledb_from_source", False) - lib_exists = libtiledb_exists(tiledb_ext.library_dirs) - do_install = False - - # Download, build and locally install TileDB if needed. - if from_source or not lib_exists: - src_dir = download_libtiledb() - prefix_dir = build_libtiledb(src_dir) - do_install = True - elif lib_exists: - prefix_dir = os.path.abspath(os.path.join(os.path.split(lib_exists)[0], "..")) - elif hasattr(tiledb_ext, "tiledb_path"): - prefix_dir = getattr(tiledb_ext, "tiledb_path") - - if wheel_build and is_windows() and lib_exists: - do_install = True - - print("prefix_dir: ", prefix_dir) - print("do_install: ", do_install) - - if do_install: - lib_subdir = "bin" if os.name == "nt" else "lib" - native_subdir = "" if is_windows() else "native" - # Copy libtiledb shared object(s) to the package directory so they can be found - # with package_data. - dest_dir = os.path.join(TILEDB_PKG_DIR, native_subdir) - for libname in libtiledb_library_names(): - src = os.path.join(prefix_dir, lib_subdir, libname) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - dest = os.path.join(dest_dir, libname) - print("Copying file {0} to {1}".format(src, dest)) - shutil.copy(src, dest) - - # Copy dependencies - if is_windows(): - - def do_copy(src, dest): - print("Copying file {0} to {1}".format(src, dest)) - shutil.copy(src, dest) - - # lib files for linking - src = os.path.join(prefix_dir, "lib", "tiledb.lib") - dest = os.path.join(dest_dir, "tiledb.lib") - do_copy(src, dest) - - # tbb - has_tbb = False - if os.path.isdir(os.path.join(prefix_dir, "bin", "tbb.dll")): - has_tbb = True - src = os.path.join(prefix_dir, "bin", "tbb.dll") - dest = os.path.join(dest_dir, "tbb.dll") - do_copy(src, dest) - src = os.path.join(prefix_dir, "lib", "tbb.lib") - dest = os.path.join(dest_dir, "tbb.lib") - do_copy(src, dest) - - # - tiledb_ext.library_dirs += [os.path.join(prefix_dir, "lib")] - main_ext.library_dirs += [os.path.join(prefix_dir, "lib")] - tiledb_cc_ext.library_dirs += [os.path.join(prefix_dir, "lib")] - - # Update the extension instances with correct build-time paths. - tiledb_ext.library_dirs += [os.path.join(prefix_dir, lib_subdir)] - tiledb_ext.include_dirs += [os.path.join(prefix_dir, "include")] - main_ext.library_dirs += [os.path.join(prefix_dir, lib_subdir)] - main_ext.include_dirs += [os.path.join(prefix_dir, "include")] - tiledb_cc_ext.library_dirs += [os.path.join(prefix_dir, lib_subdir)] - tiledb_cc_ext.include_dirs += [os.path.join(prefix_dir, "include")] - - # Update package_data so the shared object gets installed with the Python module. - libtiledb_objects = [ - os.path.join(native_subdir, libname) - for libname in libtiledb_library_names() - ] - - # Make sure the built library is usable - for shared_obj in libtiledb_objects: - if is_windows(): - continue - test_path = os.path.join(TILEDB_PKG_DIR, shared_obj) - # should only ever be 1, not sure why libtiledb_library_names -> List - try: - CDLL(test_path) - except: - print("\n-------------------") - print("Failed to load shared library: {}".format(test_path)) - print("-------------------\n") - raise - - # This needs to be a glob in order to pick up the versioned SO - libtiledb_objects = [so + "*" for so in libtiledb_objects] - - if is_windows(): - libtiledb_objects.extend( - [os.path.join(native_subdir, libname) for libname in ["tiledb.lib"]] - ) - if has_tbb: - libtiledb_objects.extend( - [ - os.path.join(native_subdir, libname) - for libname in ["tbb.lib", "tbb.dll"] - ] - ) - - print("\n-------------------") - print("libtiledb_objects: ", libtiledb_objects) - print("-------------------\n") - setuptools_cmd.distribution.package_data.update({"tiledb": libtiledb_objects}) - - libtiledb_dir = ( - os.path.dirname(os.path.dirname(lib_exists)) if lib_exists else prefix_dir - ) - version_header = os.path.join( - libtiledb_dir, "include", "tiledb", "tiledb_version.h" - ) - with open(version_header) as header: - lines = list(header)[-3:] - major, minor, patch = [int(line.split()[-1]) for line in lines] - - ext_attr_update( - "cython_compile_time_env", - { - "LIBTILEDB_VERSION_MAJOR": major, - "LIBTILEDB_VERSION_MINOR": minor, - "LIBTILEDB_VERSION_PATCH": patch, - }, - ) - - -class LazyCommandClass(dict): - """ - Lazy command class that defers operations requiring Cython and numpy until - they've actually been downloaded and installed by setup_requires. - """ - - def __contains__(self, key): - if key in ("build_ext", "bdist_wheel", "bdist_egg", "get_tiledb_version"): - return True - return super().__contains__(key) - - def __setitem__(self, key, value): - if key == "build_ext": - raise AssertionError("build_ext overridden!") - super().__setitem__(key, value) - - def __getitem__(self, key): - if key == "build_ext": - return self.make_build_ext_cmd() - elif key == "bdist_wheel": - return self.make_bdist_wheel_cmd() - elif key == "bdist_egg": - return self.make_bdist_egg_cmd() - elif key == "get_tiledb_version": - print(TILEDB_VERSION) - exit() - else: - return super().__getitem__(key) - - def make_build_ext_cmd(self): - """ - :return: A command class implementing 'build_ext'. - """ - from Cython.Distutils import build_ext as cython_build_ext - - class build_ext(cython_build_ext): - """ - Custom build_ext command that lazily adds numpy's include_dir to - extensions. - """ - - def build_extensions(self): - for ext in self.extensions: - ext.include_dirs.append(np.get_include()) - - find_or_install_libtiledb(self) - - super().build_extensions() - - return build_ext - - def make_bdist_wheel_cmd(self): - """ - :return: A command class implementing 'bdist_wheel'. - """ - from wheel.bdist_wheel import bdist_wheel - - class bdist_wheel_cmd(bdist_wheel): - def run(self): - # This may modify package_data: - find_or_install_libtiledb(self) - bdist_wheel.run(self) - - return bdist_wheel_cmd - - def make_bdist_egg_cmd(self): - """ - :return: A command class implementing 'bdist_egg'. - """ - from setuptools.command.bdist_egg import bdist_egg - - class bdist_egg_cmd(bdist_egg): - def run(self): - # This may modify package_data: - find_or_install_libtiledb(self) - bdist_egg.run(self) - - return bdist_egg_cmd - - -class get_pybind_include(object): - """Helper class to determine the pybind11 include path - The purpose of this class is to postpone importing pybind11 - until it is actually installed, so that the ``get_include()`` - method can be invoked.""" - - def __init__(self, user=False): - self.user = user - - def __str__(self): - import pybind11 - - return pybind11.get_include(self.user) - - -def cmake_available(): - """ - Checks whether CMake command is available and >= version 3.21 - Note 11/23/2021: < 3.22 temporarily due to AWS SDK imcompatibility. - :return: - """ - CMAKE_MINIMUM_MAJOR = 3 - CMAKE_MINIMUM_MINOR = 21 - CMAKE_MAXIMUM_MINOR = 22 - try: - output = subprocess.check_output(["cmake", "--version"]).split() - version = output[2].decode("utf-8").split(".") - return ( - int(version[0]) >= CMAKE_MINIMUM_MAJOR - and int(version[1]) >= CMAKE_MINIMUM_MINOR - and int(version[1]) < CMAKE_MAXIMUM_MINOR - ) - except: - return False - - -def parse_requirements(req_file): - with open(req_file) as f: - return f.read().strip().split("\n") - - -def setup_requires(): - if CONDA_FORGE_BUILD: - return [] - - if WHEEL_BUILD: - req = parse_requirements("misc/requirements_wheel.txt") - else: - req = parse_requirements("requirements_dev.txt") - req = list(filter(lambda r: not r.startswith("-r"), req)) - - req_cmake = list(filter(lambda r: "cmake" in r, req))[0] - - # Add cmake requirement if libtiledb is not found and cmake is not available. - if not libtiledb_exists(LIB_DIRS) and not cmake_available(): - req.append(req_cmake) - - return req - - -def install_requires(): - if CONDA_FORGE_BUILD: - return [] - - return parse_requirements("requirements.txt") - - -# Allow setting (lib) TileDB directory if it is installed on the system -TILEDB_PATH = os.environ.get("TILEDB_PATH", "") -print("TILEDB_PATH from env: '{}'".format(TILEDB_PATH)) -# Sources & libraries -INC_DIRS = [] -LIB_DIRS = [] -LIBS = ["tiledb"] -DEF_MACROS = [] - -# Pass command line flags to setup.py script -# handle --tiledb=[PATH] --lflags=[FLAGS] --cxxflags=[FLAGS] -args = sys.argv[:] -for arg in args: - if arg.find("--tiledb=") == 0: - TILEDB_PATH = os.path.expanduser(arg.split("=")[1]) - sys.argv.remove(arg) - if arg.find("--lflags=") == 0: - LFLAGS = arg.split("=")[1].split() - sys.argv.remove(arg) - if arg.find("--cxxflags=") == 0: - CXXFLAGS = arg.split("=")[1].split() - sys.argv.remove(arg) - if arg.find("--debug") == 0: - TILEDB_DEBUG_BUILD = True - sys.argv.remove(arg) - if arg.find("--release-symbols") == 0: - TILEDB_SYMBOLS_BUILD = True - sys.argv.remove(arg) - TILEDBPY_WERROR = False - if arg.find("--werror") == 0: - TILEDBPY_WERROR = True - sys.argv.remove(arg) - -# Global variables -CXXFLAGS = os.environ.get("CXXFLAGS", "").split() -if not is_windows(): - CXXFLAGS.append("-std=c++17") - if TILEDBPY_WERROR: - CXXFLAGS.append("-Werror") - if not TILEDB_DEBUG_BUILD: - CXXFLAGS.append("-Wno-deprecated-declarations") - elif TILEDB_DEBUG_BUILD: - CXXFLAGS.append("-g") - CXXFLAGS.append("-O0") - CXXFLAGS.append("-UNDEBUG") # defined by distutils - if TILEDB_SYMBOLS_BUILD: - CXXFLAGS.append("-g") -elif is_windows(): - CXXFLAGS.append("/std:c++17") - -LFLAGS = os.environ.get("LFLAGS", "").split() - -if TILEDB_PATH != "" and TILEDB_PATH != "source": - print("TILEDB_PATH in block before: '{}'".format(TILEDB_PATH)) - TILEDB_PATH = os.path.normpath(os.path.abspath(TILEDB_PATH)) - print("TILEDB_PATH in block after: '{}'".format(TILEDB_PATH)) - LIB_DIRS += [os.path.join(os.path.normpath(TILEDB_PATH), "lib")] - if sys.platform.startswith("linux"): - LIB_DIRS += [ - os.path.join(TILEDB_PATH, "lib64"), - os.path.join(TILEDB_PATH, "lib", "x86_64-linux-gnu"), - ] - elif os.name == "nt": - LIB_DIRS += [os.path.join(TILEDB_PATH, "bin")] - INC_DIRS += [os.path.join(TILEDB_PATH, "include")] - if sys.platform == "darwin": - LFLAGS += ["-Wl,-rpath,{}".format(p) for p in LIB_DIRS] - -# Source files for build -MODULAR_SOURCES = ["tiledb/indexing.pyx", "tiledb/libmetadata.pyx"] -MODULAR_HEADERS = ["tiledb/libtiledb.pxd", "tiledb/indexing.pxd"] - -TILEDB_MAIN_SOURCES = [ - "tiledb/main.cc", - "tiledb/core.cc", - "tiledb/npbuffer.cc", - "tiledb/fragment.cc", - "tiledb/schema_evolution.cc", - "tiledb/util.cc", - "tiledb/tests/test_metadata.cc", - "tiledb/tests/test_webp.cc", - # TODO currently included in core.cc due to dependency. - # need to un-comment after refactor. - # "tiledb/query_condition.cc", -] - -if "TILEDB_SERIALIZATION" in os.environ: - TILEDB_MAIN_SOURCES += [ - "tiledb/serialization.cc", - "tiledb/tests/test_serialization.cc", - ] - -__extensions = [ - Extension( - "tiledb.libtiledb", - include_dirs=INC_DIRS, - define_macros=DEF_MACROS, - sources=["tiledb/libtiledb.pyx"], - depends=MODULAR_HEADERS, - library_dirs=LIB_DIRS, - libraries=LIBS, - extra_link_args=LFLAGS, - extra_compile_args=[flag for flag in CXXFLAGS if flag != "-Werror"], - language="c++", - ), - Extension( - "tiledb.main", - TILEDB_MAIN_SOURCES, - include_dirs=INC_DIRS + [get_pybind_include(), get_pybind_include(user=True)], - language="c++", - library_dirs=LIB_DIRS, - libraries=LIBS, - extra_link_args=LFLAGS, - extra_compile_args=CXXFLAGS + ["-fvisibility=hidden"], - ), - Pybind11Extension( - "tiledb.cc", - sorted(glob.glob("tiledb/cc/*.cc")), # Sort source files for reproducibility - include_dirs=INC_DIRS, - define_macros=DEF_MACROS, - library_dirs=LIB_DIRS, - libraries=LIBS, - extra_link_args=LFLAGS, - extra_compile_args=CXXFLAGS + ["-fvisibility=hidden"], - language="c++", - ), -] - -__extensions[0].depends += MODULAR_SOURCES - -# Helper to set Extension attributes correctly based on python version -def ext_attr_update(attr, value): - for x in __extensions: - if isinstance(x, Pybind11Extension): - continue - setattr(x, attr, value) - - -# Monkey patches to be forwarded to cythonize -# some of these will error out if passed directly -# to Extension(..) above - -if WHEEL_BUILD: - ext_attr_update("tiledb_wheel_build", True) - - -# - build with `#line` directive annotations -# (equivalent to `emit_linenums` command line directive) -ext_attr_update("cython_line_directives", 1) - -# - generate XML debug mapping file (`cython_debug`) -if TILEDB_DEBUG_BUILD: - ext_attr_update("cython_gdb", True) - __extensions[1].depends += "tiledb/debug.cc" - -# - set rt lib dirs to get correct RPATH on unixy platforms -# note that we set rpath for darwin separately above. -if not is_windows(): - ext_attr_update("runtime_library_dirs", LIB_DIRS) - -if TILEDB_PATH == "source": - ext_attr_update("tiledb_from_source", True) -elif TILEDB_PATH != "": - ext_attr_update("tiledb_path", TILEDB_PATH) - -setup( - platforms=["any"], - ext_modules=__extensions, - setup_requires=setup_requires(), - install_requires=install_requires(), - packages=find_packages(), - cmdclass=LazyCommandClass(), - zip_safe=False, -) diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt new file mode 100644 index 0000000000..98d82e7c0f --- /dev/null +++ b/tiledb/CMakeLists.txt @@ -0,0 +1,82 @@ +# Cython + +add_custom_command( + OUTPUT libtiledb.cc + COMMAND Python::Interpreter -m cython + "${CMAKE_CURRENT_SOURCE_DIR}/libtiledb.pyx" --output-file libtiledb.cc + DEPENDS libtiledb.pyx + VERBATIM +) + +python_add_library( + libtiledb + MODULE + libtiledb.cc + WITH_SOABI +) + +target_link_libraries( + libtiledb + PUBLIC + Python::NumPy + TileDB::tiledb_shared +) + +target_compile_features( + libtiledb + PUBLIC + cxx_std_20 +) + +# Pybind11 + +pybind11_add_module( + main + main.cc + core.cc + npbuffer.cc + fragment.cc + schema_evolution.cc + util.cc + serialization.cc + tests/test_metadata.cc + tests/test_webp.cc + tests/test_serialization.cc +) + +target_link_libraries( + main + PUBLIC + TileDB::tiledb_shared +) + +target_compile_features( + main + PUBLIC + cxx_std_20 +) + +install(TARGETS main libtiledb DESTINATION tiledb) + +if(TILEDB_DOWNLOADED) + message(STATUS "Adding \"libtiledb\" into install group") + + install(IMPORTED_RUNTIME_ARTIFACTS TileDB::tiledb_shared DESTINATION tiledb) + + if (APPLE) + set_target_properties(main PROPERTIES INSTALL_RPATH "@loader_path") + set_target_properties(libtiledb PROPERTIES INSTALL_RPATH "@loader_path") + elseif(UNIX) + set_target_properties(main PROPERTIES INSTALL_RPATH "\$ORIGIN") + set_target_properties(libtiledb PROPERTIES INSTALL_RPATH "\$ORIGIN") + endif() +else() + # If using external TileDB core library force it to be linked at runtime using RPATH + get_property(TILEDB_LOCATION TARGET TileDB::tiledb_shared PROPERTY LOCATION) + get_filename_component(TILEDB_LOCATION ${TILEDB_LOCATION} DIRECTORY) + message(STATUS "Setting RPATH for targets \"main\" and \"libtiledb\" to ${TILEDB_LOCATION}") + set_target_properties(main PROPERTIES INSTALL_RPATH ${TILEDB_LOCATION}) + set_target_properties(libtiledb PROPERTIES INSTALL_RPATH ${TILEDB_LOCATION}) +endif() + +add_subdirectory(cc) \ No newline at end of file diff --git a/tiledb/__init__.py b/tiledb/__init__.py index a0f43b8d1d..1040f1488f 100644 --- a/tiledb/__init__.py +++ b/tiledb/__init__.py @@ -19,23 +19,6 @@ else: lib_name = "tiledb" -# On Windows and whl builds, we may have a shared library already linked, or -# adjacent to, the cython .pyd shared object. In this case, we can import directly -# from .libtiledb -try: - import tiledb - - from .libtiledb import Ctx - - del Ctx -except: - try: - lib_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "native") - ctypes.CDLL(os.path.join(lib_dir, lib_name)) - except OSError: - # Otherwise try loading by name only. - ctypes.CDLL(lib_name) - from .array_schema import ArraySchema from .attribute import Attr from .cc import TileDBError @@ -92,6 +75,7 @@ ) from .libtiledb import ( Array, + Ctx, DenseArrayImpl, SparseArrayImpl, ls, diff --git a/tiledb/cc/CMakeLists.txt b/tiledb/cc/CMakeLists.txt new file mode 100644 index 0000000000..ba131a856c --- /dev/null +++ b/tiledb/cc/CMakeLists.txt @@ -0,0 +1,50 @@ +pybind11_add_module( + cc + array.cc + attribute.cc + common.cc + common.h + consolidation_plan.cc + context.cc + dimension_label.cc + domain.cc + enum.cc + enumeration.cc + filestore.cc + filter.cc + group.cc + object.cc + query.cc + schema.cc + subarray.cc + tiledbcpp.cc + vfs.cc +) + +target_link_libraries( + cc + PUBLIC + TileDB::tiledb_shared +) + +target_compile_features( + cc + PUBLIC + cxx_std_20 +) + +install(TARGETS cc DESTINATION tiledb) + +if(TILEDB_DOWNLOADED) + if (APPLE) + set_target_properties(cc PROPERTIES INSTALL_RPATH "@loader_path") + elseif(UNIX) + set_target_properties(cc PROPERTIES INSTALL_RPATH "\$ORIGIN") + endif() +else() + # If using external TileDB core library force it to be linked at runtime using RPATH + get_property(TILEDB_LOCATION TARGET TileDB::tiledb_shared PROPERTY LOCATION) + get_filename_component(TILEDB_LOCATION ${TILEDB_LOCATION} DIRECTORY) + message(STATUS "Setting RPATH for target \"cc\" to ${TILEDB_LOCATION}") + set_target_properties(cc PROPERTIES INSTALL_RPATH ${TILEDB_LOCATION}) +endif() diff --git a/tiledb/tests/test_schema_evolution.py b/tiledb/tests/test_schema_evolution.py index c37babcca8..5110e8ab1e 100644 --- a/tiledb/tests/test_schema_evolution.py +++ b/tiledb/tests/test_schema_evolution.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import numpy as np import pytest @@ -42,7 +43,7 @@ def test_schema_evolution(tmp_path): with pytest.raises(tiledb.TileDBError) as excinfo: se.add_attribute(newattr) assert "Input attribute name is already there" in str(excinfo.value) - assert "tiledb/schema_evolution.cc" in str(excinfo.value) + assert str(Path("tiledb") / "schema_evolution.cc") in str(excinfo.value) se.array_evolve(uri) diff --git a/tiledb/tests/test_timestamp_overrides.py b/tiledb/tests/test_timestamp_overrides.py index d4dd66088c..f0a114ea3e 100644 --- a/tiledb/tests/test_timestamp_overrides.py +++ b/tiledb/tests/test_timestamp_overrides.py @@ -47,10 +47,10 @@ def test_timestamp_overrides(self): def helper_fragments(self, uri): start_datetime = datetime.datetime.now() - fragments = 25 + fragments = 5 A = np.zeros(fragments) - dom = tiledb.Domain(tiledb.Dim(domain=(0, 24), tile=fragments, dtype=np.int64)) + dom = tiledb.Domain(tiledb.Dim(domain=(0, 4), tile=fragments, dtype=np.int64)) att = tiledb.Attr(dtype=A.dtype) schema = tiledb.ArraySchema(domain=dom, attrs=(att,)) From c506394827ae8eadd7ad021dd007022406516c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Baran?= Date: Mon, 22 Jul 2024 16:25:36 +0200 Subject: [PATCH 2/2] Remove empty files (#2010) Remove empty files created by me incorrectly merging changes during branch syncing --- misc/azure-release.yml | 0 setup.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 misc/azure-release.yml delete mode 100644 setup.py diff --git a/misc/azure-release.yml b/misc/azure-release.yml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/setup.py b/setup.py deleted file mode 100644 index e69de29bb2..0000000000