diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index cbba3f39..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,124 +0,0 @@ -version: 2 -jobs: - checkout_code: - docker: - - image: condaforge/miniforge3 - working_directory: ~/repo - steps: - - checkout - - run: git submodule sync - - run: git submodule update --init - - save_cache: - key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} - paths: - - ~/repo - - install_conda_env: - environment: - TIKTORCH_ENV_NAME: tiktorch-server-env - TIKTORCH_ENV_PREFIX: /opt/conda/envs/tiktorch-server-env - docker: - - image: condaforge/miniforge3 - working_directory: ~/repo - steps: - - restore_cache: - keys: - - v1-repo-{{ .Environment.CIRCLE_SHA1 }} - - restore_cache: - keys: - - v11-dependencies-{{ .Environment.CIRCLE_SHA1 }} - - run: conda config --set channel_priority strict - - run: conda update -n base -c conda-forge --update-all - - run: conda install -c conda-forge conda-build make boa - - run: | - echo "Creating new environment ${TIKTORCH_ENV_NAME}" - make devenv - - save_cache: - paths: - - /opt/conda/envs - key: v11-dependencies-{{ .Environment.CIRCLE_SHA1 }} - - pre_commit_check: - docker: - - image: condaforge/miniforge3 - working_directory: ~/repo - steps: - - restore_cache: - keys: - - v1-repo-{{ .Environment.CIRCLE_SHA1 }} - - restore_cache: - keys: - - v11-dependencies-{{ .Environment.CIRCLE_SHA1 }} - - - run: - name: run pre-commit - command: | - . /opt/conda/etc/profile.d/conda.sh - conda activate tiktorch-server-env - pre-commit run --from-ref origin/${CIRCLE_BRANCH} --to-ref ${CIRCLE_BRANCH} - - tests: - docker: - - image: condaforge/miniforge3 - working_directory: ~/repo - steps: - - restore_cache: - keys: - - v1-repo-{{ .Environment.CIRCLE_SHA1 }} - - restore_cache: - keys: - - v11-dependencies-{{ .Environment.CIRCLE_SHA1 }} - - - run: - name: run tests - command: | - . /opt/conda/etc/profile.d/conda.sh - conda activate tiktorch-server-env - conda list - python -m pytest -v - - build_conda_packages: - docker: - - image: condaforge/miniforge3 - working_directory: ~/repo - steps: - - restore_cache: - keys: - - v1-repo-{{ .Environment.CIRCLE_SHA1 }} - - - run: conda config --set channel_priority strict - - run: conda install -c conda-forge conda-build anaconda-client boa - - run: - name: build packages - command: | - . /opt/conda/etc/profile.d/conda.sh - ./scripts/conda_build.sh conda-recipe - - -workflows: - version: 2 - build: - jobs: - - checkout_code: - filters: - tags: - only: /^v.*/ - - install_conda_env: - filters: - tags: - only: /^v.*/ - requires: - - checkout_code - - tests: - requires: - - install_conda_env - - pre_commit_check: - requires: - - install_conda_env - - build_conda_packages: - context: conda-upload - filters: - tags: - only: /^v.*/ - branches: - ignore: /.*/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..b6e9921c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,172 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + pre_commit_check: + name: Pre-Commit Check + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + auto-activate-base: false + activate-environment: tiktorch-server-env + environment-file: environment.yml + channel-priority: flexible + miniforge-variant: Miniforge3 + - name: Run Pre-Commit + run: | + echo $BRANCH_NAME + echo ${{ github.event.pull_request.base.ref }} + echo ${{ github.event.pull_request.head.sha }} + git fetch origin + pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref ${{ github.event.pull_request.head.sha }} + + test-dev: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + auto-activate-base: false + activate-environment: tiktorch-server-env + environment-file: environment.yml + channel-priority: flexible + miniforge-variant: Miniforge3 + - name: conda diagnostics + run: | + conda list + conda info + - name: install submodules + run: | + git submodule init + git submodule update + make install_submodules + - name: Run tests + run: | + pytest -v -s --cov=tiktorch --cov-report=xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + file: ./coverage.xml + fail_ci_if_error: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + + conda-noarch-build: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + auto-activate-base: true + channel-priority: flexible + miniforge-variant: Miniforge3 + - name: install common conda dependencies + run: conda install -n base -c conda-forge conda-build setuptools_scm -y + - name: Cache Conda Packages + uses: actions/cache@v4 + with: + path: | + pkgs/noarch + pkgs/channeldata.json + key: ${{ github.sha }}-packages + - id: version + run: | + vers=$( python setup.py --version ) + echo "version=${vers}" >> $GITHUB_OUTPUT + - name: Linux Conda Build + run: | + mkdir -p ./pkgs/noarch + conda build -c ilastik-forge -c pytorch -c conda-forge \ + conda-recipe --no-test --output-folder ./pkgs + + test-build-conda-packages: + needs: [conda-noarch-build] + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest, ubuntu-latest] + runs-on: ${{ matrix.os }} + env: + TIKTORCH_PACKAGE_NAME: tiktorch-${{ needs.conda-noarch-build.outputs.version }}-py_0.tar.bz2 + steps: + # Use GNU tar instead of BSD tar on Windows + - name: "Use GNU tar instead of BSD tar" + if: matrix.os == 'windows-latest' + shell: cmd + run: echo C:\Program Files\Git\usr\bin>>"%GITHUB_PATH%" + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + auto-activate-base: true + channel-priority: flexible + miniforge-variant: Miniforge3 + - name: install common conda dependencies + run: conda install -n base -c conda-forge conda-build setuptools_scm -y + - name: Cache Conda Packages + uses: actions/cache@v4 + with: + path: | + pkgs/noarch + pkgs/channeldata.json + key: ${{ github.sha }}-packages + enableCrossOsArchive: true + - name: Linux Test + if: matrix.os == 'ubuntu-latest' + shell: bash -l {0} + run: | + conda build --test --override-channels \ + -c ./pkgs -c ilastik-forge -c pytorch -c conda-forge \ + ./pkgs/noarch/${TIKTORCH_PACKAGE_NAME} + - name: macOS Test + if: matrix.os == 'macos-latest' + shell: bash -l {0} + run: | + conda build --test --override-channels \ + -c ./pkgs -c ilastik-forge -c pytorch -c conda-forge \ + ./pkgs/noarch/${TIKTORCH_PACKAGE_NAME} + - name: Windows Test + if: matrix.os == 'windows-latest' + # HACK: due to a bug in conda-build need to point to + # libarchive explicitly. + # https://github.com/conda/conda/issues/12563#issuecomment-1494264704 + env: + LIBARCHIVE: C:\Miniconda\Library\bin\archive.dll + shell: cmd /C CALL {0} + run: | + conda build --test --override-channels ^ + -c %CD%\pkgs -c ilastik-forge -c pytorch -c conda-forge ^ + .\pkgs\noarch\%TIKTORCH_PACKAGE_NAME% diff --git a/Makefile b/Makefile index d7e516d7..9dc9c0bf 100644 --- a/Makefile +++ b/Makefile @@ -28,5 +28,4 @@ install_submodules: remove_devenv: conda env remove --yes --name $(TIKTORCH_ENV_NAME) - .PHONY: * diff --git a/README.md b/README.md index f7dff6a6..a128daed 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![CircleCI](https://circleci.com/gh/ilastik/tiktorch.svg?style=shield)](https://circleci.com/gh/ilastik/tiktorch) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Conda](https://anaconda.org/ilastik-forge/tiktorch/badges/version.svg)](https://anaconda.org/ilastik-forge/tiktorch) +[![codecov](https://codecov.io/gh/ilastik/tiktorch/branch/main/graph/badge.svg)](https://codecov.io/gh/ilastik/tiktorch) `tiktorch` is the neural network prediction server for [`ilastik`](https://ilastik.org). The server is used in the [Neural Network Workflow](https://www.ilastik.org/documentation/nn/nn). diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index d1d33fc0..3afda61c 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -15,28 +15,26 @@ source: build: noarch: python number: 0 - script: python -m pip install --no-deps --ignore-installed . + script: + - python -m pip install . --no-deps --ignore-installed --no-cache-dir -vvv entry_points: - tiktorch-server = tiktorch.server.base:main requirements: host: - pip - - python >=3.7 + - python >=3.8 run: - - python >=3.7 - - pickle5 # [py37] + - python >=3.8 {% for dep in setup_py_data['install_requires'] %} - {{ dep.lower() }} {% endfor %} run_constrained: + - mkl <2024.1.0 # [linux] until pytorch is compatible with the current version - cudatoolkit >=10.2 {% for dep in setup_py_data['extras_require']['server-pytorch'] %} - {{ dep.lower() }} {% endfor %} - {% for dep in setup_py_data['extras_require']['server-tensorflow'] %} - - {{ dep.lower() }} - {% endfor %} about: home: https://github.com/ilastik/tiktorch @@ -52,12 +50,11 @@ test: {% for dep in setup_py_data['extras_require']['server-pytorch'] %} - {{ dep.lower() }} {% endfor %} - {% for dep in setup_py_data['extras_require']['server-tensorflow'] %} - - {{ dep.lower() }} - {% endfor %} - # this is still necessary, torchvision doesn't work properly with cpuonly mutex - - torchvision=*=*cpu + - python 3.9.* + - pytest + - pytest-grpc - cpuonly + - ilastik-pytorch-version-helper-cpu imports: # client - tiktorch @@ -70,3 +67,12 @@ test: # server - tiktorch.server.base - tiktorch.server.session + + source_files: + - tests + - tiktorch + - pytest.ini + + commands: + - pytest + diff --git a/environment.yml b/environment.yml index 94b04c80..2a44f1aa 100644 --- a/environment.yml +++ b/environment.yml @@ -13,8 +13,6 @@ dependencies: - marshmallow=3.12.* - marshmallow-jsonschema - protobuf - - pytest - - pytest-grpc - pyyaml=5.3.* - requests - ruamel.yaml @@ -22,15 +20,16 @@ dependencies: - scipy - typing-extensions - xarray + - setuptools + - pip # pytorch # remove cpuonly, add cudatoolkit and cudnn for gpu support - - pytorch=1.13.* - - inferno=v0.4.* + - pytorch=2.3.* # currently it's necessary to force the cpu version, remove # torchvision pin when going for gpu # - torchvision - #- cpuonly + - cpuonly # - cudatoolkit >=10.2 # - cudnn # - tochvision @@ -46,10 +45,14 @@ dependencies: - typer # dev stuff + - pytest + - pytest-cov + - pytest-grpc - bump2version - mypy - pre_commit - - pip - - mkl <2024.1.0 # [linux] until pytorch is compatible with the current version + + + diff --git a/pytest.ini b/pytest.ini index c2c366b2..7d2e0170 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,11 @@ [pytest] -add_opts = -s --doctest-modules +python_files = test_*.py +addopts = + -v + -s + --color=yes + --doctest-modules + --ignore=tests/data testpaths = tests #log_format = %(asctime)s.%(msecs)03d %(levelname)s %(message)s #log_date_format = %M:%S.%f diff --git a/setup.py b/setup.py index d34ab19e..a6313f3f 100644 --- a/setup.py +++ b/setup.py @@ -30,22 +30,12 @@ "bioimageio.spec==0.4.9.post5", "bioimageio.core==0.5.11", "grpcio>=1.31", - "numpy", + "numpy<2", # pytorch 2.2.2-py3.9_0 for macos is compiled with numpy 1.* "protobuf", "pyyaml", "xarray", ], - extras_require={ - "server-pytorch": [ - "inferno", - "pytorch>=1.6", - "scikit-learn", - ], - "server-tensorflow": [ - "tensorflow>=2.9", - "scikit-learn", - ], - }, + extras_require={"server-pytorch": ["pytorch>=2", "scikit-learn", "torchvision"]}, entry_points={"console_scripts": ["tiktorch=tiktorch.server.base:main"]}, # extras_require={"test": ["pytest"]}, project_urls={ # Optional diff --git a/tests/test_rpc/test_mp.py b/tests/test_rpc/test_mp.py index 2011c54e..449cf7c0 100644 --- a/tests/test_rpc/test_mp.py +++ b/tests/test_rpc/test_mp.py @@ -112,30 +112,31 @@ def __getattr__(self, name): client.fast_compute(2, 2) - client.shutdown().result() + client.shutdown() + p.join() -def test_future_timeout(client: ITestApi, log_queue): +def test_future_timeout(log_queue): child, parent = mp.Pipe() p = mp.Process(target=_srv, args=(parent, log_queue)) p.start() - client = create_client_api(iface_cls=ITestApi, conn=child, timeout=0.001) - with pytest.raises(TimeoutError): - client.compute(1, 2) + try: + with pytest.raises(TimeoutError): + client.compute(1, 2) - with pytest.raises(TimeoutError): - client.compute.async_(1, 2).result() + with pytest.raises(TimeoutError): + client.compute.async_(1, 2).result() - with pytest.raises(TimeoutError): - client.compute_fut(1, 2).result() + with pytest.raises(TimeoutError): + client.compute_fut(1, 2).result() - client.compute.async_(1, 2).result(timeout=3) - - client.shutdown() - p.join() + client.compute.async_(1, 2).result(timeout=4) + finally: + client.shutdown() + p.join() class ICancelable(RPCInterface): diff --git a/tests/test_server/test_grpc/test_fligh_control_servicer.py b/tests/test_server/test_grpc/test_fligh_control_servicer.py index a2a5080b..52f32c41 100644 --- a/tests/test_server/test_grpc/test_fligh_control_servicer.py +++ b/tests/test_server/test_grpc/test_fligh_control_servicer.py @@ -23,7 +23,7 @@ def test_shutdown_event_is_not_set_while_pings_keep_coming(): def _pinger(): while not stop_pinger.is_set(): servicer.Ping(None, None) - time.sleep(0.05) + time.sleep(0.01) pinger_thread = threading.Thread(target=_pinger, name="Pinger", daemon=True) pinger_thread.start() diff --git a/tiktorch/rpc/types.py b/tiktorch/rpc/types.py index 3adc645b..b5f7b4eb 100644 --- a/tiktorch/rpc/types.py +++ b/tiktorch/rpc/types.py @@ -107,13 +107,10 @@ def map(self, func: Callable[[T], S]) -> "RPCFuture[S]": def _checkgenericfut(type_: Type) -> bool: - # XXX: py3.7 regression isclass returns False on parametrized generics - if not isinstance(type_, (type, _GenericAlias)): - return False - - origin = getattr(type_, "__origin__", None) - - return origin and issubclass(origin, RPCFuture) + if isinstance(type_, _GenericAlias): + origin = getattr(type_, "__origin__", None) + return origin and issubclass(origin, Future) + return False def isfutureret(func: Callable):