From 4e91d7cf092c351f52969fed3d2cea4a659a52a2 Mon Sep 17 00:00:00 2001 From: Feda Curic Date: Wed, 20 Sep 2023 06:47:31 +0200 Subject: [PATCH] Replace requirement files with pyproject --- .github/workflows/annotate_cpp.yml | 45 ------- .github/workflows/annotate_python.yml | 30 +++++ .github/workflows/build.yml | 7 +- .github/workflows/coverage.yml | 3 +- .github/workflows/doctest.yml | 7 +- .github/workflows/style.yml | 6 +- .github/workflows/test_ert.yml | 6 +- .github/workflows/typing.yml | 4 +- .readthedocs.yaml | 9 +- CONTRIBUTING.md | 2 +- README.md | 2 +- ci/run_ert_ctests.sh | 3 +- ci/testkomodo.sh | 2 +- dev-requirements.txt | 33 ----- pyproject.toml | 142 +++++++++++++++++++- setup.py | 84 ------------ src/ert/dark_storage/endpoints/responses.py | 2 +- style-requirements.txt | 5 - types-requirements.txt | 10 -- 19 files changed, 188 insertions(+), 214 deletions(-) delete mode 100644 .github/workflows/annotate_cpp.yml create mode 100644 .github/workflows/annotate_python.yml delete mode 100644 dev-requirements.txt delete mode 100644 style-requirements.txt delete mode 100644 types-requirements.txt diff --git a/.github/workflows/annotate_cpp.yml b/.github/workflows/annotate_cpp.yml deleted file mode 100644 index ff06d5ef6a9..00000000000 --- a/.github/workflows/annotate_cpp.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: Annotate Cpp - -on: [pull_request] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - annotate-cpp: - strategy: - matrix: - os: ['ubuntu-latest'] - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: -1 - - - name: Install Ubuntu dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y clang-tidy - - - name: Install dependencies from PyPI - run: | - python3 -m pip install "conan<2" pybind11 ecl - - - name: Create compile commands for clib - run: | - mkdir cmake-build - cmake -S src/clib -B cmake-build -DBUILD_TESTS=ON -DCOVERAGE=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - cmake --build cmake-build - - - name: find changed files - id: find_changed_files - run: echo "changed_files=$(git diff --name-only ${{github.sha}} ${{github.event.pull_request.base.sha}} | tr ' ' '\n' | xargs ls -d 2>/dev/null | grep -E '\.(hpp|cpp)$' | tr '\n' ' ')" >> "$GITHUB_OUTPUT" - - - run: echo ::add-matcher::.github/clang-tidy-matcher.json - - - name: run clang-tidy - run: clang-tidy -p cmake-build/ ${{steps.find_changed_files.outputs.changed_files}} - if: steps.find_changed_files.outputs.changed_files != '' diff --git a/.github/workflows/annotate_python.yml b/.github/workflows/annotate_python.yml new file mode 100644 index 00000000000..319e559597f --- /dev/null +++ b/.github/workflows/annotate_python.yml @@ -0,0 +1,30 @@ +name: Annotate Python + +on: [pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + annotate-python-linting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: -1 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install flake8 + run: pip install flake8 flake8-bugbear flake8-simplify flake8-debugger flake8-pep3101 Flake8-pyproject + - name: install project + run: pip install ".[dev, types]" + - name: find changed files + id: find_changed_files + run: echo "changed_files=$(git diff --name-only ${{github.sha}} ${{github.event.pull_request.base.sha}} | tr ' ' '\n' | xargs ls -d 2>/dev/null | grep -E '.py$' | tr '\n' ' ')" >> "$GITHUB_OUTPUT" + - run: echo ::add-matcher::.github/flake8-matcher.json + - name: run flake8 + run: flake8 --exit-zero --ignore "W503, E203" ${{steps.find_changed_files.outputs.changed_files}} + if: steps.find_changed_files.outputs.changed_files != '' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a3c8623257..85d02ca5a94 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -161,9 +161,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: "pip" cache-dependency-path: | - setup.py pyproject.toml - dev-requirements.txt - name: Get wheels uses: actions/download-artifact@v3 @@ -172,10 +170,7 @@ jobs: - name: Install wheel run: | - find . -name "*.whl" -exec pip install {} \; - - - name: Install dependencies - run: pip install -r dev-requirements.txt + find . -name "*.whl" -exec pip install "{}[dev]" \; - name: Make test directory run: | diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 17252f1b2e0..bc13c7b2b21 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -36,8 +36,7 @@ jobs: - name: Install with dependencies run: | - pip install . - pip install -r dev-requirements.txt + pip install ".[dev]" - name: Test GUI if: matrix.test-type == 'gui-tests' diff --git a/.github/workflows/doctest.yml b/.github/workflows/doctest.yml index 1bb29c70392..dd5e5d33149 100644 --- a/.github/workflows/doctest.yml +++ b/.github/workflows/doctest.yml @@ -30,12 +30,7 @@ jobs: cache-dependency-path: | setup.py pyproject.toml - dev-requirements.txt - - run: pip install -e . - - - name: Install with dependencies - run: | - pip install -r dev-requirements.txt + - run: pip install -e ".[dev]" - name: Test doctest run: | diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 31c389558e8..e70b4fc6788 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -32,11 +32,10 @@ jobs: cache-dependency-path: | setup.py pyproject.toml - style-requirements.txt - - name: Install dependencies + - name: Install ERT with style dependencies run: | - pip install -r style-requirements.txt + pip install ".[style]" - name: Clang Format run: ./script/clang-format --check @@ -59,5 +58,4 @@ jobs: - name: Run pylint run: | - pip install . # We need the dependencies of ERT to avoid import-error script/pylint diff --git a/.github/workflows/test_ert.yml b/.github/workflows/test_ert.yml index 2bfb6946b79..4cf6544706d 100644 --- a/.github/workflows/test_ert.yml +++ b/.github/workflows/test_ert.yml @@ -29,7 +29,6 @@ jobs: cache-dependency-path: | setup.py pyproject.toml - dev-requirements.txt - name: Get wheels uses: actions/download-artifact@v3 @@ -38,10 +37,7 @@ jobs: - name: Install wheel run: | - find . -name "*.whl" -exec pip install {} \; - - - name: Install dependencies - run: pip install -r dev-requirements.txt + find . -name "*.whl" -exec pip install "{}[dev]" \; - name: Test GUI if: inputs.test-type == 'gui-test' diff --git a/.github/workflows/typing.yml b/.github/workflows/typing.yml index a1a87912fd2..2b7c2a95566 100644 --- a/.github/workflows/typing.yml +++ b/.github/workflows/typing.yml @@ -30,13 +30,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install ERT and dependencies run: | - pip install . + pip install ".[dev, types]" - name: Install dependencies # type checking requires protobuf stubs run: | python -m pip install --upgrade pip - python -m pip install -r types-requirements.txt - python -m pip install -r dev-requirements.txt python -m pip install grpcio-tools python -m grpc_tools.protoc -I src/_ert_com_protocol --mypy_out=src/_ert_com_protocol src/_ert_com_protocol/_schema.proto - run: echo ::add-matcher::.github/mypy-matcher.json diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f22122d0bfd..c844dd6eeeb 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,10 +1,11 @@ version: 2 python: - install: - - requirements: dev-requirements.txt - - method: pip - path: . + install: + - method: pip + path: . + extra_requirements: + - dev sphinx: builder: html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b943e44e225..f0bd4d53390 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,7 @@ noise in the review process. You can build the documentation after installation by running ```bash -pip install -r dev-requirements.txt +pip install ".[dev]" sphinx-build -n -v -E -W ./docs ./tmp/ert_docs ``` and then open the generated `./tmp/ert_docs/index.html` in a browser. diff --git a/README.md b/README.md index d39c7b06d28..a3f24948e41 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ pip install --editable . Additional development packages must be installed to run the test suite: ```sh -pip install -r dev-requirements.txt +pip install ".[dev]" pytest tests/ ``` diff --git a/ci/run_ert_ctests.sh b/ci/run_ert_ctests.sh index a2f426576e0..6d113b1870b 100644 --- a/ci/run_ert_ctests.sh +++ b/ci/run_ert_ctests.sh @@ -60,8 +60,7 @@ build_ert_clib () { build_ert_dev () { enable_environment - pip install ${ERT_SOURCE_ROOT} - pip install -r ${ERT_SOURCE_ROOT}/dev-requirements.txt + pip install "${ERT_SOURCE_ROOT}[dev]" } run_ctest () { diff --git a/ci/testkomodo.sh b/ci/testkomodo.sh index 93ab47255d0..0d5d7f451fd 100755 --- a/ci/testkomodo.sh +++ b/ci/testkomodo.sh @@ -12,7 +12,7 @@ copy_test_files () { } install_test_dependencies () { - pip install -r dev-requirements.txt + pip install "ert[dev]" } run_ert_with_opm () { diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index bf3f9701934..00000000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,33 +0,0 @@ -click -decorator -ecl_data_io -furo -flaky -grpcio-tools -hypothesis<=6.83.0;python_version=='3.8' # ipython pinned to 8.12.2 for python 3.8 support -hypothesis;python_version>='3.9' -jsonpath_ng -jupytext -oil_reservoir_synthesizer -pytest-asyncio -pytest-benchmark -pytest-cov -pytest-memray -pytest-mock -pytest-mpl -pytest-qt -pytest-raises -pytest-snapshot -pytest-timeout -pytest-xdist -pytest>6 -requests -scikit-build -setuptools_scm -sphinx<7.2 -sphinx-argparse -sphinx-autoapi -sphinx-copybutton -sphinxcontrib-plantuml -sphinxcontrib.datatemplates -testpath diff --git a/pyproject.toml b/pyproject.toml index 8e469069052..15c446cd7cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,4 @@ [build-system] - requires = [ "setuptools<64", "setuptools_scm[toml]>=6.2", @@ -12,7 +11,148 @@ requires = [ "pybind11>=2.10.0", # If this comes out of sync with the version installed by Conan please update the version in CMakeLists "grpcio-tools", ] +build-backend = "setuptools.build_meta" + +[project] +name = "ert" +authors = [ + {name = "Equinor ASA", email = "fg_sib-scout@equinor.com"}, +] +description = "Ensemble based Reservoir Tool (ERT)" +requires-python = ">=3.8" +readme = "README.md" +license = {text = "GPL-3.0"} +classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Other Environment", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Programming Language :: Python", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Physics", +] +dynamic = ["version"] +dependencies=[ + "aiofiles", + "aiohttp", + "alembic", + "ansicolors==1.1.8", + "async-generator", + "beartype > 0.11", + "cloudevents>=1.6.0", + "cloudpickle", + "tqdm>=4.62.0", + "cryptography", + "cwrap", + "dask_jobqueue", + "deprecation", + "dnspython >= 2", + "ecl >= 2.14.1", + "ert-storage >= 0.3.16", + "fastapi < 0.100.0", + "filelock", + "graphlib_backport; python_version < '3.9'", + "iterative_ensemble_smoother>=0.1.1", + "typing_extensions", + "jinja2", + "lark", + "matplotlib", + "numpy<2", + "packaging", + "pandas", + "pluggy>=1.3.0", + "protobuf", + "psutil", + "pydantic >= 1.10.8, < 2", + "PyQt5", + "pyrsistent", + "python-dateutil", + "pyyaml", + "qtpy", + "requests", + "scipy >= 1.10.1", + "sqlalchemy", + "uvicorn >= 0.17.0", + "websockets", + "httpx", + "tables", + "xarray", + "xtgeo >= 3.3.0", + "netCDF4", + "sortedcontainers" +] + +[project.scripts] +ert = "ert.__main__:main" +"job_dispatch.py" = "_ert_job_runner.job_dispatch:main" + +[project.urls] +Repository = "https://github.com/equinor/ert" + +[project.optional-dependencies] +dev = [ + "click", + "decorator", + "ecl_data_io", + "furo", + "flaky", + "grpcio-tools", + "hypothesis<=6.83.0; python_version=='3.8'", # ipython pinned to 8.12.2 for python 3.8 support + "hypothesis; python_version>='3.9'", + "jsonpath_ng", + "jupytext", + "oil_reservoir_synthesizer", + "pytest-asyncio", + "pytest-benchmark", + "pytest-cov", + "pytest-memray", + "pytest-mock", + "pytest-mpl", + "pytest-qt", + "pytest-raises", + "pytest-snapshot", + "pytest-timeout", + "pytest-xdist", + "pytest>6", + "requests", + "scikit-build", + "setuptools_scm", + "sortedcontainers", + "sphinx<7.2", + "sphinx-argparse", + "sphinx-autoapi", + "sphinx-copybutton", + "sphinxcontrib-plantuml", + "sphinxcontrib.datatemplates", + "testpath", +] +style = [ + "cmake-format", + "black", + "ruff", + "pylint", + "pylint-protobuf" +] +types = [ + "mypy", + "mypy-protobuf<3.4", + "types-aiofiles", + "types-requests", + "types-PyYAML", + "types-python-dateutil", + "types-decorator", + "types-docutils", + "types-protobuf", + "types-tqdm", +] +[tool.setuptools] +platforms = ["all"] [tool.pytest.ini_options] addopts = "-ra --strict-markers" diff --git a/setup.py b/setup.py index 3c6a59f2d6f..dead81499f4 100644 --- a/setup.py +++ b/setup.py @@ -74,18 +74,7 @@ def package_files(directory): return paths -with open("README.md") as f: - long_description = f.read() - - args = dict( - name="ert", - author="Equinor ASA", - author_email="fg_sib-scout@equinor.com", - description="Ensemble based Reservoir Tool (ERT)", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/equinor/ert", packages=find_packages(where="src"), package_dir={"": "src"}, package_data={ @@ -96,65 +85,6 @@ def package_files(directory): "job_queue/qstat_proxy.sh", ], }, - include_package_data=True, - license="GPL-3.0", - platforms="any", - python_requires=">=3.8", - install_requires=[ - "aiofiles", - "aiohttp", - "alembic", - "ansicolors==1.1.8", - "async-generator", - "beartype > 0.11", - "cloudevents>=1.6.0", - "cloudpickle", - "tqdm>=4.62.0", - "cryptography", - "cwrap", - "dask_jobqueue", - "deprecation", - "dnspython >= 2", - "ecl >= 2.14.1", - "ert-storage >= 0.3.16", - "fastapi < 0.100.0", - "filelock", - "graphlib_backport; python_version < '3.9'", - "iterative_ensemble_smoother>=0.1.1", - "typing_extensions", - "jinja2", - "lark", - "matplotlib", - "numpy<2", - "packaging", - "pandas", - "pluggy>=1.3.0", - "protobuf", - "psutil", - "pydantic >= 1.10.8, < 2", - "PyQt5", - "pyrsistent", - "python-dateutil", - "pyyaml", - "qtpy", - "requests", - "scipy >= 1.10.1", - "sortedcontainers", - "sqlalchemy", - "uvicorn >= 0.17.0", - "websockets", - "httpx", - "tables", - "xarray", - "xtgeo >= 3.3.0", - "netCDF4", - ], - entry_points={ - "console_scripts": [ - "ert=ert.__main__:main", - "job_dispatch.py = _ert_job_runner.job_dispatch:main", - ] - }, cmake_args=[ "-DBUILD_TESTS=OFF", # we can safely pass OSX_DEPLOYMENT_TARGET as it's ignored on @@ -163,20 +93,6 @@ def package_files(directory): "-DCMAKE_OSX_DEPLOYMENT_TARGET=10.15", ], cmake_source_dir="src/clib/", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Other Environment", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Natural Language :: English", - "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Physics", - ], cmdclass={ "egg_info": EggInfo, "compile_protocol_buffers": CompileProtocolBuffers, diff --git a/src/ert/dark_storage/endpoints/responses.py b/src/ert/dark_storage/endpoints/responses.py index ca417e5f9bf..96b8040eb96 100644 --- a/src/ert/dark_storage/endpoints/responses.py +++ b/src/ert/dark_storage/endpoints/responses.py @@ -19,7 +19,7 @@ async def get_ensemble_response_dataframe( res: LibresFacade = DEFAULT_LIBRESFACADE, db: StorageReader = DEFAULT_STORAGE, ensemble_id: UUID, - response_name: str + response_name: str, ) -> Response: dataframe = data_for_key(res, db.get_ensemble(ensemble_id), response_name) return Response( diff --git a/style-requirements.txt b/style-requirements.txt deleted file mode 100644 index 5d44a4eb8c9..00000000000 --- a/style-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -cmake-format -black -ruff -pylint -pylint-protobuf diff --git a/types-requirements.txt b/types-requirements.txt deleted file mode 100644 index 93fe095011e..00000000000 --- a/types-requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -mypy -mypy-protobuf<3.4 -types-aiofiles -types-requests -types-PyYAML -types-python-dateutil -types-decorator -types-docutils -types-protobuf -types-tqdm