diff --git a/README.md b/README.md index 477c2980..cea1a963 100644 --- a/README.md +++ b/README.md @@ -28,24 +28,32 @@ version >= 3.6. ## Building/Packaging -To manually build a package for distribution, run the following steps. You will -need a C++ compiler which supports `std::span` (e.g. `clang++` >= 7 or -`g++` >= 10). The steps below will generate both a `.tar.gz` and `.whl` package -under `./dist/`. +To manually build a package for distribution, follow the steps below. -```bash -# 1. Create and enter a virtual environment -python -m venv venv && . ./venv/bin/activate +### Requirements -# 2. Install development dependencies -pip install -r requirements-dev.txt +* A C++ compiler that supports C++20 and `std::span` (e.g. `clang++` >= 7 or + `g++` >= 10). +* python3 +* python3-dev +* python3-venv +* [Task][18] >= 3.35.1 -# 3. Pull all submodules in preparation for building -git submodule update --init --recursive +### Build commands -# 4. Build -python -m build --outdir build/dist -``` +* Build a Python wheel incrementally: + + ```bash + task + ``` + The command above will generate both a `.tar.gz` and `.whl` package under + `./build/dist/`. + +* Clean up the build: + + ```bash + task clean + ``` ## CLP IR Readers @@ -302,3 +310,4 @@ other package management tools such as `apt-get`: [15]: https://docs.python.org/3/library/pickle.html [16]: https://pypi.org/project/clp-ffi-py/ [17]: https://github.com/RaRe-Technologies/smart_open +[18]: https://taskfile.dev/installation/ diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 00000000..b129cde3 --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,115 @@ +version: "3" + +includes: + docs: + taskfile: "docs/tasks.yml" + +vars: + # Paths + BUILD_DIR: "{{.ROOT_DIR}}/build" + WHEEL_BUILD_DIR: "{{.BUILD_DIR}}/dist" + WHEEL_VENV_DIR: "{{.BUILD_DIR}}/wheel-venv" + +tasks: + default: + deps: ["wheel"] + + clean: + cmds: + - "rm -rf '{{.BUILD_DIR}}'" + + wheel: + vars: + CHECKSUM_FILE: "{{.BUILD_DIR}}/{{.TASK}}.md5" + OUTPUT_DIR: "{{.WHEEL_BUILD_DIR}}" + deps: + - "submodules" + - "wheel-venv" + cmds: + - "rm -rf '{{.OUTPUT_DIR}}'" + - |- + . "{{.WHEEL_VENV_DIR}}/bin/activate" + python3 -m build --outdir "{{.OUTPUT_DIR}}" + # Checksum the generated files (this command must be last) + - |- + cd "{{.OUTPUT_DIR}}" + tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum > "{{.CHECKSUM_FILE}}" + sources: + - "{{.BUILD_DIR}}/submodules.md5" + - "{{.BUILD_DIR}}/wheel-venv.md5" + - "{{.TASKFILE}}" + - "clp_ffi_py/**/*" + - "src/**/*" + status: + - "test -f '{{.CHECKSUM_FILE}}'" + - >- + diff + <(cd '{{.OUTPUT_DIR}}' && tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum) + "{{.CHECKSUM_FILE}}" + + build-venv: + internal: true + label: "{{.COMPONENT}}-{{.TASK}}" + requires: + vars: + - "COMPONENT" + - "OUTPUT_DIR" + - "REQUIREMENTS_FILE_PATH" + cmds: + - "rm -rf '{{.OUTPUT_DIR}}'" + - "python3 -m venv '{{.OUTPUT_DIR}}'" + - |- + . "{{.OUTPUT_DIR}}/bin/activate" + pip3 install -r "{{.REQUIREMENTS_FILE_PATH}}" + + submodules: + internal: true + vars: + CHECKSUM_FILE: "{{.BUILD_DIR}}/{{.TASK}}.md5" + OUTPUT_DIR: "src/clp" + preconditions: + # Ensure there are no uncommitted changes to the submodules since the submodule update + # command might undo them. Specifically, the update command will undo any uncommitted changes + # to the commits that the submodules point to. + - sh: "git diff --quiet src/clp" + msg: "There are uncommitted changes to the submodules. Commit or stash them first." + cmds: + - "git submodule update --init --recursive" + # Checksum the generated files (this command must be last) + - |- + cd "{{.OUTPUT_DIR}}" + tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum > "{{.CHECKSUM_FILE}}" + sources: + - "{{.TASKFILE}}" + - ".gitmodules" + status: + - "test -f '{{.CHECKSUM_FILE}}'" + - >- + diff + <(cd '{{.OUTPUT_DIR}}' && tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum) + "{{.CHECKSUM_FILE}}" + + wheel-venv: + internal: true + vars: + CHECKSUM_FILE: "{{.BUILD_DIR}}/{{.TASK}}.md5" + OUTPUT_DIR: "{{.WHEEL_VENV_DIR}}" + cmds: + - task: "build-venv" + vars: + COMPONENT: "wheel" + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + REQUIREMENTS_FILE_PATH: "requirements-dev.txt" + # Checksum the generated files (this command must be last) + - |- + cd "{{.OUTPUT_DIR}}" + tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum > "{{.CHECKSUM_FILE}}" + sources: + - "{{.TASKFILE}}" + - "requirements-dev.txt" + status: + - "test -f '{{.CHECKSUM_FILE}}'" + - >- + diff + <(cd '{{.OUTPUT_DIR}}' && tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum) + "{{.CHECKSUM_FILE}}" diff --git a/docs/README.md b/docs/README.md index 8ca99d4a..b5a1b670 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,49 +8,44 @@ documentation website for this package: ## Requirements -We use the following tools to build the site: - -* python3 -* python3-venv -* [Task][1] +* All the [requirements](../README.md#requirements) for building the Python + package. +* [Node.js] >= 16 to be able to [view the output](#viewing-the-output) ## Build Commands * Build the site incrementally: ```shell - task + task docs:site ``` * The output of the build will be in `../build/docs/html`. * API reference docs will be written to `src/api`. -* Clean-up the build: +* Clean up the build: ```shell - task clean + task docs:clean ``` ## Viewing the Output -You can use [Node.js' npm][2] with [http-server][3] to view the output: - ```shell -npx http-server ../build/docs/html -c-1 +task docs:serve ``` -We use `-c-1` to disable caching during development. +The command above will install [http-server] and serve the built docs site. -# Integration with [yscope-docs][4] +# Integration with [yscope-docs] -To support hosting on [docs.yscope.com][5], we need: +To support hosting on [docs.yscope.com], we need: * `build.sh` which builds a clean version of the site and outputs it to `../build/docs/html`. * `build.sh` must be runnable from any directory. -[1]: https://taskfile.dev/ -[2]: https://nodejs.org/en/download/current -[3]: https://www.npmjs.com/package/http-server -[4]: https://github.com/y-scope/yscope-docs -[5]: https://docs.yscope.com +[docs.yscope.com]: https://docs.yscope.com +[http-server]: https://www.npmjs.com/package/http-server +[Node.js]: https://nodejs.org/en/download/current +[yscope-docs]: https://github.com/y-scope/yscope-docs diff --git a/docs/Taskfile.yml b/docs/Taskfile.yml deleted file mode 100644 index e758b87f..00000000 --- a/docs/Taskfile.yml +++ /dev/null @@ -1,68 +0,0 @@ -version: "3" - -vars: - PROJECT_ROOT: "../" - PROJECT_VENV: "{{.PROJECT_ROOT}}/venv" - BUILD_DIR: "{{.PROJECT_ROOT}}/build" - DOCS_BUILD_DIR: "{{.BUILD_DIR}}/docs" - DOCS_VENV: "{{.BUILD_DIR}}/docs/venv" - PYTHON_VERSION: - sh: python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" - -tasks: - default: - deps: [ docs ] - - clean: - cmds: - - rm -rf {{.DOCS_BUILD_DIR}} - - docs: - deps: [ clp_ffi_py_install ] - cmds: - - |- - source {{.DOCS_VENV}}/bin/activate - sphinx-apidoc --force --separate --no-toc -o src/api {{.DOCS_VENV}}/lib/python{{.PYTHON_VERSION}}/site-packages/clp_ffi_py/ - sphinx-build -a -c conf -b html src {{.DOCS_BUILD_DIR}}/html - sources: - - conf/**/* - - src/**/* - - "{{.DOCS_VENV}}/lib/python3*/site-packages/clp_ffi_py-*/**/*" - generates: - - "{{.DOCS_BUILD_DIR}}/html/**/*" - - clp_ffi_py_install: - deps: [ docs_venv, project_venv ] - cmds: - - |- - source {{.PROJECT_VENV}}/bin/activate - pip3 install --upgrade --target={{.DOCS_VENV}}/lib/python{{.PYTHON_VERSION}}/site-packages/ {{.PROJECT_ROOT}} - sources: - - "{{.PROJECT_ROOT}}/clp_ffi_py/**/*" - - "{{.PROJECT_ROOT}}/src/**/*" - generates: - - "{{.DOCS_VENV}}/lib/python3*/site-packages/clp_ffi_py-*/**/*" - - docs_venv: - cmds: - - rm -rf {{.DOCS_VENV}} - - python3 -m venv {{.DOCS_VENV}} - - |- - source {{.DOCS_VENV}}/bin/activate - pip3 install --upgrade -r requirements.txt - sources: - - requirements.txt - generates: - - "{{.DOCS_VENV}}/**/*" - - project_venv: - cmds: - - rm -rf {{.PROJECT_VENV}} - - python3 -m venv {{.PROJECT_VENV}} - - |- - source {{.PROJECT_VENV}}/bin/activate - pip3 install --upgrade -r {{.PROJECT_ROOT}}/requirements-dev.txt - sources: - - "{{.PROJECT_ROOT}}/requirements-dev.txt" - generates: - - "{{.PROJECT_VENV}}/**/*" diff --git a/docs/build.sh b/docs/build.sh index 1929cbd0..b7bd518e 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -4,8 +4,8 @@ set -eu script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -pushd "$script_dir" +pushd "$script_dir/.." -task clean && task +task clean && task docs:site popd diff --git a/docs/tasks.yml b/docs/tasks.yml new file mode 100644 index 00000000..229c8e80 --- /dev/null +++ b/docs/tasks.yml @@ -0,0 +1,112 @@ +version: "3" + +vars: + # Paths + DOCS_BUILD_DIR: "{{.BUILD_DIR}}/docs/html" + DOCS_VENV_DIR: "{{.BUILD_DIR}}/docs-venv" + NODE_DEPS_DIR: "{{.BUILD_DIR}}/node" + + # Other variables + PYTHON_VERSION: + sh: python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" + +tasks: + clean: + cmds: + - "rm -rf '{{.DOCS_BUILD_DIR}}'" + + serve: + deps: + - "http-server" + - "site" + cmds: + - "npm --prefix '{{.NODE_DEPS_DIR}}' exec http-server '{{.DOCS_BUILD_DIR}}' -c-1" + + site: + dir: "{{.TASKFILE_DIR}}" + deps: ["docs-venv"] + vars: + CHECKSUM_FILE: "{{.BUILD_DIR}}/{{.TASK}}.md5" + OUTPUT_DIR: "{{.DOCS_BUILD_DIR}}" + cmds: + - |- + . "{{.DOCS_VENV_DIR}}/bin/activate" + + sphinx-apidoc \ + --force \ + --separate \ + --no-toc \ + -o src/api \ + "{{.DOCS_VENV_DIR}}/lib/python{{.PYTHON_VERSION}}/site-packages/clp_ffi_py/" + + sphinx-build -a -c conf -b html src "{{.OUTPUT_DIR}}" + # Checksum the generated files (this command must be last) + - |- + cd "{{.OUTPUT_DIR}}" + tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum > "{{.CHECKSUM_FILE}}" + sources: + - "{{.BUILD_DIR}}/docs-venv.md5" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + - "conf/**/*" + - "src/**/*" + status: + - "test -f '{{.CHECKSUM_FILE}}'" + - >- + diff + <(cd '{{.OUTPUT_DIR}}' && tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum) + "{{.CHECKSUM_FILE}}" + + docs-venv: + internal: true + deps: [":wheel"] + vars: + CHECKSUM_FILE: "{{.BUILD_DIR}}/{{.TASK}}.md5" + REQUIREMENTS_FILE_PATH: "docs/requirements.txt" + OUTPUT_DIR: "{{.DOCS_VENV_DIR}}" + cmds: + - task: ":build-venv" + vars: + COMPONENT: "docs" + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + REQUIREMENTS_FILE_PATH: "{{.REQUIREMENTS_FILE_PATH}}" + - |- + . "{{.OUTPUT_DIR}}/bin/activate" + pip3 install --upgrade "{{.WHEEL_BUILD_DIR}}"/*.whl + # Checksum the generated files (this command must be last) + - |- + cd "{{.OUTPUT_DIR}}" + tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum > "{{.CHECKSUM_FILE}}" + sources: + - "{{.BUILD_DIR}}/wheel.md5" + - "{{.REQUIREMENTS_FILE_PATH}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + status: + - "test -f '{{.CHECKSUM_FILE}}'" + - >- + diff + <(cd '{{.OUTPUT_DIR}}' && tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum) + "{{.CHECKSUM_FILE}}" + + http-server: + internal: true + vars: + CHECKSUM_FILE: "{{.BUILD_DIR}}/{{.TASK}}.md5" + OUTPUT_DIR: "{{.NODE_DEPS_DIR}}" + cmds: + - "rm -rf '{{.OUTPUT_DIR}}'" + - "npm --prefix '{{.OUTPUT_DIR}}' install http-server" + # Checksum the generated files (this command must be last) + - |- + cd "{{.OUTPUT_DIR}}" + tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum > "{{.CHECKSUM_FILE}}" + sources: + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + status: + - "test -f '{{.CHECKSUM_FILE}}'" + - >- + diff + <(cd '{{.NODE_DEPS_DIR}}' && tar cf - {{.CHECKSUM_TAR_BASE_ARGS}} . | md5sum) + "{{.CHECKSUM_FILE}}"