Package build improvements: (#181)

- Replace packager with a Taskfile to reduce complexity and support incremental builds.
- Use poetry to build Python components.

.gitignore

@@ -0,0 +1,4 @@
+.task/
+build/
+**/dist/
+**/poetry.lock

README.md

@@ -10,8 +10,8 @@ searching the compressed logs without decompression. To learn more about it, you
 
 # Getting Started
 
-You can download a release from the [releases](https://github.com/y-scope/clp/releases) page or you can build the latest by using the
-[packager](tools/packager/README.md).
+You can download a release from the [releases](https://github.com/y-scope/clp/releases) page, or
+you can build the [latest](docs/BUILDING.md).
 
 For some logs you can use to test CLP, check out our open-source 
 [datasets](docs/Datasets.md).

Taskfile.yml

@@ -0,0 +1,213 @@
+version: "3"
+
+vars:
+  BUILD_DIR: "{{.TASKFILE_DIR}}/build"
+  CORE_COMPONENT_BUILD_DIR: "{{.TASKFILE_DIR}}/build/core"
+  PACKAGE_BUILD_DIR: "{{.TASKFILE_DIR}}/build/clp-package"
+  PACKAGE_VERSION: "0.0.3-dev"
+  PYTHON_VERSION:
+    sh: "python3 -c \"import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')\""
+
+tasks:
+  default:
+    deps: ["package"]
+
+  clean:
+    cmds:
+      - "rm -rf '{{.BUILD_DIR}}'"
+      - task: "clean-python-component"
+        vars:
+          COMPONENT: "clp-package-utils"
+      - task: "clean-python-component"
+        vars:
+          COMPONENT: "clp-py-utils"
+      - task: "clean-python-component"
+        vars:
+          COMPONENT: "compression-job-handler"
+      - task: "clean-python-component"
+        vars:
+          COMPONENT: "job-orchestration"
+
+  clean-package:
+    cmds:
+      - "rm -rf '{{.PACKAGE_BUILD_DIR}}'"
+
+  package-tar:
+    deps:
+      - "package"
+    vars:
+      VERSIONED_PACKAGE_NAME:
+        sh: |
+          source /etc/os-release
+          echo "clp-package-${ID}-${VERSION_CODENAME}-$(arch)-v{{.PACKAGE_VERSION}}"
+    dir: "{{.BUILD_DIR}}"
+    method: "timestamp"
+    cmds:
+      - "rm -rf '{{.VERSIONED_PACKAGE_NAME}}' '{{.VERSIONED_PACKAGE_NAME}}.tar.gz'"
+      - "ln -s '{{.PACKAGE_BUILD_DIR}}' '{{.VERSIONED_PACKAGE_NAME}}'"
+      - "tar czf '{{.VERSIONED_PACKAGE_NAME}}.tar.gz' --dereference '{{.VERSIONED_PACKAGE_NAME}}'"
+    sources:
+      - "{{.PACKAGE_BUILD_DIR}}/**"
+    status:
+      - "test -e {{.VERSIONED_PACKAGE_NAME}}.tar.gz"
+      - "test {{.TIMESTAMP | unixEpoch}} -lt $(stat --format %Y {{.VERSIONED_PACKAGE_NAME}}.tar.gz)"
+
+  package:
+    deps:
+      - "core"
+      - "clp-package-utils"
+      - "clp-py-utils"
+      - "compression-job-handler"
+      - "job-orchestration"
+    cmds:
+      - task: "clean-package"
+      - "mkdir -p '{{.PACKAGE_BUILD_DIR}}'"
+      - "rsync -a components/package-template/src/ '{{.PACKAGE_BUILD_DIR}}'"
+      - "mkdir -p '{{.PACKAGE_BUILD_DIR}}/lib/python3/site-packages'"
+      - >-
+        pip3 install --upgrade
+        components/clp-package-utils/dist/*.whl
+        components/clp-py-utils/dist/*.whl
+        components/compression-job-handler/dist/*.whl
+        components/job-orchestration/dist/*.whl
+        -t "{{.PACKAGE_BUILD_DIR}}/lib/python3/site-packages"
+      - "mkdir -p '{{.PACKAGE_BUILD_DIR}}/bin'"
+      - >-
+        rsync -a
+        "{{.CORE_COMPONENT_BUILD_DIR}}/clg"
+        "{{.CORE_COMPONENT_BUILD_DIR}}/clo"
+        "{{.CORE_COMPONENT_BUILD_DIR}}/clp"
+        "{{.PACKAGE_BUILD_DIR}}/bin/"
+    method: "timestamp"
+    sources:
+      - "{{.CORE_COMPONENT_BUILD_DIR}}/clg"
+      - "{{.CORE_COMPONENT_BUILD_DIR}}/clo"
+      - "{{.CORE_COMPONENT_BUILD_DIR}}/clp"
+      - "{{.TASKFILE_DIR}}/Taskfile.yml"
+      - "components/clp-package-utils/dist/*.whl"
+      - "components/clp-py-utils/dist/*.whl"
+      - "components/compression-job-handler/dist/*.whl"
+      - "components/job-orchestration/dist/*.whl"
+      - "components/package-template/src/**"
+    status:
+      - "test -d {{.PACKAGE_BUILD_DIR}}"
+      - "test {{.TIMESTAMP | unixEpoch}} -lt $(stat --format %Y '{{.PACKAGE_BUILD_DIR}}')"
+
+  core:
+    deps: ["core-submodules"]
+    vars:
+      SRC_DIR: "components/core"
+    cmds:
+      - "mkdir -p '{{.CORE_COMPONENT_BUILD_DIR}}'"
+      - "cmake -S '{{.SRC_DIR}}' -B '{{.CORE_COMPONENT_BUILD_DIR}}'"
+      - "cmake --build '{{.CORE_COMPONENT_BUILD_DIR}}' --parallel --target clg clo clp"
+    method: "timestamp"
+    sources:
+      - "{{.SRC_DIR}}/cmake/**"
+      - "{{.SRC_DIR}}/CMakeLists.txt"
+      - "{{.SRC_DIR}}/src/**"
+      - "{{.SRC_DIR}}/submodules/**"
+      - "{{.TASKFILE_DIR}}/Taskfile.yml"
+    status:
+      - >-
+        test -e '{{.CORE_COMPONENT_BUILD_DIR}}/clg'
+        && test -e '{{.CORE_COMPONENT_BUILD_DIR}}/clo'
+        && test -e '{{.CORE_COMPONENT_BUILD_DIR}}/clp'
+      - >-
+        test {{.TIMESTAMP | unixEpoch}} -lt $(stat --format %Y '{{.CORE_COMPONENT_BUILD_DIR}}/clg')
+        && test {{.TIMESTAMP | unixEpoch}} -lt
+        $(stat --format %Y '{{.CORE_COMPONENT_BUILD_DIR}}/clo')
+        && test {{.TIMESTAMP | unixEpoch}} -lt
+        $(stat --format %Y '{{.CORE_COMPONENT_BUILD_DIR}}/clp')
+
+  clp-package-utils:
+    - task: "python-component"
+      vars:
+        COMPONENT: "{{.TASK}}"
+
+  clp-py-utils:
+    - task: "python-component"
+      vars:
+        COMPONENT: "{{.TASK}}"
+
+  compression-job-handler:
+    - task: "python-component"
+      vars:
+        COMPONENT: "{{.TASK}}"
+
+  job-orchestration:
+    - task: "python-component"
+      vars:
+        COMPONENT: "{{.TASK}}"
+
+  core-submodules:
+    internal: true
+    vars:
+      SRC_DIR: "components/core"
+    cmds:
+      - "{{.SRC_DIR}}/tools/scripts/deps-download/download-all.sh"
+
+  python-component:
+    internal: true
+    requires:
+      vars: ["COMPONENT"]
+    label: "{{.COMPONENT}}"
+    deps:
+      - task: "component-venv"
+        vars:
+          COMPONENT: "{{.COMPONENT}}"
+    vars:
+      PACKAGE:
+        sh: "echo {{.COMPONENT}} | tr - _"
+      VENV_DIR: "{{.BUILD_DIR}}/{{.COMPONENT}}/venv"
+    dir: "components/{{.COMPONENT}}"
+    cmds:
+      - task: "clean-python-component"
+        vars:
+          COMPONENT: "{{.COMPONENT}}"
+      - |-
+        . "{{.VENV_DIR}}/bin/activate"
+        poetry build --format wheel
+    method: "timestamp"
+    sources:
+      - "{{.PACKAGE}}/**"
+      - "{{.TASKFILE_DIR}}/requirements.txt"
+      - "{{.TASKFILE_DIR}}/Taskfile.yml"
+      - "pyproject.toml"
+    status:
+      - "ls dist/*.whl"
+      - "test {{.TIMESTAMP | unixEpoch}} -lt $(stat --format %Y dist/*.whl)"
+
+  component-venv:
+    internal: true
+    requires:
+      vars: ["COMPONENT"]
+    label: "{{.COMPONENT}}_venv"
+    dir: "components/{{.COMPONENT}}"
+    vars:
+      VENV_DIR: "{{.BUILD_DIR}}/{{.COMPONENT}}/venv"
+    cmds:
+      - "rm -rf '{{.VENV_DIR}}'"
+      - "python3 -m venv '{{.VENV_DIR}}'"
+      - |-
+        . "{{.VENV_DIR}}/bin/activate"
+        pip3 install --upgrade -r "{{.TASKFILE_DIR}}/requirements.txt"
+        poetry update
+    method: "timestamp"
+    sources:
+      - "{{.TASKFILE_DIR}}/requirements.txt"
+      - "{{.TASKFILE_DIR}}/Taskfile.yml"
+      - "pyproject.toml"
+    status:
+      - "test -d '{{.VENV_DIR}}'"
+      - "test {{.TIMESTAMP | unixEpoch}} -lt $(stat --format %Y '{{.VENV_DIR}}')"
+
+  clean-python-component:
+    internal: true
+    requires:
+      vars: ["COMPONENT"]
+    label: "clean-{{.COMPONENT}}"
+    dir: "components/{{.COMPONENT}}"
+    cmds:
+      - "rm -rf dist"
+      - "rm -rf poetry.lock"

components/clp-package-utils/README.md

@@ -0,0 +1,3 @@
+# CLP Package Utilities
+
+This python module contains utilities used by the CLP package.

components/clp-package-utils/pyproject.toml

@@ -0,0 +1,16 @@
+[tool.poetry]
+name = "clp-package-utils"
+version = "0.0.3-dev"
+description = "Utilities for the CLP package."
+authors = ["YScope Inc. <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.8 || ^3.10"
+msgpack = "^1.0.7"
+PyYAML = "^6.0.1"
+zstandard = "~0.22"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

components/clp-py-utils/README.md

@@ -1,10 +1,3 @@
 # CLP Python Utilities
 
 This python module contains utilities imported by other Python modules in the CLP package.
-
-## Installation
-
-```bash
-pip3 install -r requirements.txt --target <clp-package>/lib/python3/site-packages
-cp -R clp_py_utils <clp-package>/lib/python3/site-packages
-```

components/clp-py-utils/pyproject.toml

@@ -0,0 +1,20 @@
+[tool.poetry]
+name = "clp-py-utils"
+version = "0.0.3-dev"
+description = "Utilities for other Python packages in CLP."
+authors = ["YScope Inc. <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.8 || ^3.10"
+# mariadb version must be compatible with libmariadev installed in runtime env.
+# See https://mariadb.com/docs/server/connect/programming-languages/python/install/#Dependencies
+mariadb = "~1.0.11"
+mysql-connector-python = "^8.2.0"
+pydantic = "^1.10.13"
+python-Levenshtein = "~0.22"
+PyYAML = "^6.0.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

components/compression-job-handler/README.md

@@ -1,32 +1,3 @@
 # CLP Compression Job Handler
 
 This Python module submits compression jobs to the CLP compression scheduler.
-
-## Installation
-
-```bash
-pip3 install -r requirements.txt --target <clp-package>/lib/python3/site-packages
-cp -R compression_job_handler <clp-package>/lib/python3/site-packages
-```
-
-## Usage
-
-Below are a few ways to use this module.
-
-### Docker compression wrapper
-
-```bash
-<clp-package>/sbin/compress <parameters>
-```
-
-### Native compression wrapper
-
-```bash
-<clp-package>/sbin/native/compress <parameters>
-```
-
-### Standalone
-
-```bash
-PYTHONPATH=<clp_home/lib/python3/site-packages> python3 -m compression_job_handler <parameters>
-```

components/compression-job-handler/pyproject.toml

@@ -0,0 +1,16 @@
+[tool.poetry]
+name = "compression-job-handler"
+version = "0.0.3-dev"
+description = "Component for submitting compression jobs to CLP."
+authors = ["YScope Inc. <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.8 || ^3.10"
+msgpack = "^1.0.7"
+pydantic = "^1.10.13"
+zstandard = "~0.22"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

components/core/src/clp/version.hpp

@@ -2,7 +2,7 @@
 #define CLP_VERSION_HPP
 
 namespace clp {
-constexpr char cVersion[] = "0.2";
+constexpr char cVersion[] = "0.0.3-dev";
 }  // namespace clp
 
 #endif  // CLP_VERSION_HPP

components/job-orchestration/README.md

@@ -2,32 +2,3 @@
 
 This Python module contains CLP's scheduler and worker to handle distributed compression. 
 CLP's Compression Job Handler can be used to interface and submit compression jobs to the CLP scheduler.
-
-## Installation
-
-```bash
-pip3 install -r requirements.txt --target <clp-package>/lib/python3/site-packages
-cp -R job_orchestration <clp-package>/lib/python3/site-packages
-```
-
-## Usage
-
-### Running the `scheduler`
-
-```bash
-PYTHONPATH=<clp_home/lib/python3/site-packages> \
-  BROKER_URL=amqp://<rabbitmq_user>:<rabbitmq_password>@<rabbitmq_host>:<rabbitmq_port> \
-  python3 -m job_orchestration.scheduler.scheduler --config <clp config file path>
-```
-
-### Running the `executor`
-
-```bash
-PYTHONPATH=<clp_home/lib/python3/site-packages> \
-  CLP_HOME=<clp_home> \
-  CLP_DATA_DIR=<clp data directory> \
-  CLP_LOGS_DIR=<clp log directory> \
-  BROKER_URL=amqp://<rabbitmq_user>:<rabbitmq_password>@<rabbitmq_host>:<rabbitmq_port> \
-  RESULT_BACKEND=rpc://<rabbitmq_user>:<rabbitmq_password>@<rabbitmq_host>:<rabbitmq_port> \
-  celery -A executor worker --loglevel INFO -Q compression
-```