From 9537fde393781e6ac889d0e37a88de5debb66172 Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:47:16 -0400 Subject: [PATCH] Refactor dependency downloads: (#508) - Checksum dependencies using taskfiles rather than a separate script. - Move core's dependency download script into taskfiles. - Add support for downloading yscope-log-viewer even when no .git directory exists. --- .github/actions/clp-core-build/action.yaml | 14 +- .github/workflows/clp-core-build-macos.yaml | 13 +- .github/workflows/clp-core-build.yaml | 10 +- Taskfile.yml | 70 +-- .../clp-env-base-centos7.4/Dockerfile | 6 + .../clp-env-base-ubuntu-focal/Dockerfile | 6 + .../clp-env-base-ubuntu-jammy/Dockerfile | 6 + .../tools/scripts/deps-download/Catch2.json | 10 - .../scripts/deps-download/abseil-cpp.json | 10 - .../tools/scripts/deps-download/antlr4.json | 14 - .../tools/scripts/deps-download/date.json | 10 - .../scripts/deps-download/download-all.sh | 29 -- .../scripts/deps-download/download-dep.py | 95 ---- .../tools/scripts/deps-download/json.json | 10 - .../scripts/deps-download/log-surgeon.json | 10 - .../tools/scripts/deps-download/outcome.json | 10 - .../tools/scripts/deps-download/simdjson.json | 11 - .../tools/scripts/deps-download/sqlite3.json | 14 - .../tools/scripts/deps-download/yaml-cpp.json | 10 - .../lib_install/macos-12/install-all.sh | 2 + deps-tasks.yml | 445 ++++++++++++++++++ docs/src/dev-guide/building-package.md | 4 +- docs/src/dev-guide/components-core/index.md | 16 +- lint-tasks.yml | 2 + tools/scripts/deps-download/download-dep.py | 88 ++++ tools/scripts/deps-download/init.sh | 17 + tools/scripts/find-broken-docs-links.py | 58 +-- 27 files changed, 652 insertions(+), 338 deletions(-) delete mode 100644 components/core/tools/scripts/deps-download/Catch2.json delete mode 100644 components/core/tools/scripts/deps-download/abseil-cpp.json delete mode 100644 components/core/tools/scripts/deps-download/antlr4.json delete mode 100644 components/core/tools/scripts/deps-download/date.json delete mode 100755 components/core/tools/scripts/deps-download/download-all.sh delete mode 100644 components/core/tools/scripts/deps-download/download-dep.py delete mode 100644 components/core/tools/scripts/deps-download/json.json delete mode 100644 components/core/tools/scripts/deps-download/log-surgeon.json delete mode 100644 components/core/tools/scripts/deps-download/outcome.json delete mode 100644 components/core/tools/scripts/deps-download/simdjson.json delete mode 100644 components/core/tools/scripts/deps-download/sqlite3.json delete mode 100644 components/core/tools/scripts/deps-download/yaml-cpp.json create mode 100644 deps-tasks.yml create mode 100644 tools/scripts/deps-download/download-dep.py create mode 100755 tools/scripts/deps-download/init.sh diff --git a/.github/actions/clp-core-build/action.yaml b/.github/actions/clp-core-build/action.yaml index aed244c25..ab5070b09 100644 --- a/.github/actions/clp-core-build/action.yaml +++ b/.github/actions/clp-core-build/action.yaml @@ -18,10 +18,6 @@ inputs: runs: using: "composite" steps: - - shell: "bash" - working-directory: "./components/core" - run: "./tools/scripts/deps-download/download-all.sh" - - if: "inputs.use_published_image == 'false'" uses: "actions/download-artifact@v4" with: @@ -46,13 +42,19 @@ runs: fi shell: "bash" + - run: "./tools/scripts/deps-download/init.sh" + shell: "bash" + - run: >- docker run --user $(id -u):$(id -g) - --volume "$GITHUB_WORKSPACE/components/core":/mnt/clp + --volume "$GITHUB_WORKSPACE":/mnt/clp --workdir /mnt/clp ${{steps.get_image_props.outputs.qualified_image_name}} - /mnt/clp/tools/scripts/utils/build-and-run-unit-tests.sh . build + bash -c "task deps:core && + /mnt/clp/components/core/tools/scripts/utils/build-and-run-unit-tests.sh + /mnt/clp/components/core + /mnt/clp/components/core/build" shell: "bash" - if: "inputs.upload_binaries == 'true'" diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index c46dfc878..dc9e8209a 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -11,6 +11,9 @@ on: - "components/core/tools/scripts/lib_install/macos-12/**" - "components/core/tools/scripts/deps-download/**" - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh" + - "deps-tasks.yml" + - "Taskfile.yml" + - "tools/scripts/deps-download/**" push: paths: - ".github/workflows/clp-core-build-macos.yaml" @@ -21,6 +24,9 @@ on: - "components/core/tools/scripts/lib_install/macos-12/**" - "components/core/tools/scripts/deps-download/**" - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh" + - "deps-tasks.yml" + - "Taskfile.yml" + - "tools/scripts/deps-download/**" workflow_dispatch: concurrency: @@ -47,10 +53,11 @@ jobs: - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos-12/install-all.sh" - - name: "Download source dependencies" + - run: "./tools/scripts/deps-download/init.sh" + shell: "bash" + + - run: "task deps:core" shell: "bash" - working-directory: "./components/core" - run: "./tools/scripts/deps-download/download-all.sh" - name: "Build CLP-core and run unit tests" shell: "bash" diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index d0dbbb28a..47853375c 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -8,6 +8,9 @@ on: - ".github/workflows/clp-core-build.yaml" - ".gitmodules" - "components/core/**" + - "deps-tasks.yml" + - "Taskfile.yml" + - "tools/scripts/deps-download/**" - "!components/core/tools/scripts/lib_install/macos-12/**" push: paths: @@ -16,6 +19,9 @@ on: - ".github/workflows/clp-core-build.yaml" - ".gitmodules" - "components/core/**" + - "deps-tasks.yml" + - "Taskfile.yml" + - "tools/scripts/deps-download/**" - "!components/core/tools/scripts/lib_install/macos-12/**" workflow_dispatch: @@ -83,8 +89,10 @@ jobs: - "components/core/CMakeLists.txt" - "components/core/src/**" - "components/core/tests/**" - - "components/core/tools/scripts/deps-download/**" - "components/core/tools/scripts/utils/build-and-run-unit-tests.sh" + - "deps-tasks.yml" + - "Taskfile.yml" + - "tools/scripts/deps-download/**" centos74-deps-image: if: "needs.filter-relevant-changes.outputs.centos74_image_changed == 'true'" diff --git a/Taskfile.yml b/Taskfile.yml index 5b818c6e1..5912bd579 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,16 +1,22 @@ version: "3" includes: + deps: "deps-tasks.yml" docs: "docs/tasks.yml" lint: "lint-tasks.yml" utils: "tools/yscope-dev-utils/taskfiles/utils.yml" vars: - # Paths + # Source paths + G_COMPONENTS_DIR: "{{.ROOT_DIR}}/components" + G_CORE_COMPONENT_DIR: "{{.G_COMPONENTS_DIR}}/core" + G_CORE_COMPONENT_SUBMODULES_DIR: "{{.G_CORE_COMPONENT_DIR}}/submodules" + G_LOG_VIEWER_WEBUI_SRC_DIR: "{{.G_COMPONENTS_DIR}}/log-viewer-webui" + + # Build paths G_BUILD_DIR: "{{.ROOT_DIR}}/build" G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core" G_LOG_VIEWER_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/log-viewer-webui" - G_LOG_VIEWER_WEBUI_SRC_DIR: "{{.ROOT_DIR}}/components/log-viewer-webui" G_METEOR_BUILD_DIR: "{{.G_BUILD_DIR}}/meteor" G_NODEJS_14_BUILD_DIR: "{{.G_BUILD_DIR}}/nodejs-14" G_NODEJS_14_BIN_DIR: "{{.G_NODEJS_14_BUILD_DIR}}/bin" @@ -20,6 +26,9 @@ vars: G_PACKAGE_VENV_DIR: "{{.G_BUILD_DIR}}/package-venv" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" + # Taskfile paths + G_UTILS_TASKFILE: "{{.ROOT_DIR}}/tools/yscope-dev-utils/taskfiles/utils.yml" + # Versions G_PACKAGE_VERSION: "0.2.0-dev" @@ -171,7 +180,7 @@ tasks: vars: SRC_DIR: "components/core" sources: - - "{{.G_BUILD_DIR}}/core-submodules.md5" + - "{{.G_DEPS_CORE_CHECKSUM_FILE}}" - "{{.SRC_DIR}}/cmake/**/*" - "{{.SRC_DIR}}/CMakeLists.txt" - "{{.SRC_DIR}}/src/**/*" @@ -183,7 +192,7 @@ tasks: - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/clp-s" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server" - deps: ["core-submodules", "init"] + deps: ["deps:core", "init"] cmds: - "mkdir -p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - "cmake -S '{{.SRC_DIR}}' -B '{{.G_CORE_COMPONENT_BUILD_DIR}}'" @@ -317,31 +326,6 @@ tasks: NODEJS_VERSION: "v14.21.3" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - core-submodules: - internal: true - vars: - CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" - OUTPUT_DIR: "submodules" - sources: - - "{{.TASKFILE}}" - - ".gitmodules" - - "tools/scripts/deps-download/**/*" - dir: "components/core" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - "init" - - task: "utils:validate-checksum" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - DATA_DIR: "{{.OUTPUT_DIR}}" - cmds: - - "tools/scripts/deps-download/download-all.sh" - # This command must be last - - task: "utils:compute-checksum" - vars: - DATA_DIR: "{{.OUTPUT_DIR}}" - OUTPUT_FILE: "{{.CHECKSUM_FILE}}" - download-and-extract-tar: internal: true label: "{{.TASK}}-{{.TAR_NAME}}" @@ -399,7 +383,7 @@ tasks: PACKAGE_OUTPUT_DIR: "{{.SRC_DIR}}/node_modules" SERVER_OUTPUT_DIR: "{{.SRC_DIR}}/server/node_modules" sources: - - "{{.G_BUILD_DIR}}/log-viewer-webui-submodules.md5" + - "{{.G_DEPS_LOG_VIEWER_CHECKSUM_FILE}}" - "{{.G_BUILD_DIR}}/nodejs-22.md5" - "{{.TASKFILE}}" - "client/package.json" @@ -418,7 +402,7 @@ tasks: - "{{.PACKAGE_CHECKSUM_FILE}}" - "{{.SERVER_CHECKSUM_FILE}}" deps: - - "log-viewer-webui-submodules" + - "deps:log-viewer" - "nodejs-22" - task: "utils:validate-checksum" vars: @@ -469,30 +453,6 @@ tasks: "{{.SERVER_CHECKSUM_FILE}}" > "{{.CHECKSUM_FILE}}" - log-viewer-webui-submodules: - internal: true - vars: - CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" - OUTPUT_DIR: "yscope-log-viewer" - sources: - - "{{.TASKFILE}}" - - ".gitmodules" - dir: "components/log-viewer-webui" - generates: ["{{.CHECKSUM_FILE}}"] - deps: - - "init" - - task: "utils:validate-checksum" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - DATA_DIR: "{{.OUTPUT_DIR}}" - cmds: - - "git submodule update --init --recursive yscope-log-viewer" - # This command must be last - - task: "utils:compute-checksum" - vars: - DATA_DIR: "{{.OUTPUT_DIR}}" - OUTPUT_FILE: "{{.CHECKSUM_FILE}}" - meteor: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" diff --git a/components/core/tools/docker-images/clp-env-base-centos7.4/Dockerfile b/components/core/tools/docker-images/clp-env-base-centos7.4/Dockerfile index 1de75872a..feb3ae521 100644 --- a/components/core/tools/docker-images/clp-env-base-centos7.4/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-centos7.4/Dockerfile @@ -18,6 +18,12 @@ RUN ln -s /opt/rh/devtoolset-10/enable /etc/profile.d/devtoolset.sh # cannot be forced to use a bash shell that loads .bashrc RUN cp ./tools/docker-images/clp-env-base-centos7.4/setup-scripts/git /usr/bin/git +# NOTE: +# 1. `task` doesn't have an apt/yum package so we use its install script. +# 2. We don't want to install it using `install-prebuilt-packages.sh` since users may use that on +# their own machines and it would change their environment in a way that can't easily be undone. +RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin + # Remove cached files RUN yum clean all \ && rm -rf /tmp/* /var/tmp/* diff --git a/components/core/tools/docker-images/clp-env-base-ubuntu-focal/Dockerfile b/components/core/tools/docker-images/clp-env-base-ubuntu-focal/Dockerfile index f50ccadb4..e34deb336 100644 --- a/components/core/tools/docker-images/clp-env-base-ubuntu-focal/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-ubuntu-focal/Dockerfile @@ -16,6 +16,12 @@ RUN update-alternatives --set cpp /usr/bin/cpp-10 RUN ./tools/scripts/lib_install/ubuntu-focal/install-packages-from-source.sh +# NOTE: +# 1. `task` doesn't have an apt/yum package so we use its install script. +# 2. We don't want to install it using `install-prebuilt-packages.sh` since users may use that on +# their own machines and it would change their environment in a way that can't easily be undone. +RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin + # Remove cached files RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile b/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile index 61dec3228..affdf131d 100644 --- a/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile @@ -7,6 +7,12 @@ ADD ./tools/scripts/lib_install ./tools/scripts/lib_install RUN ./tools/scripts/lib_install/ubuntu-jammy/install-all.sh +# NOTE: +# 1. `task` doesn't have an apt/yum package so we use its install script. +# 2. We don't want to install it using `install-prebuilt-packages.sh` since users may use that on +# their own machines and it would change their environment in a way that can't easily be undone. +RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin + # Remove cached files RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/components/core/tools/scripts/deps-download/Catch2.json b/components/core/tools/scripts/deps-download/Catch2.json deleted file mode 100644 index b26eb68e8..000000000 --- a/components/core/tools/scripts/deps-download/Catch2.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "url": "https://github.com/catchorg/Catch2/archive/refs/tags/v2.13.7.zip", - "unzip": true, - "targets": [ - { - "source": "Catch2-2.13.7", - "destination": "submodules/Catch2" - } - ] -} diff --git a/components/core/tools/scripts/deps-download/abseil-cpp.json b/components/core/tools/scripts/deps-download/abseil-cpp.json deleted file mode 100644 index e38bf8bdb..000000000 --- a/components/core/tools/scripts/deps-download/abseil-cpp.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "url": "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip", - "unzip": true, - "targets": [ - { - "source": "abseil-cpp-20230802.1", - "destination": "submodules/abseil-cpp" - } - ] -} diff --git a/components/core/tools/scripts/deps-download/antlr4.json b/components/core/tools/scripts/deps-download/antlr4.json deleted file mode 100644 index ff0d4d871..000000000 --- a/components/core/tools/scripts/deps-download/antlr4.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "url": "https://www.antlr.org/download/antlr-4.13.1-complete.jar", - "unzip": false, - "hash": { - "algo": "sha3_256", - "digest": "292ba55b3be8443777737e94841cff7a343e7067747c2cb6f58830797b20be65" - }, - "targets": [ - { - "source": "antlr-4.13.1-complete.jar", - "destination": "third-party/antlr/antlr-4.13.1-complete.jar" - } - ] -} diff --git a/components/core/tools/scripts/deps-download/date.json b/components/core/tools/scripts/deps-download/date.json deleted file mode 100644 index 127275b51..000000000 --- a/components/core/tools/scripts/deps-download/date.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "url": "https://github.com/HowardHinnant/date/archive/refs/tags/v3.0.1.zip", - "unzip": true, - "targets": [ - { - "source": "date-3.0.1", - "destination": "submodules/date" - } - ] -} diff --git a/components/core/tools/scripts/deps-download/download-all.sh b/components/core/tools/scripts/deps-download/download-all.sh deleted file mode 100755 index b552f1247..000000000 --- a/components/core/tools/scripts/deps-download/download-all.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Stop on error -set -e - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -project_root_dir="$script_dir"/../../../../../ -component_root_dir="$script_dir"/../../../ - -cd "${component_root_dir}" -mkdir -p submodules - -# We don't use a git submodule for sqlite3 since that would require building the -# sqlite amalgamation -python3 "${script_dir}/download-dep.py" "${script_dir}/sqlite3.json" -python3 "${script_dir}/download-dep.py" "${script_dir}/antlr4.json" - -if [ -e "$project_root_dir/.git" ] ; then - git submodule update --init --recursive -else - python3 "${script_dir}/download-dep.py" "${script_dir}/abseil-cpp.json" - python3 "${script_dir}/download-dep.py" "${script_dir}/Catch2.json" - python3 "${script_dir}/download-dep.py" "${script_dir}/date.json" - python3 "${script_dir}/download-dep.py" "${script_dir}/json.json" - python3 "${script_dir}/download-dep.py" "${script_dir}/log-surgeon.json" - python3 "${script_dir}/download-dep.py" "${script_dir}/outcome.json" - python3 "${script_dir}/download-dep.py" "${script_dir}/simdjson.json" - python3 "${script_dir}/download-dep.py" "${script_dir}/yaml-cpp.json" -fi diff --git a/components/core/tools/scripts/deps-download/download-dep.py b/components/core/tools/scripts/deps-download/download-dep.py deleted file mode 100644 index 3a8c011a1..000000000 --- a/components/core/tools/scripts/deps-download/download-dep.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import hashlib -import json -import logging -import mmap -import pathlib -import shutil -import sys -import uuid -import urllib.parse -import urllib.request - -# Setup logging -# Create logger -logger = logging.getLogger() -logger.setLevel(logging.INFO) -# Setup console logging -logging_console_handler = logging.StreamHandler() -logging_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") -logging_console_handler.setFormatter(logging_formatter) -logger.addHandler(logging_console_handler) - - -def hash_file(algo: str, path: pathlib.Path): - if "sha3_256" == algo: - hasher = hashlib.sha3_256() - with open(path, 'rb') as f: - with mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ) as mapped_file: - hasher.update(mapped_file) - return hasher.hexdigest() - - -def main(argv): - script_dir = pathlib.Path(__file__).parent.resolve() - project_root_dir = script_dir.parent.parent.parent - - args_parser = argparse.ArgumentParser(description="Download dependency.") - args_parser.add_argument("config-file", help="Dependency configuration file.") - - parsed_args = vars(args_parser.parse_args(argv[1:])) - config_file_path = pathlib.Path(parsed_args["config-file"]).resolve() - - # Load configurations - with open(config_file_path) as f: - config = json.load(f) - - target_url = config["url"] - parsed_url = urllib.parse.urlparse(target_url) - filename = pathlib.Path(parsed_url.path).name - - extraction_dir = pathlib.Path("/") / "tmp" / str(uuid.uuid4()) - extraction_dir.mkdir(parents=True, exist_ok=True) - - # Download file - file_path = extraction_dir / filename - urllib.request.urlretrieve(target_url, file_path) - if config["unzip"]: - # NOTE: We need to convert file_path to a str since unpack_archive only - # accepts a path-like object on Python versions >= 3.7 - shutil.unpack_archive(str(file_path), extraction_dir) - - if "hash" in config: - # Verify hash - hash = hash_file(config["hash"]["algo"], file_path) - if hash != config["hash"]["digest"]: - logger.fatal("Hash mismatch.") - return -1 - - for target in config["targets"]: - target_source_path = extraction_dir / target["source"] - target_dest_path = project_root_dir / target["destination"] - - target_dest_parent = target_dest_path.parent - - # Remove destination - if target_dest_path.exists(): - shutil.rmtree(target_dest_path, ignore_errors=True) - else: - # Create destination parent - target_dest_parent.mkdir(parents=True, exist_ok=True) - - # Copy destination to target - if config["unzip"]: - shutil.copytree(target_source_path, target_dest_path) - else: - shutil.copy(target_source_path, target_dest_path) - - shutil.rmtree(extraction_dir) - - return 0 - - -if "__main__" == __name__: - sys.exit(main(sys.argv)) diff --git a/components/core/tools/scripts/deps-download/json.json b/components/core/tools/scripts/deps-download/json.json deleted file mode 100644 index 687c2697b..000000000 --- a/components/core/tools/scripts/deps-download/json.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "url": "https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.zip", - "unzip": true, - "targets": [ - { - "source": "json-3.11.3", - "destination": "submodules/json" - } - ] -} diff --git a/components/core/tools/scripts/deps-download/log-surgeon.json b/components/core/tools/scripts/deps-download/log-surgeon.json deleted file mode 100644 index 3b8951951..000000000 --- a/components/core/tools/scripts/deps-download/log-surgeon.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "url": "https://github.com/y-scope/log-surgeon/archive/895f464.zip", - "unzip": true, - "targets": [ - { - "source": "log-surgeon-895f46489b1911ab3b3aac3202afd56c96e8cd98", - "destination": "submodules/log-surgeon" - } - ] -} diff --git a/components/core/tools/scripts/deps-download/outcome.json b/components/core/tools/scripts/deps-download/outcome.json deleted file mode 100644 index 1a0725f4b..000000000 --- a/components/core/tools/scripts/deps-download/outcome.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "url": "https://github.com/ned14/outcome/archive/refs/tags/v2.2.9.zip", - "unzip": true, - "targets": [ - { - "source": "outcome-2.2.9", - "destination": "submodules/outcome" - } - ] -} diff --git a/components/core/tools/scripts/deps-download/simdjson.json b/components/core/tools/scripts/deps-download/simdjson.json deleted file mode 100644 index 8b9999961..000000000 --- a/components/core/tools/scripts/deps-download/simdjson.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "url": "https://github.com/simdjson/simdjson/archive/refs/tags/v3.6.3.zip", - "unzip": true, - "targets": [ - { - "source": "simdjson-3.6.3", - "destination": "submodules/simdjson" - } - ] -} - diff --git a/components/core/tools/scripts/deps-download/sqlite3.json b/components/core/tools/scripts/deps-download/sqlite3.json deleted file mode 100644 index 9b9391d89..000000000 --- a/components/core/tools/scripts/deps-download/sqlite3.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "url": "https://www.sqlite.org/2021/sqlite-amalgamation-3360000.zip", - "unzip": true, - "hash": { - "algo": "sha3_256", - "digest": "d25609210ec93b3c8c7da66a03cf82e2c9868cfbd2d7d866982861855e96f972" - }, - "targets": [ - { - "source": "sqlite-amalgamation-3360000", - "destination": "submodules/sqlite3" - } - ] -} diff --git a/components/core/tools/scripts/deps-download/yaml-cpp.json b/components/core/tools/scripts/deps-download/yaml-cpp.json deleted file mode 100644 index e9b4138c8..000000000 --- a/components/core/tools/scripts/deps-download/yaml-cpp.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "url": "https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.7.0.zip", - "unzip": true, - "targets": [ - { - "source": "yaml-cpp-yaml-cpp-0.7.0", - "destination": "submodules/yaml-cpp" - } - ] -} diff --git a/components/core/tools/scripts/lib_install/macos-12/install-all.sh b/components/core/tools/scripts/lib_install/macos-12/install-all.sh index c13223ba6..99e178e92 100755 --- a/components/core/tools/scripts/lib_install/macos-12/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos-12/install-all.sh @@ -10,8 +10,10 @@ brew update brew install \ boost \ cmake \ + coreutils \ fmt \ gcc \ + go-task \ java11 \ libarchive \ lz4 \ diff --git a/deps-tasks.yml b/deps-tasks.yml new file mode 100644 index 000000000..daf6186cd --- /dev/null +++ b/deps-tasks.yml @@ -0,0 +1,445 @@ +version: "3" +vars: + # Utility script path + G_DEP_DOWNLOAD_SCRIPT: "{{.ROOT_DIR}}/tools/scripts/deps-download/download-dep.py" + + # Target checksum files + G_DEPS_CORE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#core.md5" + G_DEPS_LOG_VIEWER_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#log-viewer.md5" + + # Submodule checksum files + G_ABSEIL_CPP_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#abseil-cpp.md5" + G_ANTLR4_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#antlr4.md5" + G_CATCH2_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#Catch2.md5" + G_DATE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#date.md5" + G_JSON_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#json.md5" + G_LOG_SURGEON_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#log-surgeon.md5" + G_OUTCOME_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#outcome.md5" + G_SIMDJSON_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#simdjson.md5" + G_SQLITE3_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#sqlite3.md5" + G_YAML_CPP_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#yaml-cpp.md5" + G_YSCOPE_LOG_VIEWER_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/deps#yscope-log-viewer.md5" + +tasks: + default: + cmds: + - task: "core" + - task: "log-viewer" + + core: + vars: + sources: + - "{{.G_ABSEIL_CPP_CHECKSUM_FILE}}" + - "{{.G_ANTLR4_CHECKSUM_FILE}}" + - "{{.G_CATCH2_CHECKSUM_FILE}}" + - "{{.G_DATE_CHECKSUM_FILE}}" + - "{{.G_JSON_CHECKSUM_FILE}}" + - "{{.G_LOG_SURGEON_CHECKSUM_FILE}}" + - "{{.G_OUTCOME_CHECKSUM_FILE}}" + - "{{.G_SIMDJSON_CHECKSUM_FILE}}" + - "{{.G_SQLITE3_CHECKSUM_FILE}}" + - "{{.G_YAML_CPP_CHECKSUM_FILE}}" + generates: ["{{.G_DEPS_CORE_CHECKSUM_FILE}}"] + deps: ["all-internal-deps"] + cmds: + - >- + cat + "{{.G_ABSEIL_CPP_CHECKSUM_FILE}}" + "{{.G_ANTLR4_CHECKSUM_FILE}}" + "{{.G_CATCH2_CHECKSUM_FILE}}" + "{{.G_DATE_CHECKSUM_FILE}}" + "{{.G_JSON_CHECKSUM_FILE}}" + "{{.G_LOG_SURGEON_CHECKSUM_FILE}}" + "{{.G_OUTCOME_CHECKSUM_FILE}}" + "{{.G_SIMDJSON_CHECKSUM_FILE}}" + "{{.G_SQLITE3_CHECKSUM_FILE}}" + "{{.G_YAML_CPP_CHECKSUM_FILE}}" + >> "{{.G_DEPS_CORE_CHECKSUM_FILE}}" + + log-viewer: + sources: + - "{{.G_YSCOPE_LOG_VIEWER_CHECKSUM_FILE}}" + generates: ["{{.G_DEPS_LOG_VIEWER_CHECKSUM_FILE}}"] + deps: ["all-internal-deps"] + cmds: + - >- + cat + "{{.G_YSCOPE_LOG_VIEWER_CHECKSUM_FILE}}" + >> "{{.G_DEPS_LOG_VIEWER_CHECKSUM_FILE}}" + + # NOTE: `git submodule update` doesn't support parallel invocations, so we can't use it to + # download submodules in parallel. This means: + # 1. the submodule tasks cannot be specified as a list of `deps`, because `task` always runs + # `deps` tasks in parallel. + # 2. we need to force higher level tasks, like `core` and `log-viewer`, to run their submodule + # download tasks serially. + # + # As a solution, the `all-internal-deps` task below: + # 1. uses `cmds` to sequentially download `ALL` submodules required by the clp-package. + # 2. uses `run: once`, so it executes once even if multiple higher level targets depend on it. + # + # All higher level tasks must depend on this task instead of depending on the individual + # submodule tasks in this file. + all-internal-deps: + internal: true + run: "once" + cmds: + - task: "abseil-cpp" + - task: "antlr4" + - task: "Catch2" + - task: "date" + - task: "json" + - task: "log-surgeon" + - task: "outcome" + - task: "simdjson" + - task: "sqlite3" + - task: "yaml-cpp" + - task: "yscope-log-viewer" + + abseil-cpp: + internal: true + vars: + CHECKSUM_FILE: "{{.G_ABSEIL_CPP_CHECKSUM_FILE}}" + DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/abseil-cpp" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--extract" + SRC_NAME: "abseil-cpp-20230802.1" + SRC_URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + antlr4: + internal: true + vars: + CHECKSUM_FILE: "{{.G_ANTLR4_CHECKSUM_FILE}}" + DEST_DIR: "{{.G_CORE_COMPONENT_DIR}}/third-party/antlr" + DEST: "{{.DEST_DIR}}/antlr-4.13.1-complete.jar" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST_DIR}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--no-submodule" + SRC_NAME: "antlr-4.13.1-complete.jar" + SRC_URL: "https://www.antlr.org/download/antlr-4.13.1-complete.jar" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST_DIR}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + Catch2: + internal: true + vars: + CHECKSUM_FILE: "{{.G_CATCH2_CHECKSUM_FILE}}" + DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/Catch2" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--extract" + SRC_NAME: "Catch2-2.13.7" + SRC_URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v2.13.7.zip" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + date: + internal: true + vars: + CHECKSUM_FILE: "{{.G_DATE_CHECKSUM_FILE}}" + DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/date" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--extract" + SRC_NAME: "date-3.0.1" + SRC_URL: "https://github.com/HowardHinnant/date/archive/refs/tags/v3.0.1.zip" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + json: + internal: true + vars: + CHECKSUM_FILE: "{{.G_JSON_CHECKSUM_FILE}}" + DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/json" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--extract" + SRC_NAME: "json-3.11.3" + SRC_URL: "https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.zip" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + log-surgeon: + internal: true + vars: + CHECKSUM_FILE: "{{.G_LOG_SURGEON_CHECKSUM_FILE}}" + DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/log-surgeon" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--extract" + SRC_NAME: "log-surgeon-895f46489b1911ab3b3aac3202afd56c96e8cd98" + SRC_URL: "https://github.com/y-scope/log-surgeon/archive/895f464.zip" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + outcome: + internal: true + vars: + CHECKSUM_FILE: "{{.G_OUTCOME_CHECKSUM_FILE}}" + DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/outcome" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--extract" + SRC_NAME: "outcome-2.2.9" + SRC_URL: "https://github.com/ned14/outcome/archive/refs/tags/v2.2.9.zip" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + simdjson: + internal: true + vars: + CHECKSUM_FILE: "{{.G_SIMDJSON_CHECKSUM_FILE}}" + DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/simdjson" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--extract" + SRC_NAME: "simdjson-3.6.3" + SRC_URL: "https://github.com/simdjson/simdjson/archive/refs/tags/v3.6.3.zip" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + # We don't use a git submodule for sqlite3 since that would require building the sqlite + # amalgamation + sqlite3: + internal: true + vars: + CHECKSUM_FILE: "{{.G_SQLITE3_CHECKSUM_FILE}}" + DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/sqlite3" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--extract --no-submodule" + SRC_NAME: "sqlite-amalgamation-3360000" + SRC_URL: "https://www.sqlite.org/2021/sqlite-amalgamation-3360000.zip" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + yaml-cpp: + internal: true + vars: + CHECKSUM_FILE: "{{.G_YAML_CPP_CHECKSUM_FILE}}" + DEST: "{{.G_CORE_COMPONENT_SUBMODULES_DIR}}/yaml-cpp" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--extract" + SRC_NAME: "yaml-cpp-yaml-cpp-0.7.0" + SRC_URL: "https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.7.0.zip" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + yscope-log-viewer: + internal: true + vars: + CHECKSUM_FILE: "{{.G_YSCOPE_LOG_VIEWER_CHECKSUM_FILE}}" + DEST: "{{.G_LOG_VIEWER_WEBUI_SRC_DIR}}/yscope-log-viewer" + sources: + - "{{.G_DEP_DOWNLOAD_SCRIPT}}" + - "{{.G_UTILS_TASKFILE}}" + - "{{.ROOT_DIR}}/Taskfile.yml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.DEST}}" + cmds: + - task: "download-dependency" + vars: + DEST: "{{.DEST}}" + FLAGS: "--extract" + SRC_NAME: "yscope-log-viewer-df996eac000823d02f9cd0b9eb4bb732dd634ae5" + SRC_URL: "https://github.com/y-scope/yscope-log-viewer/archive/df996ea.zip" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.DEST}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + + download-dependency: + internal: true + label: "{{.TASK}}-{{.SRC_NAME}}" + requires: + vars: ["DEST", "FLAGS", "SRC_NAME", "SRC_URL"] + deps: + - ":init" + cmds: + - >- + python3 "{{.G_DEP_DOWNLOAD_SCRIPT}}" + "{{.SRC_URL}}" + {{.SRC_NAME}} + "{{.DEST}}" + {{.FLAGS}} diff --git a/docs/src/dev-guide/building-package.md b/docs/src/dev-guide/building-package.md index 32d0ae417..4cf4e5349 100644 --- a/docs/src/dev-guide/building-package.md +++ b/docs/src/dev-guide/building-package.md @@ -16,10 +16,10 @@ prebuilt version instead, check out the [releases](https://github.com/y-scope/cl ## Setup -Download CLP core's source dependencies: +Initialize the project ```shell -components/core/tools/scripts/deps-download/download-all.sh +tools/scripts/deps-download/init.sh ``` Install CLP core's dependencies diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 62360486a..cffb0f004 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -7,17 +7,26 @@ CLP core is the low-level component that performs compression, decompression, an * We have built and tested CLP on the OSes listed [below](#native-environment). * If you have trouble building for another OS, file an issue, and we may be able to help. * A compiler that supports C++20 and std::span (e.g., gcc-10) +* [Task](https://taskfile.dev/) To build, we require some source dependencies, packages from package managers, and libraries built from source. +### Set up + +To initialize the project, run: + +```shell +tools/scripts/deps-download/init.sh +``` + ### Source Dependencies -We use both git submodules and third-party source packages. To download all, you can run this -script: +We use both git submodules and third-party source packages. To download all, run this `task` +command: ```shell -components/core/tools/scripts/deps-download/download-all.sh +task deps:core ``` This will download: @@ -27,6 +36,7 @@ This will download: * [date](https://github.com/HowardHinnant/date.git) (v3.0.1) * [json](https://github.com/nlohmann/json.git) (v3.10.4) * [log-surgeon](https://github.com/y-scope/log-surgeon) (895f464) +* [outcome](https://github.com/ned14/outcome) (v2.2.9) * [simdjson](https://github.com/simdjson/simdjson) (v3.6.3) * [SQLite3](https://www.sqlite.org/download.html) (v3.36.0) * [yaml-cpp](https://github.com/jbeder/yaml-cpp.git) (v0.7.0) diff --git a/lint-tasks.yml b/lint-tasks.yml index 9f21cd8ed..63cff497f 100644 --- a/lint-tasks.yml +++ b/lint-tasks.yml @@ -110,6 +110,7 @@ tasks: components/core/config \ components/package-template/src/etc \ docs \ + deps-tasks.yml \ lint-tasks.yml \ Taskfile.yml @@ -151,6 +152,7 @@ tasks: - "components/clp-package-utils/clp_package_utils" - "components/clp-py-utils/clp_py_utils" - "components/job-orchestration/job_orchestration" + - "tools/scripts" - "docs/conf" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" diff --git a/tools/scripts/deps-download/download-dep.py b/tools/scripts/deps-download/download-dep.py new file mode 100644 index 000000000..5866461fd --- /dev/null +++ b/tools/scripts/deps-download/download-dep.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +import argparse +import logging +import os +import shutil +import subprocess +import sys +import tempfile +import urllib.parse +import urllib.request +from pathlib import Path + +# Setup logging +# Create logger +logger = logging.getLogger() +logger.setLevel(logging.INFO) +# Setup console logging +logging_console_handler = logging.StreamHandler() +logging_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") +logging_console_handler.setFormatter(logging_formatter) +logger.addHandler(logging_console_handler) + + +def main(argv): + args_parser = argparse.ArgumentParser(description="Download dependency.") + args_parser.add_argument("source_url", help="URL of the source file.") + args_parser.add_argument("source_name", help="Name of the source file.") + args_parser.add_argument("dest_dir", help="Output directory for the download.") + args_parser.add_argument("--extract", action="store_true", help="Extract the source file.") + args_parser.add_argument( + "--no-submodule", + action="store_false", + dest="use_submodule", + help="Don't use git submodule update", + ) + + parsed_args = args_parser.parse_args(argv[1:]) + source_url = parsed_args.source_url + source_name = parsed_args.source_name + dest_dir = Path(parsed_args.dest_dir).resolve() + extract_source = parsed_args.extract + + script_path = Path(os.path.realpath(__file__)) + git_dir = script_path.parent / ".." / ".." / ".." / ".git" + if git_dir.exists() and git_dir.is_dir(): + if parsed_args.use_submodule: + cmd = ["git", "submodule", "update", "--init", "--recursive", str(dest_dir)] + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError: + logger.exception(f"Failed to update submodule '{dest_dir}'.") + return -1 + return 0 + + with tempfile.TemporaryDirectory() as work_dir_name: + work_dir = Path(work_dir_name) + + parsed_url = urllib.parse.urlparse(source_url) + filename = Path(parsed_url.path).name + file_path = work_dir / filename + + # Download file + urllib.request.urlretrieve(source_url, file_path) + if extract_source: + # NOTE: We need to convert `file_path` to a str since `unpack_archive` only accepts a + # path-like object on Python versions >= 3.7 + shutil.unpack_archive(str(file_path), work_dir) + + if dest_dir.exists(): + shutil.rmtree(dest_dir, ignore_errors=True) + else: + dest_dir.parent.mkdir(parents=True, exist_ok=True) + + source_path = work_dir / source_name + if not source_path.exists(): + logger.error(f"Source '{source_path}' doesn't exist.") + return -1 + + if extract_source: + shutil.copytree(source_path, dest_dir) + else: + shutil.copy(source_path, dest_dir) + + return 0 + + +if "__main__" == __name__: + sys.exit(main(sys.argv)) diff --git a/tools/scripts/deps-download/init.sh b/tools/scripts/deps-download/init.sh new file mode 100755 index 000000000..57fcd4594 --- /dev/null +++ b/tools/scripts/deps-download/init.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Exit on any error +set -e + +# Error on undefined variable +set -u + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +project_root_dir="$script_dir/../../../" +download_dep_script="$script_dir/download-dep.py" + +python3 "${download_dep_script}" \ + https://github.com/y-scope/yscope-dev-utils/archive/ff1611e6.zip \ + yscope-dev-utils-ff1611e6f9b116da27dc7f8f71797829c22d0b1a \ + "${project_root_dir}/tools/yscope-dev-utils" \ + --extract diff --git a/tools/scripts/find-broken-docs-links.py b/tools/scripts/find-broken-docs-links.py index 349e927ef..1a1eb7427 100644 --- a/tools/scripts/find-broken-docs-links.py +++ b/tools/scripts/find-broken-docs-links.py @@ -11,19 +11,19 @@ def main(argv): # Check for docs.yscope.com links with ".md" suffixes if _check_tracked_files( - r"docs\.yscope\.com/.+\.md", - repo_root, - repo_root, - "docs.yscope.com links cannot have \".md\" suffixes." + r"docs\.yscope\.com/.+\.md", + repo_root, + repo_root, + 'docs.yscope.com links cannot have ".md" suffixes.', ): found_violation = True # Check for sphinx :link: attributes that have ".md" suffixes if _check_tracked_files( - r":link:[[:space:]]*.+\.md", - repo_root, - repo_root / "docs", - "sphinx :link: attributes cannot have \".md\" suffixes" + r":link:[[:space:]]*.+\.md", + repo_root, + repo_root / "docs", + 'sphinx :link: attributes cannot have ".md" suffixes', ): found_violation = True @@ -35,18 +35,13 @@ def main(argv): def _get_repo_root() -> Path: path_str = subprocess.check_output( - ["git", "rev-parse", "--show-toplevel"], - cwd=Path(__file__).parent, - text=True + ["git", "rev-parse", "--show-toplevel"], cwd=Path(__file__).parent, text=True ) return Path(path_str.strip()) def _check_tracked_files( - pattern: str, - repo_root: Path, - dir_to_search: Path, - error_msg: str + pattern: str, repo_root: Path, dir_to_search: Path, error_msg: str ) -> bool: """ Check for a pattern in all tracked files in the repo (except this script). @@ -60,16 +55,16 @@ def _check_tracked_files( # NOTE: "-z" ensures the paths won't be quoted (while delimiting them using '\0') for path_str in subprocess.check_output( - [ - "git", - "ls-files", - "--cached", - "--exclude-standard", - "-z", - str(dir_to_search.relative_to(repo_root)) - ], - cwd=repo_root, - text=True, + [ + "git", + "ls-files", + "--cached", + "--exclude-standard", + "-z", + str(dir_to_search.relative_to(repo_root)), + ], + cwd=repo_root, + text=True, ).split("\0"): path = Path(path_str) @@ -79,16 +74,9 @@ def _check_tracked_files( try: for match in subprocess.check_output( - [ - "grep", - "--extended-regexp", - "--line-number", - "--with-filename", - pattern, - path - ], - cwd=repo_root, - text=True, + ["grep", "--extended-regexp", "--line-number", "--with-filename", pattern, path], + cwd=repo_root, + text=True, ).splitlines(): _parse_and_print_match(match, error_msg) found_matches = True