diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-execution-image-build/action.yaml new file mode 100644 index 000000000..45d87dff2 --- /dev/null +++ b/.github/actions/clp-execution-image-build/action.yaml @@ -0,0 +1,64 @@ +name: "clp-execution-image-build" +description: "Builds a container image that contains the dependencies necessary +to run the CLP package." + +inputs: + image_registry: + default: "ghcr.io" + description: "Container image registry" + required: false + image_registry_username: + default: "${{github.actor}}" + description: "Container image registry username" + required: false + image_registry_password: + default: "" + description: "Container image registry password" + required: false + platform_id: + description: "Platform ID of the container (e.g. ubuntu)" + required: true + platform_version_id: + description: "Platform VERSION_ID / VERSION_CODENAME of the container + (e.g. jammy, focal, etc.)" + required: true + +runs: + using: "composite" + steps: + - name: "Set up Docker Buildx" + uses: "docker/setup-buildx-action@v3" + + - name: "Login to Image Registry" + uses: "docker/login-action@v3" + with: + registry: "${{inputs.image_registry}}" + username: "${{inputs.image_registry_username}}" + password: "${{inputs.image_registry_password}}" + + - name: "Sanitize Repository Name" + id: "sanitization" + shell: "bash" + run: | + # Docker doesn't support repository names with uppercase characters, so we convert the + # name to lowercase here. + echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \ + >> "$GITHUB_OUTPUT" + + - name: "Update Metadata" + id: "meta" + uses: "docker/metadata-action@v5" + with: + images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\ + /clp-execution-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}" + + - name: "Build and Push" + if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" + uses: "docker/build-push-action@v5" + with: + context: "./" + file: "./tools/docker-images/\ + clp-execution-base-${{inputs.platform_id}}-${{inputs.platform_version_id}}/Dockerfile" + push: true + tags: "${{steps.meta.outputs.tags}}" + labels: "${{steps.meta.outputs.labels}}" diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 0dc0e532c..cdc47591d 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -3,14 +3,16 @@ name: "clp-core-build" on: pull_request: paths: - - ".github/actions/**" + - ".github/actions/clp-core-build/action.yaml" + - ".github/actions/clp-core-build-containers/action.yaml" - ".github/workflows/clp-core-build.yaml" - ".gitmodules" - "components/core/**" - "!components/core/tools/scripts/lib_install/macos-12/**" push: paths: - - ".github/actions/**" + - ".github/actions/clp-core-build/action.yaml" + - ".github/actions/clp-core-build-containers/action.yaml" - ".github/workflows/clp-core-build.yaml" - ".gitmodules" - "components/core/**" diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index 49dece398..c1fabea5d 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -3,55 +3,89 @@ name: "clp-execution-image-build" on: pull_request: paths: + - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - - "components/core/tools/scripts/lib_install/*" - - "tools/docker-images/clp-execution-base-focal/**" + - "tools/docker-images/**/*" push: paths: + - ".github/actions/clp-execution-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - - "components/core/tools/scripts/lib_install/*" - - "tools/docker-images/clp-execution-base-focal/**" + - "tools/docker-images/**/*" workflow_dispatch: -env: - CONTAINER_IMAGE_REGISTRY: "ghcr.io" +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true jobs: - build: + filter-relevant-changes: runs-on: "ubuntu-latest" + outputs: + ubuntu_focal_image_changed: "${{steps.filter.outputs.ubuntu_focal_image}}" + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" steps: - uses: "actions/checkout@v3" with: submodules: "recursive" - - name: "Workaround actions/runner-images/issues/6775" + - name: "Work around actions/runner-images/issues/6775" run: "chown $(id -u):$(id -g) -R ." + shell: "bash" - - uses: "docker/login-action@v3" + - name: "Filter relevant changes" + uses: "dorny/paths-filter@v2" + id: "filter" with: - registry: "${{env.CONTAINER_IMAGE_REGISTRY}}" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" + base: "main" + filters: | + ubuntu_focal_image: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-focal/**/*" + ubuntu_jammy_image: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" - - id: "sanitization" + ubuntu_focal_execution_image: + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_focal_image_changed" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@v3" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" shell: "bash" - run: | - # Docker doesn't support repository names with uppercase characters, so we convert the - # name to lowercase here. - echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \ - >> "$GITHUB_OUTPUT" - - - id: "meta" - uses: "docker/metadata-action@v5" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/clp-execution-image-build" with: - images: "${{env.CONTAINER_IMAGE_REGISTRY}}/${{steps.sanitization.outputs.REPOSITORY}}\ - /clp-execution-x86-ubuntu-focal" + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "focal" + + ubuntu_jammy_execution_image: + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@v3" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." - - if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" - uses: "docker/build-push-action@v5" + - uses: "./.github/actions/clp-execution-image-build" with: - context: "./" - file: "./tools/docker-images/clp-execution-base-focal/Dockerfile" - push: true - tags: "${{steps.meta.outputs.tags}}" - labels: "${{steps.meta.outputs.labels}}" + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/README.md b/README.md index 234c44d8c..a25e74260 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ YScope's Compressed Log Processor (CLP) compresses your logs, and allows you to compressed logs without decompression. CLP supports both JSON logs and unstructured (i.e., free text) logs. It also supports real-time log compression within several logging libraries. CLP also includes purpose-built web interfaces for searching and viewing the compressed logs. To learn more -about it, you can read our [paper][8]. +about it, you can read our [paper][9]. # Benchmarks @@ -27,7 +27,7 @@ index-less design, so for a fair comparison, we disabled MongoDB and PostgreSQL' left them enabled, MongoDB and PostgreSQL's compression ratio would be worse. We didn't disable indexing for Elasticsearch or Splunk since these tools are fundamentally index-based (i.e., logs cannot be searched without indexes). More details about our experimental methodology can be found in -the [CLP paper][8]. +the [CLP paper][9]. # System Overview @@ -42,25 +42,25 @@ features: (intermediate representation) produced by CLP's logging libraries. - **Real-time Compression with CLP Logging Libraries**: CLP provides logging libraries for - [Python][9] and Java ([Log4j][10] and [Logback][11]). The logging libraries compress logs in + [Python][10] and Java ([Log4j][11] and [Logback][12]). The logging libraries compress logs in real-time, so only compressed logs are written to disk or transmitted over the network. The compressed logs use CLP's intermediate representation (IR) format which achieves a higher compression ratio than general purpose compressors like Zstandard. Compressing IR into archives can further double the compression ratio and enable global search, but this requires more memory usage as it needs to buffer enough logs. More details on IR versus archives can be found in this - [Uber Engineering Blog][12]. + [Uber Engineering Blog][13]. -- **[Log Viewer][13]**: the compressed IR can be viewed in a web-based log viewer. Compared to +- **[Log Viewer][14]**: the compressed IR can be viewed in a web-based log viewer. Compared to viewing the logs in an editor, CLP's log viewer supports advanced features like filtering logs based on log level verbosity (e.g., only displaying logs with log level equal or higher than ERROR). These features are possible because CLP's logging libraries parse the logs before compressing them into IR. -- **IR Analytics Libraries**: we also provide a [Python library][14] and a [Go library][15] that can +- **IR Analytics Libraries**: we also provide a [Python library][15] and a [Go library][16] that can analyze compressed IR. -- **[Log parser][16]**: CLP also includes a custom pushdown-automata-based log parser that is 3x - faster than state-of-the-art regular expression engines like [RE2][17]. The log parser is +- **[Log parser][17]**: CLP also includes a custom pushdown-automata-based log parser that is 3x + faster than state-of-the-art regular expression engines like [RE2][18]. The log parser is available as a library that can be used by other applications. # Getting Started @@ -110,7 +110,8 @@ useful for building and running CLP: | `ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-focal:main` | The dependencies necessary to build CLP core in an Ubuntu Focal x86 environment. | [↗][3] | | `ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main` | The dependencies necessary to build CLP core in an Ubuntu Jammy x86 environment. | [↗][4] | | `ghcr.io/y-scope/clp/clp-core-x86-ubuntu-focal:main` | The CLP core binaries (`clg`, `clp`, `clp-s`, `glt`, etc.) built in an Ubuntu Focal x86 environment. | [↗][5] | -| `ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-focal:main` | The dependencies necessary to run the CLP package in an x86 environment. | [↗][6] | +| `ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-focal:main` | The dependencies necessary to run the CLP package in an Ubuntu Focal x86 environment. | [↗][6] | +| `ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main` | The dependencies necessary to run the CLP package in an Ubuntu Jammy x86 environment. | [↗][7] | # Next Steps @@ -124,7 +125,7 @@ Have an issue you want to fix or a feature you'd like to implement? We'd love to ## Linting Before submitting a PR, ensure you've run our linting tools and either fixed any violations or -suppressed the warning. To run our linting workflows locally, you'll need [Task][7]. Alternatively, +suppressed the warning. To run our linting workflows locally, you'll need [Task][8]. Alternatively, you can run the [clp-lint](.github/workflows/clp-lint.yaml) workflow in your fork. To perform the linting checks: @@ -145,14 +146,15 @@ task lint:fix [4]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-ubuntu-jammy [5]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-focal [6]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-focal -[7]: https://taskfile.dev/ -[8]: https://www.usenix.org/system/files/osdi21-rodrigues.pdf -[9]: https://github.com/y-scope/clp-loglib-py -[10]: https://github.com/y-scope/log4j1-appenders -[11]: https://github.com/y-scope/logback-appenders -[12]: https://www.uber.com/en-US/blog/reducing-logging-cost-by-two-orders-of-magnitude-using-clp -[13]: https://github.com/y-scope/yscope-log-viewer -[14]: https://github.com/y-scope/clp-ffi-py -[15]: https://github.com/y-scope/clp-ffi-go -[16]: https://github.com/y-scope/log-surgeon -[17]: https://github.com/google/re2 +[7]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-jammy +[8]: https://taskfile.dev/ +[9]: https://www.usenix.org/system/files/osdi21-rodrigues.pdf +[10]: https://github.com/y-scope/clp-loglib-py +[11]: https://github.com/y-scope/log4j1-appenders +[12]: https://github.com/y-scope/logback-appenders +[13]: https://www.uber.com/en-US/blog/reducing-logging-cost-by-two-orders-of-magnitude-using-clp +[14]: https://github.com/y-scope/yscope-log-viewer +[15]: https://github.com/y-scope/clp-ffi-py +[16]: https://github.com/y-scope/clp-ffi-go +[17]: https://github.com/y-scope/log-surgeon +[18]: https://github.com/google/re2 diff --git a/tools/docker-images/clp-execution-base-focal/Dockerfile b/tools/docker-images/clp-execution-base-focal/Dockerfile deleted file mode 100644 index 88c6e9a63..000000000 --- a/tools/docker-images/clp-execution-base-focal/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:focal AS BASE - -WORKDIR /root - -RUN mkdir -p ./tools/docker-images/clp-execution-base-focal -ADD ./tools/docker-images/clp-execution-base-focal/setup-scripts ./tools/docker-images/clp-execution-base-focal/setup-scripts - -RUN ./tools/docker-images/clp-execution-base-focal/setup-scripts/install-prebuilt-packages.sh - -# Remove cached files -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Flatten the image -FROM scratch -COPY --from=BASE / / diff --git a/tools/docker-images/clp-execution-base-jammy/Dockerfile b/tools/docker-images/clp-execution-base-jammy/Dockerfile deleted file mode 100644 index 95be22448..000000000 --- a/tools/docker-images/clp-execution-base-jammy/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:jammy AS BASE - -WORKDIR /root - -RUN mkdir -p ./tools/docker-images/clp-execution-base-jammy -ADD ./tools/docker-images/clp-execution-base-jammy/setup-scripts ./tools/docker-images/clp-execution-base-jammy/setup-scripts - -RUN ./tools/docker-images/clp-execution-base-jammy/setup-scripts/install-prebuilt-packages.sh - -# Remove cached files -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Flatten the image -FROM scratch -COPY --from=BASE / / diff --git a/tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile b/tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile new file mode 100644 index 000000000..6a7cc4318 --- /dev/null +++ b/tools/docker-images/clp-execution-base-ubuntu-focal/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:focal AS BASE + +WORKDIR /root + +RUN mkdir -p ./tools/docker-images/clp-execution-base-ubuntu-focal +ADD ./tools/docker-images/clp-execution-base-ubuntu-focal/setup-scripts ./tools/docker-images/clp-execution-base-ubuntu-focal/setup-scripts + +RUN ./tools/docker-images/clp-execution-base-ubuntu-focal/setup-scripts/install-prebuilt-packages.sh + +# Remove cached files +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Flatten the image +FROM scratch +COPY --from=BASE / / diff --git a/tools/docker-images/clp-execution-base-focal/build.sh b/tools/docker-images/clp-execution-base-ubuntu-focal/build.sh similarity index 100% rename from tools/docker-images/clp-execution-base-focal/build.sh rename to tools/docker-images/clp-execution-base-ubuntu-focal/build.sh diff --git a/tools/docker-images/clp-execution-base-focal/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-execution-base-ubuntu-focal/setup-scripts/install-prebuilt-packages.sh similarity index 100% rename from tools/docker-images/clp-execution-base-focal/setup-scripts/install-prebuilt-packages.sh rename to tools/docker-images/clp-execution-base-ubuntu-focal/setup-scripts/install-prebuilt-packages.sh diff --git a/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile b/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile new file mode 100644 index 000000000..8b1a3885f --- /dev/null +++ b/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile @@ -0,0 +1,16 @@ +FROM ubuntu:jammy AS BASE + +WORKDIR /root + +RUN mkdir -p ./tools/docker-images/clp-execution-base-ubuntu-jammy +ADD ./tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts ./tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts + +RUN ./tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh + +# Remove cached files +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Flatten the image +FROM scratch +COPY --from=BASE / / diff --git a/tools/docker-images/clp-execution-base-jammy/build.sh b/tools/docker-images/clp-execution-base-ubuntu-jammy/build.sh similarity index 100% rename from tools/docker-images/clp-execution-base-jammy/build.sh rename to tools/docker-images/clp-execution-base-ubuntu-jammy/build.sh diff --git a/tools/docker-images/clp-execution-base-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh similarity index 100% rename from tools/docker-images/clp-execution-base-jammy/setup-scripts/install-prebuilt-packages.sh rename to tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh