cicd improvements

knights-analytics · Jul 18, 2024 · 372ef1b · 372ef1b
1 parent 819435e
commit 372ef1b
Show file tree

Hide file tree

Showing 8 changed files with 204 additions and 53 deletions.
diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml
@@ -0,0 +1,50 @@
+name: Create image
+
+on: workflow_dispatch
+
+permissions:
+  contents: write
+  packages: write
+  checks: write
+  attestations: write
+  id-token: write
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -7,7 +7,14 @@ on:
 
 permissions:
   contents: write
+  packages: write
   checks: write
+  attestations: write
+  id-token: write
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
   test:
@@ -56,6 +63,43 @@ jobs:
           report_paths: './testTarget/unit/*.xml'
           fail_on_failure: true
           require_tests: true
+
+  build-and-push:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true
+
   release:
     name: Release
     runs-on: ubuntu-latest
@@ -78,11 +122,11 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: onnxruntime-linux-x64-gpu
-          path: .
+          path: ./onnxruntime-linux-x64-gpu
       - name: Display structure of downloaded files
         run: ls -R
       - uses: ncipollo/release-action@v1
         with:
-          artifacts: "libtokenizers.a, onnxruntime-linux-x64.so, onnxruntime-linux-x64-gpu.zip, hugot-cli-linux-x64"
+          artifacts: "libtokenizers.a, onnxruntime-linux-x64.so, onnxruntime-linux-x64-gpu, hugot-cli-linux-x64"
           generateReleaseNotes: true
           skipIfReleaseExists: true
diff --git a/Dockerfile b/Dockerfile
@@ -1,8 +1,10 @@
+#--- dockerfile with hugot dependencies and cli (cpu only) ---
+
 ARG GO_VERSION=1.22.5
 ARG RUST_VERSION=1.79
 ARG ONNXRUNTIME_VERSION=1.18.0
 ARG BUILD_PLATFORM=linux/amd64
-ARG CGO_LDFLAGS="-L./usr/lib/libtokenizers.a"
+
 #--- rust build of tokenizer ---
 
 FROM --platform=$BUILD_PLATFORM rust:$RUST_VERSION AS tokenizer
@@ -11,23 +13,16 @@ RUN git clone https://github.com/knights-analytics/tokenizers -b namespace && \
     cd tokenizers && \
     cargo build --release
 
-#--- build and test layer ---
+#--- build layer ---
 
 FROM --platform=$BUILD_PLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 AS hugot-build
 ARG GO_VERSION
 ARG ONNXRUNTIME_VERSION
-ARG CGO_LDFLAGS
 
 RUN dnf -y install gcc jq bash tar xz gzip glibc-static libstdc++ wget zip git && \
     ln -s /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so && \
     dnf install -y 'dnf-command(config-manager)' && \
     dnf config-manager --add-repo https://download.fedoraproject.org/pub/fedora/linux/releases/39/Everything/x86_64/os/ && \
-    # from fedora
-    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/fedora39/x86_64/cuda-fedora39.repo && \
-    dnf install -y cuda-cudart-12-4 libcublas-12-4 libcurand-12-4 libcufft-12-4 && \
-    # from rhel
-    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo && \
-    dnf install -y libcudnn8 && \
     dnf clean all
 
 # go
@@ -42,38 +37,16 @@ COPY --from=tokenizer /tokenizers/target/release/libtokenizers.a /usr/lib/libtok
 # onnxruntime cpu and gpu
 RUN curl -LO https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz && \
    tar -xzf onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz && \
-   mv ./onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION} /usr/lib64/onnxruntime.so && \
-   curl -LO https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-gpu-cuda12-${ONNXRUNTIME_VERSION}.tgz && \
-   tar -xzf onnxruntime-linux-x64-gpu-cuda12-${ONNXRUNTIME_VERSION}.tgz && \
-   mv ./onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}/lib /usr/lib64/onnxruntime-gpu
-
-# build gotestsum and test2json
-RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o test2json -ldflags="-s -w" cmd/test2json && mv test2json /usr/local/bin/test2json && \
-    curl -LO https://github.com/gotestyourself/gotestsum/releases/download/v1.12.0/gotestsum_1.12.0_linux_amd64.tar.gz && \
-    tar -xzf gotestsum_1.12.0_linux_amd64.tar.gz --directory /usr/local/bin
+   mv ./onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION} /usr/lib64/onnxruntime.so
 
 # build cli binary
 COPY . /build
 WORKDIR /build
-RUN cd ./cmd && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o ./target main.go
+RUN cd ./cmd && CGO_ENABLED=1 CGO_LDFLAGS="-L/usr/lib/" GOOS=linux GOARCH=amd64 go build -a -o ./target main.go
 
-# NON-PRIVILEDGED USER
-# create non-priviledged testuser with id: 1000
-RUN dnf install --disablerepo=* --enablerepo=amazonlinux --allowerasing -y dirmngr sudo which && dnf clean all
-RUN useradd -u 1000 -m testuser && chown -R testuser:testuser /build && usermod -a -G wheel testuser
-RUN echo "testuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/testuser
+#--- final layer ---
+FROM --platform=$BUILD_PLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 AS final
 
-# ENTRYPOINT
-COPY ./scripts/entrypoint.sh /entrypoint.sh
-# convert windows line endings if present
-RUN sed -i 's/\r//g' /entrypoint.sh
-RUN chmod +x /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]
-
-# artifacts layer
-FROM --platform=$BUILD_PLATFORM scratch AS artifacts
-
-COPY --from=hugot-build /usr/lib64/onnxruntime.so onnxruntime-linux-x64.so
-COPY --from=hugot-build /usr/lib64/onnxruntime-gpu onnxruntime-linux-x64-gpu
-COPY --from=hugot-build /usr/lib/libtokenizers.a libtokenizers.a
-COPY --from=hugot-build /build/cmd/target /hugot-cli-linux-x64
+COPY --from=tokenizer /tokenizers/target/release/libtokenizers.a /usr/lib/libtokenizers.a
+COPY --from=hugot-build /build/cmd/target /hugot-cli
+COPY --from=hugot-build /usr/lib64/onnxruntime.so /usr/lib64/onnxruntime.so
diff --git a/Dockerfile.test b/Dockerfile.test
@@ -0,0 +1,80 @@
+#--- dockerfile to test hugot  ---
+
+ARG GO_VERSION=1.22.5
+ARG RUST_VERSION=1.79
+ARG ONNXRUNTIME_VERSION=1.18.0
+ARG BUILD_PLATFORM=linux/amd64
+
+#--- rust build of tokenizer ---
+
+FROM --platform=$BUILD_PLATFORM rust:$RUST_VERSION AS tokenizer
+
+RUN git clone https://github.com/knights-analytics/tokenizers -b namespace && \
+    cd tokenizers && \
+    cargo build --release
+
+#--- build and test layer ---
+
+FROM --platform=$BUILD_PLATFORM public.ecr.aws/amazonlinux/amazonlinux:2023 AS hugot-build
+ARG GO_VERSION
+ARG ONNXRUNTIME_VERSION
+
+RUN dnf -y install gcc jq bash tar xz gzip glibc-static libstdc++ wget zip git && \
+    ln -s /usr/lib64/libstdc++.so.6 /usr/lib64/libstdc++.so && \
+    dnf install -y 'dnf-command(config-manager)' && \
+    dnf config-manager --add-repo https://download.fedoraproject.org/pub/fedora/linux/releases/39/Everything/x86_64/os/ && \
+    # from fedora
+    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/fedora39/x86_64/cuda-fedora39.repo && \
+    dnf install -y cuda-cudart-12-4 libcublas-12-4 libcurand-12-4 libcufft-12-4 && \
+    # from rhel
+    dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo && \
+    dnf install -y libcudnn8 && \
+    dnf clean all
+
+# go
+RUN curl -LO https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz && \
+    tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz && \
+    rm go${GO_VERSION}.linux-amd64.tar.gz
+ENV PATH="$PATH:/usr/local/go/bin"
+
+# tokenizer
+COPY --from=tokenizer /tokenizers/target/release/libtokenizers.a /usr/lib/libtokenizers.a
+
+# onnxruntime cpu and gpu
+RUN curl -LO https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz && \
+   tar -xzf onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz && \
+   mv ./onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION} /usr/lib64/onnxruntime.so && \
+   curl -LO https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-gpu-cuda12-${ONNXRUNTIME_VERSION}.tgz && \
+   tar -xzf onnxruntime-linux-x64-gpu-cuda12-${ONNXRUNTIME_VERSION}.tgz && \
+   mv ./onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}/lib /usr/lib64/onnxruntime-gpu
+
+# build gotestsum and test2json
+RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o test2json -ldflags="-s -w" cmd/test2json && mv test2json /usr/local/bin/test2json && \
+    curl -LO https://github.com/gotestyourself/gotestsum/releases/download/v1.12.0/gotestsum_1.12.0_linux_amd64.tar.gz && \
+    tar -xzf gotestsum_1.12.0_linux_amd64.tar.gz --directory /usr/local/bin
+
+# build cli binary
+COPY . /build
+WORKDIR /build
+RUN cd ./cmd && CGO_ENABLED=1 CGO_LDFLAGS="-L/usr/lib/" GOOS=linux GOARCH=amd64 go build -a -o ./target main.go
+
+# NON-PRIVILEDGED USER
+# create non-priviledged testuser with id: 1000
+RUN dnf install --disablerepo=* --enablerepo=amazonlinux --allowerasing -y dirmngr sudo which && dnf clean all
+RUN useradd -u 1000 -m testuser && chown -R testuser:testuser /build && usermod -a -G wheel testuser
+RUN echo "testuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/testuser
+
+# ENTRYPOINT
+COPY ./scripts/entrypoint.sh /entrypoint.sh
+# convert windows line endings if present
+RUN sed -i 's/\r//g' /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
+
+# artifacts layer
+FROM --platform=$BUILD_PLATFORM scratch AS artifacts
+
+COPY --from=hugot-build /usr/lib64/onnxruntime.so onnxruntime-linux-x64.so
+COPY --from=hugot-build /usr/lib64/onnxruntime-gpu onnxruntime-linux-x64-gpu
+COPY --from=hugot-build /usr/lib/libtokenizers.a libtokenizers.a
+COPY --from=hugot-build /build/cmd/target /hugot-cli-linux-x64
diff --git a/README.md b/README.md
@@ -45,7 +45,7 @@ Hugot can be used in two ways: as a library in your go application, or as a comm
 
 To use Hugot as a library in your application, you will need the following two dependencies on your system:
 
-- the tokenizers.a file obtained from the releases section of this page (if you want to use alternative architecture from `linux/amd64` you will have to build the tokenizers.a yourself, see [here](https://github.com/knights-analytics/tokenizers). This file should be at /usr/lib/tokenizers.a so that hugot can load it.
+- the tokenizers.a file obtained from the releases section of this page (if you want to use alternative architecture from `linux/amd64` you will have to build the tokenizers.a yourself, see [here](https://github.com/knights-analytics/tokenizers). This file should be at /usr/lib/tokenizers.a so that hugot can load it. Alternatively, you can explicitly specify the path to the folder with the `libtokenizers.a` file using the `CGO_LDFLAGS` env variable, see the [dockerfile](./Dockerfile).
 - the onnxruntime.go file obtained from the releases section of this page (if you want to use alternative architectures from `linux/amd64` you will have to download it from [the onnxruntime releases page](https://github.com/microsoft/onnxruntime/releases/), see the [dockerfile](./Dockerfile) as an example). Hugot looks for this file at /usr/lib/onnxruntime.so or /usr/lib64/onnxruntime.so by default. A different location can be specified by passing the `WithOnnxLibraryPath()` option to `NewSession()`, e.g:
 
 ```

diff --git a/compose-dev.yaml b/compose-dev.yaml
@@ -4,7 +4,7 @@ services:
     platform: linux/amd64
     build:
       context: .
-      dockerfile: ./Dockerfile
+      dockerfile: ./Dockerfile.test
       target: hugot-build
     volumes:
       - $src_dir:/home/testuser/repositories/hugot

diff --git a/compose-test.yaml b/compose-test.yaml
@@ -5,7 +5,7 @@ services:
     container_name: hugot
     build:
       context: .
-      dockerfile: ./Dockerfile
+      dockerfile: ./Dockerfile.test
       target: hugot-build
     volumes:
       - $test_folder:/test

diff --git a/scripts/run-unit-tests.sh b/scripts/run-unit-tests.sh
@@ -4,20 +4,24 @@ set -e
 
 # Directory of *this* script
 this_dir="$( cd "$( dirname "$0" )" && pwd )"
-export src_dir="$(realpath "${this_dir}/..")"
+src_dir="$(realpath "${this_dir}/..")"
+export src_dir
 
-export commit_hash=$(git rev-parse --short HEAD)
-export test_folder="$src_dir/testTarget"
-mkdir -p $test_folder
-export host_uid=$(id -u "$USER")
+commit_hash=$(git rev-parse --short HEAD)
+export commit_hash
+test_folder="$src_dir/testTarget"
+export test_folder
+mkdir -p "$test_folder"
+host_uid=$(id -u "$USER")
+export host_uid
 
 # build with compose
-docker compose -f $src_dir/compose-test.yaml build
+docker compose -f "$src_dir/compose-test.yaml" build
 echo "Running tests for commit hash: $commit_hash"
-docker compose -f $src_dir/compose-test.yaml up && \
-docker compose -f $src_dir/compose-test.yaml logs --no-color >& $test_folder/logs.txt
-docker compose -f $src_dir/compose-test.yaml rm -fsv
+docker compose -f "$src_dir/compose-test.yaml" up && \
+docker compose -f "$src_dir/compose-test.yaml" logs --no-color >& "$test_folder/logs.txt"
+docker compose -f "$src_dir/compose-test.yaml" rm -fsv
 
 echo "Extracting lib artifacts"
-docker build . --output "$src_dir/artifacts" --target artifacts
+docker build -f ./Dockerfile.test . --output "$src_dir/artifacts" --target artifacts
 echo "lib artifacts extracted"