diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml new file mode 100644 index 0000000..97d9a49 --- /dev/null +++ b/.github/workflows/build-push.yaml @@ -0,0 +1,50 @@ +name: Create image + +on: workflow_dispatch + +permissions: + contents: write + packages: write + checks: write + attestations: write + id-token: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: push + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7bb4abf..e1c4b6a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,10 +4,18 @@ on: push: tags: - v* + pull_request: permissions: contents: write + packages: write checks: write + attestations: write + id-token: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} jobs: test: @@ -56,6 +64,40 @@ jobs: report_paths: './testTarget/unit/*.xml' fail_on_failure: true require_tests: true + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: push + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true release: name: Release runs-on: ubuntu-latest @@ -78,11 +120,11 @@ jobs: - uses: actions/download-artifact@v4 with: name: onnxruntime-linux-x64-gpu - path: . + path: ./onnxruntime-linux-x64-gpu - name: Display structure of downloaded files run: ls -R - uses: ncipollo/release-action@v1 with: - artifacts: "libtokenizers.a, onnxruntime-linux-x64.so, onnxruntime-linux-x64-gpu.zip, hugot-cli-linux-x64" + artifacts: "libtokenizers.a, onnxruntime-linux-x64.so, onnxruntime-linux-x64-gpu, hugot-cli-linux-x64" generateReleaseNotes: true skipIfReleaseExists: true diff --git a/Dockerfile b/Dockerfile index b33e884..aa3d31d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG GO_VERSION=1.22.5 ARG RUST_VERSION=1.79 ARG ONNXRUNTIME_VERSION=1.18.0 ARG BUILD_PLATFORM=linux/amd64 -ARG CGO_LDFLAGS="-L./usr/lib/libtokenizers.a" + #--- rust build of tokenizer --- FROM --platform=$BUILD_PLATFORM rust:$RUST_VERSION AS tokenizer @@ -55,7 +55,7 @@ RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o test2json -ldflags="-s -w" # build cli binary COPY . /build WORKDIR /build -RUN cd ./cmd && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o ./target main.go +RUN cd ./cmd && CGO_ENABLED=1 CGO_LDFLAGS="-L/usr/lib/" GOOS=linux GOARCH=amd64 go build -a -o ./target main.go # NON-PRIVILEDGED USER # create non-priviledged testuser with id: 1000 diff --git a/README.md b/README.md index b822f37..299e43d 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Hugot can be used in two ways: as a library in your go application, or as a comm To use Hugot as a library in your application, you will need the following two dependencies on your system: -- the tokenizers.a file obtained from the releases section of this page (if you want to use alternative architecture from `linux/amd64` you will have to build the tokenizers.a yourself, see [here](https://github.com/knights-analytics/tokenizers). This file should be at /usr/lib/tokenizers.a so that hugot can load it. +- the tokenizers.a file obtained from the releases section of this page (if you want to use alternative architecture from `linux/amd64` you will have to build the tokenizers.a yourself, see [here](https://github.com/knights-analytics/tokenizers). This file should be at /usr/lib/tokenizers.a so that hugot can load it. Alternatively, you can explicitly specify the path to the folder with the `libtokenizers.a` file using the `CGO_LDFLAGS` env variable, see the [dockerfile](./Dockerfile). - the onnxruntime.go file obtained from the releases section of this page (if you want to use alternative architectures from `linux/amd64` you will have to download it from [the onnxruntime releases page](https://github.com/microsoft/onnxruntime/releases/), see the [dockerfile](./Dockerfile) as an example). Hugot looks for this file at /usr/lib/onnxruntime.so or /usr/lib64/onnxruntime.so by default. A different location can be specified by passing the `WithOnnxLibraryPath()` option to `NewSession()`, e.g: ``` diff --git a/scripts/run-unit-tests.sh b/scripts/run-unit-tests.sh index eff8bcf..185e0ad 100755 --- a/scripts/run-unit-tests.sh +++ b/scripts/run-unit-tests.sh @@ -4,19 +4,23 @@ set -e # Directory of *this* script this_dir="$( cd "$( dirname "$0" )" && pwd )" -export src_dir="$(realpath "${this_dir}/..")" +src_dir="$(realpath "${this_dir}/..")" +export src_dir -export commit_hash=$(git rev-parse --short HEAD) -export test_folder="$src_dir/testTarget" -mkdir -p $test_folder -export host_uid=$(id -u "$USER") +commit_hash=$(git rev-parse --short HEAD) +export commit_hash +test_folder="$src_dir/testTarget" +export test_folder +mkdir -p "$test_folder" +host_uid=$(id -u "$USER") +export host_uid # build with compose -docker compose -f $src_dir/compose-test.yaml build +docker compose -f "$src_dir/compose-test.yaml" build echo "Running tests for commit hash: $commit_hash" -docker compose -f $src_dir/compose-test.yaml up && \ -docker compose -f $src_dir/compose-test.yaml logs --no-color >& $test_folder/logs.txt -docker compose -f $src_dir/compose-test.yaml rm -fsv +docker compose -f "$src_dir/compose-test.yaml" up && \ +docker compose -f "$src_dir/compose-test.yaml" logs --no-color >& "$test_folder/logs.txt" +docker compose -f "$src_dir/compose-test.yaml" rm -fsv echo "Extracting lib artifacts" docker build . --output "$src_dir/artifacts" --target artifacts