From aa6bf9d29318bcb46998c5f972f2e587ab72b48d Mon Sep 17 00:00:00 2001 From: Cromefire_ Date: Sat, 25 Nov 2023 21:29:37 +0100 Subject: [PATCH 1/8] Added build configurations for Intel and AMD hardware --- Dockerfile | 5 ++- crates/llama-cpp-bindings/Cargo.toml | 2 + crates/llama-cpp-bindings/build.rs | 18 ++++++++ crates/tabby/Cargo.toml | 2 + oneapi.Dockerfile | 61 ++++++++++++++++++++++++++++ rocm.Dockerfile | 61 ++++++++++++++++++++++++++++ 6 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 oneapi.Dockerfile create mode 100644 rocm.Dockerfile diff --git a/Dockerfile b/Dockerfile index 69ac755a59fa..170fa641420f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ARG BASE_CUDA_RUN_CONTAINER=nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_V FROM ${BASE_CUDA_DEV_CONTAINER} as build # Rust toolchain version -ARG RUST_TOOLCHAIN stable +ARG RUST_TOOLCHAIN=stable ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ @@ -29,12 +29,13 @@ RUN curl https://sh.rustup.rs -sSf | bash -s -- --default-toolchain ${RUST_TOOLC ENV PATH="/root/.cargo/bin:${PATH}" WORKDIR /root/workspace -COPY . . RUN mkdir -p /opt/tabby/bin RUN mkdir -p /opt/tabby/lib RUN mkdir -p target +COPY . . + RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/root/workspace/target \ cargo build --features cuda --release --package tabby && \ diff --git a/crates/llama-cpp-bindings/Cargo.toml b/crates/llama-cpp-bindings/Cargo.toml index 5dbf46f3019f..5cccfe8ff3e9 100644 --- a/crates/llama-cpp-bindings/Cargo.toml +++ b/crates/llama-cpp-bindings/Cargo.toml @@ -5,6 +5,8 @@ edition = "2021" [features] cuda = [] +rocm = [] +oneapi = [] [build-dependencies] cxx-build = "1.0" diff --git a/crates/llama-cpp-bindings/build.rs b/crates/llama-cpp-bindings/build.rs index 488df009664f..59266f632b8c 100644 --- a/crates/llama-cpp-bindings/build.rs +++ b/crates/llama-cpp-bindings/build.rs @@ -31,6 +31,24 @@ fn main() { println!("cargo:rustc-link-lib=cublas"); println!("cargo:rustc-link-lib=cublasLt"); } + if cfg!(feature = "rocm") { + config.define("LLAMA_HIPBLAS", "ON"); + config.define("CMAKE_C_COMPILER", "/opt/rocm/llvm/bin/clang"); + config.define("CMAKE_CXX_COMPILER", "/opt/rocm/llvm/bin/clang++"); + println!("cargo:rustc-link-search=native=/opt/rocm/hipblas/lib"); + println!("cargo:rustc-link-search=native=/opt/rocm/hipblaslt/lib"); + println!("cargo:rustc-link-lib=hipblas"); + println!("cargo:rustc-link-lib=hipblaslt"); + } + if cfg!(feature = "oneapi") { + config.define("LLAMA_BLAS", "ON"); + config.define("LLAMA_BLAS_VENDOR", "Intel10_64lp"); + println!("cargo:rustc-link-search=native=/opt/intel/oneapi/mkl/latest/lib"); + println!("cargo:rustc-link-lib=mkl_rt"); + println!("cargo:rustc-link-lib=pthread"); + println!("cargo:rustc-link-lib=m"); + println!("cargo:rustc-link-lib=dl"); + } let dst = config.build(); println!("cargo:rustc-link-search=native={}/build", dst.display()); diff --git a/crates/tabby/Cargo.toml b/crates/tabby/Cargo.toml index 2c804e36f487..ce9835523e89 100644 --- a/crates/tabby/Cargo.toml +++ b/crates/tabby/Cargo.toml @@ -7,6 +7,8 @@ edition = "2021" default = ["ee"] ee = ["dep:tabby-webserver"] cuda = ["llama-cpp-bindings/cuda"] +rocm = ["llama-cpp-bindings/rocm"] +oneapi = ["llama-cpp-bindings/oneapi"] experimental-http = ["dep:http-api-bindings"] [dependencies] diff --git a/oneapi.Dockerfile b/oneapi.Dockerfile new file mode 100644 index 000000000000..0ded05d3822d --- /dev/null +++ b/oneapi.Dockerfile @@ -0,0 +1,61 @@ +ARG UBUNTU_VERSION=22.04 +# This needs to generally match the container host's environment. +ARG ONEAPI_VERSION=2024.0.0 +# Target the CUDA build image +ARG BASE_ONEAPI_DEV_CONTAINER="intel/oneapi-basekit:${ONEAPI_VERSION}-devel-ubuntu${UBUNTU_VERSION}" +# Target the CUDA runtime image +ARG BASE_ONEAPI_RUN_CONTAINER="intel/oneapi-runtime:${ONEAPI_VERSION}-devel-ubuntu${UBUNTU_VERSION}" + +FROM ${BASE_ONEAPI_DEV_CONTAINER} as build + +# Rust toolchain version +ARG RUST_TOOLCHAIN=stable + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + pkg-config \ + libssl-dev \ + protobuf-compiler \ + git \ + cmake \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# setup rust. +RUN curl https://sh.rustup.rs -sSf | bash -s -- --default-toolchain ${RUST_TOOLCHAIN} -y +ENV PATH="/root/.cargo/bin:${PATH}" + +WORKDIR /root/workspace + +RUN mkdir -p /opt/tabby/bin +RUN mkdir -p /opt/tabby/lib +RUN mkdir -p target + +COPY . . + +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/root/workspace/target \ + cargo build --features oneapi --release --package tabby && \ + cp target/release/tabby /opt/tabby/bin/ + +FROM ${BASE_ONEAPI_RUN_CONTAINER} as runtime + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Disable safe directory in docker +# Context: https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9 +RUN git config --system --add safe.directory "*" + +COPY --from=build /opt/tabby /opt/tabby + +ENV TABBY_ROOT=/data + +ENTRYPOINT ["/opt/tabby/bin/tabby"] diff --git a/rocm.Dockerfile b/rocm.Dockerfile new file mode 100644 index 000000000000..f5314376958a --- /dev/null +++ b/rocm.Dockerfile @@ -0,0 +1,61 @@ +ARG UBUNTU_VERSION=22.04 +# This needs to generally match the container host's environment. +ARG ROCM_VERSION=5.7 +# Target the CUDA build image +ARG BASE_ROCM_DEV_CONTAINER="rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}" +# Target the CUDA runtime image +ARG BASE_ROCM_RUN_CONTAINER="rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}" + +FROM ${BASE_ROCM_DEV_CONTAINER} as build + +# Rust toolchain version +ARG RUST_TOOLCHAIN=stable + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + pkg-config \ + libssl-dev \ + protobuf-compiler \ + git \ + cmake \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# setup rust. +RUN curl https://sh.rustup.rs -sSf | bash -s -- --default-toolchain ${RUST_TOOLCHAIN} -y +ENV PATH="/root/.cargo/bin:${PATH}" + +WORKDIR /root/workspace + +RUN mkdir -p /opt/tabby/bin +RUN mkdir -p /opt/tabby/lib +RUN mkdir -p target + +COPY . . + +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/root/workspace/target \ + cargo build --features rocm --release --package tabby && \ + cp target/release/tabby /opt/tabby/bin/ + +FROM ${BASE_ROCM_RUN_CONTAINER} as runtime + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Disable safe directory in docker +# Context: https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9 +RUN git config --system --add safe.directory "*" + +COPY --from=build /opt/tabby /opt/tabby + +ENV TABBY_ROOT=/data + +ENTRYPOINT ["/opt/tabby/bin/tabby"] From 5732d7ddf0916c6bc15f59cbfe59d8b1f9cb9731 Mon Sep 17 00:00:00 2001 From: Cromefire_ Date: Sat, 25 Nov 2023 21:55:33 +0100 Subject: [PATCH 2/8] Improved rocm build --- .dockerignore | 4 ++++ crates/llama-cpp-bindings/build.rs | 3 +-- rocm.Dockerfile | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index de70e0d16772..dc662c389cfd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,6 @@ +ci +clients +.github +python **/target **/node_modules diff --git a/crates/llama-cpp-bindings/build.rs b/crates/llama-cpp-bindings/build.rs index 59266f632b8c..431e388615bd 100644 --- a/crates/llama-cpp-bindings/build.rs +++ b/crates/llama-cpp-bindings/build.rs @@ -35,10 +35,9 @@ fn main() { config.define("LLAMA_HIPBLAS", "ON"); config.define("CMAKE_C_COMPILER", "/opt/rocm/llvm/bin/clang"); config.define("CMAKE_CXX_COMPILER", "/opt/rocm/llvm/bin/clang++"); + println!("cargo:rustc-link-arg=-Wl,--copy-dt-needed-entries"); println!("cargo:rustc-link-search=native=/opt/rocm/hipblas/lib"); - println!("cargo:rustc-link-search=native=/opt/rocm/hipblaslt/lib"); println!("cargo:rustc-link-lib=hipblas"); - println!("cargo:rustc-link-lib=hipblaslt"); } if cfg!(feature = "oneapi") { config.define("LLAMA_BLAS", "ON"); diff --git a/rocm.Dockerfile b/rocm.Dockerfile index f5314376958a..87ccd51acf11 100644 --- a/rocm.Dockerfile +++ b/rocm.Dockerfile @@ -2,9 +2,9 @@ ARG UBUNTU_VERSION=22.04 # This needs to generally match the container host's environment. ARG ROCM_VERSION=5.7 # Target the CUDA build image -ARG BASE_ROCM_DEV_CONTAINER="rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}" +ARG BASE_ROCM_DEV_CONTAINER="rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}-complete" # Target the CUDA runtime image -ARG BASE_ROCM_RUN_CONTAINER="rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}" +ARG BASE_ROCM_RUN_CONTAINER="rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}-complete" FROM ${BASE_ROCM_DEV_CONTAINER} as build From 1e05350d71868c911c34fa5d1d4eebd790405f81 Mon Sep 17 00:00:00 2001 From: Cromefire_ Date: Sat, 25 Nov 2023 22:48:26 +0100 Subject: [PATCH 3/8] Added options for OneAPI and ROCm --- crates/tabby/src/main.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/tabby/src/main.rs b/crates/tabby/src/main.rs index fe7c4a4ad7b0..a4d6fc1c6a8c 100644 --- a/crates/tabby/src/main.rs +++ b/crates/tabby/src/main.rs @@ -69,6 +69,14 @@ pub enum Device { #[strum(serialize = "cuda")] Cuda, + #[cfg(feature = "rocm")] + #[strum(serialize = "rocm")] + Rocm, + + #[cfg(feature = "oneapi")] + #[strum(serialize = "oneapi")] + OneApi, + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] #[strum(serialize = "metal")] Metal, @@ -89,7 +97,17 @@ impl Device { *self == Device::Cuda } - #[cfg(not(any(all(target_os = "macos", target_arch = "aarch64"), feature = "cuda")))] + #[cfg(feature = "rocm")] + pub fn ggml_use_gpu(&self) -> bool { + *self == Device::Rocm + } + + #[cfg(feature = "oneapi")] + pub fn ggml_use_gpu(&self) -> bool { + *self == Device::OneApi + } + + #[cfg(not(any(all(target_os = "macos", target_arch = "aarch64"), feature = "cuda", feature = "rocm", feature = "oneapi")))] pub fn ggml_use_gpu(&self) -> bool { false } From 3496bb9b863758ee1eb2d32a7ce56fef3a1c5fe1 Mon Sep 17 00:00:00 2001 From: Cromefire_ Date: Sun, 26 Nov 2023 01:47:01 +0100 Subject: [PATCH 4/8] Build llama using icx --- crates/llama-cpp-bindings/build.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/crates/llama-cpp-bindings/build.rs b/crates/llama-cpp-bindings/build.rs index 431e388615bd..e79b1875e7a8 100644 --- a/crates/llama-cpp-bindings/build.rs +++ b/crates/llama-cpp-bindings/build.rs @@ -1,3 +1,4 @@ +use std::env::var; use std::path::Path; use cmake::Config; @@ -40,10 +41,29 @@ fn main() { println!("cargo:rustc-link-lib=hipblas"); } if cfg!(feature = "oneapi") { + let mkl_root = var("MKLROOT").expect("MKLROOT needs to be defined to compile for oneAPI (use setvars.sh to set)"); + let compiler_root = var("CMPLR_ROOT").expect("CMPLR_ROOT needs to be defined to compile for oneAPI (use setvars.sh to set)"); config.define("LLAMA_BLAS", "ON"); config.define("LLAMA_BLAS_VENDOR", "Intel10_64lp"); - println!("cargo:rustc-link-search=native=/opt/intel/oneapi/mkl/latest/lib"); - println!("cargo:rustc-link-lib=mkl_rt"); + config.define("C_FLAGS", "-fiopenmp -fopenmp-targets=spir64 -m64 -DMKL_ILP64"); + config.define("CXX_FLAGS", "-fiopenmp -fopenmp-targets=spir64 -m64 -DMKL_ILP64"); + config.define("CMAKE_C_COMPILER", format!("{}/bin/icx", compiler_root)); + config.define("CMAKE_CXX_COMPILER", format!("{}/bin/icpx", compiler_root)); + println!("cargo:rustc-link-arg=-fiopenmp"); + println!("cargo:rustc-link-arg=-fopenmp-targets=spir64"); + println!("cargo:rustc-link-arg=-fsycl"); + println!("cargo:rustc-link-arg=-Wl,--no-as-needed"); + println!("cargo:rustc-link-search=native={}/lib", compiler_root); + println!("cargo:rustc-link-search=native={}/lib", mkl_root); + println!("cargo:rustc-link-lib=intlc"); + println!("cargo:rustc-link-lib=svml"); + println!("cargo:rustc-link-lib=mkl_sycl_blas"); + println!("cargo:rustc-link-lib=mkl_intel_ilp64"); + println!("cargo:rustc-link-lib=mkl_intel_thread"); + println!("cargo:rustc-link-lib=mkl_core"); + println!("cargo:rustc-link-lib=iomp5"); + println!("cargo:rustc-link-lib=sycl"); + println!("cargo:rustc-link-lib=stdc++"); println!("cargo:rustc-link-lib=pthread"); println!("cargo:rustc-link-lib=m"); println!("cargo:rustc-link-lib=dl"); From 609847512c147e2591cb492cac1f2c6947cc4168 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Sun, 26 Nov 2023 02:49:49 +0000 Subject: [PATCH 5/8] [autofix.ci] apply automated fixes --- crates/llama-cpp-bindings/build.rs | 19 +++++++++++++------ crates/tabby/src/main.rs | 7 ++++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/crates/llama-cpp-bindings/build.rs b/crates/llama-cpp-bindings/build.rs index e79b1875e7a8..78b8a8c6bcf5 100644 --- a/crates/llama-cpp-bindings/build.rs +++ b/crates/llama-cpp-bindings/build.rs @@ -1,5 +1,4 @@ -use std::env::var; -use std::path::Path; +use std::{env::var, path::Path}; use cmake::Config; @@ -41,12 +40,20 @@ fn main() { println!("cargo:rustc-link-lib=hipblas"); } if cfg!(feature = "oneapi") { - let mkl_root = var("MKLROOT").expect("MKLROOT needs to be defined to compile for oneAPI (use setvars.sh to set)"); - let compiler_root = var("CMPLR_ROOT").expect("CMPLR_ROOT needs to be defined to compile for oneAPI (use setvars.sh to set)"); + let mkl_root = var("MKLROOT") + .expect("MKLROOT needs to be defined to compile for oneAPI (use setvars.sh to set)"); + let compiler_root = var("CMPLR_ROOT") + .expect("CMPLR_ROOT needs to be defined to compile for oneAPI (use setvars.sh to set)"); config.define("LLAMA_BLAS", "ON"); config.define("LLAMA_BLAS_VENDOR", "Intel10_64lp"); - config.define("C_FLAGS", "-fiopenmp -fopenmp-targets=spir64 -m64 -DMKL_ILP64"); - config.define("CXX_FLAGS", "-fiopenmp -fopenmp-targets=spir64 -m64 -DMKL_ILP64"); + config.define( + "C_FLAGS", + "-fiopenmp -fopenmp-targets=spir64 -m64 -DMKL_ILP64", + ); + config.define( + "CXX_FLAGS", + "-fiopenmp -fopenmp-targets=spir64 -m64 -DMKL_ILP64", + ); config.define("CMAKE_C_COMPILER", format!("{}/bin/icx", compiler_root)); config.define("CMAKE_CXX_COMPILER", format!("{}/bin/icpx", compiler_root)); println!("cargo:rustc-link-arg=-fiopenmp"); diff --git a/crates/tabby/src/main.rs b/crates/tabby/src/main.rs index a4d6fc1c6a8c..b626abed97ac 100644 --- a/crates/tabby/src/main.rs +++ b/crates/tabby/src/main.rs @@ -107,7 +107,12 @@ impl Device { *self == Device::OneApi } - #[cfg(not(any(all(target_os = "macos", target_arch = "aarch64"), feature = "cuda", feature = "rocm", feature = "oneapi")))] + #[cfg(not(any( + all(target_os = "macos", target_arch = "aarch64"), + feature = "cuda", + feature = "rocm", + feature = "oneapi" + )))] pub fn ggml_use_gpu(&self) -> bool { false } From 7d38fc49f3a5bc538319ad91f6f9b65268b8a52c Mon Sep 17 00:00:00 2001 From: Cromefire_ Date: Sun, 26 Nov 2023 03:59:59 +0100 Subject: [PATCH 6/8] Fixed rocm image --- .dockerignore | 1 + crates/llama-cpp-bindings/build.rs | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index dc662c389cfd..ca65e5c7b99b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +.idea ci clients .github diff --git a/crates/llama-cpp-bindings/build.rs b/crates/llama-cpp-bindings/build.rs index 78b8a8c6bcf5..4bdb5bea3991 100644 --- a/crates/llama-cpp-bindings/build.rs +++ b/crates/llama-cpp-bindings/build.rs @@ -32,11 +32,18 @@ fn main() { println!("cargo:rustc-link-lib=cublasLt"); } if cfg!(feature = "rocm") { + let rocm_root = "/opt/rocm"; + //config.generator("Ninja"); config.define("LLAMA_HIPBLAS", "ON"); - config.define("CMAKE_C_COMPILER", "/opt/rocm/llvm/bin/clang"); - config.define("CMAKE_CXX_COMPILER", "/opt/rocm/llvm/bin/clang++"); + config.define("CMAKE_C_COMPILER", format!("{}/llvm/bin/clang", rocm_root)); + config.define("CMAKE_CXX_COMPILER", format!("{}/llvm/bin/clang++", rocm_root)); + config.define("AMDGPU_TARGETS", "gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102"); println!("cargo:rustc-link-arg=-Wl,--copy-dt-needed-entries"); - println!("cargo:rustc-link-search=native=/opt/rocm/hipblas/lib"); + println!("cargo:rustc-link-search=native={}/hip/lib", rocm_root); + println!("cargo:rustc-link-search=native={}/rocblas/lib", rocm_root); + println!("cargo:rustc-link-search=native={}/hipblas/lib", rocm_root); + println!("cargo:rustc-link-lib=amdhip64"); + println!("cargo:rustc-link-lib=rocblas"); println!("cargo:rustc-link-lib=hipblas"); } if cfg!(feature = "oneapi") { @@ -70,7 +77,6 @@ fn main() { println!("cargo:rustc-link-lib=mkl_core"); println!("cargo:rustc-link-lib=iomp5"); println!("cargo:rustc-link-lib=sycl"); - println!("cargo:rustc-link-lib=stdc++"); println!("cargo:rustc-link-lib=pthread"); println!("cargo:rustc-link-lib=m"); println!("cargo:rustc-link-lib=dl"); From 4f90c98e6ab3fe3ab4d921ba610b42bc168f850a Mon Sep 17 00:00:00 2001 From: Cromefire_ Date: Sun, 26 Nov 2023 04:45:19 +0100 Subject: [PATCH 7/8] Build ROCm --- .github/workflows/docker.yml | 45 +++++++++++++++++++++++++++++++---- Dockerfile => cuda.Dockerfile | 0 2 files changed, 40 insertions(+), 5 deletions(-) rename Dockerfile => cuda.Dockerfile (100%) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1e7482119d18..58ee2b6b4a3b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -50,7 +50,10 @@ jobs: # Workaround: https://github.com/docker/build-push-action/issues/461 - name: Setup Docker buildx - uses: docker/setup-buildx-action@v2.0.0 + uses: docker/setup-buildx-action@v3.0.0 + with: + # Needed to support OCI annotations + version: v0.12.0 # Login against a Docker registry except on PR # https://github.com/docker/login-action @@ -78,7 +81,7 @@ jobs: - name: Docker meta id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5.0.0 with: # list of Docker images to use as base name for tags images: | @@ -91,17 +94,50 @@ jobs: type=schedule,pattern={{date 'YYYYMMDD'}} type=semver,pattern={{version}} + - name: Docker meta for ROCm + id: meta-rocm + uses: docker/metadata-action@v5.0.0 + with: + # list of Docker images to use as base name for tags + images: | + ghcr.io/${{ env.IMAGE_NAME }}/rocm + ${{ env.IMAGE_NAME }}-rocm + # generate Docker tags based on the following events/attributes + variant: rocm + tags: | + type=raw,value={{branch}}-{{sha}},enable=${{ startsWith(github.ref, 'refs/heads') }} + type=schedule,pattern=nightly + type=schedule,pattern={{date 'YYYYMMDD'}} + type=semver,pattern={{version}} + # Build and push Docker image with Buildx (don't push on PR) # https://github.com/docker/build-push-action - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v3.1.1 + uses: docker/build-push-action@v5.1.0 with: - file: Dockerfile + file: cuda.Dockerfile push: true context: . tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.labels }} + cache-from: ${{ steps.cache.outputs.cache-from }} + cache-to: ${{ steps.cache.outputs.cache-to }} + build-args: RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image for ROCm + id: build-and-push-rocm + uses: docker/build-push-action@v5.1.0 + with: + file: rocm.Dockerfile + push: true + context: . + tags: ${{ steps.meta-rocm.outputs.tags }} + labels: ${{ steps.meta-rocm.outputs.labels }} + annotations: ${{ steps.meta-rocm.outputs.labels }} cache-from: ${{ steps.cache.outputs.cache-from }} cache-to: ${{ steps.cache.outputs.cache-to }} build-args: RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} @@ -112,4 +148,3 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} repository: tabbyml/tabby - diff --git a/Dockerfile b/cuda.Dockerfile similarity index 100% rename from Dockerfile rename to cuda.Dockerfile From c0804cb96c9c05e742b23c2e115e0ed18a4441e4 Mon Sep 17 00:00:00 2001 From: Cromefire_ Date: Sun, 26 Nov 2023 19:46:10 +0100 Subject: [PATCH 8/8] Tried to adjust compile flags for SYCL --- crates/llama-cpp-bindings/build.rs | 62 +++++++++++++++++++++--------- crates/tabby/build.rs | 30 +++++++++++++++ oneapi.Dockerfile | 15 ++++++-- 3 files changed, 86 insertions(+), 21 deletions(-) diff --git a/crates/llama-cpp-bindings/build.rs b/crates/llama-cpp-bindings/build.rs index 4bdb5bea3991..66183d453e24 100644 --- a/crates/llama-cpp-bindings/build.rs +++ b/crates/llama-cpp-bindings/build.rs @@ -4,6 +4,20 @@ use cmake::Config; fn main() { const LLAMA_CMAKE_PATH: &str = "llama.cpp/CMakeLists.txt"; + let intel_compile_flags: Vec<&str> = vec![ + "-fsycl", + "-fsycl-targets=spir64_gen", + "-fiopenmp", + "-fopenmp-targets=spir64_gen", + "-m64", + "-DMKL_ILP64", + "-qopt-report=3", + "-O3", + "-Xs", + "-device skl", + //"-device *", + ]; + const AMDGPU_TARGETS: &str = "gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102"; assert!( Path::new(LLAMA_CMAKE_PATH).exists(), @@ -33,11 +47,13 @@ fn main() { } if cfg!(feature = "rocm") { let rocm_root = "/opt/rocm"; - //config.generator("Ninja"); config.define("LLAMA_HIPBLAS", "ON"); config.define("CMAKE_C_COMPILER", format!("{}/llvm/bin/clang", rocm_root)); - config.define("CMAKE_CXX_COMPILER", format!("{}/llvm/bin/clang++", rocm_root)); - config.define("AMDGPU_TARGETS", "gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx940;gfx941;gfx942;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102"); + config.define( + "CMAKE_CXX_COMPILER", + format!("{}/llvm/bin/clang++", rocm_root), + ); + config.define("AMDGPU_TARGETS", AMDGPU_TARGETS); println!("cargo:rustc-link-arg=-Wl,--copy-dt-needed-entries"); println!("cargo:rustc-link-search=native={}/hip/lib", rocm_root); println!("cargo:rustc-link-search=native={}/rocblas/lib", rocm_root); @@ -53,25 +69,25 @@ fn main() { .expect("CMPLR_ROOT needs to be defined to compile for oneAPI (use setvars.sh to set)"); config.define("LLAMA_BLAS", "ON"); config.define("LLAMA_BLAS_VENDOR", "Intel10_64lp"); - config.define( - "C_FLAGS", - "-fiopenmp -fopenmp-targets=spir64 -m64 -DMKL_ILP64", - ); - config.define( - "CXX_FLAGS", - "-fiopenmp -fopenmp-targets=spir64 -m64 -DMKL_ILP64", - ); + config.define("C_FLAGS", intel_compile_flags.join(" ")); + config.define("CXX_FLAGS", intel_compile_flags.join(" ")); config.define("CMAKE_C_COMPILER", format!("{}/bin/icx", compiler_root)); config.define("CMAKE_CXX_COMPILER", format!("{}/bin/icpx", compiler_root)); println!("cargo:rustc-link-arg=-fiopenmp"); - println!("cargo:rustc-link-arg=-fopenmp-targets=spir64"); + println!("cargo:rustc-link-arg=-fopenmp-targets=spir64_gen"); println!("cargo:rustc-link-arg=-fsycl"); println!("cargo:rustc-link-arg=-Wl,--no-as-needed"); println!("cargo:rustc-link-search=native={}/lib", compiler_root); println!("cargo:rustc-link-search=native={}/lib", mkl_root); - println!("cargo:rustc-link-lib=intlc"); println!("cargo:rustc-link-lib=svml"); println!("cargo:rustc-link-lib=mkl_sycl_blas"); + println!("cargo:rustc-link-lib=mkl_sycl_lapack"); + println!("cargo:rustc-link-lib=mkl_sycl_dft"); + println!("cargo:rustc-link-lib=mkl_sycl_sparse"); + println!("cargo:rustc-link-lib=mkl_sycl_vm"); + println!("cargo:rustc-link-lib=mkl_sycl_rng"); + println!("cargo:rustc-link-lib=mkl_sycl_stats"); + println!("cargo:rustc-link-lib=mkl_sycl_data_fitting"); println!("cargo:rustc-link-lib=mkl_intel_ilp64"); println!("cargo:rustc-link-lib=mkl_intel_thread"); println!("cargo:rustc-link-lib=mkl_core"); @@ -85,10 +101,20 @@ fn main() { let dst = config.build(); println!("cargo:rustc-link-search=native={}/build", dst.display()); - cxx_build::bridge("src/lib.rs") + let crate_dir = var("CARGO_MANIFEST_DIR").unwrap(); + + let mut build = cxx_build::bridge("src/lib.rs"); + if cfg!(feature = "oneapi") { + let compiler_root = var("CMPLR_ROOT").unwrap(); + build.compiler(format!("{}/bin/icpx", compiler_root)); + for flag in intel_compile_flags { + build.flag(flag); + } + } + build .file("src/engine.cc") - .flag_if_supported("-Iinclude") - .flag_if_supported("-Illama.cpp") - .flag_if_supported("-std=c++14") - .compile("cxxbridge"); + .include(format!("{}/include", crate_dir)) + .include("llama.cpp") + .flag_if_supported("-std=c++14"); + build.compile("cxxbridge"); } diff --git a/crates/tabby/build.rs b/crates/tabby/build.rs index dc138d230671..4c876d1429f3 100644 --- a/crates/tabby/build.rs +++ b/crates/tabby/build.rs @@ -1,8 +1,38 @@ +use std::env; use std::error::Error; use vergen::EmitBuilder; fn main() -> Result<(), Box> { + if cfg!(feature = "oneapi") { + let mkl_root = env::var("MKLROOT") + .expect("MKLROOT needs to be defined to compile for oneAPI (use setvars.sh to set)"); + let compiler_root = env::var("CMPLR_ROOT") + .expect("CMPLR_ROOT needs to be defined to compile for oneAPI (use setvars.sh to set)"); + println!("cargo:rustc-link-arg=-fiopenmp"); + println!("cargo:rustc-link-arg=-fopenmp-targets=spir64_gen"); + println!("cargo:rustc-link-arg=-fsycl"); + println!("cargo:rustc-link-arg=-Wl,--no-as-needed"); + println!("cargo:rustc-link-search=native={}/lib", compiler_root); + println!("cargo:rustc-link-search=native={}/lib", mkl_root); + println!("cargo:rustc-link-lib=svml"); + println!("cargo:rustc-link-lib=mkl_sycl_blas"); + println!("cargo:rustc-link-lib=mkl_sycl_lapack"); + println!("cargo:rustc-link-lib=mkl_sycl_dft"); + println!("cargo:rustc-link-lib=mkl_sycl_sparse"); + println!("cargo:rustc-link-lib=mkl_sycl_vm"); + println!("cargo:rustc-link-lib=mkl_sycl_rng"); + println!("cargo:rustc-link-lib=mkl_sycl_stats"); + println!("cargo:rustc-link-lib=mkl_sycl_data_fitting"); + println!("cargo:rustc-link-lib=mkl_intel_ilp64"); + println!("cargo:rustc-link-lib=mkl_intel_thread"); + println!("cargo:rustc-link-lib=mkl_core"); + println!("cargo:rustc-link-lib=iomp5"); + println!("cargo:rustc-link-lib=sycl"); + println!("cargo:rustc-link-lib=pthread"); + println!("cargo:rustc-link-lib=m"); + println!("cargo:rustc-link-lib=dl"); + } // touch EmitBuilder::builder() .all_build() diff --git a/oneapi.Dockerfile b/oneapi.Dockerfile index 0ded05d3822d..263be55489d8 100644 --- a/oneapi.Dockerfile +++ b/oneapi.Dockerfile @@ -20,6 +20,7 @@ RUN apt-get update && \ protobuf-compiler \ git \ cmake \ + intel-opencl-icd \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -28,6 +29,12 @@ RUN apt-get update && \ RUN curl https://sh.rustup.rs -sSf | bash -s -- --default-toolchain ${RUST_TOOLCHAIN} -y ENV PATH="/root/.cargo/bin:${PATH}" +RUN update-alternatives --install "/usr/bin/cc" "cc" "${CMPLR_ROOT}/bin/icx" 100 +RUN update-alternatives --install "/usr/bin/c++" "c++" "${CMPLR_ROOT}/bin/icpx" 100 +RUN update-alternatives --install "/usr/bin/ld" "ld" "${CMPLR_ROOT}/bin/compiler/ld.lld" 100 +ENV RUSTFLAGS="-C link-args=-fPIC -C link-args=-fsycl -C link-args=-L${CMPLR_ROOT}/lib -C link-args=-lsycl -C link-args=-lintlc" +ENV OPENSSL_NO_VENDOR=1 + WORKDIR /root/workspace RUN mkdir -p /opt/tabby/bin @@ -36,9 +43,11 @@ RUN mkdir -p target COPY . . -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/root/workspace/target \ - cargo build --features oneapi --release --package tabby && \ +#RUN --mount=type=cache,target=/usr/local/cargo/registry \ +# --mount=type=cache,target=/root/workspace/target \ +# cargo build --features oneapi --release --package tabby && \ +# cp target/release/tabby /opt/tabby/bin/ +RUN cargo build --features oneapi --release --package tabby && \ cp target/release/tabby /opt/tabby/bin/ FROM ${BASE_ONEAPI_RUN_CONTAINER} as runtime