-
Notifications
You must be signed in to change notification settings - Fork 125
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Skip rebuild from scratch after cmake is ran
By commiting the patch changes, this commit prevents the libraries to be built from scratch every time cmake is ran. This will save additional build time. In addition to this, this commit separates out the different functionality to make the build system more readable. We can continue to iterate on this in the future. Signed-off-by: John Mazanec <[email protected]>
- Loading branch information
1 parent
a2bb6cc
commit 9724245
Showing
9 changed files
with
144 additions
and
160 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,21 +37,10 @@ jobs: | |
| uses: actions/checkout@v1 | ||
| with: | ||
| submodules: true | ||
|
|
||
| # Git functionality in CMAKE file does not work with given ubuntu image. Therefore, handling it here. | ||
| - name: Apply Git Patch | ||
| # Deleting file at the end to skip `git apply` inside CMAKE file | ||
| - name: Setup default git user for patches | ||
| run: | | ||
| cd jni/external/faiss | ||
| git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch | ||
| rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch | ||
| git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch | ||
| rm ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch | ||
| cd ../nmslib | ||
| git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch | ||
| rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch | ||
| working-directory: ${{ github.workspace }} | ||
|
|
||
| git config user.name "opensearch-ci" | ||
| git config user.email "[email protected]" | ||
| - name: Setup Java ${{ matrix.java }} | ||
| uses: actions/setup-java@v1 | ||
| with: | ||
|
|
@@ -60,6 +49,8 @@ jobs: | |
| - name: Run build | ||
| # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. | ||
| run: | | ||
| git config -l | grep "user.name" | ||
| git config -l | grep "user.email" | ||
| chown -R 1000:1000 `pwd` | ||
| if lscpu | grep -i avx2 | ||
| then | ||
|
|
@@ -88,7 +79,10 @@ jobs: | |
| steps: | ||
| - name: Checkout k-NN | ||
| uses: actions/checkout@v1 | ||
|
|
||
| - name: Setup default git user for patches | ||
| run: | | ||
| git config --global user.name "opensearch-ci" | ||
| git config --global user.email "[email protected]" | ||
| - name: Setup Java ${{ matrix.java }} | ||
| uses: actions/setup-java@v1 | ||
| with: | ||
|
|
@@ -122,7 +116,10 @@ jobs: | |
| steps: | ||
| - name: Checkout k-NN | ||
| uses: actions/checkout@v1 | ||
|
|
||
| - name: Setup default git user for patches | ||
| run: | | ||
| git config --global user.name "opensearch-ci" | ||
| git config --global user.email "[email protected]" | ||
| - name: Setup Java ${{ matrix.java }} | ||
| uses: actions/setup-java@v1 | ||
| with: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -35,7 +35,10 @@ jobs: | |
| steps: | ||
| - name: Checkout k-NN | ||
| uses: actions/checkout@v1 | ||
|
|
||
| - name: Setup default git user for patches | ||
| run: | | ||
| git config --global user.name "opensearch-ci" | ||
| git config --global user.email "[email protected]" | ||
| - name: Setup Java ${{ matrix.java }} | ||
| uses: actions/setup-java@v1 | ||
| with: | ||
|
|
@@ -99,7 +102,10 @@ jobs: | |
| steps: | ||
| - name: Checkout k-NN | ||
| uses: actions/checkout@v1 | ||
|
|
||
| - name: Setup default git user for patches | ||
| run: | | ||
| git config --global user.name "opensearch-ci" | ||
| git config --global user.email "[email protected]" | ||
| - name: Setup Java ${{ matrix.java }} | ||
| uses: actions/setup-java@v1 | ||
| with: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,21 +37,10 @@ jobs: | |
| uses: actions/checkout@v1 | ||
| with: | ||
| submodules: true | ||
|
|
||
| # Git functionality in CMAKE file does not work with given ubuntu image. Therefore, handling it here. | ||
| - name: Apply Git Patch | ||
| # Deleting file at the end to skip `git apply` inside CMAKE file | ||
| - name: Setup default git user for patches | ||
| run: | | ||
| cd jni/external/faiss | ||
| git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch | ||
| rm ../../patches/faiss/0001-Custom-patch-to-support-multi-vector.patch | ||
| git apply --ignore-space-change --ignore-whitespace --3way ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch | ||
| rm ../../patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch | ||
| cd ../nmslib | ||
| git apply --ignore-space-change --ignore-whitespace --3way ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch | ||
| rm ../../patches/nmslib/0001-Initialize-maxlevel-during-add-from-enterpoint-level.patch | ||
| working-directory: ${{ github.workspace }} | ||
|
|
||
| git config --global user.name "opensearch-ci" | ||
| git config --global user.email "[email protected]" | ||
| - name: Setup Java ${{ matrix.java }} | ||
| uses: actions/setup-java@v1 | ||
| with: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| # | ||
| # Copyright OpenSearch Contributors | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # | ||
|
|
||
| # Check if faiss exists | ||
| find_path(FAISS_REPO_DIR NAMES faiss PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss NO_DEFAULT_PATH) | ||
|
|
||
| # If not, pull the updated submodule | ||
| if (NOT EXISTS ${FAISS_REPO_DIR}) | ||
| message(STATUS "Could not find faiss. Pulling updated submodule.") | ||
| execute_process(COMMAND git submodule update --init -- external/faiss WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) | ||
| endif () | ||
|
|
||
| # Check if patch exist, this is to skip git apply during CI build. See CI.yml with ubuntu. | ||
| find_path(PATCH_FILE NAMES 0001-Custom-patch-to-support-multi-vector.patch 0002-Enable-precomp-table-to-be-shared-ivfpq.patch PATHS ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss NO_DEFAULT_PATH) | ||
|
|
||
| # If it exists, apply patches | ||
| if (EXISTS ${PATCH_FILE}) | ||
| message(STATUS "Applying custom patches.") | ||
| execute_process(COMMAND git am --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0001-Custom-patch-to-support-multi-vector.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) | ||
| execute_process(COMMAND git am --3way ${CMAKE_CURRENT_SOURCE_DIR}/patches/faiss/0002-Enable-precomp-table-to-be-shared-ivfpq.patch WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/external/faiss ERROR_VARIABLE ERROR_MSG RESULT_VARIABLE RESULT_CODE) | ||
| if(RESULT_CODE) | ||
| message(FATAL_ERROR "Failed to apply patch:\n${ERROR_MSG}") | ||
| endif() | ||
| endif() | ||
|
|
||
| if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin) | ||
| if(CMAKE_C_COMPILER_ID MATCHES "Clang\$") | ||
| set(OpenMP_C_FLAGS "-Xpreprocessor -fopenmp") | ||
| set(OpenMP_C_LIB_NAMES "omp") | ||
| set(OpenMP_omp_LIBRARY /usr/local/opt/libomp/lib/libomp.dylib) | ||
| endif() | ||
|
|
||
| if(CMAKE_CXX_COMPILER_ID MATCHES "Clang\$") | ||
| set(OpenMP_CXX_FLAGS "-Xpreprocessor -fopenmp -I/usr/local/opt/libomp/include") | ||
| set(OpenMP_CXX_LIB_NAMES "omp") | ||
| set(OpenMP_omp_LIBRARY /usr/local/opt/libomp/lib/libomp.dylib) | ||
| endif() | ||
| endif() | ||
|
|
||
| find_package(ZLIB REQUIRED) | ||
| find_package(BLAS REQUIRED) | ||
| enable_language(Fortran) | ||
| find_package(LAPACK REQUIRED) | ||
|
|
||
| # Set relevant properties | ||
| set(BUILD_TESTING OFF) # Avoid building faiss tests | ||
| set(BLA_STATIC ON) # Statically link BLAS | ||
| set(FAISS_ENABLE_GPU OFF) | ||
| set(FAISS_ENABLE_PYTHON OFF) | ||
|
|
||
| if(NOT DEFINED SIMD_ENABLED) | ||
| set(SIMD_ENABLED true) # set default value as true if the argument is not set | ||
| endif() | ||
|
|
||
| if(${CMAKE_SYSTEM_NAME} STREQUAL Windows OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64" OR ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm64" OR NOT ${SIMD_ENABLED}) | ||
| set(FAISS_OPT_LEVEL generic) # Keep optimization level as generic on Windows OS as it is not supported due to MINGW64 compiler issue. Also, on aarch64 avx2 is not supported. | ||
| set(TARGET_LINK_FAISS_LIB faiss) | ||
| else() | ||
| set(FAISS_OPT_LEVEL avx2) # Keep optimization level as avx2 to improve performance on Linux and Mac. | ||
| set(TARGET_LINK_FAISS_LIB faiss_avx2) | ||
| string(PREPEND LIB_EXT "_avx2") # Prepend "_avx2" to lib extension to create the library as "libopensearchknn_faiss_avx2.so" on linux and "libopensearchknn_faiss_avx2.jnilib" on mac | ||
| endif() | ||
|
|
||
| add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/faiss EXCLUDE_FROM_ALL) |
Oops, something went wrong.