From 6b3205988285aca3e301ea8fc2737669401058ff Mon Sep 17 00:00:00 2001 From: Derick M <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:04:58 -0400 Subject: [PATCH 1/6] refactor: tsp plugin --- .gitignore | 5 +- or-tools/install.sh | 28 ++++++--- or-tools/{ => src}/tsp/CMakeLists.txt | 4 +- or-tools/{tsp/tsp.cc => src/tsp/koji-tsp.cc} | 7 ++- or-tools/src/tsp/memory_limit.cc | 65 ++++++++++++++++++++ or-tools/src/tsp/memory_limit.h | 7 +++ 6 files changed, 100 insertions(+), 16 deletions(-) rename or-tools/{ => src}/tsp/CMakeLists.txt (97%) rename or-tools/{tsp/tsp.cc => src/tsp/koji-tsp.cc} (98%) create mode 100644 or-tools/src/tsp/memory_limit.cc create mode 100644 or-tools/src/tsp/memory_limit.h diff --git a/.gitignore b/.gitignore index 8bbae861..64b649bc 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,6 @@ server/algorithms/src/**/plugins/**/* .DS_Store # or-tools -or-tools/**/* -!or-tools/tsp -!or-tools/install.sh +or-tools/* +!or-tools/src or-tools.tar.gz diff --git a/or-tools/install.sh b/or-tools/install.sh index 905f1735..3ed4b5c8 100755 --- a/or-tools/install.sh +++ b/or-tools/install.sh @@ -2,6 +2,11 @@ os=$OSTYPE cpu=$(uname -p) +minor_version="9.10" +patch_version="4067" +full_version="${minor_version}.${patch_version}" + +echo Installing or-tools v${full_version} if [[ $PWD != *"or-tools"* ]]; then cd ./or-tools @@ -9,7 +14,7 @@ fi echo Found $os $cpu -if [ ! -d "source" ]; then +if [ ! -d "${full_version}" ]; then url="" if [[ "$os" == "linux-gnu"* ]]; then distro=$(cat /etc/os-release | grep "^ID=" | cut -d '=' -f 2) @@ -17,12 +22,12 @@ if [ ! -d "source" ]; then distro=${distro//\"/} version=${version//\"/} echo Found distro $distro $version - url="https://github.com/google/or-tools/releases/download/v9.5/or-tools_amd64_${distro}-${version}_cpp_v9.5.2237.tar.gz" + url="https://github.com/google/or-tools/releases/download/v${minor_version}/or-tools_amd64_${distro}-${version}_cpp_v${full_version}.tar.gz" elif [[ "$os" == "darwin"* ]]; then if [[ "$cpu" == "arm" ]]; then cpu="arm64" fi - url="https://github.com/google/or-tools/releases/download/v9.5/or-tools_${cpu}_macOS-13.0.1_cpp_v9.5.2237.tar.gz" + url="https://github.com/google/or-tools/releases/download/v${minor_version}/or-tools_${cpu}_macOS-14.4.1_cpp_v${full_version}.tar.gz" else echo "Unsupported OS, contact the developer" exit 1 @@ -30,12 +35,15 @@ if [ ! -d "source" ]; then echo Downloading $url curl -L $url -o ortools.tar.gz tar -xzf ortools.tar.gz - mv ./or-tools_* ./source + mv ./or-tools_* ./${full_version} rm ortools.tar.gz fi -cd source -mkdir examples/koji -cp ../tsp/tsp.cc ./examples/koji/koji.cc -cp ../tsp/CMakeLists.txt ./examples/koji/CMakeLists.txt -make build SOURCE=examples/koji/koji.cc -mv ./examples/koji/build/bin/koji ../../server/algorithms/src/routing/plugins/tsp + +# check if folder exists and remove if so +if [ -d "./${full_version}/examples/koji-tsp" ]; then + rm -r ./${full_version}/examples/koji-tsp +fi +cp -r ./src/tsp ./${full_version}/examples/koji-tsp +cd $full_version +make build SOURCE=examples/koji-tsp/koji-tsp.cc +mv ./examples/koji-tsp/build/bin/koji-tsp ../../server/algorithms/src/routing/plugins/tsp diff --git a/or-tools/tsp/CMakeLists.txt b/or-tools/src/tsp/CMakeLists.txt similarity index 97% rename from or-tools/tsp/CMakeLists.txt rename to or-tools/src/tsp/CMakeLists.txt index 132dbdc4..463db030 100644 --- a/or-tools/tsp/CMakeLists.txt +++ b/or-tools/src/tsp/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.18) option(CMAKE_EXPORT_COMPILE_COMMANDS "Export compile command" OFF) -project(koji VERSION 1.0 LANGUAGES CXX) +project(koji-tsp VERSION 1.0 LANGUAGES CXX) message(STATUS "${PROJECT_NAME} version: ${PROJECT_VERSION}") #message(STATUS "major: ${PROJECT_VERSION_MAJOR}") #message(STATUS "minor: ${PROJECT_VERSION_MINOR}") @@ -68,7 +68,7 @@ endif() find_package(ortools REQUIRED CONFIG) -add_executable(${PROJECT_NAME} koji.cc) +add_executable(${PROJECT_NAME} koji-tsp.cc memory_limit.cc) target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) #target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17) target_link_libraries(${PROJECT_NAME} PRIVATE ortools::ortools) diff --git a/or-tools/tsp/tsp.cc b/or-tools/src/tsp/koji-tsp.cc similarity index 98% rename from or-tools/tsp/tsp.cc rename to or-tools/src/tsp/koji-tsp.cc index 0de76326..fcf9f2fc 100644 --- a/or-tools/tsp/tsp.cc +++ b/or-tools/src/tsp/koji-tsp.cc @@ -13,6 +13,8 @@ #include "ortools/constraint_solver/routing_index_manager.h" #include "ortools/constraint_solver/routing_parameters.h" +#include "memory_limit.h" + typedef std::vector> DistanceMatrix; typedef std::vector> RawInput; typedef std::vector RawOutput; @@ -168,6 +170,9 @@ std::vector split(const std::string &s, char delimiter) int main(int argc, char *argv[]) { + set_memory_limit(); + // LOG(INFO) << "Memory Set To: " << mem_limit; + std::map args; RawInput points; std::vector stringPoints; @@ -207,4 +212,4 @@ int main(int argc, char *argv[]) } return EXIT_SUCCESS; -} \ No newline at end of file +} diff --git a/or-tools/src/tsp/memory_limit.cc b/or-tools/src/tsp/memory_limit.cc new file mode 100644 index 00000000..707f2020 --- /dev/null +++ b/or-tools/src/tsp/memory_limit.cc @@ -0,0 +1,65 @@ +#include "memory_limit.h" +#include +#include + +#if defined(_WIN32) || defined(_WIN64) +#include +#else +#include +#if defined(__APPLE__) +#include +#else +#include +#endif +#endif + +unsigned long get_total_memory() +{ +#if defined(_WIN32) || defined(_WIN64) + MEMORYSTATUSEX status; + status.dwLength = sizeof(status); + GlobalMemoryStatusEx(&status); + return status.ullTotalPhys; +#elif defined(__APPLE__) + int mib[2] = {CTL_HW, HW_MEMSIZE}; + unsigned long total_memory; + size_t length = sizeof(total_memory); + if (sysctl(mib, 2, &total_memory, &length, NULL, 0) != 0) + { + std::cerr << "Failed to get system memory size" << std::endl; + } + return total_memory; +#else // Linux + struct sysinfo info; + if (sysinfo(&info) != 0) + { + std::cerr << "Failed to get system info" << std::endl; + } + return info.totalram * info.mem_unit; +#endif +} + +void set_memory_limit() +{ + unsigned long total_memory = get_total_memory(); + unsigned long limit_50_percent = total_memory / 2; + unsigned long max_limit = 20UL * 1024 * 1024 * 1024; + unsigned long memory_limit = std::min(max_limit, limit_50_percent); + +#if defined(_WIN32) || defined(_WIN64) + SIZE_T minimumWorkingSetSize = memory_limit; + SIZE_T maximumWorkingSetSize = memory_limit; + if (!SetProcessWorkingSetSize(GetCurrentProcess(), minimumWorkingSetSize, maximumWorkingSetSize)) + { + std::cerr << "Failed to set memory limit" << std::endl; + } +#else + struct rlimit rl; + rl.rlim_cur = memory_limit; + rl.rlim_max = memory_limit; + if (setrlimit(RLIMIT_AS, &rl) != 0) + { + std::cerr << "Failed to set memory limit" << std::endl; + } +#endif +} diff --git a/or-tools/src/tsp/memory_limit.h b/or-tools/src/tsp/memory_limit.h new file mode 100644 index 00000000..5326c986 --- /dev/null +++ b/or-tools/src/tsp/memory_limit.h @@ -0,0 +1,7 @@ +#ifndef MEMORY_LIMIT_H +#define MEMORY_LIMIT_H + +unsigned long get_total_memory(); +void set_memory_limit(); + +#endif // MEMORY_LIMIT_H From a598ee0feb02ad6f56fd4ec18bc57f7ded485096 Mon Sep 17 00:00:00 2001 From: Derick M <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:18:03 -0400 Subject: [PATCH 2/6] fix: docker build --- Dockerfile | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9a471162..30ff6f36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,12 +21,5 @@ RUN apt install curl -y RUN apt install -y build-essential cmake lsb-release RUN mkdir -p /algorithms/src/routing/plugins COPY ./or-tools . -RUN curl -L https://github.com/google/or-tools/releases/download/v9.5/or-tools_amd64_debian-11_cpp_v9.5.2237.tar.gz -o ortools.tar.gz -RUN cat ortools.tar.gz | tar -xzf - && \ - cd or-tools_* && \ - mkdir examples/koji && \ - cp /tsp/tsp.cc ./examples/koji/koji.cc && \ - cp /tsp/CMakeLists.txt ./examples/koji/CMakeLists.txt && \ - make build SOURCE=examples/koji/koji.cc && \ - mv ./examples/koji/build/bin/koji /algorithms/src/routing/plugins/tsp +RUN ./or-tools/install.sh CMD koji From 619473cef3b67e2dce86079dd7436a8ffa0b8f19 Mon Sep 17 00:00:00 2001 From: Derick M <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:37:55 -0400 Subject: [PATCH 3/6] fix: more dockerfile fixes --- Dockerfile | 25 +++++++++++++------ or-tools/install.sh | 18 ++++++++----- or-tools/src/tsp/CMakeLists.txt | 7 ++---- or-tools/src/tsp/{koji-tsp.cc => koji_tsp.cc} | 0 4 files changed, 31 insertions(+), 19 deletions(-) rename or-tools/src/tsp/{koji-tsp.cc => koji_tsp.cc} (100%) diff --git a/Dockerfile b/Dockerfile index 30ff6f36..3482b817 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-alpine as client +FROM node:22-alpine as client WORKDIR /app COPY ./client . RUN yarn install @@ -11,15 +11,24 @@ COPY ./server . RUN apt-get update && apt-get install -y RUN cargo install --path . --locked -FROM debian:bullseye-slim +FROM debian:bullseye-slim as or-tools +RUN mkdir -p /algorithms/src/routing/plugins RUN apt-get update && apt-get install -y -RUN apt install -y build-essential cmake lsb-release +RUN apt install -y curl build-essential cmake lsb-release RUN ldconfig +COPY ./or-tools/src ./src +RUN curl -L https://github.com/google/or-tools/releases/download/v9.10/or-tools_amd64_debian-11_cpp_v9.10.4067.tar.gz -o ortools.tar.gz +RUN cat ortools.tar.gz | tar -xzf - && \ + mv or-tools_* source && \ + cd source && \ + mv /src/tsp/ ./examples/koji_tsp && \ + make build SOURCE=examples/koji_tsp/koji_tsp.cc && \ + mv ./examples/koji_tsp/build/bin/koji_tsp /algorithms/src/routing/plugins/tsp + +FROM debian:bullseye-slim +COPY --from=or-tools /algorithms . +COPY --from=or-tools /source . COPY --from=client /app/dist ./dist COPY --from=server /usr/local/cargo/bin/koji /usr/local/bin/koji -RUN apt install curl -y -RUN apt install -y build-essential cmake lsb-release -RUN mkdir -p /algorithms/src/routing/plugins -COPY ./or-tools . -RUN ./or-tools/install.sh + CMD koji diff --git a/or-tools/install.sh b/or-tools/install.sh index 3ed4b5c8..b221eb2d 100755 --- a/or-tools/install.sh +++ b/or-tools/install.sh @@ -1,5 +1,5 @@ #!/bin/bash - +# add output as the first arg os=$OSTYPE cpu=$(uname -p) minor_version="9.10" @@ -40,10 +40,16 @@ if [ ! -d "${full_version}" ]; then fi # check if folder exists and remove if so -if [ -d "./${full_version}/examples/koji-tsp" ]; then - rm -r ./${full_version}/examples/koji-tsp +if [ -d "./${full_version}/examples/koji_tsp" ]; then + rm -r ./${full_version}/examples/koji_tsp fi -cp -r ./src/tsp ./${full_version}/examples/koji-tsp +cp -r ./src/tsp ./${full_version}/examples/koji_tsp cd $full_version -make build SOURCE=examples/koji-tsp/koji-tsp.cc -mv ./examples/koji-tsp/build/bin/koji-tsp ../../server/algorithms/src/routing/plugins/tsp +make build SOURCE=examples/koji_tsp + +if [ $# -eq 0 ]; then + echo hi + # mv ./examples/koji_tsp/build/bin/koji_tsp ../../server/algorithms/src/routing/plugins/tsp + # else + # mv ./${full_version}/examples/koji_tsp/build/bin/koji_tsp /algorithms/src/routing/plugins/tsp +fi diff --git a/or-tools/src/tsp/CMakeLists.txt b/or-tools/src/tsp/CMakeLists.txt index 463db030..07d99398 100644 --- a/or-tools/src/tsp/CMakeLists.txt +++ b/or-tools/src/tsp/CMakeLists.txt @@ -3,11 +3,8 @@ cmake_minimum_required(VERSION 3.18) option(CMAKE_EXPORT_COMPILE_COMMANDS "Export compile command" OFF) -project(koji-tsp VERSION 1.0 LANGUAGES CXX) +project(koji_tsp VERSION 1.0 LANGUAGES CXX) message(STATUS "${PROJECT_NAME} version: ${PROJECT_VERSION}") -#message(STATUS "major: ${PROJECT_VERSION_MAJOR}") -#message(STATUS "minor: ${PROJECT_VERSION_MINOR}") -#message(STATUS "patch: ${PROJECT_VERSION_PATCH}") if(MSVC) set(CMAKE_CXX_STANDARD 20) @@ -68,7 +65,7 @@ endif() find_package(ortools REQUIRED CONFIG) -add_executable(${PROJECT_NAME} koji-tsp.cc memory_limit.cc) +add_executable(${PROJECT_NAME} koji_tsp.cc memory_limit.cc) target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) #target_compile_features(${PROJECT_NAME} PRIVATE cxx_std_17) target_link_libraries(${PROJECT_NAME} PRIVATE ortools::ortools) diff --git a/or-tools/src/tsp/koji-tsp.cc b/or-tools/src/tsp/koji_tsp.cc similarity index 100% rename from or-tools/src/tsp/koji-tsp.cc rename to or-tools/src/tsp/koji_tsp.cc From 7e7c69dd69c45ddd428620b3d8975ffcef21258c Mon Sep 17 00:00:00 2001 From: Derick M <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:58:20 -0400 Subject: [PATCH 4/6] Update Dockerfile --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3482b817..dd6a376d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,15 +19,15 @@ RUN ldconfig COPY ./or-tools/src ./src RUN curl -L https://github.com/google/or-tools/releases/download/v9.10/or-tools_amd64_debian-11_cpp_v9.10.4067.tar.gz -o ortools.tar.gz RUN cat ortools.tar.gz | tar -xzf - && \ - mv or-tools_* source && \ - cd source && \ + mv or-tools_* or-tools && \ + cd or-tools && \ mv /src/tsp/ ./examples/koji_tsp && \ make build SOURCE=examples/koji_tsp/koji_tsp.cc && \ mv ./examples/koji_tsp/build/bin/koji_tsp /algorithms/src/routing/plugins/tsp -FROM debian:bullseye-slim +FROM debian:bullseye-slim as runner COPY --from=or-tools /algorithms . -COPY --from=or-tools /source . +COPY --from=or-tools /or-tools . COPY --from=client /app/dist ./dist COPY --from=server /usr/local/cargo/bin/koji /usr/local/bin/koji From e187149e08ded48b5636d1b1f7cccbe1f37be577 Mon Sep 17 00:00:00 2001 From: Derick M <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 2 Jul 2024 13:09:19 -0400 Subject: [PATCH 5/6] Update Dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index dd6a376d..c42019ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,8 @@ RUN cat ortools.tar.gz | tar -xzf - && \ mv ./examples/koji_tsp/build/bin/koji_tsp /algorithms/src/routing/plugins/tsp FROM debian:bullseye-slim as runner -COPY --from=or-tools /algorithms . -COPY --from=or-tools /or-tools . +COPY --from=or-tools /algorithms ./algorithms +COPY --from=or-tools /or-tools ./or-tools COPY --from=client /app/dist ./dist COPY --from=server /usr/local/cargo/bin/koji /usr/local/bin/koji From dced337af4172fb55611b14103181440badac624 Mon Sep 17 00:00:00 2001 From: Derick M <58572875+TurtIeSocks@users.noreply.github.com> Date: Tue, 2 Jul 2024 13:16:05 -0400 Subject: [PATCH 6/6] Update install.sh --- or-tools/install.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/or-tools/install.sh b/or-tools/install.sh index b221eb2d..8b2884b2 100755 --- a/or-tools/install.sh +++ b/or-tools/install.sh @@ -1,5 +1,5 @@ #!/bin/bash -# add output as the first arg + os=$OSTYPE cpu=$(uname -p) minor_version="9.10" @@ -46,10 +46,4 @@ fi cp -r ./src/tsp ./${full_version}/examples/koji_tsp cd $full_version make build SOURCE=examples/koji_tsp - -if [ $# -eq 0 ]; then - echo hi - # mv ./examples/koji_tsp/build/bin/koji_tsp ../../server/algorithms/src/routing/plugins/tsp - # else - # mv ./${full_version}/examples/koji_tsp/build/bin/koji_tsp /algorithms/src/routing/plugins/tsp -fi +mv ./examples/koji_tsp/build/bin/koji_tsp ../../server/algorithms/src/routing/plugins/tsp