diff --git a/Core/CMakeLists.txt b/Core/CMakeLists.txt index c92e26d..dda4b48 100644 --- a/Core/CMakeLists.txt +++ b/Core/CMakeLists.txt @@ -34,3 +34,7 @@ add_executable(${PROJECT_NAME} if(APPLE) target_link_libraries(xMind PUBLIC "-framework CoreFoundation") endif() + +if(NOT (WIN32)) +target_link_libraries(${PROJECT_NAME} pthread dl) +endif() \ No newline at end of file diff --git a/Core/Common/utility.cpp b/Core/Common/utility.cpp index fb7c693..a3e418d 100644 --- a/Core/Common/utility.cpp +++ b/Core/Common/utility.cpp @@ -28,6 +28,7 @@ limitations under the License. #include #include #include +#include #if defined(__APPLE__) #include diff --git a/Core/Executor/PythonWorkerManager.h b/Core/Executor/PythonWorkerManager.h new file mode 100644 index 0000000..eed3378 --- /dev/null +++ b/Core/Executor/PythonWorkerManager.h @@ -0,0 +1,274 @@ +/* +Copyright (C) 2024 The XLang Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#include +#include +#endif + +class PythonWorkerManager { +public: + PythonWorkerManager() : monitoringThreadRunning(false) {} + + ~PythonWorkerManager() { + Shutdown(); + } + + // Adds a task and starts a Python worker with the given moduleId and pyfilePath + void AddTask(int moduleId, const std::string& pyfilePath) { + std::lock_guard lock(processMutex_); + + ProcessKey key = std::make_pair(moduleId, pyfilePath); + + // Add task to the tasks set + tasks_.insert(key); + + // If process is already running, do nothing + if (processes_.count(key)) { + return; + } + + ProcessHandle process = StartProcess(moduleId, pyfilePath); + if (!IsValidProcessHandle(process)) { + std::cerr << "Failed to start process for moduleId: " << moduleId << std::endl; + return; + } + + processes_[key] = process; + + // Start the monitoring thread if not already running + if (!monitoringThreadRunning) { + monitoringThreadRunning = true; + monitorThread_ = std::thread(&PythonWorkerManager::MonitorProcesses, this); + } + } + + // Removes a task and kills the corresponding process + void RemoveTask(int moduleId, const std::string& pyfilePath) { + std::lock_guard lock(processMutex_); + + ProcessKey key = std::make_pair(moduleId, pyfilePath); + + // Remove task from the tasks set + tasks_.erase(key); + + // If process is running, kill it + auto it = processes_.find(key); + if (it != processes_.end()) { + ProcessHandle process = it->second; + KillProcess(process); + CloseProcessHandle(process); + processes_.erase(it); + } + } + + // Shuts down all running Python worker instances + void Shutdown() { + { + std::lock_guard lock(processMutex_); + monitoringThreadRunning = false; + } + + // Wait for the monitoring thread to finish + if (monitorThread_.joinable()) { + monitorThread_.join(); + } + + // Terminate all running processes + std::lock_guard lock(processMutex_); + for (auto& entry : processes_) { + ProcessHandle process = entry.second; + KillProcess(process); + CloseProcessHandle(process); + } + processes_.clear(); + tasks_.clear(); + } + +private: + using ProcessKey = std::pair; + +#ifdef _WIN32 + using ProcessHandle = HANDLE; + static constexpr ProcessHandle INVALID_PROCESS_HANDLE = NULL; +#else + using ProcessHandle = pid_t; + static constexpr ProcessHandle INVALID_PROCESS_HANDLE = -1; +#endif + + // Monitors all running processes and restarts them if they crash + void MonitorProcesses() { + while (true) { + { + std::lock_guard lock(processMutex_); + if (!monitoringThreadRunning) { + break; + } + + for (auto it = processes_.begin(); it != processes_.end();) { + ProcessHandle process = it->second; + ProcessKey key = it->first; + + if (IsProcessRunning(process)) { + // Process is still running + ++it; + } + else { + // Process has terminated + CloseProcessHandle(process); + it = processes_.erase(it); // Remove from processes_ + + // Check if the task is still supposed to be running + if (tasks_.count(key)) { + // Restart the process + int moduleId = key.first; + std::string pyfilePath = key.second; + ProcessHandle newProcess = StartProcess(moduleId, pyfilePath); + if (IsValidProcessHandle(newProcess)) { + processes_[key] = newProcess; + } + else { + std::cerr << "Failed to restart process for moduleId: " << moduleId << std::endl; + } + } + // Else, task has been removed; do not restart + } + } + } + // Sleep before next check + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + } + + // Helper function to start a process + ProcessHandle StartProcess(int moduleId, const std::string& pyfilePath) { +#ifdef _WIN32 + STARTUPINFOA si; + PROCESS_INFORMATION pi; + ZeroMemory(&si, sizeof(si)); + si.cb = sizeof(si); + ZeroMemory(&pi, sizeof(pi)); + + // Build the command line: python pyfilePath moduleId=moduleId + std::string cmd = "python \"" + pyfilePath + "\" moduleId=" + std::to_string(moduleId); + + // Create the child process. + if (!CreateProcessA(NULL, // No module name (use command line) + cmd.data(), // Command line + NULL, // Process handle not inheritable + NULL, // Thread handle not inheritable + FALSE, // Set handle inheritance to FALSE + 0, // No creation flags + NULL, // Use parent's environment block + NULL, // Use parent's starting directory + &si, // Pointer to STARTUPINFO structure + &pi) // Pointer to PROCESS_INFORMATION structure + ) + { + // Failed to create process + return INVALID_PROCESS_HANDLE; + } + + // Close thread handle; we don't need it + CloseHandle(pi.hThread); + + return pi.hProcess; +#else + pid_t pid = fork(); + if (pid == -1) { + // Fork failed + return INVALID_PROCESS_HANDLE; + } + else if (pid == 0) { + // Child process: execute the Python script + execlp("python", "python", pyfilePath.c_str(), ("moduleId=" + std::to_string(moduleId)).c_str(), (char*)NULL); + // If execlp returns, execution failed + std::cerr << "Failed to execute Python script: " << pyfilePath << std::endl; + exit(EXIT_FAILURE); + } + else { + // Parent process: return the child PID + return pid; + } +#endif + } + + // Helper function to check if a process is running + bool IsProcessRunning(ProcessHandle process) { +#ifdef _WIN32 + DWORD exitCode; + if (GetExitCodeProcess(process, &exitCode)) { + return (exitCode == STILL_ACTIVE); + } + return false; +#else + int status; + pid_t result = waitpid(process, &status, WNOHANG); + if (result == 0) { + // Process is still running + return true; + } + return false; +#endif + } + + // Helper function to kill a process + void KillProcess(ProcessHandle process) { +#ifdef _WIN32 + TerminateProcess(process, 0); + WaitForSingleObject(process, INFINITE); +#else + kill(process, SIGTERM); // Send termination signal + waitpid(process, nullptr, 0); // Wait for process to terminate +#endif + } + + // Helper function to close process handle + void CloseProcessHandle(ProcessHandle process) { +#ifdef _WIN32 + CloseHandle(process); +#endif + } + + // Helper function to check if process handle is valid + bool IsValidProcessHandle(ProcessHandle process) { +#ifdef _WIN32 + return process != INVALID_HANDLE_VALUE && process != NULL; +#else + return process != -1; +#endif + } + + std::set tasks_; + std::map processes_; + std::mutex processMutex_; + std::thread monitorThread_; + bool monitoringThreadRunning; +}; diff --git a/Docker/Dockerfile b/Docker/Dockerfile new file mode 100644 index 0000000..a63957a --- /dev/null +++ b/Docker/Dockerfile @@ -0,0 +1,77 @@ +# Use Ubuntu 22.04 as the base image +FROM ubuntu:22.04 + +# Set non-interactive mode for apt-get +ENV DEBIAN_FRONTEND=noninteractive + +# Update package lists and install prerequisites +RUN apt-get update && apt-get install -y \ + software-properties-common \ + build-essential \ + wget \ + curl \ + git \ + ca-certificates \ + cmake \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + libncurses5-dev \ + libgdbm-dev \ + libnss3-dev \ + libffi-dev \ + liblzma-dev \ + tk-dev \ + uuid-dev \ + xz-utils + +# Add the deadsnakes PPA to install Python 3.12 +RUN add-apt-repository ppa:deadsnakes/ppa -y && \ + apt-get update && \ + apt-get install -y python3.12 python3.12-dev python3.12-venv python3-pip + +# Verify OpenSSL version (should be >= 3.0) +RUN openssl version + +# Install GCC 13 and G++ 13 from the Ubuntu Toolchain PPA +RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ + apt-get update && \ + apt-get install -y gcc-13 g++-13 && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 + +# Set Python 3.12 as the default python3 and ensure pip is installed +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 && \ + python3.12 -m ensurepip --upgrade && \ + python3.12 -m pip install --upgrade pip + +# Install NumPy using pip +RUN python3.12 -m pip install numpy + +# Set up Python environment variables for CMake to find Python 3.12 +ENV PYTHON_EXECUTABLE=/usr/bin/python3.12 +ENV PYTHON_INCLUDE_DIR=/usr/include/python3.12 +ENV PYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython3.12.so + +# Clone the xMind repository +RUN git clone -b shawn-framework https://github.com/xlang-foundation/xMind.git + +# Clone xlang into xMind/ThirdParty +RUN git clone -b Jit https://github.com/xlang-foundation/xlang.git xMind/ThirdParty/xlang + +# Set working directory to xMind +WORKDIR /xMind + +# Build the project +RUN mkdir build && cd build && cmake .. && make -j$(nproc) + +# Copy the install script to build/bin, make it executable, and run it +WORKDIR /xMind/build/bin +RUN cp /xMind/ThirdParty/xlang/PyEng/install_py_xlang.sh . && \ + chmod +x install_py_xlang.sh && \ + ./install_py_xlang.sh + +# Set entrypoint (optional) +CMD ["/bin/bash"] diff --git a/Examples/PySimple/simple.py b/Examples/PySimple/simple.py index 8b49b9d..d5e816a 100644 --- a/Examples/PySimple/simple.py +++ b/Examples/PySimple/simple.py @@ -14,7 +14,9 @@ # limitations under the License. # import sys,os -sys.path.append(os.path.abspath('../../Scripts')) +script_dir = os.path.dirname(os.path.abspath(__file__)) +normalized_path = os.path.normpath(os.path.join(script_dir, '../../Scripts')) +sys.path.append(normalized_path) from xMind import xMind import time diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt index a3700f6..54230f0 100644 --- a/cli/CMakeLists.txt +++ b/cli/CMakeLists.txt @@ -21,4 +21,6 @@ add_executable (${PROJECT_NAME} ${AppEntry_SRC} ) - \ No newline at end of file +if(NOT (WIN32)) +target_link_libraries(${PROJECT_NAME} pthread dl) +endif() \ No newline at end of file