diff --git a/justfile b/justfile index 9096b7c0..f719d4e6 100644 --- a/justfile +++ b/justfile @@ -69,12 +69,41 @@ run-client name port="auto" server="http://localhost:5001": # --------------------------------------------------------------------------------------------------------------------- +# Run a local syftbox app command +[group('app')] +run-app name command subcommand="": + #!/bin/bash + set -eou pipefail + + # generate a local email from name, but if it looks like an email, then use it as is + EMAIL="{{ name }}@openmined.org" + if [[ "{{ name }}" == *@*.* ]]; then EMAIL="{{ name }}"; fi + + # Working directory for client is .clients/ + CONFIG_DIR=.clients/$EMAIL/config + SYNC_DIR=.clients/$EMAIL/sync + mkdir -p $CONFIG_DIR $SYNC_DIR + + echo -e "Config Dir : $CONFIG_DIR" + + uv run syftbox/main.py app {{ command }} {{ subcommand }} --config_path=$CONFIG_DIR/config.json + +# --------------------------------------------------------------------------------------------------------------------- + # Build syftbox wheel [group('build')] build: rm -rf dist uv build + +# Build syftbox wheel +[group('install')] +install: + rm -rf dist + uv build + uv tool install $(ls /Users/madhavajay/dev/syft/dist/*.whl) --reinstall + # Bump version, commit and tag [group('build')] bump-version level="patch": diff --git a/pyproject.toml b/pyproject.toml index 734d0b2f..f60b16d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,8 @@ dependencies = [ "pydantic-settings>=2.5.2", "pytest>=8.3.3", "httpx>=0.27.2", + "pyyaml>=6.0.2", + "psutil>=6.0.0", ] [project.optional-dependencies] diff --git a/syftbox/app/__init__.py b/syftbox/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/syftbox/app/install.py b/syftbox/app/install.py new file mode 100644 index 00000000..e9834460 --- /dev/null +++ b/syftbox/app/install.py @@ -0,0 +1,774 @@ +import argparse +import json +import os +import platform +import re +import shutil +import subprocess +from pathlib import Path +from tempfile import mkdtemp +from types import SimpleNamespace +from typing import Any, Tuple + +from syftbox.lib.lib import ClientConfig + + +def is_git_installed() -> bool: + """ + Checks if Git is installed on the system. + + Returns: + bool: `True` if Git is installed, `False` otherwise. + + Functionality: + - Runs the `git --version` command to check if Git is installed. + - If the command runs successfully, returns `True`. + - If the command fails (e.g., Git is not installed), returns `False`. + + Example: + ```python + if is_git_installed(): + print("Git is installed on this system.") + else: + print("Git is not installed. Please install Git to proceed.") + ``` + This will print a message indicating whether Git is installed or not. + """ + try: + subprocess.run( + ["git", "--version"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return True + except subprocess.CalledProcessError: + return False + + +def sanitize_git_path(path: str) -> str: + """ + Validates and sanitizes a Git repository path, ensuring it matches the required format. + + Args: + path (str): The Git repository path to validate. + + Returns: + str: The sanitized Git repository path if it matches the valid pattern. + + Raises: + ValueError: If the provided path does not match the expected format for a Git repository. + + Functionality: + - Uses a regular expression pattern to ensure that the given path follows the format `owner/repository`. + - If the path matches the pattern, returns it as a valid Git path. + - If the path does not match the pattern, raises a `ValueError` with a descriptive message. + + Example: + Suppose you have a GitHub path like `OpenMined/logged_in` and want to validate it: + ```python + try: + sanitized_path = sanitize_git_path("OpenMined/logged_in") + except ValueError as e: + print(e) + ``` + If the path is valid, `sanitized_path` will contain the validated GitHub path. If it is not valid, the error message + "Invalid Git repository path format. (eg: OpenMined/logged_in)" will be printed. + """ + + if path.startswith("http://"): + path = path.replace("http://", "") + + if path.startswith("https://"): + path = path.replace("https://", "") + + if path.startswith("github.com/"): + path = path.replace("github.com/", "") + + # Define a regex pattern for a valid GitHub path + pattern = r"^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$" + + # Check if the path matches the pattern + if re.match(pattern, path): + return path + else: + raise ValueError( + "Invalid Git repository path format. (eg: OpenMined/logged_in)" + ) + + +def delete_folder_if_exists(folder_path: str) -> None: + """ + Deletes a folder if it exists at the specified path. + + Args: + folder_path (str): The path to the folder to be deleted. + + Returns: + None: This function does not return any value. + + Functionality: + - Checks if the folder exists at the given path. + - If the folder exists and is a directory, deletes it and all of its contents using `shutil.rmtree()`. + + Example: + Suppose you want to delete a folder located at `/tmp/old_clone` if it exists: + ```python + delete_folder_if_exists("/tmp/old_clone") + ``` + This will delete the folder and all of its contents if it exists. + """ + if os.path.exists(folder_path) and os.path.isdir(folder_path): + shutil.rmtree(folder_path) + + +def is_repo_accessible(repo_url: str) -> bool: + """ + Checks if the specified Git repository is accessible. + + Args: + repo_url (str): The URL of the Git repository to check. + + Returns: + bool: `True` if the repository is accessible, `False` otherwise. + + Functionality: + - Uses the `git ls-remote` command to check if the Git repository is accessible. + - If the command succeeds, returns `True`. + - If the command fails or times out, returns `False`. + + Example: + Suppose you want to check if a repository located at `https://github.com/example/repo.git` is accessible. + You can call the function like this: + ```python + is_accessible = is_repo_accessible("https://github.com/example/repo.git") + ``` + This will return `True` if the repository is accessible, or `False` if it is not. + """ + try: + subprocess.run( + ["git", "ls-remote", repo_url], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.DEVNULL, + timeout=2, + ) + return True + except subprocess.CalledProcessError: + return False + except subprocess.TimeoutExpired: + return False + + +def clone_repository(sanitized_git_path: str) -> str: + """ + Clones a Git repository from GitHub to a temporary directory. + + Args: + sanitized_git_path (str): The Git repository path in the format `owner/repository`. + + Returns: + str: The path to the cloned repository. + + Raises: + Exception: If Git is not installed on the system. + ValueError: If the provided repository path is not accessible. + CalledProcessError: If there is an error during the cloning process. + + Functionality: + - Checks if Git is installed on the system by calling `is_git_installed()`. + - Forms the GitHub repository URL from the provided `sanitized_git_path`. + - Checks if the repository is accessible by calling `is_repo_accessible()`. + - Clones the repository to a temporary directory (`/tmp`). + - Deletes any existing folder in `/tmp` with the same name before cloning. + - If cloning is successful, returns the path to the cloned repository. + - If any error occurs during cloning, raises the corresponding exception. + + Example: + Suppose you want to clone a repository located at `OpenMined/PySyft` to a temporary directory. + You can call the function like this: + ```python + try: + clone_path = clone_repository("OpenMined/PySyft") + print(f"Repository cloned to: {clone_path}") + except Exception as e: + print(e) + ``` + This will clone the repository to `/tmp/PySyft` if successful, or print an error message if any issues occur. + """ + if not is_git_installed(): + raise Exception( + "git cli isn't installed. Please, follow the instructions" + + " to install git according to your OS. (eg. brew install git)" + ) + + repo_url = f"https://github.com/{sanitized_git_path}.git" + if not is_repo_accessible(repo_url): + raise ValueError( + "The provided repository path doesn't seems to be accessible. Please check it out." + ) + # Clone repository in /tmp + tmp_path = mkdtemp(prefix="syftbox_app_") + temp_clone_path = Path(tmp_path, sanitized_git_path.split("/")[-1]) + + # Delete if there's already an existent repository folder in /tmp path. + delete_folder_if_exists(temp_clone_path) + + try: + subprocess.run( + ["git", "clone", repo_url, temp_clone_path], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return temp_clone_path + except subprocess.CalledProcessError as e: + raise e + + +def dict_to_namespace(data: Any) -> Any: + """ + Converts a dictionary (or nested dictionary) to a SimpleNamespace object. + + Args: + data (dict or list): The data to convert. Can be a dictionary, list of dictionaries, or other types. + + Returns: + SimpleNamespace or list: A SimpleNamespace object representing the dictionary data, + or a list of SimpleNamespace objects if the input is a list. + If the input is not a dictionary or list, returns the input as-is. + + Functionality: + - Recursively converts dictionaries to SimpleNamespace objects. + - If the data is a list, each item in the list is recursively converted. + - If the data is neither a dictionary nor a list, returns the data unchanged. + + Example: + Suppose you have a dictionary with nested data: + ```python + data = { + "user": { + "name": "Alice", + "age": 30, + "address": { + "city": "Wonderland", + "zipcode": "12345" + } + }, + "active": True + } + namespace_data = dict_to_namespace(data) + print(namespace_data.user.name) # Output: Alice + print(namespace_data.user.address.city) # Output: Wonderland + ``` + This will allow you to access dictionary values using dot notation like attributes. + """ + if isinstance(data, dict): + return SimpleNamespace( + **{key: dict_to_namespace(value) for key, value in data.items()} + ) + elif isinstance(data, list): + return [dict_to_namespace(item) for item in data] + else: + return data + + +def load_config(path: str) -> SimpleNamespace: + """ + Loads a JSON configuration file and converts it to a SimpleNamespace object. + + Args: + path (str): The file path to the JSON configuration file. + + Returns: + SimpleNamespace: A SimpleNamespace object representing the configuration data. + + Raises: + ValueError: If the file does not exist, is not in JSON format, or does not contain a dictionary. + + Functionality: + - Checks if the provided file path exists. If not, raises a `ValueError` indicating the file is not found. + - Opens and reads the JSON file. If the file cannot be decoded or does not contain a dictionary, raises a `ValueError`. + - Converts the loaded dictionary to a SimpleNamespace object for easy attribute-based access. + + Example: + Suppose you have a JSON configuration file at `/path/to/config.json` with the following content: + ```json + { + "version": "0.1.0", + "app": { + "version": "1.0" + "env": { + "TEST_ENV": "testing", + }, + }, + } + ``` + You can load the configuration and access its fields using dot notation: + ```python + try: + config = load_config("/path/to/config.json") + print(config.app.version) # Output: MyApp + print(config.app.env.TEST_ENV) # Output: True + except ValueError as e: + print(e) + ``` + This will load the configuration and allow access to its values using attribute access. + """ + if not os.path.exists(path): + raise ValueError("Couln't find the json config file for this path.") + try: + error_msg = "File isn't in JSON format." + with open(path, "r") as f: + data = json.load(f) + if not isinstance(data, dict): + raise ValueError(error_msg) + except json.JSONDecodeError: + raise ValueError(error_msg) + return dict_to_namespace(data) + + +def create_symbolic_link( + client_config: ClientConfig, app_path: str, sanitized_path: str +): + """ + Creates a symbolic link from the application directory in the Syftbox directory to the user's sync folder. + + Args: + client_config (ClientConfig): The configuration object for the client, which contains the sync folder path. + app_path (str): The actual path of the application directory. + sanitized_path (str): The sanitized Git repository path in the format `owner/repository`. + + Returns: + None: This function does not return any value. + + Functionality: + - Constructs the symbolic link path within the user's sync folder (`apps` folder). + - If a symlink already exists at the target location, deletes it to avoid conflicts. + - Creates a new symbolic link pointing from the sync folder to the application directory. + + Example: + Suppose you want to create a symbolic link for an application located at `/home/user/.syftbox/apps/PySyft`: + ```python + create_symbolic_link( + client_config=client_config, + app_path="/home/user/.syftbox/apps/PySyft", + sanitized_path="OpenMined/PySyft" + ) + ``` + This will create a symbolic link at `/apps/PySyft` pointing to the application directory. + """ + # TODO: Create a Symlink function + # - Handles if path doesn't exists. + target_symlink_path = ( + f"{str(client_config.sync_folder)}/apps/{sanitized_path.split('/')[-1]}" + ) + + # Create the symlink + if os.path.exists(target_symlink_path) and os.path.islink(target_symlink_path): + os.unlink(target_symlink_path) + + if not os.path.exists(target_symlink_path): + os.symlink(app_path, target_symlink_path) + else: + raise Exception(f"Path exists and isn't a symlink: {target_symlink_path}") + + +def move_repository_to_syftbox( + client_config: ClientConfig, tmp_clone_path: str, sanitized_path: str +) -> str: + """ + Moves a cloned Git repository to the Syftbox directory. + + Args: + tmp_clone_path (str): The file path to the temporarily cloned Git repository. + sanitized_path (str): The sanitized Git repository path in the format `owner/repository`. + + Returns: + str: The final destination path of the moved repository. + + Functionality: + - Constructs the destination path within the Syftbox configuration directory (`apps` folder). + - Deletes any existing folder at the destination path to avoid conflicts. + - Moves the repository from the temporary clone path to the destination path. + - Returns the new path of the moved repository. + + Example: + Suppose you have cloned a repository to a temporary path `/tmp/syftbox` and want to move it to the Syftbox directory: + ```python + output_path = move_repository_to_syftbox("/tmp/PySyft", "OpenMined/PySyft") + print(output_path) # Output: /path/to/config/apps/PySyft + ``` + This will move the cloned repository to the Syftbox `apps` directory and return the final destination path. + """ + output_path = f"{client_config.sync_folder}/apps/{sanitized_path.split('/')[-1]}" + delete_folder_if_exists(output_path) + shutil.move(tmp_clone_path, output_path) + return output_path + + +def run_pre_install(app_config: SimpleNamespace): + """ + Runs pre-installation commands specified in the application configuration. + + Args: + app_config (SimpleNamespace): The configuration object for the application, which is expected to have an `app` + attribute with a `pre_install` attribute containing a list of commands to run. + + Returns: + None: This function does not return any value. + + Functionality: + - Checks if the `pre_install` attribute exists and contains commands in the application configuration. + - If the `pre_install` attribute is empty or does not exist, the function returns without executing any command. + - If there are pre-installation commands, runs them using `subprocess.run()`. + + Example: + Suppose you have an application configuration that specifies a pre-installation command to install dependencies: + ```python + app_config = SimpleNamespace( + app=SimpleNamespace(pre_install=["echo", "Installing dependencies..."]) + ) + run_pre_install(app_config) + ``` + This will run the specified pre-installation command using `subprocess.run()`. + """ + if len(getattr(app_config.app, "pre_install", [])) == 0: + return + + subprocess.run( + app_config.app.pre_install, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def run_post_install(app_config: SimpleNamespace): + """ + Runs post-installation commands specified in the application configuration. + + Args: + app_config (SimpleNamespace): The configuration object for the application, which is expected to have an `app` + attribute with a `post_install` attribute containing a list of commands to run. + + Returns: + None: This function does not return any value. + + Functionality: + - Checks if the `post_install` attribute exists and contains commands in the application configuration. + - If the `post_install` attribute is empty or does not exist, the function returns without executing any command. + - If there are post-installation commands, runs them using `subprocess.run()`. + + Example: + Suppose you have an application configuration that specifies a post-installation command to perform cleanup: + ```python + app_config = SimpleNamespace( + app=SimpleNamespace(post_install=["echo", "Performing post-installation cleanup..."]) + ) + run_post_install(app_config) + ``` + This will run the specified post-installation command using `subprocess.run()`. + """ + if len(getattr(app_config.app, "post_install", [])) == 0: + return + + subprocess.run( + app_config.app.post_install, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def check_os_compatibility(app_config) -> None: + """ + Checks whether the current operating system is compatible with the application based on the configuration. + + Args: + app_config: The configuration object for the application, which is expected to have an `app` attribute + with a `platforms` attribute containing a list of supported operating systems. + + Returns: + None: This function does not return any value. + + Raises: + OSError: If the current operating system is not supported by the application. + + Functionality: + - Uses the `platform.system()` function to determine the current operating system. + - Checks the application's configuration (`app_config`) for a list of supported operating systems. + - If no platforms are defined in the configuration, the function simply returns without doing anything. + - If the current operating system is not in the list of supported platforms, raises an `OSError`. + + Example: + Suppose you have an application configuration that specifies supported platforms as `['Windows', 'Linux']`. + The function will determine the current operating system and raise an `OSError` if it is not supported: + ```python + try: + check_os_compatibility(app_config) + except OSError as e: + print(e) + ``` + If the current OS is not in the supported platforms list, the message "Your OS isn't supported by this app." will be printed. + """ + os_name = platform.system().lower() + supported_os = getattr(app_config.app, "platforms", []) + + # If there's no platforms field in config.json, just ignore it. + if len(supported_os) == 0: + return + + is_compatible = False + for operational_system in supported_os: + if operational_system.lower() == os_name: + is_compatible = True + + if not is_compatible: + raise OSError("Your OS isn't supported by this app.") + + +def get_current_commit(app_path: str) -> str: + """ + Retrieves the current commit hash for a Git repository located at the specified path. + + Args: + app_path (str): The file path to the Git repository. + + Returns: + str: The current commit hash of the repository if the command is successful. + If an error occurs, returns an error message describing the failure. + + Functionality: + - Uses the `git rev-parse HEAD` command to get the current commit hash. + - If the command succeeds, returns the commit hash as a string. + - If the command fails (e.g., if the provided path is not a valid Git repository), + returns an error message detailing what went wrong. + + Example: + Suppose you have a Git repository at `/path/to/repo` and want to retrieve its current commit hash. + You can call the function like this: + ```python + commit_hash = get_current_commit("/path/to/repo") + ``` + This will return the commit hash if the repository exists and the command runs successfully, + or an error message if there is an issue with the command. + """ + try: + # Navigate to the repository path and get the current commit hash + commit_hash = ( + subprocess.check_output( + ["git", "-C", app_path, "rev-parse", "HEAD"], stderr=subprocess.STDOUT + ) + .strip() + .decode("utf-8") + ) + return commit_hash + except subprocess.CalledProcessError as e: + return f"Error: {e.output.decode('utf-8')}" + + +def update_app_config_file(app_path: str, sanitized_git_path: str, app_config) -> None: + """ + Updates the `app.json` configuration file with the current commit and version information of an application. + + Args: + app_path (str): The file path of the application. + sanitized_git_path (str): The sanitized path representing the Git repository. + app_config: The configuration object for the application, which is expected to have an `app` attribute + with a `version` attribute, if available. + + Returns: + None: This function modifies the `app.json` configuration file in place and returns nothing. + + Functionality: + - Normalizes the provided application path. + - Determines the configuration directory by navigating two levels up from the application path. + - Checks if an `app.json` file exists in the configuration directory. + - If it exists, loads its contents into a dictionary. + - If it does not exist, creates an empty dictionary for new configuration entries. + - Retrieves the current commit information of the application using the `get_current_commit` function. + - If the application version is available from the `app_config` object, includes it in the configuration. + - Updates the `app.json` configuration file with the new commit and version information under the key + specified by `sanitized_git_path`. + - Writes the updated configuration back to the `app.json` file with indentation for readability. + + Example: + Suppose you have an application located at `/path/to/app` and you want to update the `app.json` file + with the latest commit and version. You can call the function like this: + ```python + update_app_config_file("/path/to/app", "my_sanitized_git_path", app_config) + ``` + This will update or create entries in `app.json` for the given Git path, storing commit and version details. + """ + normalized_app_path = os.path.normpath(app_path) + + conf_path = os.path.dirname(os.path.dirname(normalized_app_path)) + + app_json_path = conf_path + "/app.json" + app_json_config = {} + if os.path.exists(app_json_path): + # Read from it. + app_json_config = vars(load_config(app_json_path)) + + app_version = None + if getattr(app_config.app, "version", None) is not None: + app_version = app_config.app.version + + app_json_config[sanitized_git_path] = { + "commit": get_current_commit(normalized_app_path), + "version": app_version, + } + + with open(app_json_path, "w") as json_file: + json.dump(app_json_config, json_file, indent=4) + + +def check_app_config(tmp_clone_path) -> SimpleNamespace | None: + try: + app_config_path = tmp_clone_path / "config.json" + if os.path.exists(app_config_path): + app_config = load_config(app_config_path) + step = "Loading config.json" + print(step) + # NOTE: + # Check OS platform compatibility + # Handles if app isn't compatible with the target os system. + step = "Checking platform compatibility." + print(step) + check_os_compatibility(app_config) + + return app_config + except Exception as e: + print("No app config", e) + return None + + +def install(client_config: ClientConfig) -> None | Tuple[str, Exception]: + """ + Installs an application by cloning the repository, checking compatibility, and running installation scripts. + + Args: + client_config (ClientConfig): The configuration object for the client, which is used during the installation process. + + Returns: + None: If the installation is successful. + Tuple[str, Exception]: If an error occurs during any installation step, returns a tuple with the step description and the exception raised. + + Functionality: + - Parses command-line arguments to get the Git repository to install. + - Performs a series of steps to install the application, including: + 1. Sanitizing the Git repository path. + 2. Cloning the repository to a temporary directory. + 3. Loading the application's configuration (`config.json`). + 4. Checking platform compatibility. + 5. Moving the repository to the Syftbox directory. + 6. Creating a symbolic link on the user's desktop. + 7. Running pre-installation commands. + 8. Running post-installation commands. + 9. Updating the `apps.json` file to include the installed application. + - If any step fails, returns the step description and the exception raised. + + Example: + Suppose you have a client configuration and want to install an application from a repository: + ```python + try: + result = install(client_config) + if result is not None: + step, error = result + print(f"Error during step '{step}': {error}") + else: + print("Installation completed successfully.") + except Exception as e: + print(f"Unexpected error: {e}") + ``` + This will install the application, and if an error occurs, it will indicate the step where the failure happened. + """ + parser = argparse.ArgumentParser(description="Run FastAPI server") + + parser.add_argument("repository", type=str, help="App repository") + + args = parser.parse_args() + step = "" + try: + # NOTE: + # Sanitize git repository path + # Handles: bad format repository path. + # Returns: Sanitized repository path. + step = "Checking app name" + sanitized_path = sanitize_git_path(args.repository) + + # NOTE: + # Clones the app repository + # Handles: Git cli tool not installed. + # Handles: Repository path doesn't exits / isn't public. + # Handles: If /tmp/apps/ already exists (replaces it) + # Returns: Path where the repository folder was cloned temporarily. + step = "Pulling App" + tmp_clone_path = clone_repository(sanitized_path) + + # NOTE: + # Load config.json + # Handles: config.json doesn't exist in the pulled repository + # Handles: config.json version is different from syftbox config version. + # Returns: Loaded app config as SimpleNamespace instance. + + # make optional + app_config = check_app_config(tmp_clone_path) + + # NOTE: + # Moves the repository from /tmp to ~/.syftbox/apps/ + # Handles: If ~/.syftbox/apps/ already exists (replaces it) + app_config_path = move_repository_to_syftbox( + client_config, tmp_clone_path=tmp_clone_path, sanitized_path=sanitized_path + ) + + # NOTE: + # Creates a Symbolic Link ( ~/Desktop/Syftbox/app/ -> ~/.syftbox/apps/) + # Handles: If ~/.syftbox/apps/ already exists (replaces it) + # step = "Creating Symbolic Link" + # print(step) + # create_symbolic_link( + # client_config=client_config, + # app_path=app_config_path, + # sanitized_path=sanitized_path, + # ) + + # NOTE: + # Executes config.json pre-install command list + # Handles: Exceptions from pre-install command execution + if app_config: + step = "Running pre-install commands" + run_pre_install(app_config) + + # NOTE: + # Executes config.json post-install command list + # Handles: Exceptions from post-install command execution + if app_config: + step = "Running post-install commands" + run_post_install(app_config) + + # NOTE: + # Updates the apps.json file + # Handles: If apps.json file doesn't exist yet. + # Handles: If apps.json already have the repository_name app listed. + # Handles: If apps.json exists but doesn't have the repository_name app listed. + if app_config: + step = "Updating apps.json config" + update_app_config_file(app_config_path, sanitized_path, app_config) + + app_name = app_config_path.split("/")[-1] + parts = app_config_path.split("/apps/") + print( + "------------------------------------\n" + f"Datasite Path: {parts[0]}\n" + f"In SyftBox: /apps/{parts[-1]}\n" + "------------------------------------\n\n" + f"✅ App {app_name} installed\n" + ) + except Exception as e: + return (step, e) diff --git a/syftbox/app/manager.py b/syftbox/app/manager.py new file mode 100644 index 00000000..87b517a9 --- /dev/null +++ b/syftbox/app/manager.py @@ -0,0 +1,108 @@ +import argparse +import os +import sys +from collections import namedtuple +from pathlib import Path + +from ..lib import DEFAULT_CONFIG_PATH, ClientConfig +from .install import install + +config_path = os.environ.get("SYFTBOX_CLIENT_CONFIG_PATH", None) + + +def list_app(client_config: ClientConfig, silent: bool = False) -> list[str]: + apps_path = Path(client_config.sync_folder + "/" + "apps") + apps = [] + if os.path.exists(apps_path): + files_and_folders = os.listdir(apps_path) + apps = [app for app in files_and_folders if os.path.isdir(apps_path / app)] + + if len(apps): + if not silent: + print("\nInstalled apps:") + for app in apps: + print(f"✅ {app}") + else: + if not silent: + print( + "\nYou have no apps installed.\n\n" + f"Try:\nsyftbox app install OpenMined/github_app_updater\n\nor copy an app to: {apps_path}" + ) + return apps + + +def uninstall_app(client_config: ClientConfig) -> None: + print("Uninstalling Apps") + + +def update_app(client_config: ClientConfig) -> None: + print("Updating Apps") + + +def upgrade_app(client_config: ClientConfig) -> None: + print("Upgrading Apps") + + +Commands = namedtuple("Commands", ["description", "execute"]) + + +def make_commands() -> dict[str, Commands]: + return { + "list": Commands( + "List all currently installed apps in your syftbox.", list_app + ), + "install": Commands("Install a new app in your syftbox.", install), + "uninstall": Commands("Uninstall a certain app.", uninstall_app), + "update": Commands("Check for app updates.", update_app), + "upgrade": Commands("Upgrade an app.", upgrade_app), + } + + +class CustomHelpFormatter(argparse.HelpFormatter): + def add_arguments(self, actions): + for action in actions: + if action.dest == "command": + commands = make_commands() + action.choices = commands.keys() + # Build help message with descriptions + action.help = "\n".join( + [f"{cmd}: {commands[cmd].description}" for cmd in commands] + ) + super().add_arguments(actions) + + +# Parsing arguments and initializing shared state +def parse_args(): + parser = argparse.ArgumentParser( + description="Run the web application with plugins.", + ) + + commands = make_commands() + + # Add a subparser to the "app" parser to handle different actions + parser.add_argument( + "command", choices=commands.keys(), help="The command to execute" + ) + + parser.add_argument( + "--config_path", type=str, default=DEFAULT_CONFIG_PATH, help="config path" + ) + args, remaining_args = parser.parse_known_args() + return args, remaining_args + + +def main(parser, args_list) -> None: + args, remaining_args = parse_args() + client_config = ClientConfig.load(args.config_path) + + commands = make_commands() + # Handle the subcommands as needed + if args.command: + command = commands[args.command] + sys.argv = [sys.argv[0]] + remaining_args + error = command.execute(client_config) + if error is not None: + step, exception = error + print(f"Error during {step}: ", str(exception)) + else: + parser.print_help() diff --git a/syftbox/app/utils.py b/syftbox/app/utils.py new file mode 100644 index 00000000..89febe4d --- /dev/null +++ b/syftbox/app/utils.py @@ -0,0 +1,22 @@ +import os + +config_path = os.environ.get( + "SYFTBOX_CLIENT_CONFIG_PATH", os.path.expanduser("~/.syftbox/client_config.json") +) + + +def get_config_path() -> str: + config_path = os.environ.get("SYFTBOX_CLIENT_CONFIG_PATH", None) + if config_path is None: + config_path = ( + str( + input( + "Path to your syftbox config.json file (eg: /home/user/SyftBox/config/): " + ) + ) + + "/" + ) + os.environ["SYFTBOX_CLIENT_CONFIG_PATH"] = ( + config_path + "/" + ) # Add a / just in case it's not passed. + return config_path diff --git a/syftbox/client/__init__.py b/syftbox/client/__init__.py index 11b3f2c6..2b685cf6 100644 --- a/syftbox/client/__init__.py +++ b/syftbox/client/__init__.py @@ -1 +1,3 @@ from syftbox import __version__ + +from .fsevents import AnyFileSystemEventHandler, FileSystemEvent, FSWatchdog diff --git a/syftbox/client/client.py b/syftbox/client/client.py index e8223848..e6550938 100644 --- a/syftbox/client/client.py +++ b/syftbox/client/client.py @@ -3,7 +3,6 @@ import contextlib import importlib import os -import platform import sys import time import traceback @@ -11,7 +10,7 @@ from dataclasses import dataclass from functools import partial from pathlib import Path -from typing import Any, Optional +from typing import Any import uvicorn from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore @@ -32,8 +31,12 @@ FileSystemEvent, FSWatchdog, ) -from syftbox.client.utils import macos -from syftbox.lib import ClientConfig, SharedState, validate_email +from syftbox.lib import ( + DEFAULT_CONFIG_PATH, + ClientConfig, + SharedState, + load_or_create_config, +) class CustomFastAPI(FastAPI): @@ -56,8 +59,8 @@ class CustomFastAPI(FastAPI): sys.path.insert(0, os.path.dirname(PLUGINS_DIR)) DEFAULT_SYNC_FOLDER = os.path.expanduser("~/Desktop/SyftBox") -DEFAULT_PORT = 8082 -DEFAULT_CONFIG_PATH = os.path.expanduser("~/.syftbox/client_config.json") + + ASSETS_FOLDER = current_dir.parent / "assets" ICON_FOLDER = ASSETS_FOLDER / "icon" @@ -72,81 +75,6 @@ class Plugin: description: str -def load_or_create_config(args) -> ClientConfig: - syft_config_dir = os.path.abspath(os.path.expanduser("~/.syftbox")) - os.makedirs(syft_config_dir, exist_ok=True) - - client_config = None - try: - client_config = ClientConfig.load(args.config_path) - except Exception: - pass - - if client_config is None and args.config_path: - config_path = os.path.abspath(os.path.expanduser(args.config_path)) - client_config = ClientConfig(config_path=config_path) - - if client_config is None: - # config_path = get_user_input("Path to config file?", DEFAULT_CONFIG_PATH) - config_path = os.path.abspath(os.path.expanduser(config_path)) - client_config = ClientConfig(config_path=config_path) - - if args.sync_folder: - sync_folder = os.path.abspath(os.path.expanduser(args.sync_folder)) - client_config.sync_folder = sync_folder - - if client_config.sync_folder is None: - sync_folder = get_user_input( - "Where do you want to Sync SyftBox to?", - DEFAULT_SYNC_FOLDER, - ) - sync_folder = os.path.abspath(os.path.expanduser(sync_folder)) - client_config.sync_folder = sync_folder - - if args.server: - client_config.server_url = args.server - - if not os.path.exists(client_config.sync_folder): - os.makedirs(client_config.sync_folder, exist_ok=True) - - if platform.system() == "Darwin": - macos.copy_icon_file(ICON_FOLDER, client_config.sync_folder) - - if args.email: - client_config.email = args.email - - if client_config.email is None: - email = get_user_input("What is your email address? ") - if not validate_email(email): - raise Exception(f"Invalid email: {email}") - client_config.email = email - - if args.port: - client_config.port = args.port - - if client_config.port is None: - port = int(get_user_input("Enter the port to use", DEFAULT_PORT)) - client_config.port = port - - email_token = os.environ.get("EMAIL_TOKEN", None) - if email_token: - client_config.email_token = email_token - - # Migrate Old Server URL to HTTPS - if client_config.server_url == "http://20.168.10.234:8080": - client_config.server_url = "https://syftbox.openmined.org" - - client_config.save(args.config_path) - return client_config - - -def get_user_input(prompt, default: Optional[str] = None): - if default: - prompt = f"{prompt} (default: {default}): " - user_input = input(prompt).strip() - return user_input if user_input else default - - def process_folder_input(user_input, default_path): if not user_input: return default_path diff --git a/syftbox/client/plugins/apps.py b/syftbox/client/plugins/apps.py index e43aacee..014c2f57 100644 --- a/syftbox/client/plugins/apps.py +++ b/syftbox/client/plugins/apps.py @@ -1,7 +1,10 @@ +import json import logging import os -import shutil import subprocess +import threading +from types import SimpleNamespace +from typing import Any from syftbox.lib import ( SyftPermission, @@ -53,27 +56,48 @@ def find_and_run_script(task_path, extra_args): DEFAULT_SCHEDULE = 10000 DESCRIPTION = "Runs Apps" +RUNNING_APPS = {} DEFAULT_APPS_PATH = os.path.abspath( os.path.join(os.path.dirname(__file__), "..", "..", "..", "default_apps") ) -def copy_default_apps(apps_path): - if not os.path.exists(DEFAULT_APPS_PATH): - print(f"Default apps directory not found: {DEFAULT_APPS_PATH}") - return +# def copy_default_apps(apps_path): +# if not os.path.exists(DEFAULT_APPS_PATH): +# print(f"Default apps directory not found: {DEFAULT_APPS_PATH}") +# return - for app in os.listdir(DEFAULT_APPS_PATH): - src_app_path = os.path.join(DEFAULT_APPS_PATH, app) - dst_app_path = os.path.join(apps_path, app) +# for app in os.listdir(DEFAULT_APPS_PATH): +# src_app_path = os.path.join(DEFAULT_APPS_PATH, app) +# dst_app_path = os.path.join(apps_path, app) - if os.path.isdir(src_app_path): - if os.path.exists(dst_app_path): - print(f"App already installed at: {dst_app_path}") - # shutil.rmtree(dst_app_path) - else: - shutil.copytree(src_app_path, dst_app_path) - print(f"Copied default app: {app}") +# if os.path.isdir(src_app_path): +# if os.path.exists(dst_app_path): +# print(f"App already installed at: {dst_app_path}") +# # shutil.rmtree(dst_app_path) +# else: +# shutil.copytree(src_app_path, dst_app_path) +# print(f"Copied default app: {app}") + + +def dict_to_namespace(data) -> SimpleNamespace | list | Any: + if isinstance(data, dict): + return SimpleNamespace( + **{key: dict_to_namespace(value) for key, value in data.items()} + ) + elif isinstance(data, list): + return [dict_to_namespace(item) for item in data] + else: + return data + + +def load_config(path: str) -> None | SimpleNamespace: + try: + with open(path, "r") as f: + data = json.load(f) + return dict_to_namespace(data) + except Exception: + return None def run_apps(client_config): @@ -82,7 +106,7 @@ def run_apps(client_config): os.makedirs(apps_path, exist_ok=True) # Copy default apps if they don't exist - copy_default_apps(apps_path) + # copy_default_apps(apps_path) # add the first perm file file_path = perm_file_path(apps_path) @@ -100,7 +124,17 @@ def run_apps(client_config): for app in apps: app_path = os.path.abspath(apps_path + "/" + app) if os.path.isdir(app_path): - run_app(client_config, app_path) + app_config = load_config(app_path + "/" + "config.json") + if app_config is None: + run_app(client_config, app_path) + elif RUNNING_APPS.get(app, None) is None: + print("⏱ Scheduling a new app run.") + thread = threading.Thread( + target=run_custom_app_config, + args=(client_config, app_config, app_path), + ) + thread.start() + RUNNING_APPS[app] = thread def output_published(app_output, published_output) -> bool: @@ -111,6 +145,30 @@ def output_published(app_output, published_output) -> bool: ) +def run_custom_app_config(client_config, app_config, path): + import time + + env = os.environ.copy() # Copy the current environment + app_name = os.path.basename(path) + + app_envs = getattr(app_config.app, "env", {}) + if not isinstance(app_envs, dict): + app_envs = vars(app_envs) + + env.update(app_envs) + while True: + print(f"👟 Running {app_name}") + _ = subprocess.run( + app_config.app.run.command, + cwd=path, + check=True, + capture_output=True, + text=True, + env=env, + ) + time.sleep(int(app_config.app.run.interval)) + + def run_app(client_config, path): app_name = os.path.basename(path) diff --git a/syftbox/lib/lib.py b/syftbox/lib/lib.py index 71971cea..08f958f2 100644 --- a/syftbox/lib/lib.py +++ b/syftbox/lib/lib.py @@ -4,6 +4,7 @@ import hashlib import json import os +import platform import re import threading import zlib @@ -11,14 +12,22 @@ from datetime import datetime from pathlib import Path from threading import Lock -from typing import Any +from typing import Any, Optional import httpx import requests from typing_extensions import Self +from syftbox.client.utils import macos from syftbox.server.models import get_file_hash, get_file_last_modified +current_dir = Path(__file__).parent +ASSETS_FOLDER = current_dir.parent / "assets" +DEFAULT_PORT = 8082 +ICON_FOLDER = ASSETS_FOLDER / "icon" +DEFAULT_SYNC_FOLDER = os.path.expanduser("~/Desktop/SyftBox") +DEFAULT_CONFIG_PATH = os.path.expanduser("~/.syftbox/client_config.json") + USER_GROUP_GLOBAL = "GLOBAL" ICON_FILE = "Icon" # special @@ -586,3 +595,78 @@ def create_public_folder(self, path: str): public_read = SyftPermission.mine_with_public_read(email=self.datasite) public_read.save(full_path) return Path(full_path) + + +def get_user_input(prompt, default: Optional[str] = None): + if default: + prompt = f"{prompt} (default: {default}): " + user_input = input(prompt).strip() + return user_input if user_input else default + + +def load_or_create_config(args) -> ClientConfig: + syft_config_dir = os.path.abspath(os.path.expanduser("~/.syftbox")) + os.makedirs(syft_config_dir, exist_ok=True) + + client_config = None + try: + client_config = ClientConfig.load(args.config_path) + except Exception: + pass + + if client_config is None and args.config_path: + config_path = os.path.abspath(os.path.expanduser(args.config_path)) + client_config = ClientConfig(config_path=config_path) + + if client_config is None: + # config_path = get_user_input("Path to config file?", DEFAULT_CONFIG_PATH) + config_path = os.path.abspath(os.path.expanduser(config_path)) + client_config = ClientConfig(config_path=config_path) + + if args.sync_folder: + sync_folder = os.path.abspath(os.path.expanduser(args.sync_folder)) + client_config.sync_folder = sync_folder + + if client_config.sync_folder is None: + sync_folder = get_user_input( + "Where do you want to Sync SyftBox to?", + DEFAULT_SYNC_FOLDER, + ) + sync_folder = os.path.abspath(os.path.expanduser(sync_folder)) + client_config.sync_folder = sync_folder + + if args.server: + client_config.server_url = args.server + + if not os.path.exists(client_config.sync_folder): + os.makedirs(client_config.sync_folder, exist_ok=True) + + if platform.system() == "Darwin": + macos.copy_icon_file(ICON_FOLDER, client_config.sync_folder) + + if args.email: + client_config.email = args.email + + if client_config.email is None: + email = get_user_input("What is your email address? ") + if not validate_email(email): + raise Exception(f"Invalid email: {email}") + client_config.email = email + + if args.port: + client_config.port = args.port + + if client_config.port is None: + port = int(get_user_input("Enter the port to use", DEFAULT_PORT)) + client_config.port = port + + email_token = os.environ.get("EMAIL_TOKEN", None) + if email_token: + client_config.email_token = email_token + + # Migrate Old Server URL to HTTPS + if client_config.server_url == "http://20.168.10.234:8080": + client_config.server_url = "https://syftbox.openmined.org" + + client_config.save(args.config_path) + return client_config diff --git a/syftbox/main.py b/syftbox/main.py index 864277d7..3c606db2 100644 --- a/syftbox/main.py +++ b/syftbox/main.py @@ -1,13 +1,78 @@ import argparse import sys +from syftbox import __version__ +from syftbox.app.manager import list_app +from syftbox.app.manager import main as app_manager_main from syftbox.client.client import main as client_main from syftbox.server.server import main as server_main +def print_debug(): + try: + import os + import platform + import shutil + + import psutil + import yaml + + from syftbox.lib import DEFAULT_CONFIG_PATH, ClientConfig + + config_path = os.environ.get("SYFTBOX_CLIENT_CONFIG_PATH", DEFAULT_CONFIG_PATH) + client_config = None + apps = [] + try: + client_config = ClientConfig.load(config_path) + apps = list_app(client_config, silent=True) + client_config = client_config.to_dict() + except Exception: + pass + + syftbox_path = shutil.which("syftbox") + + debug_info = { + "system": { + "resources": { + "cpus": psutil.cpu_count(logical=True), + "architecture": platform.machine(), + "ram": f"{psutil.virtual_memory().total / (1024**3):.2f} GB", + }, + "operating_system": { + "name": "macOS" + if platform.system() == "Darwin" + else platform.system(), + "version": platform.release(), + }, + "python": { + "version": platform.python_version(), + "binary_location": sys.executable, + }, + }, + "syftbox": { + "command": syftbox_path or "syftbox executable not found in PATH", + "apps": apps, + "client_config_path": config_path, + "client_config": client_config, + }, + "syftbox_env": { + key: value + for key, value in os.environ.items() + if key.startswith("SYFT") + }, + } + print(yaml.dump(debug_info, default_flow_style=False)) + except Exception as e: + print(e) + + def main(): parser = argparse.ArgumentParser(description="Syftbox CLI") - subparsers = parser.add_subparsers(dest="command") + subparsers = parser.add_subparsers( + dest="command", + description="Valid syftbox commands", + help="subcommand to run", + ) # Define the client command subparsers.add_parser("client", help="Run the Syftbox client") @@ -15,6 +80,19 @@ def main(): # Define the server command subparsers.add_parser("server", help="Run the Syftbox server") + # Define the install + app_parser = subparsers.add_parser( + "app", help="Manage SyftBox apps.", description="Manages SyftBox Apps" + ) + + app_parser = subparsers.add_parser( + "version", help="Show SyftBox version", description="Shows the version" + ) + + app_parser = subparsers.add_parser( + "debug", help="Show SyftBox debug info", description="Shows the debug info" + ) + args, remaining_args = parser.parse_known_args() if args.command == "client": @@ -25,6 +103,13 @@ def main(): # Modify sys.argv to exclude the subcommand sys.argv = [sys.argv[0]] + remaining_args server_main() + elif args.command == "app": + sys.argv = [sys.argv[0]] + remaining_args + app_manager_main(app_parser, remaining_args) + elif args.command == "version": + print(f"{__version__}") + elif args.command == "debug": + print_debug() else: parser.print_help() diff --git a/tests/client/app_manager/install_test.py b/tests/client/app_manager/install_test.py new file mode 100644 index 00000000..3e8c5ff4 --- /dev/null +++ b/tests/client/app_manager/install_test.py @@ -0,0 +1,153 @@ +import json +import os +import shutil +from types import SimpleNamespace + +import pytest + +from syftbox.app.install import ( + check_os_compatibility, + clone_repository, + load_config, + sanitize_git_path, +) + + +def test_valid_git_path(): + path = "Example/Repository" + output_path = sanitize_git_path(path) + assert path == output_path + + +def test_valid_git_url(): + path = "Example/Repository" + http_url = f"http://github.com/{path}" + output_path = sanitize_git_path(http_url) + assert path == output_path + + https_url = f"https://github.com/{path}" + output_path = sanitize_git_path(https_url) + assert path == output_path + + +def test_invalid_git_path(): + path = "..Example/../Repository" + with pytest.raises(ValueError) as excpt: + _ = sanitize_git_path(path) + assert excpt.value == "Invalid Git repository path format." + + +def test_second_invalid_git_path(): + path = "http://example.com" + with pytest.raises(ValueError) as excpt: + _ = sanitize_git_path(path) + assert excpt.value == "Invalid Git repository path format." + + +def test_clone_valid_repository(): + path = "OpenMined/logged_in" + temp_path = clone_repository(path) + assert os.path.exists(temp_path) + shutil.rmtree(temp_path) + + +def test_clone_repository_to_an_existent_path(): + # First call will make the repository path exist + path = "OpenMined/logged_in" + temp_path = clone_repository(path) + assert os.path.exists(temp_path) + + # Second call must clone it again without any exception (replaces the old one). + temp_path = clone_repository(path) + shutil.rmtree(temp_path) + + +def test_clone_invalid_repository(): + path = "InvalidUser/InvalidRepo" + with pytest.raises(ValueError) as excpt: + _ = clone_repository(path) + assert ( + excpt.value + == "The provided repository path doesn't seems to be accessible. Please check it out." + ) + + +def test_load_app_config(): + valid_json_config = { + "version": "0.1.0", + "app": { + "version": "0.1.0", + "run": {"command": ["python", "main.py"], "interval": "10"}, + "env": {}, + "platforms": ["linux"], + "pre_install": ["pip", "install", "psutil"], + "post_install": [], + "pre_update": [], + "post_update": [], + }, + } + with open("app.json", "w") as app_json_file: + json.dump(valid_json_config, app_json_file, indent=4) + + app_config = load_config("app.json") + assert app_config.version == valid_json_config["version"] + assert app_config.app.version == valid_json_config["app"]["version"] + assert app_config.app.run.command == valid_json_config["app"]["run"]["command"] + assert vars(app_config.app.env) == valid_json_config["app"]["env"] + assert app_config.app.platforms == valid_json_config["app"]["platforms"] + assert app_config.app.pre_install == valid_json_config["app"]["pre_install"] + assert app_config.app.pre_update == valid_json_config["app"]["pre_update"] + assert app_config.app.post_update == valid_json_config["app"]["post_update"] + os.remove("app.json") + + +def test_load_invalid_app_config(): + with open("app.json", "w") as app_json_file: + json.dump("\nHello World: \n Test", app_json_file, indent=4) + + with pytest.raises(ValueError) as expt: + load_config("app.json") + assert expt.value == "File isn't in JSON format" + + os.remove("app.json") + + +def test_load_inexistent_app_config(): + with pytest.raises(ValueError) as expt: + load_config("inexistent_app.json") + assert expt.value == "Couln't find the json config file for this path." + + +def test_os_compatibility_compatible(): + app_config_mock = SimpleNamespace( + **{ + "app": SimpleNamespace( + **{ + "platforms": ["darwin", "linux"], + } + ), + } + ) + + check_os_compatibility(app_config_mock) + + +def test_os_compatibility_incompatible(): + app_config_mock = SimpleNamespace( + **{ + "app": SimpleNamespace( + **{ + "platforms": ["different_os"], + } + ), + } + ) + with pytest.raises(OSError) as e: + check_os_compatibility(app_config_mock) + assert e.value == "Your OS isn't supported by this app." + + +def test_os_compatibility_without_config(): + app_config_mock = SimpleNamespace(**{"app": {}}) + + check_os_compatibility(app_config_mock) diff --git a/uv.lock b/uv.lock index a9ebad75..9f6c4918 100644 --- a/uv.lock +++ b/uv.lock @@ -1190,8 +1190,10 @@ dependencies = [ { name = "markdown" }, { name = "pandas" }, { name = "postmarker" }, + { name = "psutil" }, { name = "pydantic-settings" }, { name = "pytest" }, + { name = "pyyaml" }, { name = "requests" }, { name = "setuptools" }, { name = "sqlalchemy" }, @@ -1220,8 +1222,10 @@ requires-dist = [ { name = "markdown", specifier = ">=3.7" }, { name = "pandas", specifier = ">=2.2.2" }, { name = "postmarker", specifier = ">=1.0" }, + { name = "psutil", specifier = ">=6.0.0" }, { name = "pydantic-settings", specifier = ">=2.5.2" }, { name = "pytest", specifier = ">=8.3.3" }, + { name = "pyyaml", specifier = ">=6.0.2" }, { name = "requests", specifier = ">=2.32.3" }, { name = "setuptools", specifier = ">=75.1.0" }, { name = "sqlalchemy", specifier = ">=2.0.34" },