From e0e91faf4e397f2a0bf8cbfaf288cf1085a75602 Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 2 Oct 2024 23:23:09 -0300 Subject: [PATCH 01/17] feat(syftbox/app_manager): ADD new syftbox/app_manager submodule Adds a submodule to perform syftbox install --- syftbox/app_manager/__init__.py | 0 syftbox/app_manager/manager.py | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 syftbox/app_manager/__init__.py create mode 100644 syftbox/app_manager/manager.py diff --git a/syftbox/app_manager/__init__.py b/syftbox/app_manager/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/syftbox/app_manager/manager.py b/syftbox/app_manager/manager.py new file mode 100644 index 00000000..468b946c --- /dev/null +++ b/syftbox/app_manager/manager.py @@ -0,0 +1,46 @@ +import argparse +import os +import subprocess +import sys + +from ..lib import ClientConfig + +config_path = os.environ.get( + "SYFTBOX_CLIENT_CONFIG_PATH", os.path.expanduser("~/.syftbox/client_config.json") +) + + +def is_git_installed(): + try: + subprocess.run( + ["git", "--version"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return True + except subprocess.CalledProcessError: + return False + + +def clone_repository(repo_url, clone_dir): + try: + subprocess.run(["git", "clone", repo_url, clone_dir], check=True) + print(f"Repository cloned into {clone_dir}") + except subprocess.CalledProcessError as e: + print(f"Error cloning repository: {e}") + + +def main() -> None: + client_config = ClientConfig.load(config_path) + parser = argparse.ArgumentParser(description="Install syftbox app.") + parser.add_argument("app", type=str, help="App repository name.") + + args = parser.parse_args() + + if is_git_installed(): + repo_url = f"https://github.com/{args.app}.git" + clone_dir = f"{str(client_config.sync_folder)}/apps/{args.app.split('/')[-1]}" + clone_repository(repo_url, clone_dir) + else: + print("Git is not installed. Please install Git and try again.") From ca7680fa6a66e085c1c1a27a9807e81db994787f Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 2 Oct 2024 23:24:38 -0300 Subject: [PATCH 02/17] feat(syftbox/main.py): ADDS install as a valid arg for syftbox CLI syftbox CLI will use app_manager to install app repositories --- syftbox/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/syftbox/main.py b/syftbox/main.py index 864277d7..40ee0fc6 100644 --- a/syftbox/main.py +++ b/syftbox/main.py @@ -1,6 +1,7 @@ import argparse import sys +from syftbox.app_manager.manager import main as app_manager_main from syftbox.client.client import main as client_main from syftbox.server.server import main as server_main @@ -15,6 +16,9 @@ def main(): # Define the server command subparsers.add_parser("server", help="Run the Syftbox server") + # Define the install + subparsers.add_parser("install", help="Install a new app in your syftbox.") + args, remaining_args = parser.parse_known_args() if args.command == "client": @@ -25,6 +29,9 @@ def main(): # Modify sys.argv to exclude the subcommand sys.argv = [sys.argv[0]] + remaining_args server_main() + elif args.command == "install": + sys.argv = [sys.argv[0]] + remaining_args + app_manager_main() else: parser.print_help() From c5bd408df679f2a36b3d72c42dfc0c56867dcadf Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 09:35:55 -0300 Subject: [PATCH 03/17] feat(syftbox/main.py): ADD new app command to syftbox cli app command will be responsible for installing and managing new syftbox apps --- syftbox/main.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/syftbox/main.py b/syftbox/main.py index 40ee0fc6..01da47a8 100644 --- a/syftbox/main.py +++ b/syftbox/main.py @@ -1,14 +1,18 @@ import argparse import sys -from syftbox.app_manager.manager import main as app_manager_main +from syftbox.app.manager import main as app_manager_main from syftbox.client.client import main as client_main from syftbox.server.server import main as server_main def main(): parser = argparse.ArgumentParser(description="Syftbox CLI") - subparsers = parser.add_subparsers(dest="command") + subparsers = parser.add_subparsers( + dest="command", + description="Valid syftbox commands", + help="subcommand to run", + ) # Define the client command subparsers.add_parser("client", help="Run the Syftbox client") @@ -17,7 +21,9 @@ def main(): subparsers.add_parser("server", help="Run the Syftbox server") # Define the install - subparsers.add_parser("install", help="Install a new app in your syftbox.") + app_parser = subparsers.add_parser( + "app", help="Manage SyftBox apps.", description="Manages SyftBox Apps" + ) args, remaining_args = parser.parse_known_args() @@ -29,9 +35,9 @@ def main(): # Modify sys.argv to exclude the subcommand sys.argv = [sys.argv[0]] + remaining_args server_main() - elif args.command == "install": + elif args.command == "app": sys.argv = [sys.argv[0]] + remaining_args - app_manager_main() + app_manager_main(app_parser, remaining_args) else: parser.print_help() From 10d7fadc758f565784082912f3db5fb276836388 Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 09:37:16 -0300 Subject: [PATCH 04/17] refactor(syftbox/app_manager): RENAME app_manager submodule to just app Rename app manager submodule to app --- syftbox/{app_manager => app}/__init__.py | 0 syftbox/app/manager.py | 69 ++++++++++++++++++++++++ syftbox/app_manager/manager.py | 46 ---------------- 3 files changed, 69 insertions(+), 46 deletions(-) rename syftbox/{app_manager => app}/__init__.py (100%) create mode 100644 syftbox/app/manager.py delete mode 100644 syftbox/app_manager/manager.py diff --git a/syftbox/app_manager/__init__.py b/syftbox/app/__init__.py similarity index 100% rename from syftbox/app_manager/__init__.py rename to syftbox/app/__init__.py diff --git a/syftbox/app/manager.py b/syftbox/app/manager.py new file mode 100644 index 00000000..db1717cc --- /dev/null +++ b/syftbox/app/manager.py @@ -0,0 +1,69 @@ +import argparse +import os +import subprocess +import sys +from collections import namedtuple + +from ..lib import ClientConfig +from .app_install import install + +base_path = os.path.expanduser("~/.syftbox/") +config_path = os.environ.get( + "SYFTBOX_CLIENT_CONFIG_PATH", os.path.expanduser("~/.syftbox/client_config.json") +) + + +def list_app(client_config: ClientConfig) -> None: + print("Listing apps") + + +def uninstall_app(client_config: ClientConfig) -> None: + print("Uninstalling Apps") + + +def update_app(client_config: ClientConfig) -> None: + print("Updating Apps") + + +def upgrade_app(client_config: ClientConfig) -> None: + print("Upgrading Apps") + + +def main(parser, args_list) -> None: + client_config = ClientConfig.load(config_path) + + Commands = namedtuple("Commands", ["description", "execute"]) + commands = { + "list": Commands( + "List all currently installed apps in your syftbox.", list_app + ), + "install": Commands("Install a new app in your syftbox.", install), + "uninstall": Commands("Uninstall a certain app.", uninstall_app), + "update": Commands("Check for app updates.", update_app), + "upgrade": Commands("Upgrade an app.", upgrade_app), + } + + # Add a subparser to the "app" parser to handle different actions + app_subparsers = parser.add_subparsers( + title="App Commands", + dest="subcommand", + ) + + # Add all the commands to the argparser + for command, cmd_info in commands.items(): + app_subparsers.add_parser(command, help=cmd_info.description) + + # Parse the remaining args using the parser with subparsers added + # args = parser.parse_args(args_list) + args, remaining_args = parser.parse_known_args() + + # Handle the subcommands as needed + if args.subcommand: + command = commands[args.subcommand] + sys.argv = [sys.argv[0]] + remaining_args + error = command.execute(client_config) + if error is not None: + step, exception = error + print(f"Error during {step}: ", str(exception)) + else: + parser.print_help() diff --git a/syftbox/app_manager/manager.py b/syftbox/app_manager/manager.py deleted file mode 100644 index 468b946c..00000000 --- a/syftbox/app_manager/manager.py +++ /dev/null @@ -1,46 +0,0 @@ -import argparse -import os -import subprocess -import sys - -from ..lib import ClientConfig - -config_path = os.environ.get( - "SYFTBOX_CLIENT_CONFIG_PATH", os.path.expanduser("~/.syftbox/client_config.json") -) - - -def is_git_installed(): - try: - subprocess.run( - ["git", "--version"], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - return True - except subprocess.CalledProcessError: - return False - - -def clone_repository(repo_url, clone_dir): - try: - subprocess.run(["git", "clone", repo_url, clone_dir], check=True) - print(f"Repository cloned into {clone_dir}") - except subprocess.CalledProcessError as e: - print(f"Error cloning repository: {e}") - - -def main() -> None: - client_config = ClientConfig.load(config_path) - parser = argparse.ArgumentParser(description="Install syftbox app.") - parser.add_argument("app", type=str, help="App repository name.") - - args = parser.parse_args() - - if is_git_installed(): - repo_url = f"https://github.com/{args.app}.git" - clone_dir = f"{str(client_config.sync_folder)}/apps/{args.app.split('/')[-1]}" - clone_repository(repo_url, clone_dir) - else: - print("Git is not installed. Please install Git and try again.") From 32bf89373a03bbe8480917eb2767885ca828dd20 Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 09:38:03 -0300 Subject: [PATCH 05/17] feat(syftbox/install.py): ADD a new source file responsible for app installation workflow --- syftbox/app/install.py | 229 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 syftbox/app/install.py diff --git a/syftbox/app/install.py b/syftbox/app/install.py new file mode 100644 index 00000000..3f1bd46d --- /dev/null +++ b/syftbox/app/install.py @@ -0,0 +1,229 @@ +import argparse +import json +import os +import re +import shutil +import subprocess +from os.path import islink +from sys import exception +from types import SimpleNamespace +from typing import Tuple + +from ..lib import ClientConfig +from .utils import base_path + +TEMP_PATH = "/tmp/apps/" + + +def is_git_installed(): + try: + subprocess.run( + ["git", "--version"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return True + except subprocess.CalledProcessError: + return False + + +def sanitize_git_path(path): + # Define a regex pattern for a valid GitHub path + pattern = r"^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$" + + # Check if the path matches the pattern + if re.match(pattern, path): + return path + else: + raise ValueError("Invalid Git repository path format.") + + +def delete_folder_if_exists(folder_path: str): + # Check if temp clone path already exists, if so, delete it. + if os.path.exists(folder_path) and os.path.isdir(folder_path): + shutil.rmtree(folder_path) + + +def clone_repository(sanitized_git_path: str) -> str: + if not is_git_installed(): + raise Exception("Git isn't installed.") + + # Clone repository in /tmp + repo_url = f"https://github.com/{sanitized_git_path}.git" + temp_clone_path = f"{TEMP_PATH}/{sanitized_git_path.split('/')[-1]}" + + # Delete if there's already an existent repository folder in /tmp path. + delete_folder_if_exists(temp_clone_path) + + try: + subprocess.run( + ["git", "clone", repo_url, temp_clone_path], check=True, text=True + ) + return temp_clone_path + except subprocess.CalledProcessError as e: + print(f"Error cloning repository: {e}") + raise e + + +def dict_to_namespace(data): + if isinstance(data, dict): + return SimpleNamespace( + **{key: dict_to_namespace(value) for key, value in data.items()} + ) + elif isinstance(data, list): + return [dict_to_namespace(item) for item in data] + else: + return data + + +def set_environment_values(config, app_path: str): + env_namespace = getattr(config.app, "env", None) + if env_namespace is None: + return + + env_vars = vars(env_namespace) + + with open(f"{app_path}/.env", "w") as envfile: + for key, val in env_vars.items(): + envfile.write(f"export {key}={val}\n") + try: + subprocess.run(["source", f"{app_path}/.env"], check=True, text=True) + except subprocess.CalledProcessError as e: + print(f"Error cloning repository: {e}") + raise e + + +def create_symbolic_link( + client_config: ClientConfig, app_path: str, sanitized_path: str +): + # TODO: Create a Symlink function + # - Handles if symlink already exists + # - Handles if path doesn't exists. + target_symlink_path = ( + f"{str(client_config.sync_folder)}/apps/{sanitized_path.split('/')[-1]}" + ) + + # Create the symlink + try: + if os.path.islink(target_symlink_path): + os.unlink(target_symlink_path) + os.symlink(app_path, target_symlink_path) + print(f"Symlink created: {target_symlink_path} -> {app_path}") + except FileExistsError: + print(f"Symlink already exists: {target_symlink_path}") + + +def load_config(path: str): + with open(path, "r") as f: + data = json.load(f) + return dict_to_namespace(data) + + +def move_repository_to_syftbox(tmp_clone_path: str, sanitized_path: str): + output_path = f"{base_path}/apps/{sanitized_path.split('/')[-1]}" + # Check and delete if there's already the same repository + # name in ~/.syftbox/apps directory. + delete_folder_if_exists(output_path) + shutil.move(tmp_clone_path, output_path) + return output_path + + +def run_pre_install(config): + if len(getattr(config.app, "pre_install", [])) == 0: + return + + try: + subprocess.run(config.app.pre_install, check=True, text=True) + except subprocess.CalledProcessError: + return False + + +def run_post_install(config): + if len(getattr(config.app, "post_install", [])) == 0: + return + + try: + subprocess.run(config.app.post_install, check=True, text=True) + except subprocess.CalledProcessError: + return False + + +def update_app_config_file(): + pass + + +def install(client_config: ClientConfig) -> None | Tuple[str, Exception]: + parser = argparse.ArgumentParser(description="Run FastAPI server") + + parser.add_argument("repository", type=str, help="App repository") + + args = parser.parse_args() + step = "" + try: + # NOTE: + # Sanitize git repository path + # Handles: bad format repository path. + # Returns: Sanitized repository path. + step = "Checking app name" + sanitized_path = sanitize_git_path(args.repository) + + # NOTE: + # Clones the app repository + # Handles: Git cli tool not installed. + # Handles: Repository path doesn't exits / isn't public. + # Handles: If /tmp/apps/ already exists (replaces it) + # Returns: Path where the repository folder was cloned temporarily. + step = "Pulling App" + tmp_clone_path = clone_repository(sanitized_path) + + # NOTE: + # Load config.json + # Handles: config.json doesn't exist in the pulled repository + # Handles: config.json version is different from syftbox config version. + # Returns: Loaded app config as SimpleNamespace instance. + step = "Loading config.json" + app_config = load_config(tmp_clone_path + "/config.json") + + # NOTE: + # Moves the repository from /tmp to ~/.syftbox/apps/ + # Handles: If ~/.syftbox/apps/ already exists (replaces it) + app_path = move_repository_to_syftbox( + tmp_clone_path=tmp_clone_path, sanitized_path=sanitized_path + ) + + # NOTE: + # Creates a Symbolic Link ( ~/Desktop/Syftbox/app/ -> ~/.syftbox/apps/) + # Handles: If ~/.syftbox/apps/ already exists (replaces it) + step = "Creating Symbolic Link" + create_symbolic_link( + client_config=client_config, + app_path=app_path, + sanitized_path=sanitized_path, + ) + + # NOTE: + # Set app environment variables. + # set_environment_values(app_config, app_path) + + # NOTE: + # Executes config.json pre-install command list + # Handles: Exceptions from pre-install command execution + step = "Running pre-install commands" + run_pre_install(app_config) + + # NOTE: + # Executes config.json post-install command list + # Handles: Exceptions from post-install command execution + step = "Running post-install commands" + run_post_install(app_config) + + # NOTE: + # Updates the apps.json file + # Handles: If apps.json file doesn't exist yet. + # Handles: If apps.json already have the repository_name app listed. + # Handles: If apps.json exists but doesn't have the repository_name app listed. + step = "Updating apps.json config" + update_app_config_file() + except Exception as e: + return (step, e) From e839d4612d62b71a1a1e5cbf0332fedfe2050b2e Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 09:39:25 -0300 Subject: [PATCH 06/17] feat(syftbox/app/utils.py): ADD small syftbox/app/utils.py source file Add a utils.py source file to share common variables/functions in the module --- syftbox/app/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 syftbox/app/utils.py diff --git a/syftbox/app/utils.py b/syftbox/app/utils.py new file mode 100644 index 00000000..7525a601 --- /dev/null +++ b/syftbox/app/utils.py @@ -0,0 +1,6 @@ +import os + +base_path = os.path.expanduser("~/.syftbox/") +config_path = os.environ.get( + "SYFTBOX_CLIENT_CONFIG_PATH", os.path.expanduser("~/.syftbox/client_config.json") +) From d4039ba458dcac330c2524f88d23528c4dd8f504 Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 10:38:44 -0300 Subject: [PATCH 07/17] fix(syftbox/app/manager.py): FIX SYFT_CLIENT_CONFIG path exception Handle when SYFT_CLIENT_CONFIG_PATH exception when the env var isn't set --- syftbox/app/manager.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/syftbox/app/manager.py b/syftbox/app/manager.py index db1717cc..17d5a2fa 100644 --- a/syftbox/app/manager.py +++ b/syftbox/app/manager.py @@ -5,11 +5,11 @@ from collections import namedtuple from ..lib import ClientConfig -from .app_install import install +from .install import install base_path = os.path.expanduser("~/.syftbox/") config_path = os.environ.get( - "SYFTBOX_CLIENT_CONFIG_PATH", os.path.expanduser("~/.syftbox/client_config.json") + "SYFTBOX_CLIENT_CONFIG_PATH", None # os.path.expanduser("~/.syftbox/config.json") ) @@ -17,6 +17,16 @@ def list_app(client_config: ClientConfig) -> None: print("Listing apps") +def get_config_path() -> str: + config_path = os.environ.get( + "SYFTBOX_CLIENT_CONFIG_PATH", None + ) + if config_path is None: + config_path = str(input("Path to your syftbox config.json file: ")) + os.environ["SYFTBOX_CLIENT_CONFIG_PATH"] = config_path + return config_path + + def uninstall_app(client_config: ClientConfig) -> None: print("Uninstalling Apps") @@ -30,6 +40,7 @@ def upgrade_app(client_config: ClientConfig) -> None: def main(parser, args_list) -> None: + config_path = get_config_path() client_config = ClientConfig.load(config_path) Commands = namedtuple("Commands", ["description", "execute"]) From 62a2d7adc7bf86e76df452efd5085f49af31f202 Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 15:13:08 -0300 Subject: [PATCH 08/17] refactor(syftbox/app/manager.py-syftbox/app/utils.py): REMOVE unused constant, move shared function to utils.py The base_path constant isn't required anymore, instead we must use the get_config_path --- syftbox/app/manager.py | 19 +++---------------- syftbox/app/utils.py | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/syftbox/app/manager.py b/syftbox/app/manager.py index 17d5a2fa..b9b27d21 100644 --- a/syftbox/app/manager.py +++ b/syftbox/app/manager.py @@ -1,32 +1,19 @@ -import argparse import os -import subprocess import sys from collections import namedtuple from ..lib import ClientConfig from .install import install +from .utils import get_config_path -base_path = os.path.expanduser("~/.syftbox/") config_path = os.environ.get( - "SYFTBOX_CLIENT_CONFIG_PATH", None # os.path.expanduser("~/.syftbox/config.json") + "SYFTBOX_CLIENT_CONFIG_PATH", None ) def list_app(client_config: ClientConfig) -> None: print("Listing apps") - -def get_config_path() -> str: - config_path = os.environ.get( - "SYFTBOX_CLIENT_CONFIG_PATH", None - ) - if config_path is None: - config_path = str(input("Path to your syftbox config.json file: ")) - os.environ["SYFTBOX_CLIENT_CONFIG_PATH"] = config_path - return config_path - - def uninstall_app(client_config: ClientConfig) -> None: print("Uninstalling Apps") @@ -41,7 +28,7 @@ def upgrade_app(client_config: ClientConfig) -> None: def main(parser, args_list) -> None: config_path = get_config_path() - client_config = ClientConfig.load(config_path) + client_config = ClientConfig.load(config_path + "config.json") Commands = namedtuple("Commands", ["description", "execute"]) commands = { diff --git a/syftbox/app/utils.py b/syftbox/app/utils.py index 7525a601..efd9e3e8 100644 --- a/syftbox/app/utils.py +++ b/syftbox/app/utils.py @@ -1,6 +1,19 @@ import os -base_path = os.path.expanduser("~/.syftbox/") config_path = os.environ.get( "SYFTBOX_CLIENT_CONFIG_PATH", os.path.expanduser("~/.syftbox/client_config.json") ) + +def get_config_path() -> str: + config_path = os.environ.get( + "SYFTBOX_CLIENT_CONFIG_PATH", None + ) + if config_path is None: + config_path = str( + input( + "Path to your syftbox config.json file (eg: /home/user/SyftBox/config/): " + ) + ) + "/" + os.environ["SYFTBOX_CLIENT_CONFIG_PATH"] = config_path + "/" # Add a / just in case it's not passed. + return config_path + From d185fc811ce196f2a6521a2b6ba395baeebff350 Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 15:14:20 -0300 Subject: [PATCH 09/17] feat(syftbox/apps/install.py): ADD the update_app_json at the end of app installation workflow This app.json will be responsible to keep track of which apps are installed and up to date --- syftbox/app/install.py | 134 +++++++++++++++++++++++++++-------------- 1 file changed, 89 insertions(+), 45 deletions(-) diff --git a/syftbox/app/install.py b/syftbox/app/install.py index 3f1bd46d..c98c78d3 100644 --- a/syftbox/app/install.py +++ b/syftbox/app/install.py @@ -4,13 +4,11 @@ import re import shutil import subprocess -from os.path import islink -from sys import exception +import platform from types import SimpleNamespace from typing import Tuple - +from .utils import get_config_path from ..lib import ClientConfig -from .utils import base_path TEMP_PATH = "/tmp/apps/" @@ -47,7 +45,10 @@ def delete_folder_if_exists(folder_path: str): def clone_repository(sanitized_git_path: str) -> str: if not is_git_installed(): - raise Exception("Git isn't installed.") + raise Exception( + "git cli isn't installed. Please, follow the instructions" + + " to install git according to your OS. (eg. brew install git)" + ) # Clone repository in /tmp repo_url = f"https://github.com/{sanitized_git_path}.git" @@ -58,7 +59,11 @@ def clone_repository(sanitized_git_path: str) -> str: try: subprocess.run( - ["git", "clone", repo_url, temp_clone_path], check=True, text=True + ["git", "clone", repo_url, temp_clone_path], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) return temp_clone_path except subprocess.CalledProcessError as e: @@ -77,23 +82,6 @@ def dict_to_namespace(data): return data -def set_environment_values(config, app_path: str): - env_namespace = getattr(config.app, "env", None) - if env_namespace is None: - return - - env_vars = vars(env_namespace) - - with open(f"{app_path}/.env", "w") as envfile: - for key, val in env_vars.items(): - envfile.write(f"export {key}={val}\n") - try: - subprocess.run(["source", f"{app_path}/.env"], check=True, text=True) - except subprocess.CalledProcessError as e: - print(f"Error cloning repository: {e}") - raise e - - def create_symbolic_link( client_config: ClientConfig, app_path: str, sanitized_path: str ): @@ -105,13 +93,9 @@ def create_symbolic_link( ) # Create the symlink - try: - if os.path.islink(target_symlink_path): - os.unlink(target_symlink_path) - os.symlink(app_path, target_symlink_path) - print(f"Symlink created: {target_symlink_path} -> {app_path}") - except FileExistsError: - print(f"Symlink already exists: {target_symlink_path}") + if os.path.islink(target_symlink_path): + os.unlink(target_symlink_path) + os.symlink(app_path, target_symlink_path) def load_config(path: str): @@ -121,7 +105,7 @@ def load_config(path: str): def move_repository_to_syftbox(tmp_clone_path: str, sanitized_path: str): - output_path = f"{base_path}/apps/{sanitized_path.split('/')[-1]}" + output_path = f"{get_config_path()}/apps/{sanitized_path.split('/')[-1]}" # Check and delete if there's already the same repository # name in ~/.syftbox/apps directory. delete_folder_if_exists(output_path) @@ -133,24 +117,82 @@ def run_pre_install(config): if len(getattr(config.app, "pre_install", [])) == 0: return - try: - subprocess.run(config.app.pre_install, check=True, text=True) - except subprocess.CalledProcessError: - return False + subprocess.run( + config.app.pre_install, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) def run_post_install(config): if len(getattr(config.app, "post_install", [])) == 0: return + subprocess.run( + config.app.post_install, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def check_os_compatibility(app_config): + os_name = platform.system().lower() + supported_os = getattr(app_config.app, "platforms", []) + + # If there's no platforms field in config.json, just ignore it. + if len(supported_os) == 0: + return + + is_compatible = False + for operational_system in supported_os: + if operational_system.lower() == os_name: + is_compatible = True + + if not is_compatible: + raise Exception("Your OS isn't supported by this app.") + + +def get_current_commit(app_path): try: - subprocess.run(config.app.post_install, check=True, text=True) - except subprocess.CalledProcessError: - return False + # Navigate to the repository path and get the current commit hash + commit_hash = ( + subprocess.check_output( + ["git", "-C", app_path, "rev-parse", "HEAD"], stderr=subprocess.STDOUT + ) + .strip() + .decode("utf-8") + ) + return commit_hash + except subprocess.CalledProcessError as e: + return f"Error: {e.output.decode('utf-8')}" + + +def update_app_config_file(app_path: str, sanitized_git_path: str, app_config) -> None: + normalized_app_path = os.path.normpath(app_path) + + conf_path = os.path.dirname(os.path.dirname(normalized_app_path)) + + app_json_path = conf_path + "/app.json" + app_json_config = {} + if os.path.exists(app_json_path): + # Read from it. + app_json_config = vars(load_config(app_json_path)) + app_version = None + if getattr(app_config.app, "version", None) is not None: + app_version = app_config.app.version -def update_app_config_file(): - pass + app_json_config[sanitized_git_path] = { + "commit": get_current_commit(normalized_app_path), + "version": app_version, + } + + with open(app_json_path, "w") as json_file: + json.dump(app_json_config, json_file, indent=4) def install(client_config: ClientConfig) -> None | Tuple[str, Exception]: @@ -185,6 +227,12 @@ def install(client_config: ClientConfig) -> None | Tuple[str, Exception]: step = "Loading config.json" app_config = load_config(tmp_clone_path + "/config.json") + # NOTE: + # Check OS platform compatibility + # Handles if app isn't compatible with the target os system. + step = "Checking platform compatibility." + check_os_compatibility(app_config) + # NOTE: # Moves the repository from /tmp to ~/.syftbox/apps/ # Handles: If ~/.syftbox/apps/ already exists (replaces it) @@ -202,10 +250,6 @@ def install(client_config: ClientConfig) -> None | Tuple[str, Exception]: sanitized_path=sanitized_path, ) - # NOTE: - # Set app environment variables. - # set_environment_values(app_config, app_path) - # NOTE: # Executes config.json pre-install command list # Handles: Exceptions from pre-install command execution @@ -224,6 +268,6 @@ def install(client_config: ClientConfig) -> None | Tuple[str, Exception]: # Handles: If apps.json already have the repository_name app listed. # Handles: If apps.json exists but doesn't have the repository_name app listed. step = "Updating apps.json config" - update_app_config_file() + update_app_config_file(app_path, sanitized_path, app_config) except Exception as e: return (step, e) From 2e235e56dc975de2f5f6db6a32d4957607a545cf Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 15:16:12 -0300 Subject: [PATCH 10/17] feat(syftbox/client/plugins/app.py): Empower each app to have their own interval schedule defined by their app.json config file Most of apps will have different requirements in terms of refresh rate. To achieve that, we must empower the app config to tell the system how often they must run. --- syftbox/client/plugins/apps.py | 66 +++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/syftbox/client/plugins/apps.py b/syftbox/client/plugins/apps.py index 13905517..a2e6fcbe 100644 --- a/syftbox/client/plugins/apps.py +++ b/syftbox/client/plugins/apps.py @@ -1,8 +1,11 @@ import logging import os +import json import shutil import subprocess - +import threading +from types import SimpleNamespace +from typing import Any from syftbox.lib import ( SyftPermission, get_file_hash, @@ -53,7 +56,7 @@ def find_and_run_script(task_path, extra_args): DEFAULT_SCHEDULE = 10000 DESCRIPTION = "Runs Apps" - +RUNNING_APPS = {} DEFAULT_APPS_PATH = os.path.abspath( os.path.join(os.path.dirname(__file__), "..", "..", "..", "default_apps") ) @@ -77,13 +80,33 @@ def copy_default_apps(apps_path): print(f"Copied default app: {app}") +def dict_to_namespace(data) -> SimpleNamespace | list | Any: + if isinstance(data, dict): + return SimpleNamespace( + **{key: dict_to_namespace(value) for key, value in data.items()} + ) + elif isinstance(data, list): + return [dict_to_namespace(item) for item in data] + else: + return data + + +def load_config(path: str) -> None | SimpleNamespace: + try: + with open(path, "r") as f: + data = json.load(f) + return dict_to_namespace(data) + except Exception: + return None + + def run_apps(client_config): # create the directory apps_path = client_config.sync_folder + "/" + "apps" os.makedirs(apps_path, exist_ok=True) # Copy default apps if they don't exist - copy_default_apps(apps_path) + # copy_default_apps(apps_path) # add the first perm file file_path = perm_file_path(apps_path) @@ -101,8 +124,17 @@ def run_apps(client_config): for app in apps: app_path = os.path.abspath(apps_path + "/" + app) if os.path.isdir(app_path): - run_app(client_config, app_path) - + app_config = load_config(app_path + "/" + "config.json") + if app_config is None: + run_app(client_config, app_path) + elif RUNNING_APPS.get(app, None) is None: + print("⏱ Scheduling a new app run.") + thread = threading.Thread( + target=run_custom_app_config, + args=(client_config, app_config, app_path), + ) + thread.start() + RUNNING_APPS[app] = thread def output_published(app_output, published_output) -> bool: return ( @@ -112,6 +144,30 @@ def output_published(app_output, published_output) -> bool: ) +def run_custom_app_config(client_config, app_config, path): + import time + + env = os.environ.copy() # Copy the current environment + app_name = os.path.basename(path) + + app_envs = getattr(app_config.app, "env", {}) + if not isinstance(app_envs, dict): + app_envs = vars(app_envs) + + env.update(app_envs) + while True: + print(f"👟 Running {app_name}") + result = subprocess.run( + app_config.app.run.command, + cwd=path, + check=True, + capture_output=True, + text=True, + env=env, + ) + time.sleep(int(app_config.app.run.interval)) + + def run_app(client_config, path): app_name = os.path.basename(path) From 4816283bf2d22cf8acf3f2fbb05d42b0800ac7ce Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 16:36:59 -0300 Subject: [PATCH 11/17] fix(syftbox/app/install.py): Fix small uncovered workflows and handle them properly Cover use cases where the given repository path doesn't exist / app.json is invalid or isn't found --- syftbox/app/install.py | 53 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/syftbox/app/install.py b/syftbox/app/install.py index c98c78d3..4f7abadb 100644 --- a/syftbox/app/install.py +++ b/syftbox/app/install.py @@ -34,7 +34,9 @@ def sanitize_git_path(path): if re.match(pattern, path): return path else: - raise ValueError("Invalid Git repository path format.") + raise ValueError( + "Invalid Git repository path format. (eg: OpenMined/logged_in)" + ) def delete_folder_if_exists(folder_path: str): @@ -43,6 +45,24 @@ def delete_folder_if_exists(folder_path: str): shutil.rmtree(folder_path) +def is_repo_accessible(repo_url): + """Check if the Git repository is accessible.""" + try: + subprocess.run( + ["git", "ls-remote", repo_url], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.DEVNULL, + timeout=2, + ) + return True + except subprocess.CalledProcessError as e: + return False + except subprocess.TimeoutExpired: + return False + + def clone_repository(sanitized_git_path: str) -> str: if not is_git_installed(): raise Exception( @@ -50,8 +70,12 @@ def clone_repository(sanitized_git_path: str) -> str: + " to install git according to your OS. (eg. brew install git)" ) - # Clone repository in /tmp repo_url = f"https://github.com/{sanitized_git_path}.git" + if not is_repo_accessible(repo_url): + raise ValueError( + "The provided repository path doesn't seems to be accessible. Please check it out." + ) + # Clone repository in /tmp temp_clone_path = f"{TEMP_PATH}/{sanitized_git_path.split('/')[-1]}" # Delete if there's already an existent repository folder in /tmp path. @@ -82,6 +106,20 @@ def dict_to_namespace(data): return data +def load_config(path: str): + if not os.path.exists(path): + raise ValueError("Couln't find the json config file for this path.") + try: + error_msg = "File isn't in JSON format." + with open(path, "r") as f: + data = json.load(f) + if not isinstance(data, dict): + raise ValueError("File isn't in JSON format.") + except json.JSONDecodeError: + raise ValueError("File isn't in JSON format.") + return dict_to_namespace(data) + + def create_symbolic_link( client_config: ClientConfig, app_path: str, sanitized_path: str ): @@ -98,12 +136,6 @@ def create_symbolic_link( os.symlink(app_path, target_symlink_path) -def load_config(path: str): - with open(path, "r") as f: - data = json.load(f) - return dict_to_namespace(data) - - def move_repository_to_syftbox(tmp_clone_path: str, sanitized_path: str): output_path = f"{get_config_path()}/apps/{sanitized_path.split('/')[-1]}" # Check and delete if there's already the same repository @@ -139,10 +171,11 @@ def run_post_install(config): ) -def check_os_compatibility(app_config): +def check_os_compatibility(app_config) -> None: os_name = platform.system().lower() supported_os = getattr(app_config.app, "platforms", []) + print("\n\n\nHere\n\n\n: ", supported_os, app_config) # If there's no platforms field in config.json, just ignore it. if len(supported_os) == 0: return @@ -153,7 +186,7 @@ def check_os_compatibility(app_config): is_compatible = True if not is_compatible: - raise Exception("Your OS isn't supported by this app.") + raise OSError("Your OS isn't supported by this app.") def get_current_commit(app_path): From 240e132e18842c376c5e0dd579b5eb9cfd233f67 Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 16:37:48 -0300 Subject: [PATCH 12/17] test(tests/client/app_manager/install_test.py): ADD a set of tests to cover syftbox app install workflow --- tests/client/app_manager/install_test.py | 139 +++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 tests/client/app_manager/install_test.py diff --git a/tests/client/app_manager/install_test.py b/tests/client/app_manager/install_test.py new file mode 100644 index 00000000..8ffe7dd0 --- /dev/null +++ b/tests/client/app_manager/install_test.py @@ -0,0 +1,139 @@ +import pytest +from syftbox.app.install import sanitize_git_path +from syftbox.app.install import clone_repository +from syftbox.app.install import check_os_compatibility +from syftbox.app.install import load_config +import json +import shutil +import os +from types import SimpleNamespace + + +def test_valid_git_path(): + path = "Example/Repository" + output_path = sanitize_git_path(path) + assert path == output_path + + +def test_invalid_git_path(): + path = "..Example/../Repository" + with pytest.raises(ValueError) as excpt: + output_path = sanitize_git_path(path) + assert excpt.value == "Invalid Git repository path format." + + +def test_second_invalid_git_path(): + path = "http://example.com" + with pytest.raises(ValueError) as excpt: + output_path = sanitize_git_path(path) + assert excpt.value == "Invalid Git repository path format." + + +def test_clone_valid_repository(): + path = "OpenMined/logged_in" + temp_path = clone_repository(path) + assert os.path.exists(temp_path) + shutil.rmtree(temp_path) + + +def test_clone_repository_to_an_existent_path(): + # First call will make the repository path exist + path = "OpenMined/logged_in" + temp_path = clone_repository(path) + assert os.path.exists(temp_path) + + # Second call must clone it again without any exception (replaces the old one). + temp_path = clone_repository(path) + shutil.rmtree(temp_path) + + +def test_clone_invalid_repository(): + path = "InvalidUser/InvalidRepo" + with pytest.raises(ValueError) as excpt: + temp_path = clone_repository(path) + assert ( + excpt.value + == "The provided repository path doesn't seems to be accessible. Please check it out." + ) + + +def test_load_app_config(): + valid_json_config = { + "version": "0.1.0", + "app": { + "version": "0.1.0", + "run": {"command": ["python", "main.py"], "interval": "10"}, + "env": {}, + "platforms": ["linux"], + "pre_install": ["pip", "install", "psutil"], + "post_install": [], + "pre_update": [], + "post_update": [], + }, + } + with open("app.json", "w") as app_json_file: + json.dump(valid_json_config, app_json_file, indent=4) + + app_config = load_config("app.json") + assert app_config.version == valid_json_config["version"] + assert app_config.app.version == valid_json_config["app"]["version"] + assert app_config.app.run.command == valid_json_config["app"]["run"]["command"] + assert vars(app_config.app.env) == valid_json_config["app"]["env"] + assert app_config.app.platforms == valid_json_config["app"]["platforms"] + assert app_config.app.pre_install == valid_json_config["app"]["pre_install"] + assert app_config.app.pre_update == valid_json_config["app"]["pre_update"] + assert app_config.app.post_update == valid_json_config["app"]["post_update"] + os.remove("app.json") + + +def test_load_invalid_app_config(): + with open("app.json", "w") as app_json_file: + json.dump("\nHello World: \n Test", app_json_file, indent=4) + + with pytest.raises(ValueError) as expt: + load_config("app.json") + assert expt.value == "File isn't in JSON format" + + os.remove("app.json") + + +def test_load_inexistent_app_config(): + + with pytest.raises(ValueError) as expt: + load_config("inexistent_app.json") + assert expt.value == "Couln't find the json config file for this path." + + +def test_os_compatibility_compatible(): + app_config_mock = SimpleNamespace( + **{ + "app": SimpleNamespace( + **{ + "platforms": ["darwin", "linux"], + } + ), + } + ) + + check_os_compatibility(app_config_mock) + + +def test_os_compatibility_incompatible(): + app_config_mock = SimpleNamespace( + **{ + "app": SimpleNamespace( + **{ + "platforms": ["different_os"], + } + ), + } + ) + with pytest.raises(OSError) as e: + check_os_compatibility(app_config_mock) + assert e.value == "Your OS isn't supported by this app." + + +def test_os_compatibility_without_config(): + app_config_mock = SimpleNamespace(**{"app": {}}) + + check_os_compatibility(app_config_mock) From 9c4edb3bd9b2506aa8db02f5050e789c8ebb6836 Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 16:43:40 -0300 Subject: [PATCH 13/17] style(syftbox/app/install.py): REMOVE unused/uncessary print statements --- syftbox/app/install.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/syftbox/app/install.py b/syftbox/app/install.py index 4f7abadb..0673e6bf 100644 --- a/syftbox/app/install.py +++ b/syftbox/app/install.py @@ -91,7 +91,6 @@ def clone_repository(sanitized_git_path: str) -> str: ) return temp_clone_path except subprocess.CalledProcessError as e: - print(f"Error cloning repository: {e}") raise e @@ -175,7 +174,6 @@ def check_os_compatibility(app_config) -> None: os_name = platform.system().lower() supported_os = getattr(app_config.app, "platforms", []) - print("\n\n\nHere\n\n\n: ", supported_os, app_config) # If there's no platforms field in config.json, just ignore it. if len(supported_os) == 0: return From 7793ae820e878f2eb20a9068fc59d49bd8fe9194 Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 16:45:55 -0300 Subject: [PATCH 14/17] fix(syftbox/client/plugins/app.py): Uncomment the copy_default_apps function During development process i turned off this function to track app installation. --- syftbox/client/plugins/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syftbox/client/plugins/apps.py b/syftbox/client/plugins/apps.py index a2e6fcbe..53a9d4ef 100644 --- a/syftbox/client/plugins/apps.py +++ b/syftbox/client/plugins/apps.py @@ -106,7 +106,7 @@ def run_apps(client_config): os.makedirs(apps_path, exist_ok=True) # Copy default apps if they don't exist - # copy_default_apps(apps_path) + copy_default_apps(apps_path) # add the first perm file file_path = perm_file_path(apps_path) From bdbc09233de5be382adb62e8ea06b08d4150d44c Mon Sep 17 00:00:00 2001 From: IonesioJunior Date: Wed, 9 Oct 2024 17:30:08 -0300 Subject: [PATCH 15/17] docs(syftbox/apps/install.py): ADD docstrings to install.py functions --- syftbox/app/install.py | 465 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 445 insertions(+), 20 deletions(-) diff --git a/syftbox/app/install.py b/syftbox/app/install.py index 0673e6bf..5d708cad 100644 --- a/syftbox/app/install.py +++ b/syftbox/app/install.py @@ -7,13 +7,34 @@ import platform from types import SimpleNamespace from typing import Tuple +from typing import Any from .utils import get_config_path from ..lib import ClientConfig TEMP_PATH = "/tmp/apps/" -def is_git_installed(): +def is_git_installed() -> bool: + """ + Checks if Git is installed on the system. + + Returns: + bool: `True` if Git is installed, `False` otherwise. + + Functionality: + - Runs the `git --version` command to check if Git is installed. + - If the command runs successfully, returns `True`. + - If the command fails (e.g., Git is not installed), returns `False`. + + Example: + ```python + if is_git_installed(): + print("Git is installed on this system.") + else: + print("Git is not installed. Please install Git to proceed.") + ``` + This will print a message indicating whether Git is installed or not. + """ try: subprocess.run( ["git", "--version"], @@ -26,7 +47,35 @@ def is_git_installed(): return False -def sanitize_git_path(path): +def sanitize_git_path(path: str) -> str: + """ + Validates and sanitizes a Git repository path, ensuring it matches the required format. + + Args: + path (str): The Git repository path to validate. + + Returns: + str: The sanitized Git repository path if it matches the valid pattern. + + Raises: + ValueError: If the provided path does not match the expected format for a Git repository. + + Functionality: + - Uses a regular expression pattern to ensure that the given path follows the format `owner/repository`. + - If the path matches the pattern, returns it as a valid Git path. + - If the path does not match the pattern, raises a `ValueError` with a descriptive message. + + Example: + Suppose you have a GitHub path like `OpenMined/logged_in` and want to validate it: + ```python + try: + sanitized_path = sanitize_git_path("OpenMined/logged_in") + except ValueError as e: + print(e) + ``` + If the path is valid, `sanitized_path` will contain the validated GitHub path. If it is not valid, the error message + "Invalid Git repository path format. (eg: OpenMined/logged_in)" will be printed. + """ # Define a regex pattern for a valid GitHub path pattern = r"^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$" @@ -39,14 +88,54 @@ def sanitize_git_path(path): ) -def delete_folder_if_exists(folder_path: str): - # Check if temp clone path already exists, if so, delete it. +def delete_folder_if_exists(folder_path: str) -> None: + """ + Deletes a folder if it exists at the specified path. + + Args: + folder_path (str): The path to the folder to be deleted. + + Returns: + None: This function does not return any value. + + Functionality: + - Checks if the folder exists at the given path. + - If the folder exists and is a directory, deletes it and all of its contents using `shutil.rmtree()`. + + Example: + Suppose you want to delete a folder located at `/tmp/old_clone` if it exists: + ```python + delete_folder_if_exists("/tmp/old_clone") + ``` + This will delete the folder and all of its contents if it exists. + """ if os.path.exists(folder_path) and os.path.isdir(folder_path): shutil.rmtree(folder_path) -def is_repo_accessible(repo_url): - """Check if the Git repository is accessible.""" +def is_repo_accessible(repo_url: str) -> bool: + """ + Checks if the specified Git repository is accessible. + + Args: + repo_url (str): The URL of the Git repository to check. + + Returns: + bool: `True` if the repository is accessible, `False` otherwise. + + Functionality: + - Uses the `git ls-remote` command to check if the Git repository is accessible. + - If the command succeeds, returns `True`. + - If the command fails or times out, returns `False`. + + Example: + Suppose you want to check if a repository located at `https://github.com/example/repo.git` is accessible. + You can call the function like this: + ```python + is_accessible = is_repo_accessible("https://github.com/example/repo.git") + ``` + This will return `True` if the repository is accessible, or `False` if it is not. + """ try: subprocess.run( ["git", "ls-remote", repo_url], @@ -57,13 +146,48 @@ def is_repo_accessible(repo_url): timeout=2, ) return True - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: return False except subprocess.TimeoutExpired: return False def clone_repository(sanitized_git_path: str) -> str: + """ + Clones a Git repository from GitHub to a temporary directory. + + Args: + sanitized_git_path (str): The Git repository path in the format `owner/repository`. + + Returns: + str: The path to the cloned repository. + + Raises: + Exception: If Git is not installed on the system. + ValueError: If the provided repository path is not accessible. + CalledProcessError: If there is an error during the cloning process. + + Functionality: + - Checks if Git is installed on the system by calling `is_git_installed()`. + - Forms the GitHub repository URL from the provided `sanitized_git_path`. + - Checks if the repository is accessible by calling `is_repo_accessible()`. + - Clones the repository to a temporary directory (`/tmp`). + - Deletes any existing folder in `/tmp` with the same name before cloning. + - If cloning is successful, returns the path to the cloned repository. + - If any error occurs during cloning, raises the corresponding exception. + + Example: + Suppose you want to clone a repository located at `OpenMined/PySyft` to a temporary directory. + You can call the function like this: + ```python + try: + clone_path = clone_repository("OpenMined/PySyft") + print(f"Repository cloned to: {clone_path}") + except Exception as e: + print(e) + ``` + This will clone the repository to `/tmp/PySyft` if successful, or print an error message if any issues occur. + """ if not is_git_installed(): raise Exception( "git cli isn't installed. Please, follow the instructions" @@ -94,7 +218,43 @@ def clone_repository(sanitized_git_path: str) -> str: raise e -def dict_to_namespace(data): +def dict_to_namespace(data: Any) -> Any: + """ + Converts a dictionary (or nested dictionary) to a SimpleNamespace object. + + Args: + data (dict or list): The data to convert. Can be a dictionary, list of dictionaries, or other types. + + Returns: + SimpleNamespace or list: A SimpleNamespace object representing the dictionary data, + or a list of SimpleNamespace objects if the input is a list. + If the input is not a dictionary or list, returns the input as-is. + + Functionality: + - Recursively converts dictionaries to SimpleNamespace objects. + - If the data is a list, each item in the list is recursively converted. + - If the data is neither a dictionary nor a list, returns the data unchanged. + + Example: + Suppose you have a dictionary with nested data: + ```python + data = { + "user": { + "name": "Alice", + "age": 30, + "address": { + "city": "Wonderland", + "zipcode": "12345" + } + }, + "active": True + } + namespace_data = dict_to_namespace(data) + print(namespace_data.user.name) # Output: Alice + print(namespace_data.user.address.city) # Output: Wonderland + ``` + This will allow you to access dictionary values using dot notation like attributes. + """ if isinstance(data, dict): return SimpleNamespace( **{key: dict_to_namespace(value) for key, value in data.items()} @@ -105,7 +265,48 @@ def dict_to_namespace(data): return data -def load_config(path: str): +def load_config(path: str) -> SimpleNamespace: + """ + Loads a JSON configuration file and converts it to a SimpleNamespace object. + + Args: + path (str): The file path to the JSON configuration file. + + Returns: + SimpleNamespace: A SimpleNamespace object representing the configuration data. + + Raises: + ValueError: If the file does not exist, is not in JSON format, or does not contain a dictionary. + + Functionality: + - Checks if the provided file path exists. If not, raises a `ValueError` indicating the file is not found. + - Opens and reads the JSON file. If the file cannot be decoded or does not contain a dictionary, raises a `ValueError`. + - Converts the loaded dictionary to a SimpleNamespace object for easy attribute-based access. + + Example: + Suppose you have a JSON configuration file at `/path/to/config.json` with the following content: + ```json + { + "version": "0.1.0", + "app": { + "version": "1.0" + "env": { + "TEST_ENV": "testing", + }, + }, + } + ``` + You can load the configuration and access its fields using dot notation: + ```python + try: + config = load_config("/path/to/config.json") + print(config.app.version) # Output: MyApp + print(config.app.env.TEST_ENV) # Output: True + except ValueError as e: + print(e) + ``` + This will load the configuration and allow access to its values using attribute access. + """ if not os.path.exists(path): raise ValueError("Couln't find the json config file for this path.") try: @@ -122,8 +323,34 @@ def load_config(path: str): def create_symbolic_link( client_config: ClientConfig, app_path: str, sanitized_path: str ): + """ + Creates a symbolic link from the application directory in the Syftbox directory to the user's sync folder. + + Args: + client_config (ClientConfig): The configuration object for the client, which contains the sync folder path. + app_path (str): The actual path of the application directory. + sanitized_path (str): The sanitized Git repository path in the format `owner/repository`. + + Returns: + None: This function does not return any value. + + Functionality: + - Constructs the symbolic link path within the user's sync folder (`apps` folder). + - If a symlink already exists at the target location, deletes it to avoid conflicts. + - Creates a new symbolic link pointing from the sync folder to the application directory. + + Example: + Suppose you want to create a symbolic link for an application located at `/home/user/.syftbox/apps/PySyft`: + ```python + create_symbolic_link( + client_config=client_config, + app_path="/home/user/.syftbox/apps/PySyft", + sanitized_path="OpenMined/PySyft" + ) + ``` + This will create a symbolic link at `/apps/PySyft` pointing to the application directory. + """ # TODO: Create a Symlink function - # - Handles if symlink already exists # - Handles if path doesn't exists. target_symlink_path = ( f"{str(client_config.sync_folder)}/apps/{sanitized_path.split('/')[-1]}" @@ -135,21 +362,68 @@ def create_symbolic_link( os.symlink(app_path, target_symlink_path) -def move_repository_to_syftbox(tmp_clone_path: str, sanitized_path: str): +def move_repository_to_syftbox(tmp_clone_path: str, sanitized_path: str) -> str: + """ + Moves a cloned Git repository to the Syftbox directory. + + Args: + tmp_clone_path (str): The file path to the temporarily cloned Git repository. + sanitized_path (str): The sanitized Git repository path in the format `owner/repository`. + + Returns: + str: The final destination path of the moved repository. + + Functionality: + - Constructs the destination path within the Syftbox configuration directory (`apps` folder). + - Deletes any existing folder at the destination path to avoid conflicts. + - Moves the repository from the temporary clone path to the destination path. + - Returns the new path of the moved repository. + + Example: + Suppose you have cloned a repository to a temporary path `/tmp/syftbox` and want to move it to the Syftbox directory: + ```python + output_path = move_repository_to_syftbox("/tmp/PySyft", "OpenMined/PySyft") + print(output_path) # Output: /path/to/config/apps/PySyft + ``` + This will move the cloned repository to the Syftbox `apps` directory and return the final destination path. + """ output_path = f"{get_config_path()}/apps/{sanitized_path.split('/')[-1]}" - # Check and delete if there's already the same repository - # name in ~/.syftbox/apps directory. delete_folder_if_exists(output_path) shutil.move(tmp_clone_path, output_path) return output_path -def run_pre_install(config): - if len(getattr(config.app, "pre_install", [])) == 0: +def run_pre_install(app_config: SimpleNamespace): + """ + Runs pre-installation commands specified in the application configuration. + + Args: + app_config (SimpleNamespace): The configuration object for the application, which is expected to have an `app` + attribute with a `pre_install` attribute containing a list of commands to run. + + Returns: + None: This function does not return any value. + + Functionality: + - Checks if the `pre_install` attribute exists and contains commands in the application configuration. + - If the `pre_install` attribute is empty or does not exist, the function returns without executing any command. + - If there are pre-installation commands, runs them using `subprocess.run()`. + + Example: + Suppose you have an application configuration that specifies a pre-installation command to install dependencies: + ```python + app_config = SimpleNamespace( + app=SimpleNamespace(pre_install=["echo", "Installing dependencies..."]) + ) + run_pre_install(app_config) + ``` + This will run the specified pre-installation command using `subprocess.run()`. + """ + if len(getattr(app_config.app, "pre_install", [])) == 0: return subprocess.run( - config.app.pre_install, + app_config.app.pre_install, check=True, text=True, stdout=subprocess.PIPE, @@ -157,12 +431,37 @@ def run_pre_install(config): ) -def run_post_install(config): - if len(getattr(config.app, "post_install", [])) == 0: +def run_post_install(app_config: SimpleNamespace): + """ + Runs post-installation commands specified in the application configuration. + + Args: + app_config (SimpleNamespace): The configuration object for the application, which is expected to have an `app` + attribute with a `post_install` attribute containing a list of commands to run. + + Returns: + None: This function does not return any value. + + Functionality: + - Checks if the `post_install` attribute exists and contains commands in the application configuration. + - If the `post_install` attribute is empty or does not exist, the function returns without executing any command. + - If there are post-installation commands, runs them using `subprocess.run()`. + + Example: + Suppose you have an application configuration that specifies a post-installation command to perform cleanup: + ```python + app_config = SimpleNamespace( + app=SimpleNamespace(post_install=["echo", "Performing post-installation cleanup..."]) + ) + run_post_install(app_config) + ``` + This will run the specified post-installation command using `subprocess.run()`. + """ + if len(getattr(app_config.app, "post_install", [])) == 0: return subprocess.run( - config.app.post_install, + app_config.app.post_install, check=True, text=True, stdout=subprocess.PIPE, @@ -171,6 +470,36 @@ def run_post_install(config): def check_os_compatibility(app_config) -> None: + """ + Checks whether the current operating system is compatible with the application based on the configuration. + + Args: + app_config: The configuration object for the application, which is expected to have an `app` attribute + with a `platforms` attribute containing a list of supported operating systems. + + Returns: + None: This function does not return any value. + + Raises: + OSError: If the current operating system is not supported by the application. + + Functionality: + - Uses the `platform.system()` function to determine the current operating system. + - Checks the application's configuration (`app_config`) for a list of supported operating systems. + - If no platforms are defined in the configuration, the function simply returns without doing anything. + - If the current operating system is not in the list of supported platforms, raises an `OSError`. + + Example: + Suppose you have an application configuration that specifies supported platforms as `['Windows', 'Linux']`. + The function will determine the current operating system and raise an `OSError` if it is not supported: + ```python + try: + check_os_compatibility(app_config) + except OSError as e: + print(e) + ``` + If the current OS is not in the supported platforms list, the message "Your OS isn't supported by this app." will be printed. + """ os_name = platform.system().lower() supported_os = getattr(app_config.app, "platforms", []) @@ -187,7 +516,32 @@ def check_os_compatibility(app_config) -> None: raise OSError("Your OS isn't supported by this app.") -def get_current_commit(app_path): +def get_current_commit(app_path: str) -> str: + """ + Retrieves the current commit hash for a Git repository located at the specified path. + + Args: + app_path (str): The file path to the Git repository. + + Returns: + str: The current commit hash of the repository if the command is successful. + If an error occurs, returns an error message describing the failure. + + Functionality: + - Uses the `git rev-parse HEAD` command to get the current commit hash. + - If the command succeeds, returns the commit hash as a string. + - If the command fails (e.g., if the provided path is not a valid Git repository), + returns an error message detailing what went wrong. + + Example: + Suppose you have a Git repository at `/path/to/repo` and want to retrieve its current commit hash. + You can call the function like this: + ```python + commit_hash = get_current_commit("/path/to/repo") + ``` + This will return the commit hash if the repository exists and the command runs successfully, + or an error message if there is an issue with the command. + """ try: # Navigate to the repository path and get the current commit hash commit_hash = ( @@ -203,6 +557,38 @@ def get_current_commit(app_path): def update_app_config_file(app_path: str, sanitized_git_path: str, app_config) -> None: + """ + Updates the `app.json` configuration file with the current commit and version information of an application. + + Args: + app_path (str): The file path of the application. + sanitized_git_path (str): The sanitized path representing the Git repository. + app_config: The configuration object for the application, which is expected to have an `app` attribute + with a `version` attribute, if available. + + Returns: + None: This function modifies the `app.json` configuration file in place and returns nothing. + + Functionality: + - Normalizes the provided application path. + - Determines the configuration directory by navigating two levels up from the application path. + - Checks if an `app.json` file exists in the configuration directory. + - If it exists, loads its contents into a dictionary. + - If it does not exist, creates an empty dictionary for new configuration entries. + - Retrieves the current commit information of the application using the `get_current_commit` function. + - If the application version is available from the `app_config` object, includes it in the configuration. + - Updates the `app.json` configuration file with the new commit and version information under the key + specified by `sanitized_git_path`. + - Writes the updated configuration back to the `app.json` file with indentation for readability. + + Example: + Suppose you have an application located at `/path/to/app` and you want to update the `app.json` file + with the latest commit and version. You can call the function like this: + ```python + update_app_config_file("/path/to/app", "my_sanitized_git_path", app_config) + ``` + This will update or create entries in `app.json` for the given Git path, storing commit and version details. + """ normalized_app_path = os.path.normpath(app_path) conf_path = os.path.dirname(os.path.dirname(normalized_app_path)) @@ -227,6 +613,45 @@ def update_app_config_file(app_path: str, sanitized_git_path: str, app_config) - def install(client_config: ClientConfig) -> None | Tuple[str, Exception]: + """ + Installs an application by cloning the repository, checking compatibility, and running installation scripts. + + Args: + client_config (ClientConfig): The configuration object for the client, which is used during the installation process. + + Returns: + None: If the installation is successful. + Tuple[str, Exception]: If an error occurs during any installation step, returns a tuple with the step description and the exception raised. + + Functionality: + - Parses command-line arguments to get the Git repository to install. + - Performs a series of steps to install the application, including: + 1. Sanitizing the Git repository path. + 2. Cloning the repository to a temporary directory. + 3. Loading the application's configuration (`config.json`). + 4. Checking platform compatibility. + 5. Moving the repository to the Syftbox directory. + 6. Creating a symbolic link on the user's desktop. + 7. Running pre-installation commands. + 8. Running post-installation commands. + 9. Updating the `apps.json` file to include the installed application. + - If any step fails, returns the step description and the exception raised. + + Example: + Suppose you have a client configuration and want to install an application from a repository: + ```python + try: + result = install(client_config) + if result is not None: + step, error = result + print(f"Error during step '{step}': {error}") + else: + print("Installation completed successfully.") + except Exception as e: + print(f"Unexpected error: {e}") + ``` + This will install the application, and if an error occurs, it will indicate the step where the failure happened. + """ parser = argparse.ArgumentParser(description="Run FastAPI server") parser.add_argument("repository", type=str, help="App repository") From 7fb9ffb492dff17c0b7fab16544e9f7bae53d3e8 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Thu, 10 Oct 2024 23:24:08 +0530 Subject: [PATCH 16/17] potential fix for /tmp/ in macos --- syftbox/app/install.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/syftbox/app/install.py b/syftbox/app/install.py index 5d708cad..22a8a643 100644 --- a/syftbox/app/install.py +++ b/syftbox/app/install.py @@ -1,17 +1,17 @@ import argparse import json import os +import platform import re import shutil import subprocess -import platform +from pathlib import Path +from tempfile import mkdtemp from types import SimpleNamespace -from typing import Tuple -from typing import Any -from .utils import get_config_path -from ..lib import ClientConfig +from typing import Any, Tuple -TEMP_PATH = "/tmp/apps/" +from syftbox.app.utils import get_config_path +from syftbox.lib.lib import ClientConfig def is_git_installed() -> bool: @@ -200,7 +200,8 @@ def clone_repository(sanitized_git_path: str) -> str: "The provided repository path doesn't seems to be accessible. Please check it out." ) # Clone repository in /tmp - temp_clone_path = f"{TEMP_PATH}/{sanitized_git_path.split('/')[-1]}" + tmp_path = mkdtemp(prefix="syftbox_app_") + temp_clone_path = Path(tmp_path, sanitized_git_path.split("/")[-1]) # Delete if there's already an existent repository folder in /tmp path. delete_folder_if_exists(temp_clone_path) @@ -314,9 +315,9 @@ def load_config(path: str) -> SimpleNamespace: with open(path, "r") as f: data = json.load(f) if not isinstance(data, dict): - raise ValueError("File isn't in JSON format.") + raise ValueError(error_msg) except json.JSONDecodeError: - raise ValueError("File isn't in JSON format.") + raise ValueError(error_msg) return dict_to_namespace(data) From 493ebba3b50b964cca8e198ccf309249846a5422 Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Fri, 11 Oct 2024 12:24:17 +1000 Subject: [PATCH 17/17] Made app install less strict and tested against github_app_updater - added support for full github urls - added version command - added debug command - added just app - added just install --- justfile | 29 +++++++ pyproject.toml | 2 + syftbox/app/install.py | 96 +++++++++++++++++------- syftbox/app/manager.py | 82 +++++++++++++++----- syftbox/client/__init__.py | 2 + syftbox/client/client.py | 90 +++------------------- syftbox/client/plugins/apps.py | 31 ++++---- syftbox/lib/lib.py | 86 ++++++++++++++++++++- syftbox/main.py | 72 ++++++++++++++++++ tests/client/app_manager/install_test.py | 11 +++ uv.lock | 4 + 11 files changed, 361 insertions(+), 144 deletions(-) diff --git a/justfile b/justfile index 9096b7c0..f719d4e6 100644 --- a/justfile +++ b/justfile @@ -69,12 +69,41 @@ run-client name port="auto" server="http://localhost:5001": # --------------------------------------------------------------------------------------------------------------------- +# Run a local syftbox app command +[group('app')] +run-app name command subcommand="": + #!/bin/bash + set -eou pipefail + + # generate a local email from name, but if it looks like an email, then use it as is + EMAIL="{{ name }}@openmined.org" + if [[ "{{ name }}" == *@*.* ]]; then EMAIL="{{ name }}"; fi + + # Working directory for client is .clients/ + CONFIG_DIR=.clients/$EMAIL/config + SYNC_DIR=.clients/$EMAIL/sync + mkdir -p $CONFIG_DIR $SYNC_DIR + + echo -e "Config Dir : $CONFIG_DIR" + + uv run syftbox/main.py app {{ command }} {{ subcommand }} --config_path=$CONFIG_DIR/config.json + +# --------------------------------------------------------------------------------------------------------------------- + # Build syftbox wheel [group('build')] build: rm -rf dist uv build + +# Build syftbox wheel +[group('install')] +install: + rm -rf dist + uv build + uv tool install $(ls /Users/madhavajay/dev/syft/dist/*.whl) --reinstall + # Bump version, commit and tag [group('build')] bump-version level="patch": diff --git a/pyproject.toml b/pyproject.toml index 734d0b2f..f60b16d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,8 @@ dependencies = [ "pydantic-settings>=2.5.2", "pytest>=8.3.3", "httpx>=0.27.2", + "pyyaml>=6.0.2", + "psutil>=6.0.0", ] [project.optional-dependencies] diff --git a/syftbox/app/install.py b/syftbox/app/install.py index 22a8a643..e9834460 100644 --- a/syftbox/app/install.py +++ b/syftbox/app/install.py @@ -10,7 +10,6 @@ from types import SimpleNamespace from typing import Any, Tuple -from syftbox.app.utils import get_config_path from syftbox.lib.lib import ClientConfig @@ -76,6 +75,16 @@ def sanitize_git_path(path: str) -> str: If the path is valid, `sanitized_path` will contain the validated GitHub path. If it is not valid, the error message "Invalid Git repository path format. (eg: OpenMined/logged_in)" will be printed. """ + + if path.startswith("http://"): + path = path.replace("http://", "") + + if path.startswith("https://"): + path = path.replace("https://", "") + + if path.startswith("github.com/"): + path = path.replace("github.com/", "") + # Define a regex pattern for a valid GitHub path pattern = r"^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$" @@ -358,12 +367,18 @@ def create_symbolic_link( ) # Create the symlink - if os.path.islink(target_symlink_path): + if os.path.exists(target_symlink_path) and os.path.islink(target_symlink_path): os.unlink(target_symlink_path) - os.symlink(app_path, target_symlink_path) + + if not os.path.exists(target_symlink_path): + os.symlink(app_path, target_symlink_path) + else: + raise Exception(f"Path exists and isn't a symlink: {target_symlink_path}") -def move_repository_to_syftbox(tmp_clone_path: str, sanitized_path: str) -> str: +def move_repository_to_syftbox( + client_config: ClientConfig, tmp_clone_path: str, sanitized_path: str +) -> str: """ Moves a cloned Git repository to the Syftbox directory. @@ -388,7 +403,7 @@ def move_repository_to_syftbox(tmp_clone_path: str, sanitized_path: str) -> str: ``` This will move the cloned repository to the Syftbox `apps` directory and return the final destination path. """ - output_path = f"{get_config_path()}/apps/{sanitized_path.split('/')[-1]}" + output_path = f"{client_config.sync_folder}/apps/{sanitized_path.split('/')[-1]}" delete_folder_if_exists(output_path) shutil.move(tmp_clone_path, output_path) return output_path @@ -613,6 +628,26 @@ def update_app_config_file(app_path: str, sanitized_git_path: str, app_config) - json.dump(app_json_config, json_file, indent=4) +def check_app_config(tmp_clone_path) -> SimpleNamespace | None: + try: + app_config_path = tmp_clone_path / "config.json" + if os.path.exists(app_config_path): + app_config = load_config(app_config_path) + step = "Loading config.json" + print(step) + # NOTE: + # Check OS platform compatibility + # Handles if app isn't compatible with the target os system. + step = "Checking platform compatibility." + print(step) + check_os_compatibility(app_config) + + return app_config + except Exception as e: + print("No app config", e) + return None + + def install(client_config: ClientConfig) -> None | Tuple[str, Exception]: """ Installs an application by cloning the repository, checking compatibility, and running installation scripts. @@ -681,50 +716,59 @@ def install(client_config: ClientConfig) -> None | Tuple[str, Exception]: # Handles: config.json doesn't exist in the pulled repository # Handles: config.json version is different from syftbox config version. # Returns: Loaded app config as SimpleNamespace instance. - step = "Loading config.json" - app_config = load_config(tmp_clone_path + "/config.json") - # NOTE: - # Check OS platform compatibility - # Handles if app isn't compatible with the target os system. - step = "Checking platform compatibility." - check_os_compatibility(app_config) + # make optional + app_config = check_app_config(tmp_clone_path) # NOTE: # Moves the repository from /tmp to ~/.syftbox/apps/ # Handles: If ~/.syftbox/apps/ already exists (replaces it) - app_path = move_repository_to_syftbox( - tmp_clone_path=tmp_clone_path, sanitized_path=sanitized_path + app_config_path = move_repository_to_syftbox( + client_config, tmp_clone_path=tmp_clone_path, sanitized_path=sanitized_path ) # NOTE: # Creates a Symbolic Link ( ~/Desktop/Syftbox/app/ -> ~/.syftbox/apps/) # Handles: If ~/.syftbox/apps/ already exists (replaces it) - step = "Creating Symbolic Link" - create_symbolic_link( - client_config=client_config, - app_path=app_path, - sanitized_path=sanitized_path, - ) + # step = "Creating Symbolic Link" + # print(step) + # create_symbolic_link( + # client_config=client_config, + # app_path=app_config_path, + # sanitized_path=sanitized_path, + # ) # NOTE: # Executes config.json pre-install command list # Handles: Exceptions from pre-install command execution - step = "Running pre-install commands" - run_pre_install(app_config) + if app_config: + step = "Running pre-install commands" + run_pre_install(app_config) # NOTE: # Executes config.json post-install command list # Handles: Exceptions from post-install command execution - step = "Running post-install commands" - run_post_install(app_config) + if app_config: + step = "Running post-install commands" + run_post_install(app_config) # NOTE: # Updates the apps.json file # Handles: If apps.json file doesn't exist yet. # Handles: If apps.json already have the repository_name app listed. # Handles: If apps.json exists but doesn't have the repository_name app listed. - step = "Updating apps.json config" - update_app_config_file(app_path, sanitized_path, app_config) + if app_config: + step = "Updating apps.json config" + update_app_config_file(app_config_path, sanitized_path, app_config) + + app_name = app_config_path.split("/")[-1] + parts = app_config_path.split("/apps/") + print( + "------------------------------------\n" + f"Datasite Path: {parts[0]}\n" + f"In SyftBox: /apps/{parts[-1]}\n" + "------------------------------------\n\n" + f"✅ App {app_name} installed\n" + ) except Exception as e: return (step, e) diff --git a/syftbox/app/manager.py b/syftbox/app/manager.py index e077909e..87b517a9 100644 --- a/syftbox/app/manager.py +++ b/syftbox/app/manager.py @@ -1,16 +1,34 @@ +import argparse import os import sys from collections import namedtuple +from pathlib import Path -from ..lib import ClientConfig +from ..lib import DEFAULT_CONFIG_PATH, ClientConfig from .install import install -from .utils import get_config_path config_path = os.environ.get("SYFTBOX_CLIENT_CONFIG_PATH", None) -def list_app(client_config: ClientConfig) -> None: - print("Listing apps") +def list_app(client_config: ClientConfig, silent: bool = False) -> list[str]: + apps_path = Path(client_config.sync_folder + "/" + "apps") + apps = [] + if os.path.exists(apps_path): + files_and_folders = os.listdir(apps_path) + apps = [app for app in files_and_folders if os.path.isdir(apps_path / app)] + + if len(apps): + if not silent: + print("\nInstalled apps:") + for app in apps: + print(f"✅ {app}") + else: + if not silent: + print( + "\nYou have no apps installed.\n\n" + f"Try:\nsyftbox app install OpenMined/github_app_updater\n\nor copy an app to: {apps_path}" + ) + return apps def uninstall_app(client_config: ClientConfig) -> None: @@ -25,12 +43,11 @@ def upgrade_app(client_config: ClientConfig) -> None: print("Upgrading Apps") -def main(parser, args_list) -> None: - config_path = get_config_path() - client_config = ClientConfig.load(config_path + "config.json") +Commands = namedtuple("Commands", ["description", "execute"]) + - Commands = namedtuple("Commands", ["description", "execute"]) - commands = { +def make_commands() -> dict[str, Commands]: + return { "list": Commands( "List all currently installed apps in your syftbox.", list_app ), @@ -40,23 +57,48 @@ def main(parser, args_list) -> None: "upgrade": Commands("Upgrade an app.", upgrade_app), } - # Add a subparser to the "app" parser to handle different actions - app_subparsers = parser.add_subparsers( - title="App Commands", - dest="subcommand", + +class CustomHelpFormatter(argparse.HelpFormatter): + def add_arguments(self, actions): + for action in actions: + if action.dest == "command": + commands = make_commands() + action.choices = commands.keys() + # Build help message with descriptions + action.help = "\n".join( + [f"{cmd}: {commands[cmd].description}" for cmd in commands] + ) + super().add_arguments(actions) + + +# Parsing arguments and initializing shared state +def parse_args(): + parser = argparse.ArgumentParser( + description="Run the web application with plugins.", ) - # Add all the commands to the argparser - for command, cmd_info in commands.items(): - app_subparsers.add_parser(command, help=cmd_info.description) + commands = make_commands() - # Parse the remaining args using the parser with subparsers added - # args = parser.parse_args(args_list) + # Add a subparser to the "app" parser to handle different actions + parser.add_argument( + "command", choices=commands.keys(), help="The command to execute" + ) + + parser.add_argument( + "--config_path", type=str, default=DEFAULT_CONFIG_PATH, help="config path" + ) args, remaining_args = parser.parse_known_args() + return args, remaining_args + + +def main(parser, args_list) -> None: + args, remaining_args = parse_args() + client_config = ClientConfig.load(args.config_path) + commands = make_commands() # Handle the subcommands as needed - if args.subcommand: - command = commands[args.subcommand] + if args.command: + command = commands[args.command] sys.argv = [sys.argv[0]] + remaining_args error = command.execute(client_config) if error is not None: diff --git a/syftbox/client/__init__.py b/syftbox/client/__init__.py index 11b3f2c6..2b685cf6 100644 --- a/syftbox/client/__init__.py +++ b/syftbox/client/__init__.py @@ -1 +1,3 @@ from syftbox import __version__ + +from .fsevents import AnyFileSystemEventHandler, FileSystemEvent, FSWatchdog diff --git a/syftbox/client/client.py b/syftbox/client/client.py index e8223848..e6550938 100644 --- a/syftbox/client/client.py +++ b/syftbox/client/client.py @@ -3,7 +3,6 @@ import contextlib import importlib import os -import platform import sys import time import traceback @@ -11,7 +10,7 @@ from dataclasses import dataclass from functools import partial from pathlib import Path -from typing import Any, Optional +from typing import Any import uvicorn from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore @@ -32,8 +31,12 @@ FileSystemEvent, FSWatchdog, ) -from syftbox.client.utils import macos -from syftbox.lib import ClientConfig, SharedState, validate_email +from syftbox.lib import ( + DEFAULT_CONFIG_PATH, + ClientConfig, + SharedState, + load_or_create_config, +) class CustomFastAPI(FastAPI): @@ -56,8 +59,8 @@ class CustomFastAPI(FastAPI): sys.path.insert(0, os.path.dirname(PLUGINS_DIR)) DEFAULT_SYNC_FOLDER = os.path.expanduser("~/Desktop/SyftBox") -DEFAULT_PORT = 8082 -DEFAULT_CONFIG_PATH = os.path.expanduser("~/.syftbox/client_config.json") + + ASSETS_FOLDER = current_dir.parent / "assets" ICON_FOLDER = ASSETS_FOLDER / "icon" @@ -72,81 +75,6 @@ class Plugin: description: str -def load_or_create_config(args) -> ClientConfig: - syft_config_dir = os.path.abspath(os.path.expanduser("~/.syftbox")) - os.makedirs(syft_config_dir, exist_ok=True) - - client_config = None - try: - client_config = ClientConfig.load(args.config_path) - except Exception: - pass - - if client_config is None and args.config_path: - config_path = os.path.abspath(os.path.expanduser(args.config_path)) - client_config = ClientConfig(config_path=config_path) - - if client_config is None: - # config_path = get_user_input("Path to config file?", DEFAULT_CONFIG_PATH) - config_path = os.path.abspath(os.path.expanduser(config_path)) - client_config = ClientConfig(config_path=config_path) - - if args.sync_folder: - sync_folder = os.path.abspath(os.path.expanduser(args.sync_folder)) - client_config.sync_folder = sync_folder - - if client_config.sync_folder is None: - sync_folder = get_user_input( - "Where do you want to Sync SyftBox to?", - DEFAULT_SYNC_FOLDER, - ) - sync_folder = os.path.abspath(os.path.expanduser(sync_folder)) - client_config.sync_folder = sync_folder - - if args.server: - client_config.server_url = args.server - - if not os.path.exists(client_config.sync_folder): - os.makedirs(client_config.sync_folder, exist_ok=True) - - if platform.system() == "Darwin": - macos.copy_icon_file(ICON_FOLDER, client_config.sync_folder) - - if args.email: - client_config.email = args.email - - if client_config.email is None: - email = get_user_input("What is your email address? ") - if not validate_email(email): - raise Exception(f"Invalid email: {email}") - client_config.email = email - - if args.port: - client_config.port = args.port - - if client_config.port is None: - port = int(get_user_input("Enter the port to use", DEFAULT_PORT)) - client_config.port = port - - email_token = os.environ.get("EMAIL_TOKEN", None) - if email_token: - client_config.email_token = email_token - - # Migrate Old Server URL to HTTPS - if client_config.server_url == "http://20.168.10.234:8080": - client_config.server_url = "https://syftbox.openmined.org" - - client_config.save(args.config_path) - return client_config - - -def get_user_input(prompt, default: Optional[str] = None): - if default: - prompt = f"{prompt} (default: {default}): " - user_input = input(prompt).strip() - return user_input if user_input else default - - def process_folder_input(user_input, default_path): if not user_input: return default_path diff --git a/syftbox/client/plugins/apps.py b/syftbox/client/plugins/apps.py index fc3f83cd..014c2f57 100644 --- a/syftbox/client/plugins/apps.py +++ b/syftbox/client/plugins/apps.py @@ -1,7 +1,6 @@ import json import logging import os -import shutil import subprocess import threading from types import SimpleNamespace @@ -63,22 +62,22 @@ def find_and_run_script(task_path, extra_args): ) -def copy_default_apps(apps_path): - if not os.path.exists(DEFAULT_APPS_PATH): - print(f"Default apps directory not found: {DEFAULT_APPS_PATH}") - return +# def copy_default_apps(apps_path): +# if not os.path.exists(DEFAULT_APPS_PATH): +# print(f"Default apps directory not found: {DEFAULT_APPS_PATH}") +# return - for app in os.listdir(DEFAULT_APPS_PATH): - src_app_path = os.path.join(DEFAULT_APPS_PATH, app) - dst_app_path = os.path.join(apps_path, app) +# for app in os.listdir(DEFAULT_APPS_PATH): +# src_app_path = os.path.join(DEFAULT_APPS_PATH, app) +# dst_app_path = os.path.join(apps_path, app) - if os.path.isdir(src_app_path): - if os.path.exists(dst_app_path): - print(f"App already installed at: {dst_app_path}") - # shutil.rmtree(dst_app_path) - else: - shutil.copytree(src_app_path, dst_app_path) - print(f"Copied default app: {app}") +# if os.path.isdir(src_app_path): +# if os.path.exists(dst_app_path): +# print(f"App already installed at: {dst_app_path}") +# # shutil.rmtree(dst_app_path) +# else: +# shutil.copytree(src_app_path, dst_app_path) +# print(f"Copied default app: {app}") def dict_to_namespace(data) -> SimpleNamespace | list | Any: @@ -107,7 +106,7 @@ def run_apps(client_config): os.makedirs(apps_path, exist_ok=True) # Copy default apps if they don't exist - copy_default_apps(apps_path) + # copy_default_apps(apps_path) # add the first perm file file_path = perm_file_path(apps_path) diff --git a/syftbox/lib/lib.py b/syftbox/lib/lib.py index 71971cea..08f958f2 100644 --- a/syftbox/lib/lib.py +++ b/syftbox/lib/lib.py @@ -4,6 +4,7 @@ import hashlib import json import os +import platform import re import threading import zlib @@ -11,14 +12,22 @@ from datetime import datetime from pathlib import Path from threading import Lock -from typing import Any +from typing import Any, Optional import httpx import requests from typing_extensions import Self +from syftbox.client.utils import macos from syftbox.server.models import get_file_hash, get_file_last_modified +current_dir = Path(__file__).parent +ASSETS_FOLDER = current_dir.parent / "assets" +DEFAULT_PORT = 8082 +ICON_FOLDER = ASSETS_FOLDER / "icon" +DEFAULT_SYNC_FOLDER = os.path.expanduser("~/Desktop/SyftBox") +DEFAULT_CONFIG_PATH = os.path.expanduser("~/.syftbox/client_config.json") + USER_GROUP_GLOBAL = "GLOBAL" ICON_FILE = "Icon" # special @@ -586,3 +595,78 @@ def create_public_folder(self, path: str): public_read = SyftPermission.mine_with_public_read(email=self.datasite) public_read.save(full_path) return Path(full_path) + + +def get_user_input(prompt, default: Optional[str] = None): + if default: + prompt = f"{prompt} (default: {default}): " + user_input = input(prompt).strip() + return user_input if user_input else default + + +def load_or_create_config(args) -> ClientConfig: + syft_config_dir = os.path.abspath(os.path.expanduser("~/.syftbox")) + os.makedirs(syft_config_dir, exist_ok=True) + + client_config = None + try: + client_config = ClientConfig.load(args.config_path) + except Exception: + pass + + if client_config is None and args.config_path: + config_path = os.path.abspath(os.path.expanduser(args.config_path)) + client_config = ClientConfig(config_path=config_path) + + if client_config is None: + # config_path = get_user_input("Path to config file?", DEFAULT_CONFIG_PATH) + config_path = os.path.abspath(os.path.expanduser(config_path)) + client_config = ClientConfig(config_path=config_path) + + if args.sync_folder: + sync_folder = os.path.abspath(os.path.expanduser(args.sync_folder)) + client_config.sync_folder = sync_folder + + if client_config.sync_folder is None: + sync_folder = get_user_input( + "Where do you want to Sync SyftBox to?", + DEFAULT_SYNC_FOLDER, + ) + sync_folder = os.path.abspath(os.path.expanduser(sync_folder)) + client_config.sync_folder = sync_folder + + if args.server: + client_config.server_url = args.server + + if not os.path.exists(client_config.sync_folder): + os.makedirs(client_config.sync_folder, exist_ok=True) + + if platform.system() == "Darwin": + macos.copy_icon_file(ICON_FOLDER, client_config.sync_folder) + + if args.email: + client_config.email = args.email + + if client_config.email is None: + email = get_user_input("What is your email address? ") + if not validate_email(email): + raise Exception(f"Invalid email: {email}") + client_config.email = email + + if args.port: + client_config.port = args.port + + if client_config.port is None: + port = int(get_user_input("Enter the port to use", DEFAULT_PORT)) + client_config.port = port + + email_token = os.environ.get("EMAIL_TOKEN", None) + if email_token: + client_config.email_token = email_token + + # Migrate Old Server URL to HTTPS + if client_config.server_url == "http://20.168.10.234:8080": + client_config.server_url = "https://syftbox.openmined.org" + + client_config.save(args.config_path) + return client_config diff --git a/syftbox/main.py b/syftbox/main.py index 01da47a8..3c606db2 100644 --- a/syftbox/main.py +++ b/syftbox/main.py @@ -1,11 +1,71 @@ import argparse import sys +from syftbox import __version__ +from syftbox.app.manager import list_app from syftbox.app.manager import main as app_manager_main from syftbox.client.client import main as client_main from syftbox.server.server import main as server_main +def print_debug(): + try: + import os + import platform + import shutil + + import psutil + import yaml + + from syftbox.lib import DEFAULT_CONFIG_PATH, ClientConfig + + config_path = os.environ.get("SYFTBOX_CLIENT_CONFIG_PATH", DEFAULT_CONFIG_PATH) + client_config = None + apps = [] + try: + client_config = ClientConfig.load(config_path) + apps = list_app(client_config, silent=True) + client_config = client_config.to_dict() + except Exception: + pass + + syftbox_path = shutil.which("syftbox") + + debug_info = { + "system": { + "resources": { + "cpus": psutil.cpu_count(logical=True), + "architecture": platform.machine(), + "ram": f"{psutil.virtual_memory().total / (1024**3):.2f} GB", + }, + "operating_system": { + "name": "macOS" + if platform.system() == "Darwin" + else platform.system(), + "version": platform.release(), + }, + "python": { + "version": platform.python_version(), + "binary_location": sys.executable, + }, + }, + "syftbox": { + "command": syftbox_path or "syftbox executable not found in PATH", + "apps": apps, + "client_config_path": config_path, + "client_config": client_config, + }, + "syftbox_env": { + key: value + for key, value in os.environ.items() + if key.startswith("SYFT") + }, + } + print(yaml.dump(debug_info, default_flow_style=False)) + except Exception as e: + print(e) + + def main(): parser = argparse.ArgumentParser(description="Syftbox CLI") subparsers = parser.add_subparsers( @@ -25,6 +85,14 @@ def main(): "app", help="Manage SyftBox apps.", description="Manages SyftBox Apps" ) + app_parser = subparsers.add_parser( + "version", help="Show SyftBox version", description="Shows the version" + ) + + app_parser = subparsers.add_parser( + "debug", help="Show SyftBox debug info", description="Shows the debug info" + ) + args, remaining_args = parser.parse_known_args() if args.command == "client": @@ -38,6 +106,10 @@ def main(): elif args.command == "app": sys.argv = [sys.argv[0]] + remaining_args app_manager_main(app_parser, remaining_args) + elif args.command == "version": + print(f"{__version__}") + elif args.command == "debug": + print_debug() else: parser.print_help() diff --git a/tests/client/app_manager/install_test.py b/tests/client/app_manager/install_test.py index 0e1ad8bd..3e8c5ff4 100644 --- a/tests/client/app_manager/install_test.py +++ b/tests/client/app_manager/install_test.py @@ -19,6 +19,17 @@ def test_valid_git_path(): assert path == output_path +def test_valid_git_url(): + path = "Example/Repository" + http_url = f"http://github.com/{path}" + output_path = sanitize_git_path(http_url) + assert path == output_path + + https_url = f"https://github.com/{path}" + output_path = sanitize_git_path(https_url) + assert path == output_path + + def test_invalid_git_path(): path = "..Example/../Repository" with pytest.raises(ValueError) as excpt: diff --git a/uv.lock b/uv.lock index a9ebad75..9f6c4918 100644 --- a/uv.lock +++ b/uv.lock @@ -1190,8 +1190,10 @@ dependencies = [ { name = "markdown" }, { name = "pandas" }, { name = "postmarker" }, + { name = "psutil" }, { name = "pydantic-settings" }, { name = "pytest" }, + { name = "pyyaml" }, { name = "requests" }, { name = "setuptools" }, { name = "sqlalchemy" }, @@ -1220,8 +1222,10 @@ requires-dist = [ { name = "markdown", specifier = ">=3.7" }, { name = "pandas", specifier = ">=2.2.2" }, { name = "postmarker", specifier = ">=1.0" }, + { name = "psutil", specifier = ">=6.0.0" }, { name = "pydantic-settings", specifier = ">=2.5.2" }, { name = "pytest", specifier = ">=8.3.3" }, + { name = "pyyaml", specifier = ">=6.0.2" }, { name = "requests", specifier = ">=2.32.3" }, { name = "setuptools", specifier = ">=75.1.0" }, { name = "sqlalchemy", specifier = ">=2.0.34" },