Merge pull request #74 from OpenMined/eelco/server-settings

add server settings, remove server globals

pyproject.toml

@@ -18,6 +18,7 @@ dependencies = [
     "setuptools>=75.1.0",
     "postmarker>=1.0",
     "watchdog>=5.0.2",
+    "pydantic-settings>=2.5.2",
 ]
 
 [project.optional-dependencies]
@@ -37,6 +38,7 @@ build-backend = "setuptools.build_meta"
 # this will be completely ignored in the built wheel
 dev-dependencies = [
     "bump2version>=1.0.1",
+    "pre-commit>=4.0.1",
     "pytest-cov>=5.0.0",
     "pytest-xdist[psutil]>=3.6.1",
     "pytest>=8.3.3",

syftbox/server/server.py

@@ -9,7 +9,7 @@
 from typing import Optional
 
 import uvicorn
-from fastapi import FastAPI, Request
+from fastapi import Depends, FastAPI, Request
 from fastapi.responses import (
     FileResponse,
     HTMLResponse,
@@ -31,17 +31,11 @@
     hash_dir,
     strtobin,
 )
+from syftbox.server.settings import ServerSettings, get_server_settings
 
 current_dir = Path(__file__).parent
 
 
-DATA_FOLDER = "data"
-SNAPSHOT_FOLDER = f"{DATA_FOLDER}/snapshot"
-USER_FILE_PATH = f"{DATA_FOLDER}/users.json"
-
-FOLDERS = [DATA_FOLDER, SNAPSHOT_FOLDER]
-
-
 def load_list(cls, filepath: str) -> list[Any]:
     try:
         with open(filepath) as f:
@@ -94,20 +88,21 @@ class User(Jsonable):
 
 
 class Users:
-    def __init__(self) -> None:
+    def __init__(self, path: Path) -> None:
+        self.path = path
         self.users = {}
         self.load()
 
     def load(self):
-        if os.path.exists(USER_FILE_PATH):
-            users = load_dict(User, USER_FILE_PATH)
+        if os.path.exists(str(self.path)):
+            users = load_dict(User, str(self.path))
         else:
             users = None
         if users:
             self.users = users
 
     def save(self):
-        save_dict(self.users, USER_FILE_PATH)
+        save_dict(self.users, str(self.path))
 
     def get_user(self, email: str) -> Optional[User]:
         if email not in self.users:
@@ -131,14 +126,9 @@ def __repr__(self) -> str:
             string += f"{email}: {user}"
         return string
 
-    # def key_for_email(self, email: str) -> int | None:
-    #     user = self.get_user(email)
-    #     if user:
-    #         return user.public_key
-    #     return None
-
 
-USERS = Users()
+def get_users(request: Request) -> Users:
+    return request.state.users
 
 
 def create_folders(folders: list[str]) -> None:
@@ -150,12 +140,21 @@ def create_folders(folders: list[str]) -> None:
 async def lifespan(app: FastAPI):
     # Startup
     print("> Starting Server")
+    settings = ServerSettings()
+    print(settings)
+
     print("> Creating Folders")
-    create_folders(FOLDERS)
+
+    create_folders(settings.folders)
+
+    users = Users(path=settings.user_file_path)
     print("> Loading Users")
-    print(USERS)
+    print(users)
 
-    yield  # Run the application
+    yield {
+        "server_settings": settings,
+        "users": users,
+    }
 
     print("> Shutting down server")
 
@@ -204,7 +203,10 @@ async def get_wheel(request: Request, path: str):
     return filename
 
 
-def get_file_list(directory="."):
+def get_file_list(directory: str | Path = ".") -> list[dict[str, Any]]:
+    # TODO rewrite with pathlib
+    directory = str(directory)
+
     file_list = []
     for item in os.listdir(directory):
         item_path = os.path.join(directory, item)
@@ -222,9 +224,10 @@ def get_file_list(directory="."):
 
 
 @app.get("/datasites", response_class=HTMLResponse)
-async def list_datasites(request: Request):
-    datasite_path = os.path.join(SNAPSHOT_FOLDER)
-    files = get_file_list(datasite_path)
+async def list_datasites(
+    request: Request, server_settings: ServerSettings = Depends(get_server_settings)
+):
+    files = get_file_list(server_settings.snapshot_folder)
     template_path = current_dir / "templates" / "datasites.html"
     html = ""
     with open(template_path) as f:
@@ -242,17 +245,22 @@ async def list_datasites(request: Request):
 
 
 @app.get("/datasites/{path:path}", response_class=HTMLResponse)
-async def browse_datasite(request: Request, path: str):
+async def browse_datasite(
+    request: Request,
+    path: str,
+    server_settings: ServerSettings = Depends(get_server_settings),
+):
     if path == "":  # Check if path is empty (meaning "/datasites/")
         return RedirectResponse(url="/datasites")
 
+    snapshot_folder = str(server_settings.snapshot_folder)
     datasite_part = path.split("/")[0]
-    datasites = get_datasites(SNAPSHOT_FOLDER)
+    datasites = get_datasites(snapshot_folder)
     if datasite_part in datasites:
         slug = path[len(datasite_part) :]
         if slug == "":
             slug = "/"
-        datasite_path = os.path.join(SNAPSHOT_FOLDER, datasite_part)
+        datasite_path = os.path.join(snapshot_folder, datasite_part)
         datasite_public = datasite_path + "/public"
         if not os.path.exists(datasite_public):
             return "No public datasite"
@@ -301,24 +309,26 @@ async def browse_datasite(request: Request, path: str):
 
 
 @app.post("/register")
-async def register(request: Request):
+async def register(request: Request, users: Users = Depends(get_users)):
     data = await request.json()
     email = data["email"]
-    token = USERS.create_user(email)
+    token = users.create_user(email)
     print(f"> {email} registering: {token}")
     return JSONResponse({"status": "success", "token": token}, status_code=200)
 
 
 @app.post("/write")
-async def write(request: Request):
+async def write(
+    request: Request, server_settings: ServerSettings = Depends(get_server_settings)
+):
     try:
         data = await request.json()
         email = data["email"]
         change_dict = data["change"]
         change_dict["kind"] = FileChangeKind(change_dict["kind"])
         change = FileChange(**change_dict)
 
-        change.sync_folder = os.path.abspath(SNAPSHOT_FOLDER)
+        change.sync_folder = os.path.abspath(str(server_settings.snapshot_folder))
         result = True
         accepted = True
         if change.newer():
@@ -366,13 +376,15 @@ async def write(request: Request):
 
 
 @app.post("/read")
-async def read(request: Request):
+async def read(
+    request: Request, server_settings: ServerSettings = Depends(get_server_settings)
+):
     data = await request.json()
     email = data["email"]
     change_dict = data["change"]
     change_dict["kind"] = FileChangeKind(change_dict["kind"])
     change = FileChange(**change_dict)
-    change.sync_folder = os.path.abspath(SNAPSHOT_FOLDER)
+    change.sync_folder = os.path.abspath(str(server_settings.snapshot_folder))
 
     json_dict = {"change": change.to_dict()}
 
@@ -395,13 +407,16 @@ async def read(request: Request):
 
 
 @app.post("/dir_state")
-async def dir_state(request: Request):
+async def dir_state(
+    request: Request, server_settings: ServerSettings = Depends(get_server_settings)
+):
     try:
         data = await request.json()
         email = data["email"]
         sub_path = data["sub_path"]
-        full_path = os.path.join(SNAPSHOT_FOLDER, sub_path)
-        remote_dir_state = hash_dir(SNAPSHOT_FOLDER, sub_path)
+        snapshot_folder = str(server_settings.snapshot_folder)
+        full_path = os.path.join(snapshot_folder, sub_path)
+        remote_dir_state = hash_dir(snapshot_folder, sub_path)
 
         # get the top level perm file
         perm_tree = PermissionTree.from_path(full_path)
@@ -419,8 +434,10 @@ async def dir_state(request: Request):
 
 
 @app.get("/list_datasites")
-async def datasites(request: Request):
-    datasites = get_datasites(SNAPSHOT_FOLDER)
+async def datasites(
+    request: Request, server_settings: ServerSettings = Depends(get_server_settings)
+):
+    datasites = get_datasites(server_settings.snapshot_folder)
     response_json = {"datasites": datasites}
     if datasites:
         return JSONResponse({"status": "success"} | response_json, status_code=200)

syftbox/server/settings.py

@@ -0,0 +1,30 @@
+from pathlib import Path
+
+from fastapi import Request
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class ServerSettings(BaseSettings):
+    """
+    Reads the server settings from the environment variables, using the prefix SYFTBOX_.
+
+    example:
+    `export SYFTBOX_DATA_FOLDER=data/data_folder`
+    will set the server_settings.data_folder to `data/data_folder`
+
+    see: https://docs.pydantic.dev/latest/concepts/pydantic_settings/#parsing-environment-variable-values
+    """
+
+    model_config = SettingsConfigDict(env_prefix="SYFTBOX_")
+
+    data_folder: Path = Path("data")
+    snapshot_folder: Path = Path("data/snapshot")
+    user_file_path: Path = Path("data/users.json")
+
+    @property
+    def folders(self) -> list[Path]:
+        return [self.data_folder, self.snapshot_folder]
+
+
+def get_server_settings(request: Request) -> ServerSettings:
+    return request.state.server_settings

tests/server/settings_test.py

@@ -0,0 +1,16 @@
+import os
+from pathlib import Path
+
+from syftbox.server.settings import ServerSettings
+
+
+def test_server_settings_from_env():
+    os.environ["SYFTBOX_DATA_FOLDER"] = "data_folder"
+    os.environ["SYFTBOX_SNAPSHOT_FOLDER"] = "data_folder/snapshot_folder"
+    os.environ["SYFTBOX_USER_FILE_PATH"] = "data_folder/user_file_path.json"
+
+    settings = ServerSettings()
+    print(settings)
+    assert settings.data_folder == Path("data_folder")
+    assert settings.snapshot_folder == Path("data_folder/snapshot_folder")
+    assert settings.user_file_path == Path("data_folder/user_file_path.json")