Merge branch 'master' into bug/SK-1040-bug-in-example-monai-2D-mednist

docs/conf.py

@@ -11,7 +11,7 @@
 author = "Scaleout Systems AB"
 
 # The full version, including alpha/beta/rc tags
-release = "0.15.0"
+release = "0.16.0"
 
 # Add any Sphinx extension module names here, as strings
 extensions = [

docs/introduction.rst

@@ -71,4 +71,11 @@ For professionals / Enteprise, we offer `Dedicated support <https://www.scaleout
 
 .. meta::
     :description lang=en:
-        In contrast to traditional machine learning setups where data is collected and stored centrally, Federated Learning allows for collaborative model training while keeping data local with the data owner or device.
+        In contrast to traditional machine learning setups where data is collected and stored centrally, Federated Learning allows for collaborative model training while keeping data local with the data owner or device.
+    :keywords: Federated Learning, Machine Learning, What is federated machine learning, Federated Learning Framework, Federated Learning Platform
+    :robots: index, follow
+    :og:title: What is Federated Learning?
+    :og:description: Federated Learning is a novel approach to address challenges related to data privacy, security, and decentralized data distribution.
+    :og:image: https://fedn.scaleoutsystems.com/static/images/scaleout_black.png
+    :og:url: https://fedn.scaleoutsystems.com/docs/introduction.html
+    :og:type: article

docs/quickstart.rst

@@ -204,5 +204,11 @@ to learn how to set up an all-in-one development environment using Docker and do
 :ref:`developer-label`
 
 .. meta::
-   :description lang=en:
-      This tutorial is a quickstart guide to FEDn based on a pre-made FEDn Project. It is designed to serve as a starting point for new developers. The first step is to start the server side (aggregator, controller). We do this by setting up a new Project in FEDn Studio.
+   :description lang=en: This tutorial is a quickstart guide to FEDn based on a pre-made FEDn Project. It is designed to serve as a starting point for new developers. The first step is to start the server side (aggregator, controller). We do this by setting up a new Project in FEDn Studio.
+   :keywords: Getting started with Federated Learning, Federated Learning, Federated Learning Framework, Federated Learning Platform
+   :robots: index, follow
+   :og:title: Getting started with FEDn
+   :og:description: This tutorial is a quickstart guide to FEDn based on a pre-made FEDn Project. It is designed to serve as a starting point for new developers.
+   :og:image: https://fedn.scaleoutsystems.com/static/images/scaleout_black.png
+   :og:url: https://fedn.scaleoutsystems.com/docs/quickstart.html
+   :og:type: website

examples/FedSimSiam/client/python_env.yaml

@@ -4,12 +4,12 @@ build_dependencies:
   - setuptools
   - wheel==0.37.1
 dependencies:
-  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   # PyTorch macOS x86 builds deprecation
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9")
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
   - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
   - numpy==1.24.4; python_version == "3.8"
   - fedn

examples/flower-client/client/python_env.yaml

@@ -5,12 +5,12 @@ build_dependencies:
   - wheel==0.37.1
 dependencies:
   - fedn[flower]
-  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   # PyTorch macOS x86 builds deprecation
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9")
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
   - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
   - numpy==1.24.4; python_version == "3.8"
   - fire==0.3.1

examples/huggingface/client/python_env.yaml

@@ -4,12 +4,12 @@ build_dependencies:
   - setuptools
   - wheel
 dependencies:
-  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   # PyTorch macOS x86 builds deprecation
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9")
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
   - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
   - numpy==1.24.4; python_version == "3.8"
   - transformers

examples/mnist-pytorch-DPSGD/client/python_env.yaml

@@ -5,12 +5,12 @@ build_dependencies:
   - wheel
 dependencies:
   - fedn
-  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   # PyTorch macOS x86 builds deprecation
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9")
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
   - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
   - numpy==1.24.4; python_version == "3.8"
   - opacus

examples/mnist-pytorch/client/python_env.yaml

@@ -5,11 +5,11 @@ build_dependencies:
   - wheel
 dependencies:
   - fedn
-  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   # PyTorch macOS x86 builds deprecation
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9")
+  - numpy==2.0.2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux" and python_version >= "3.9")
   - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")
   - numpy==1.24.4; python_version == "3.8"

examples/monai-2D-mednist/client/python_env.yaml

@@ -5,10 +5,10 @@ build_dependencies:
   - wheel
 dependencies:
   - fedn
-  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torch==2.4.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   # PyTorch macOS x86 builds deprecation
   - torch==2.2.2; sys_platform == "darwin" and platform_machine == "x86_64"
-  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win" or sys_platform == "linux")
+  - torchvision==0.19.1; (sys_platform == "darwin" and platform_machine == "arm64") or (sys_platform == "win32" or sys_platform == "win64" or sys_platform == "linux")
   - torchvision==0.17.2; sys_platform == "darwin" and platform_machine == "x86_64"
   - numpy<2; (sys_platform == "darwin" and platform_machine == "arm64" and python_version >= "3.9") or (sys_platform == "win" or sys_platform == "linux" and python_version >= "3.9")
   - numpy==1.26.4; (sys_platform == "darwin" and platform_machine == "x86_64" and python_version >= "3.9")

fedn/cli/client_cmd.py

@@ -3,11 +3,10 @@
 import click
 import requests
 
-from fedn.common.exceptions import InvalidClientConfig
-from fedn.network.clients.client import Client
-
 from fedn.cli.main import main
 from fedn.cli.shared import CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, print_response
+from fedn.common.exceptions import InvalidClientConfig
+from fedn.network.clients.client import Client
 
 
 def validate_client_config(config):
@@ -17,18 +16,20 @@ def validate_client_config(config):
     """
     try:
         if config["discover_host"] is None or config["discover_host"] == "":
-            raise InvalidClientConfig("Missing required configuration: discover_host")
+            if config["combiner"] is None or config["combiner"] == "":
+                raise InvalidClientConfig("Missing required configuration: discover_host or combiner")
         if "discover_port" not in config.keys():
             config["discover_port"] = None
+        if config["remote_compute_context"] and config["discover_host"] is None:
+            raise InvalidClientConfig("Remote compute context requires discover_host")
     except Exception:
         raise InvalidClientConfig("Could not load config from file. Check config")
 
 
 @main.group("client")
 @click.pass_context
 def client_cmd(ctx):
-    """:param ctx:
-    """
+    """:param ctx:"""
     pass
 
 
@@ -79,7 +80,10 @@ def list_clients(ctx, protocol: str, host: str, port: str, token: str = None, n_
 @click.option("-s", "--secure", required=False, default=False)
 @click.option("-pc", "--preshared-cert", required=False, default=False)
 @click.option("-v", "--verify", is_flag=True, help="Verify SSL/TLS for REST service")
-@click.option("-c", "--preferred-combiner", type=str,required=False, default="",help="name of the preferred combiner")
+@click.option("-c", "--preferred-combiner", type=str, required=False, default="", help="name of the preferred combiner")
+@click.option("--combiner", type=str, required=False, default="", help="Skip combiner assignment from discover service and attatch directly to combiner host.")
+@click.option("--combiner-port", type=str, required=False, default="12080", help="Combiner port, need to be used with --combiner")
+@click.option("--proxy-server", type=str, required=False, default="", help="gRPC proxy server, need to be used together with --combiner")
 @click.option("-va", "--validator", required=False, default=True)
 @click.option("-tr", "--trainer", required=False, default=True)
 @click.option("-in", "--init", required=False, default=None, help="Set to a filename to (re)init client from file state.")
@@ -102,6 +106,9 @@ def client_cmd(
     preshared_cert,
     verify,
     preferred_combiner,
+    combiner,
+    combiner_port,
+    proxy_server,
     validator,
     trainer,
     init,
@@ -143,6 +150,9 @@ def client_cmd(
         "preshared_cert": preshared_cert,
         "verify": verify,
         "preferred_combiner": preferred_combiner,
+        "combiner": combiner,
+        "combiner_port": combiner_port,
+        "proxy_server": proxy_server,
         "validator": validator,
         "trainer": trainer,
         "logfile": logfile,
@@ -156,6 +166,11 @@ def client_cmd(
         click.echo(f"\nClient configuration loaded from file: {init}")
         click.echo("Values set in file override defaults and command line arguments...\n")
 
+    # proxy_server needs combiner check
+    if config["proxy_server"]:
+        if not config["combiner"]:
+            click.echo("--proxy-server/proxy_server requires a combiner host in --combiner/combiner")
+            return
     try:
         validate_client_config(config)
     except InvalidClientConfig as e:

fedn/common/config.py

@@ -16,9 +16,10 @@
 FEDN_CUSTOM_URL_PREFIX = os.environ.get("FEDN_CUSTOM_URL_PREFIX", "")
 
 
-FEDN_ALLOW_LOCAL_PACKAGE = os.environ.get("FEDN_ALLOW_LOCAL_PACKAGE", False)
 FEDN_PACKAGE_EXTRACT_DIR = os.environ.get("FEDN_PACKAGE_EXTRACT_DIR", "package")
 
+FEDN_COMPUTE_PACKAGE_DIR = os.environ.get("FEDN_COMPUTE_PACKAGE_DIR", "/app/client/package/")
+
 
 def get_environment_config():
     """Get the configuration from environment variables."""

fedn/network/api/interface.py

@@ -7,7 +7,7 @@
 from werkzeug.security import safe_join
 from werkzeug.utils import secure_filename
 
-from fedn.common.config import FEDN_ALLOW_LOCAL_PACKAGE, get_controller_config, get_network_config
+from fedn.common.config import FEDN_COMPUTE_PACKAGE_DIR, get_controller_config, get_network_config
 from fedn.common.log_config import logger
 from fedn.network.combiner.interfaces import CombinerUnavailableError
 from fedn.network.state import ReducerState, ReducerStateToString
@@ -230,7 +230,7 @@ def set_compute_package(self, file, helper_type: str, name: str = None, descript
         file_name = file.filename
         storage_file_name = secure_filename(f"{str(uuid.uuid4())}.{extension}")
 
-        file_path = safe_join(os.getcwd(), storage_file_name)
+        file_path = safe_join(FEDN_COMPUTE_PACKAGE_DIR, storage_file_name)
         file.save(file_path)
 
         self.control.set_compute_package(storage_file_name, file_path)
@@ -370,22 +370,20 @@ def download_compute_package(self, name):
         try:
             mutex = threading.Lock()
             mutex.acquire()
-            # TODO: make configurable, perhaps in config.py or package.py
-            return send_from_directory(os.getcwd(), name, as_attachment=True)
+
+            return send_from_directory(FEDN_COMPUTE_PACKAGE_DIR, name, as_attachment=True)
         except Exception:
             try:
                 data = self.control.get_compute_package(name)
                 # TODO: make configurable, perhaps in config.py or package.py
-                file_path = safe_join(os.getcwd(), name)
+                file_path = safe_join(FEDN_COMPUTE_PACKAGE_DIR, name)
                 with open(file_path, "wb") as fh:
                     fh.write(data)
                 # TODO: make configurable, perhaps in config.py or package.py
-                return send_from_directory(os.getcwd(), name, as_attachment=True)
+                return send_from_directory(FEDN_COMPUTE_PACKAGE_DIR, name, as_attachment=True)
             except Exception:
                 raise
         finally:
-            # Delete the file after it has been saved
-            os.remove(file_path)
             mutex.release()
 
     def _create_checksum(self, name=None):
@@ -522,10 +520,6 @@ def add_client(self, client_id, preferred_combiner, remote_addr, name, package):
         :return: A json response with combiner assignment config.
         :rtype: :class:`flask.Response`
         """
-        local_package = FEDN_ALLOW_LOCAL_PACKAGE
-        if local_package:
-            local_package = True
-
         if package == "remote":
             package_object = self.statestore.get_compute_package()
             if package_object is None:
@@ -540,18 +534,8 @@ def add_client(self, client_id, preferred_combiner, remote_addr, name, package):
                     203,
                 )
             helper_type = self.control.statestore.get_helper()
-        elif package == "local" and local_package is False:
-            print("Local package not allowed. Set FEDN_ALLOW_LOCAL_PACKAGE=True in controller config.")
-            return (
-                jsonify(
-                    {
-                        "success": False,
-                        "message": "Local package not allowed. Set FEDN_ALLOW_LOCAL_PACKAGE=True in controller config.",
-                    }
-                ),
-                400,
-            )
-        elif package == "local" and local_package is True:
+        else:
+            # Else package is "local":
             helper_type = ""
 
         # Assign client to combiner
@@ -582,6 +566,7 @@ def add_client(self, client_id, preferred_combiner, remote_addr, name, package):
             "combiner": combiner.name,
             "ip": remote_addr,
             "status": "available",
+            "package": package,
         }
         # Add client to network
         self.control.network.add_client(client_config)