Refactor/SK-936 | Clean up certs and remove /add_combiner logic

.github/workflows/code-checks.yaml

@@ -26,6 +26,8 @@ jobs:
           --exclude-dir='flower-client'
           --exclude='tests.py'
           --exclude='controller_cmd.py'
+          --exclude='combiner_cmd.py'
+          --exclude='run_cmd.py'
           --exclude='README.rst'
           '^[ \t]+(import|from) ' -I . 
 

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.rst
+include fedn/common/settings-controller.yaml.template

config/settings-client.yaml.local.template

@@ -0,0 +1,3 @@
+network_id: fedn-network
+discover_host: localhost
+discover_port: 8092

config/settings-combiner.yaml.local.template

@@ -0,0 +1,31 @@
+network_id: fedn-network
+
+name: combiner
+host: localhost
+address: localhost
+port: 12080
+max_clients: 30
+
+cert_path: tmp/server.crt
+key_path: tmp/server.key
+
+statestore:
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: localhost
+    port: 6534
+
+storage:
+  storage_type: S3
+  storage_config:
+    storage_hostname: localhost
+    storage_port: 9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False 
+
+

config/settings-combiner.yaml.template

@@ -7,4 +7,23 @@ host: combiner
 port: 12080
 max_clients: 30
 
+statestore:
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: mongo
+    port: 6534
+
+storage:
+  storage_type: S3
+  storage_config:
+    storage_hostname: minio
+    storage_port: 9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False
+
 

config/settings-controller.yaml.local.template

@@ -0,0 +1,24 @@
+network_id: fedn-network
+controller:
+  host: localhost
+  port: 8092
+  debug: True
+
+statestore:
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: localhost
+    port: 6534
+
+storage:
+  storage_type: S3
+  storage_config:
+    storage_hostname: localhost
+    storage_port: 9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False 

docker-compose.yaml

@@ -87,6 +87,8 @@ services:
     environment:
       - PYTHONUNBUFFERED=0
       - GET_HOSTS_FROM=dns
+      - STATESTORE_CONFIG=/app/config/settings-combiner.yaml
+      - MODELSTORAGE_CONFIG=/app/config/settings-combiner.yaml
     build:
       context: .
       args:

examples/mnist-pytorch/client/python_env.yaml

@@ -4,6 +4,6 @@ build_dependencies:
   - setuptools
   - wheel
 dependencies:
-  - torch==2.3.1
-  - torchvision==0.18.1
+  - torch
+  - torchvision
   - fedn

fedn/cli/combiner_cmd.py

@@ -3,17 +3,14 @@
 import click
 import requests
 
-from fedn.network.combiner.combiner import Combiner
-
 from .main import main
 from .shared import CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token, print_response
 
 
 @main.group("combiner")
 @click.pass_context
 def combiner_cmd(ctx):
-    """:param ctx:
-    """
+    """:param ctx:"""
     pass
 
 
@@ -60,6 +57,8 @@ def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, se
         click.echo(f"\nCombiner configuration loaded from file: {init}")
         click.echo("Values set in file override defaults and command line arguments...\n")
 
+    from fedn.network.combiner.combiner import Combiner
+
     combiner = Combiner(config)
     combiner.run()
 

fedn/cli/run_cmd.py

@@ -4,15 +4,15 @@
 
 import click
 import yaml
+
+from fedn.cli.client_cmd import validate_client_config
+from fedn.cli.main import main
+from fedn.cli.shared import apply_config
 from fedn.common.exceptions import InvalidClientConfig
 from fedn.common.log_config import logger
 from fedn.network.clients.client import Client
-from fedn.network.combiner.combiner import Combiner
 from fedn.utils.dispatcher import Dispatcher, _read_yaml_file
 
-from fedn.cli.client_cmd import validate_client_config
-from fedn.cli.main import main
-from fedn.cli.shared import apply_config
 
 def get_statestore_config_from_file(init):
     """:param init:
@@ -35,6 +35,7 @@ def check_helper_config_file(config):
         exit(-1)
     return helper
 
+
 def check_yaml_exists(path):
     """Check if fedn.yaml exists in the given path."""
     yaml_file = os.path.join(path, "fedn.yaml")
@@ -43,21 +44,26 @@ def check_yaml_exists(path):
         click.echo(f"Could not find fedn.yaml in {path}")
         exit(-1)
     return yaml_file
+
+
 def delete_virtual_environment(dispatcher):
     if dispatcher.python_env_path:
         logger.info(f"Removing virtualenv {dispatcher.python_env_path}")
         shutil.rmtree(dispatcher.python_env_path)
     else:
         logger.warning("No virtualenv found to remove.")
+
+
 @main.group("run")
 @click.pass_context
 def run_cmd(ctx):
-    """:param ctx:
-    """
+    """:param ctx:"""
     pass
+
+
 @run_cmd.command("validate")
 @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml")
-@click.option("-i", "--input", required=True, help="Path to input model" )
+@click.option("-i", "--input", required=True, help="Path to input model")
 @click.option("-o", "--output", required=True, help="Path to write the output JSON containing validation metrics")
 @click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)")
 @click.pass_context
@@ -82,9 +88,11 @@ def validate_cmd(ctx, path, input, output, keep_venv):
     dispatcher.run_cmd("validate {} {}".format(input, output))
     if not keep_venv:
         delete_virtual_environment(dispatcher)
+
+
 @run_cmd.command("train")
 @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml")
-@click.option("-i", "--input", required=True, help="Path to input model parameters" )
+@click.option("-i", "--input", required=True, help="Path to input model parameters")
 @click.option("-o", "--output", required=True, help="Path to write the updated model parameters ")
 @click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)")
 @click.pass_context
@@ -109,6 +117,8 @@ def train_cmd(ctx, path, input, output, keep_venv):
     dispatcher.run_cmd("train {} {}".format(input, output))
     if not keep_venv:
         delete_virtual_environment(dispatcher)
+
+
 @run_cmd.command("startup")
 @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml")
 @click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)")
@@ -134,6 +144,7 @@ def startup_cmd(ctx, path, keep_venv):
     if not keep_venv:
         delete_virtual_environment(dispatcher)
 
+
 @run_cmd.command("build")
 @click.option("-p", "--path", required=True, help="Path to package directory containing fedn.yaml")
 @click.option("-v", "--keep-venv", is_flag=True, required=False, help="Use flag to keep the python virtual environment (python_env in fedn.yaml)")
@@ -173,7 +184,7 @@ def build_cmd(ctx, path, keep_venv):
 @click.option("-s", "--secure", required=False, default=False)
 @click.option("-pc", "--preshared-cert", required=False, default=False)
 @click.option("-v", "--verify", is_flag=True, help="Verify SSL/TLS for REST service")
-@click.option("-c", "--preferred-combiner", required=False,type=str, default="",help="url to the combiner or name of the preferred combiner")
+@click.option("-c", "--preferred-combiner", required=False, type=str, default="", help="url to the combiner or name of the preferred combiner")
 @click.option("-va", "--validator", required=False, default=True)
 @click.option("-tr", "--trainer", required=False, default=True)
 @click.option("-in", "--init", required=False, default=None, help="Set to a filename to (re)init client from file state.")
@@ -310,5 +321,7 @@ def combiner_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn,
         click.echo(f"\nCombiner configuration loaded from file: {init}")
         click.echo("Values set in file override defaults and command line arguments...\n")
 
+    from fedn.network.combiner.combiner import Combiner
+
     combiner = Combiner(config)
     combiner.run()

fedn/common/certificate/certificate.py

@@ -9,24 +9,27 @@
 
 
 class Certificate:
-    """Utility to generate unsigned certificates.
-
-    """
+    """Utility to generate unsigned certificates."""
 
     CERT_NAME = "cert.pem"
     KEY_NAME = "key.pem"
     BITS = 2048
 
-    def __init__(self, cwd, name=None, key_name="key.pem", cert_name="cert.pem", create_dirs=True):
-        try:
-            os.makedirs(cwd)
-        except OSError:
-            logger.info("Directory exists, will store all cert and keys here.")
+    def __init__(self, name=None, key_path="", cert_path="", create_dirs=False):
+        if create_dirs:
+            try:
+                cwd = os.getcwd()
+                os.makedirs(cwd)
+            except OSError:
+                logger.info("Directory exists, will store all cert and keys here.")
+            else:
+                logger.info("Successfully created the directory to store cert and keys in {}".format(cwd))
+
+            self.key_path = os.path.join(cwd, "key.pem")
+            self.cert_path = os.path.join(cwd, "cert.pem")
         else:
-            logger.info("Successfully created the directory to store cert and keys in {}".format(cwd))
-
-        self.key_path = os.path.join(cwd, key_name)
-        self.cert_path = os.path.join(cwd, cert_name)
+            self.key_path = key_path
+            self.cert_path = cert_path
 
         if name:
             self.name = name
@@ -36,9 +39,7 @@ def __init__(self, cwd, name=None, key_name="key.pem", cert_name="cert.pem", cre
     def gen_keypair(
         self,
     ):
-        """Generate keypair.
-
-        """
+        """Generate keypair."""
         key = crypto.PKey()
         key.generate_key(crypto.TYPE_RSA, 2048)
         cert = crypto.X509()
@@ -73,25 +74,22 @@ def set_keypair_raw(self, certificate, privatekey):
             certfile.write(crypto.dump_certificate(crypto.FILETYPE_PEM, certificate))
 
     def get_keypair_raw(self):
-        """:return:
-        """
+        """:return:"""
         with open(self.key_path, "rb") as keyfile:
             key_buf = keyfile.read()
         with open(self.cert_path, "rb") as certfile:
             cert_buf = certfile.read()
         return copy.deepcopy(cert_buf), copy.deepcopy(key_buf)
 
     def get_key(self):
-        """:return:
-        """
+        """:return:"""
         with open(self.key_path, "rb") as keyfile:
             key_buf = keyfile.read()
         key = crypto.load_privatekey(crypto.FILETYPE_PEM, key_buf)
         return key
 
     def get_cert(self):
-        """:return:
-        """
+        """:return:"""
         with open(self.cert_path, "rb") as certfile:
             cert_buf = certfile.read()
         cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_buf)

fedn/common/config.py

@@ -15,6 +15,8 @@
 FEDN_AUTH_REFRESH_TOKEN = os.environ.get("FEDN_AUTH_REFRESH_TOKEN", False)
 FEDN_CUSTOM_URL_PREFIX = os.environ.get("FEDN_CUSTOM_URL_PREFIX", "")
 
+
+FEDN_ALLOW_LOCAL_PACKAGE = os.environ.get("FEDN_ALLOW_LOCAL_PACKAGE", False)
 FEDN_PACKAGE_EXTRACT_DIR = os.environ.get("FEDN_PACKAGE_EXTRACT_DIR", "package")