Update Tensorflow, gRPC, Protobuf dependencies (securefederatedai#868)

* Update Tensorflow to latest, finally update grpcio/protobuf Signed-off-by: Patrick Foley <[email protected]> * Lint issue fix and missing tf reference Signed-off-by: Patrick Foley <[email protected]> * pyzmq version fixed * fix taskrunner tests for windows Signed-off-by: Mansi Sharma <[email protected]> * fix taskrunner test syntax for windows Signed-off-by: Mansi Sharma <[email protected]> * adding user option to workspace pip install requirements for windows Signed-off-by: Mansi Sharma <[email protected]> * fix windows CI test Signed-off-by: Mansi Sharma <[email protected]> * testing virtual env for windows github actions Signed-off-by: Mansi Sharma <[email protected]> * testing virtual env for windows github actions Signed-off-by: Mansi Sharma <[email protected]> * testing virtual env for windows github actions Signed-off-by: Mansi Sharma <[email protected]> * testing venv for windows Signed-off-by: Mansi Sharma <[email protected]> * test venv for windows * test venv for windows * Added new KerasSerializer. Fixed other Interactive API experiments * Update taskrunner.yml * Update taskrunner.yml * Update workspace.py * Update workspace.py * Update taskrunner.yml * Remove get_model import from global namespace so dependencies are not loaded into memory unnecessarily (breaking windows build) * Refactoring and cleaning up imports to support Windows install * Fixed logger import paths * Fix missing imports * Fix native import * Fix lint errors * Fix keras optimizer patch. Remove irrelevant unit test * Format logs in UTF-8 for windows * Update interactive-kvasir.yml * Consolidate github actions python versions to single file * Update python versions * Update python versions * Update python versions * Reduce # of DataLoader workers for Pytorch Kvasir CI test * Fix Windows encoding * Fix Windows encoding and limit rounds so Github Actions CI doesn't run out of memory Signed-off-by: Patrick Foley <[email protected]> * Fix windows encoding * Fix Windows encoding --------- Signed-off-by: Patrick Foley <[email protected]> Signed-off-by: Mansi Sharma <[email protected]> Co-authored-by: Mansi Sharma <[email protected]> Co-authored-by: Mansi Sharma <[email protected]> Signed-off-by: Parth Mandaliya <[email protected]>
ParthMandaliya · Sep 28, 2023 · 1c7ccc6 · 1c7ccc6
1 parent 1b06e60
commit 1c7ccc6
Show file tree

Hide file tree

Showing 45 changed files with 238 additions and 265 deletions.
diff --git a/.github/workflows/interactive-tensorflow.yml b/.github/workflows/interactive-tensorflow.yml
@@ -27,5 +27,5 @@ jobs:
     - name: Interactive API - tensorflow_mnist
       run: |
         python setup.py build_grpc
-        pip install tensorflow==2.11
+        pip install tensorflow==2.13
         python -m tests.github.interactive_api_director.experiments.tensorflow_mnist.run
diff --git a/.github/workflows/taskrunner.yml b/.github/workflows/taskrunner.yml
@@ -12,19 +12,24 @@ permissions:
 
 jobs:
   build:
-
     strategy:
       matrix:
        os: ['ubuntu-latest', 'windows-latest']
+       python-version: ['3.8','3.9','3.10','3.11']
     runs-on: ${{ matrix.os }}
-
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v3
+    - name: Set up Python 
+      uses: actions/setup-python@v4
       with:
-        python-version: "3.8"
-    - name: Install dependencies
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies ubuntu
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        python -m pip install --upgrade pip
+        pip install .
+    - name: Install dependencies windows
+      if: matrix.os == 'windows-latest'
       run: |
         python -m pip install --upgrade pip
         pip install .

diff --git a/.github/workflows/taskrunner_python_3.10.yml b/.github/workflows/taskrunner_python_3.10.yml
diff --git a/.github/workflows/taskrunner_python_3.9.yml b/.github/workflows/taskrunner_python_3.9.yml
diff --git a/openfl-tutorials/Federated_Keras_MNIST_Tutorial.ipynb b/openfl-tutorials/Federated_Keras_MNIST_Tutorial.ipynb
@@ -16,10 +16,10 @@
    "outputs": [],
    "source": [
     "#Install Tensorflow and MNIST dataset if not installed\n",
-    "!pip install tensorflow==2.7.0\n",
+    "!pip install tensorflow==2.13\n",
     "\n",
     "#Alternatively you could use the intel-tensorflow build\n",
-    "# !pip install intel-tensorflow==2.3.0"
+    "# !pip install intel-tensorflow==2.13"
    ]
   },
   {

diff --git a/openfl-tutorials/interactive_api/Flax_CNN_CIFAR/requirements.txt b/openfl-tutorials/interactive_api/Flax_CNN_CIFAR/requirements.txt
@@ -1,4 +1,4 @@
-tensorflow==2.11.1
+tensorflow==2.13
 tensorflow-datasets==4.6.0
 jax
 --find-links https://storage.googleapis.com/jax-releases/jax_cuda_releases.html

diff --git a/openfl-tutorials/interactive_api/Tensorflow_CIFAR_tfdata/workspace/Tensorflow_CIFAR.ipynb b/openfl-tutorials/interactive_api/Tensorflow_CIFAR_tfdata/workspace/Tensorflow_CIFAR.ipynb
@@ -16,8 +16,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Install TF if not already. We recommend TF2.7 or greater.\n",
-    "# !pip install tensorflow==2.8"
+    "# Install TF if not already. We recommend TF2.13 or greater.\n",
+    "# !pip install tensorflow==2.13"
    ]
   },
   {
@@ -157,7 +157,7 @@
     "model.summary()\n",
     "\n",
     "# Define optimizer\n",
-    "optimizer = tf.optimizers.Adam(learning_rate=1e-4)\n",
+    "optimizer = tf.keras.optimizers.legacy.Adam(learning_rate=1e-4)\n",
     "\n",
     "# Loss and metrics. These will be used later.\n",
     "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
@@ -327,7 +327,7 @@
    "source": [
     "# create an experimnet in federation\n",
     "experiment_name = 'cifar10_experiment'\n",
-    "fl_experiment = FLExperiment(federation=federation, experiment_name=experiment_name)"
+    "fl_experiment = FLExperiment(federation=federation, experiment_name=experiment_name,serializer_plugin='openfl.plugins.interface_serializer.keras_serializer.KerasSerializer)"
    ]
   },
   {

diff --git a/openfl-tutorials/interactive_api/Tensorflow_MNIST/workspace/Tensorflow_MNIST.ipynb b/openfl-tutorials/interactive_api/Tensorflow_MNIST/workspace/Tensorflow_MNIST.ipynb
@@ -404,7 +404,7 @@
    "source": [
     "# create an experimnet in federation\n",
     "experiment_name = 'mnist_experiment'\n",
-    "fl_experiment = FLExperiment(federation=federation, experiment_name=experiment_name)"
+    "fl_experiment = FLExperiment(federation=federation, experiment_name=experiment_name,serializer_plugin='openfl.plugins.interface_serializer.keras_serializer.KerasSerializer)"
    ]
   },
   {

diff --git a/...als/interactive_api/Tensorflow_Word_Prediction/workspace/Tensorflow_Word_Prediction.ipynb b/...als/interactive_api/Tensorflow_Word_Prediction/workspace/Tensorflow_Word_Prediction.ipynb
@@ -233,7 +233,7 @@
    "source": [
     "import tensorflow as tf\n",
     "from tensorflow.keras.layers import LSTM, Dense\n",
-    "from tensorflow.keras.optimizers import Adam\n",
+    "from tensorflow.keras.legacy.optimizers import Adam\n",
     "from tensorflow.keras.metrics import TopKCategoricalAccuracy\n",
     "from tensorflow.keras.losses import CategoricalCrossentropy\n",
     "from tensorflow.keras.models import Sequential\n",
@@ -363,7 +363,7 @@
    "source": [
     "# create an experimnet in federation\n",
     "experiment_name = 'word_prediction_test_experiment'\n",
-    "fl_experiment = FLExperiment(federation=federation, experiment_name=experiment_name)"
+    "fl_experiment = FLExperiment(federation=federation, experiment_name=experiment_name,serializer_plugin='openfl.plugins.interface_serializer.keras_serializer.KerasSerializer)"
    ]
   },
   {

diff --git a/openfl-tutorials/interactive_api/Tensorflow_Word_Prediction/workspace/requirements.txt b/openfl-tutorials/interactive_api/Tensorflow_Word_Prediction/workspace/requirements.txt
@@ -1,2 +1,2 @@
-tensorflow==2.9.3
+tensorflow==2.13
 numpy==1.22.2
diff --git a/openfl-workspace/keras_cnn_mnist/requirements.txt b/openfl-workspace/keras_cnn_mnist/requirements.txt
@@ -1 +1 @@
-tensorflow==2.11.1
+tensorflow==2.13
diff --git a/openfl-workspace/keras_cnn_with_compression/requirements.txt b/openfl-workspace/keras_cnn_with_compression/requirements.txt
@@ -1 +1 @@
-tensorflow==2.11.1
+tensorflow==2.13
diff --git a/openfl-workspace/keras_nlp/requirements.txt b/openfl-workspace/keras_nlp/requirements.txt
@@ -1 +1 @@
-tensorflow==2.11.1
+tensorflow==2.13
diff --git a/openfl-workspace/keras_nlp_gramine_ready/requirements.txt b/openfl-workspace/keras_nlp_gramine_ready/requirements.txt
@@ -1 +1 @@
-tensorflow-cpu==2.11.1
+tensorflow-cpu==2.13
diff --git a/openfl-workspace/tf_2dunet/requirements.txt b/openfl-workspace/tf_2dunet/requirements.txt
@@ -1,3 +1,3 @@
 nibabel
-tensorflow==2.11.1
+tensorflow==2.13
 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
diff --git a/openfl-workspace/tf_cnn_histology/requirements.txt b/openfl-workspace/tf_cnn_histology/requirements.txt
@@ -1,3 +1,3 @@
 pillow
-tensorflow==2.9.3
+tensorflow==2.13
 tensorflow-datasets
diff --git a/openfl/__init__.py b/openfl/__init__.py
@@ -3,4 +3,4 @@
 """openfl base package."""
 from .__version__ import __version__
 # flake8: noqa
-from .interface.model import get_model
+#from .interface.model import get_model
diff --git a/openfl/federated/plan/plan.py b/openfl/federated/plan/plan.py
@@ -274,7 +274,9 @@ def get_assigner(self):
             aggregation_functions_by_task = self.restore_object('aggregation_function_obj.pkl')
             assigner_function = self.restore_object('task_assigner_obj.pkl')
         except Exception as exc:
-            self.logger.error(f'Failed to load aggregation and assigner functions: {exc}')
+            self.logger.error(
+                f'Failed to load aggregation and assigner functions: {exc}'
+            )
             self.logger.info('Using Task Runner API workflow')
         if assigner_function:
             self.assigner_ = Assigner(

diff --git a/openfl/federated/task/runner_gandlf.py b/openfl/federated/task/runner_gandlf.py
@@ -11,7 +11,7 @@
 from typing import Union
 import yaml
 
-from openfl.utilities import split_tensor_dict_for_holdouts
+from openfl.utilities.split import split_tensor_dict_for_holdouts
 from openfl.utilities import TensorKey
 
 from .runner import TaskRunner

diff --git a/openfl/federated/task/runner_keras.py b/openfl/federated/task/runner_keras.py
@@ -13,7 +13,7 @@
 
 from openfl.utilities import change_tags
 from openfl.utilities import Metric
-from openfl.utilities import split_tensor_dict_for_holdouts
+from openfl.utilities.split import split_tensor_dict_for_holdouts
 from openfl.utilities import TensorKey
 from .runner import TaskRunner
 

diff --git a/openfl/federated/task/runner_pt.py b/openfl/federated/task/runner_pt.py
@@ -14,7 +14,7 @@
 
 from openfl.utilities import change_tags
 from openfl.utilities import Metric
-from openfl.utilities import split_tensor_dict_for_holdouts
+from openfl.utilities.split import split_tensor_dict_for_holdouts
 from openfl.utilities import TensorKey
 from .runner import TaskRunner
 

diff --git a/openfl/federated/task/runner_tf.py b/openfl/federated/task/runner_tf.py
@@ -7,7 +7,7 @@
 import tensorflow.compat.v1 as tf
 from tqdm import tqdm
 
-from openfl.utilities import split_tensor_dict_for_holdouts
+from openfl.utilities.split import split_tensor_dict_for_holdouts
 from openfl.utilities import TensorKey
 from .runner import TaskRunner
 

diff --git a/openfl/federated/task/task_runner.py b/openfl/federated/task/task_runner.py
@@ -7,7 +7,7 @@
 import numpy as np
 
 from openfl.utilities import change_tags
-from openfl.utilities import split_tensor_dict_for_holdouts
+from openfl.utilities.split import split_tensor_dict_for_holdouts
 from openfl.utilities import TensorKey
 
 

diff --git a/openfl/interface/cli.py b/openfl/interface/cli.py
@@ -16,9 +16,8 @@
 from click import pass_context
 from click import style
 import time
-
-from openfl.utilities import add_log_level
 import sys
+from openfl.utilities import add_log_level
 
 
 def setup_logging(level='info', log_file=None):

diff --git a/openfl/interface/director.py b/openfl/interface/director.py
@@ -14,9 +14,7 @@
 from click import Path as ClickPath
 from dynaconf import Validator
 
-from openfl.component.director import Director
 from openfl.interface.cli_helper import WORKSPACE
-from openfl.transport import DirectorGRPCServer
 from openfl.utilities import merge_configs
 from openfl.utilities.path_check import is_directory_traversal
 from openfl.interface.cli import review_plan_callback
@@ -47,6 +45,10 @@ def director(context):
         help='Path to a signed certificate')
 def start(director_config_path, tls, root_certificate, private_key, certificate):
     """Start the director service."""
+
+    from openfl.component.director import Director
+    from openfl.transport import DirectorGRPCServer
+
     director_config_path = Path(director_config_path).absolute()
     logger.info('🧿 Starting the Director Service.')
     if is_directory_traversal(director_config_path):

diff --git a/openfl/interface/envoy.py b/openfl/interface/envoy.py
@@ -15,7 +15,6 @@
 from click import Path as ClickPath
 from dynaconf import Validator
 
-from openfl.component.envoy.envoy import Envoy
 from openfl.interface.cli import review_plan_callback
 from openfl.interface.cli_helper import WORKSPACE
 from openfl.utilities import click_types
@@ -52,6 +51,9 @@ def envoy(context):
 def start_(shard_name, director_host, director_port, tls, envoy_config_path,
            root_certificate, private_key, certificate):
     """Start the Envoy."""
+
+    from openfl.component.envoy.envoy import Envoy
+
     logger.info('🧿 Starting the Envoy.')
     if is_directory_traversal(envoy_config_path):
         click.echo('The shard config path is out of the openfl workspace scope.')

diff --git a/openfl/interface/interactive_api/experiment.py b/openfl/interface/interactive_api/experiment.py
@@ -22,7 +22,7 @@
 from openfl.interface.cli import setup_logging
 from openfl.interface.cli_helper import WORKSPACE
 from openfl.native import update_plan
-from openfl.utilities import split_tensor_dict_for_holdouts
+from openfl.utilities.split import split_tensor_dict_for_holdouts
 from openfl.utilities.workspace import dump_requirements_file
 
 
@@ -81,7 +81,9 @@ def _initialize_plan(self):
     def _assert_experiment_submitted(self):
         """Assure experiment is sent to director and accepted."""
         if not self.experiment_submitted:
-            self.logger.error('The experiment was not submitted to a Director service.')
+            self.logger.error(
+                'The experiment was not submitted to a Director service.'
+            )
             self.logger.error(
                 'Report the experiment first: '
                 'use the Experiment.start() method.')
@@ -145,7 +147,8 @@ def stream_metrics(self, tensorboard_logs: bool = True) -> None:
                 f'Round {metric_message_dict["round"]}, '
                 f'collaborator {metric_message_dict["metric_origin"]} '
                 f'{metric_message_dict["task_name"]} result '
-                f'{metric_message_dict["metric_name"]}:\t{metric_message_dict["metric_value"]:f}')
+                f'{metric_message_dict["metric_name"]}:\t{metric_message_dict["metric_value"]:f}'
+            )
 
             if tensorboard_logs:
                 self.write_tensorboard_metric(metric_message_dict)

diff --git a/openfl/interface/model.py b/openfl/interface/model.py
@@ -11,12 +11,6 @@
 from logging import getLogger
 from pathlib import Path
 
-from openfl.federated import Plan
-from openfl.federated import TaskRunner
-from openfl.protocols import utils
-from openfl.pipelines import NoCompressionPipeline
-from openfl.utilities.workspace import set_directory
-
 logger = getLogger(__name__)
 
 
@@ -68,7 +62,7 @@ def get_model(
     cols_config: str,
     data_config: str,
     model_protobuf_path: str
-) -> TaskRunner:
+):
     """
     Initialize TaskRunner and load it with provided model.pbuf.
 
@@ -77,6 +71,11 @@ def get_model(
     the diversity of the ways we store models in our template workspaces.
     """
 
+    from openfl.federated import Plan
+    from openfl.pipelines import NoCompressionPipeline
+    from openfl.protocols import utils
+    from openfl.utilities.workspace import set_directory
+
     # Here we change cwd to the experiment workspace folder
     # because plan.yaml usually contains relative paths to components.
     workspace_path = Path(plan_config).resolve().parent.parent