Merge pull request #227 from rizoudal/rizoudal

Rafactor(NeuralNetwork): remove workers, batch_queue_size, multiprocessing parameters

aucmedi/automl/block_pred.py

@@ -74,9 +74,6 @@ def block_predict(config):
     # Define neural network parameters
     nn_paras = {"n_labels": 1,                                  # placeholder
                 "channels": 1,                                  # placeholder
-                "workers": config["workers"],
-                "batch_queue_size": 4,
-                "multiprocessing": False,
     }
     # Select input shape for 3D
     if meta_training["three_dim"]:
@@ -123,7 +120,7 @@ def block_predict(config):
                                  standardize_mode=model.meta_standardize,
                                  **paras_datagen)
         # Load model
-        path_model = os.path.join(config["path_modeldir"], "model.last.hdf5")
+        path_model = os.path.join(config["path_modeldir"], "model.last.keras")
         model.load(path_model)
         # Start model inference
         preds = model.predict(prediction_generator=pred_gen)
@@ -142,7 +139,7 @@ def block_predict(config):
                                  **paras_datagen)
         # Load model
         path_model = os.path.join(config["path_modeldir"],
-                                  "model.best_loss.hdf5")
+                                  "model.best_loss.keras")
         model.load(path_model)
         # Start model inference via Augmenting
         preds = predict_augmenting(model, pred_gen)

aucmedi/automl/block_train.py

@@ -105,7 +105,7 @@ def block_train(config):
     callbacks = []
     if config["analysis"] == "standard":
         cb_loss = ModelCheckpoint(os.path.join(config["path_modeldir"],
-                                               "model.best_loss.hdf5"),
+                                               "model.best_loss.keras"),
                                   monitor="val_loss", verbose=1,
                                   save_best_only=True)
         callbacks.append(cb_loss)
@@ -136,12 +136,9 @@ def block_train(config):
     # Define neural network parameters
     nn_paras = {"n_labels": class_n,
                 "channels": 3,
-                "workers": config["workers"],
-                "batch_queue_size": 4,
                 "loss": loss,
                 "metrics": [AUC(100)],
                 "pretrained_weights": True,
-                "multiprocessing": False,
     }
     # Select input shape for 3D
     if config["three_dim"] : nn_paras["input_shape"] = config["shape_3D"]
@@ -217,7 +214,7 @@ def block_train(config):
         # Start model training
         hist = model.train(training_generator=train_gen, **paras_train)
         # Store model
-        path_model = os.path.join(config["path_modeldir"], "model.last.hdf5")
+        path_model = os.path.join(config["path_modeldir"], "model.last.keras")
         model.dump(path_model)
     elif config["analysis"] == "standard":
         # Setup neural network
@@ -250,7 +247,7 @@ def block_train(config):
                            validation_generator=val_gen,
                            **paras_train)
         # Store model
-        path_model = os.path.join(config["path_modeldir"], "model.last.hdf5")
+        path_model = os.path.join(config["path_modeldir"], "model.last.keras")
         model.dump(path_model)
     else:
         # Sanity check of architecutre config

aucmedi/data_processing/augmentation/aug_image.py

@@ -22,6 +22,7 @@
 # External libraries
 from albumentations import Compose
 import albumentations.augmentations as ai
+import cv2
 import warnings
 import numpy as np
 import random
@@ -291,11 +292,9 @@ def apply(self, image):
         aug_image = self.operator(image=image)["image"]
         # Perform padding & cropping if image shape changed
         if self.refine and aug_image.shape != org_shape:
-            aug_image = ai.pad(aug_image, org_shape[0], org_shape[1])
-            offset = (random.random(), random.random())
-            aug_image = ai.random_crop(aug_image,
-                                       org_shape[0], org_shape[1],
-                                       offset[0], offset[1])
+            aug_image = ai.pad(aug_image, org_shape[0], org_shape[1], border_mode=cv2.BORDER_REPLICATE, 
+                                value=0)
+            aug_image = ai.RandomCrop(height=org_shape[0], width=org_shape[1])(image=aug_image)["image"]
         # Perform clipping if image is out of grayscale/RGB encodings
         if self.refine and (np.min(aug_image) < 0 or np.max(aug_image) > 255):
             aug_image = np.clip(aug_image, a_min=0, a_max=255)

aucmedi/data_processing/data_generator.py

@@ -301,7 +301,7 @@ def _get_batches_of_transformed_samples(self, index_array):
         # Stack images and optional metadata together into a batch
         input_stack = np.stack(batch_stack[0], axis=0)
         if self.metadata is not None:
-            input_stack = [input_stack, self.metadata[index_array]]
+            input_stack = (input_stack, self.metadata[index_array])
         batch = (input_stack, )
         # Stack classifications together into a batch if available
         if self.labels is not None:

aucmedi/ensemble/bagging.py

@@ -164,7 +164,7 @@ def train(self, training_generator, epochs=20, iterations=None,
             # Extend Callback list
             cb_mc = ModelCheckpoint(os.path.join(self.cache_dir.name,
                                                  "cv_" + str(i) + \
-                                                 ".model.hdf5"),
+                                                 ".model.keras"),
                                     monitor="val_loss", verbose=1,
                                     save_best_only=True, mode="min")
             cb_cl = CSVLogger(os.path.join(self.cache_dir.name,
@@ -186,9 +186,6 @@ def train(self, training_generator, epochs=20, iterations=None,
                 "fcl_dropout": self.model_template.fcl_dropout,
                 "meta_variables": self.model_template.meta_variables,
                 "learning_rate": self.model_template.learning_rate,
-                "batch_queue_size": self.model_template.batch_queue_size,
-                "workers": self.model_template.workers,
-                "multiprocessing": self.model_template.multiprocessing,
             }
 
             # Gather DataGenerator parameters
@@ -309,7 +306,7 @@ def predict(self, prediction_generator, aggregate="mean",
         for i in range(self.k_fold):
             # Identify path to fitted model
             path_model = os.path.join(path_model_dir,
-                                      "cv_" + str(i) + ".model.hdf5")
+                                      "cv_" + str(i) + ".model.keras")
 
             # Gather NeuralNetwork parameters
             model_paras = {
@@ -324,9 +321,6 @@ def predict(self, prediction_generator, aggregate="mean",
                 "fcl_dropout": self.model_template.fcl_dropout,
                 "meta_variables": self.model_template.meta_variables,
                 "learning_rate": self.model_template.learning_rate,
-                "batch_queue_size": self.model_template.batch_queue_size,
-                "workers": self.model_template.workers,
-                "multiprocessing": self.model_template.multiprocessing,
             }
 
             # Start inference process for fold i
@@ -391,7 +385,7 @@ def load(self, directory_path):
         # Check model existence
         for i in range(self.k_fold):
             path_model = os.path.join(directory_path,
-                                      "cv_" + str(i) + ".model.hdf5")
+                                      "cv_" + str(i) + ".model.keras")
             if not os.path.exists(path_model):
                 raise FileNotFoundError("Bagging model for fold " + str(i) + \
                                         " does not exist!", path_model)

aucmedi/ensemble/composite.py

@@ -220,7 +220,7 @@ def train(self, training_generator, epochs=20, iterations=None,
             callbacks_model = callbacks.copy()
             # Extend Callback list
             path_model = os.path.join(self.cache_dir.name,
-                                      "cv_" + str(i) + ".model.hdf5")
+                                      "cv_" + str(i) + ".model.keras")
             cb_mc = ModelCheckpoint(path_model,
                                     monitor="val_loss", verbose=1,
                                     save_best_only=True, mode="min")
@@ -243,9 +243,6 @@ def train(self, training_generator, epochs=20, iterations=None,
                 "fcl_dropout": self.model_list[i].fcl_dropout,
                 "meta_variables": self.model_list[i].meta_variables,
                 "learning_rate": self.model_list[i].learning_rate,
-                "batch_queue_size": self.model_list[i].batch_queue_size,
-                "workers": self.model_list[i].workers,
-                "multiprocessing": self.model_list[i].multiprocessing,
             }
 
             # Gather DataGenerator parameters
@@ -339,7 +336,7 @@ def train_metalearner(self, training_generator):
         for i in range(len(self.model_list)):
             # Load current model
             path_model = os.path.join(path_model_dir,
-                                      "cv_" + str(i) + ".model.hdf5")
+                                      "cv_" + str(i) + ".model.keras")
 
             # Gather NeuralNetwork parameters
             model_paras = {
@@ -354,9 +351,6 @@ def train_metalearner(self, training_generator):
                 "fcl_dropout": self.model_list[i].fcl_dropout,
                 "meta_variables": self.model_list[i].meta_variables,
                 "learning_rate": self.model_list[i].learning_rate,
-                "batch_queue_size": self.model_list[i].batch_queue_size,
-                "workers": self.model_list[i].workers,
-                "multiprocessing": self.model_list[i].multiprocessing,
             }
 
             # Gather DataGenerator parameters
@@ -453,7 +447,7 @@ def predict(self, prediction_generator, return_ensemble=False):
         # Sequentially iterate over model list
         for i in range(len(self.model_list)):
             path_model = os.path.join(path_model_dir,
-                                      "cv_" + str(i) + ".model.hdf5")
+                                      "cv_" + str(i) + ".model.keras")
 
             # Gather NeuralNetwork parameters
             model_paras = {
@@ -468,9 +462,6 @@ def predict(self, prediction_generator, return_ensemble=False):
                 "fcl_dropout": self.model_list[i].fcl_dropout,
                 "meta_variables": self.model_list[i].meta_variables,
                 "learning_rate": self.model_list[i].learning_rate,
-                "batch_queue_size": self.model_list[i].batch_queue_size,
-                "workers": self.model_list[i].workers,
-                "multiprocessing": self.model_list[i].multiprocessing,
             }
 
             # Gather DataGenerator parameters
@@ -563,7 +554,7 @@ def load(self, directory_path):
         # Check model existence
         for i in range(len(self.model_list)):
             path_model = os.path.join(directory_path,
-                                      "cv_" + str(i) + ".model.hdf5")
+                                      "cv_" + str(i) + ".model.keras")
             if not os.path.exists(path_model):
                 raise FileNotFoundError("Composite model " + str(i) + \
                                         " does not exist!", path_model)

aucmedi/ensemble/stacking.py

@@ -208,7 +208,7 @@ def train(self, training_generator, epochs=20, iterations=None,
             callbacks_model = callbacks.copy()
             # Extend Callback list
             path_model = os.path.join(self.cache_dir.name,
-                                      "nn_" + str(i) + ".model.hdf5")
+                                      "nn_" + str(i) + ".model.keras")
             cb_mc = ModelCheckpoint(path_model,
                                     monitor="val_loss", verbose=1,
                                     save_best_only=True, mode="min")
@@ -231,9 +231,6 @@ def train(self, training_generator, epochs=20, iterations=None,
                 "fcl_dropout": self.model_list[i].fcl_dropout,
                 "meta_variables": self.model_list[i].meta_variables,
                 "learning_rate": self.model_list[i].learning_rate,
-                "batch_queue_size": self.model_list[i].batch_queue_size,
-                "workers": self.model_list[i].workers,
-                "multiprocessing": self.model_list[i].multiprocessing,
             }
 
             # Gather DataGenerator parameters
@@ -327,7 +324,7 @@ def train_metalearner(self, training_generator):
         for i in range(len(self.model_list)):
             #  Load current model
             path_model = os.path.join(path_model_dir,
-                                      "nn_" + str(i) + ".model.hdf5")
+                                      "nn_" + str(i) + ".model.keras")
 
             # Gather NeuralNetwork parameters
             model_paras = {
@@ -342,9 +339,6 @@ def train_metalearner(self, training_generator):
                 "fcl_dropout": self.model_list[i].fcl_dropout,
                 "meta_variables": self.model_list[i].meta_variables,
                 "learning_rate": self.model_list[i].learning_rate,
-                "batch_queue_size": self.model_list[i].batch_queue_size,
-                "workers": self.model_list[i].workers,
-                "multiprocessing": self.model_list[i].multiprocessing,
             }
 
             # Gather DataGenerator parameters
@@ -440,7 +434,7 @@ def predict(self, prediction_generator, return_ensemble=False):
         # Sequentially iterate over model list
         for i in range(len(self.model_list)):
             path_model = os.path.join(path_model_dir,
-                                      "nn_" + str(i) + ".model.hdf5")
+                                      "nn_" + str(i) + ".model.keras")
 
             # Gather NeuralNetwork parameters
             model_paras = {
@@ -455,9 +449,6 @@ def predict(self, prediction_generator, return_ensemble=False):
                 "fcl_dropout": self.model_list[i].fcl_dropout,
                 "meta_variables": self.model_list[i].meta_variables,
                 "learning_rate": self.model_list[i].learning_rate,
-                "batch_queue_size": self.model_list[i].batch_queue_size,
-                "workers": self.model_list[i].workers,
-                "multiprocessing": self.model_list[i].multiprocessing,
             }
 
             # Gather DataGenerator parameters
@@ -550,7 +541,7 @@ def load(self, directory_path):
         # Check model existence
         for i in range(len(self.model_list)):
             path_model = os.path.join(directory_path,
-                                      "nn_" + str(i) + ".model.hdf5")
+                                      "nn_" + str(i) + ".model.keras")
             if not os.path.exists(path_model):
                 raise FileNotFoundError("Stacking model " + str(i) + \
                                         " does not exist!", path_model)

aucmedi/neural_network/architectures/classifier.py

@@ -75,7 +75,7 @@ class Classifier:
     ???+ example
         ```python
         # Recommended way (automatic creation in NeuralNetwork)
-        model = NeuralNetwork(n_labels=20, channels=3, batch_queue_size=1,
+        model = NeuralNetwork(n_labels=20, channels=3,
                               input_shape=(32, 32), activation_output="sigmoid",
                               fcl_dropout=False)