Sort directory listings where necessary; minor documentation improvem…

README.md

@@ -1,6 +1,6 @@
 # ADI MAX78000/MAX78002 Model Training and Synthesis
 
-March 13, 2024
+March 28, 2024
 
 ADI’s MAX78000/MAX78002 project is comprised of five repositories:
 
@@ -1163,7 +1163,7 @@ The MAX78002 hardware does not support arbitrary network parameters. Specificall
 
 * Streaming mode:
 
-  * When using data greater than 143×143 (20,480 pixels per channel in HWC mode), or 286×286 (81,920 pixels in CHW mode), and [Data Folding](#data-folding) techniques are not used, then `streaming` mode must be used.
+  * When using data greater than 20,480 pixels per channel in HWC mode (143×143 when height = width), or 81,920 pixels in CHW mode (286×286 when height = width), and [Data Folding](#data-folding) techniques are not used, then `streaming` mode must be used.
   * When using `streaming` mode, the product of any layer’s input width, input height, and input channels divided by 64 rounded up must not exceed 2^21: $width * height * ⌈\frac{channels}{64}⌉ < 2^{21}$; _width_ and _height_ must not exceed 2047.
   * Streaming is limited to 8 consecutive layers or fewer, and is limited to four FIFOs (up to 4 input channels in CHW and up to 16 channels in HWC format), see [FIFOs](#fifos).
   * Layers that use 1×1 kernels without padding are automatically replaced with equivalent layers that use 3×3 kernels with padding.
@@ -1174,7 +1174,7 @@ The MAX78002 hardware does not support arbitrary network parameters. Specificall
   When using more than 64 input or output channels, weight memory is shared, and effective capacity decreases.
   Weights must be arranged according to specific rules detailed in [Layers and Weight Memory](#layers-and-weight-memory).
 
-* The total of [1,280 KiB of data memory](docs/AHBAddresses.md) is split into 16 sections of 80 KiB each. When not using streaming mode, any data channel (input, intermediate, or output) must completely fit into one memory instance. This limits the first-layer input to 286×286 pixels per channel in the CHW format. However, when using more than one input channel, the HWC format may be preferred, and all layer outputs are in HWC format as well. In those cases, it is required that four channels fit into a single memory section — or 143×143 pixels per channel.
+* The total of [1,280 KiB of data memory](docs/AHBAddresses.md) is split into 16 sections of 80 KiB each. When not using streaming mode, any data channel (input, intermediate, or output) must completely fit into one memory instance. This limits the first-layer input to 81,920 pixels per channel in CHW format (286×286 when height = width). However, when using more than one input channel, the HWC format may be preferred, and all layer outputs are in HWC format as well. In those cases, it is required that four channels fit into a single memory section — or 20,480 pixels per channel (143×143 when height = width).
   Note that the first layer commonly creates a wide expansion (i.e., a large number of output channels) that needs to fit into data memory, so the input size limit is mostly theoretical. In many cases, [Data Folding](#data-folding) (distributing the input data across multiple channels) can effectively increase both the input dimensions as well as improve model performance.
 
 * The hardware supports 1D and 2D convolution layers, 2D transposed convolution layers (upsampling), element-wise addition, subtraction, binary OR, binary XOR as well as fully connected layers (`Linear`), which are implemented using 1×1 convolutions on 1×1 data:

datasets/aisegment.py

@@ -1,10 +1,9 @@
 ###################################################################################################
 #
 # Copyright (C) 2021-2023 Maxim Integrated Products, Inc. All Rights Reserved.
+# Copyright (C) 2024 Analog Devices, Inc. All Rights Reserved.
 #
-# Maxim Integrated Products, Inc. Default Copyright Notice:
-# https://www.maximintegrated.com/en/aboutus/legal/copyrights.html
-#
+# This software is proprietary to Analog Devices, Inc. and its licensors.
 ###################################################################################################
 """
 Classes and functions used to create AISegment dataset.
@@ -242,7 +241,8 @@ def __check_pkl_files_exist(self):
 
         if os.path.exists(self.processed_folder_path) and \
            os.path.isdir(self.processed_folder_path):
-            pkl_files = [f for f in os.listdir(self.processed_folder_path) if f.endswith('.pkl')]
+            pkl_files = [f for f in sorted(os.listdir(self.processed_folder_path))
+                         if f.endswith('.pkl')]
         else:
             pkl_files = []
         return len(pkl_files) > 0

datasets/msnoise.py

@@ -275,7 +275,7 @@ def __gen_datasets(self, exp_len=16384, row_len=128, overlap_ratio=0):
 
             for i, label in enumerate(labels):
                 for folder in train_test_folders:
-                    for record_name in os.listdir(folder):
+                    for record_name in sorted(os.listdir(folder)):
                         if record_name.split('_')[0] in label:
                             record_path = os.path.join(folder, record_name)
                             record, _ = librosa.load(record_path, offset=0, sr=None)

datasets/samplemotordatalimerick.py

@@ -165,7 +165,7 @@ def gen_dataframe(self):
             os.path.join(self.root, self.__class__.__name__, file_name)
 
         if os.path.isfile(df_path):
-            print(f'\n{file_name} file already exists\n')
+            print(f'\nFile {file_name} already exists\n')
             main_df = pd.read_pickle(df_path)
 
             return main_df
@@ -176,20 +176,26 @@ def gen_dataframe(self):
                                        "SpectraQuest_Rig_Data_Voyager_3/")
         data_dir = os.path.join(actual_root_dir, 'Data_ADXL356C')
 
-        if not os.listdir(data_dir):
-            print('\nDataset directory is empty.\n')
+        if not os.path.isdir(data_dir):
+            print(f'\nDataset directory {data_dir} does not exist.\n')
+            return None
+
+        with os.scandir(data_dir) as it:
+            if not any(it):
+                print(f'\nDataset directory {data_dir} is empty.\n')
+                return None
 
         rpm_prefixes = ('0600', '1200', '1800', '2400', '3000')
 
-        sensor_sr_Hz = 20000   # Hz
+        sensor_sr_Hz = 20000  # Hz
 
         faulty_data_list = []
         healthy_data_list = []
 
         df_normals = self.create_common_empty_df()
         df_anormals = self.create_common_empty_df()
 
-        for file in os.listdir(data_dir):
+        for file in sorted(os.listdir(data_dir)):
             full_path = os.path.join(data_dir, file)
             speed = int(file.split("_")[0]) / 60  # Hz
             load = int(file.split("_")[-1][0:2])  # LBS
@@ -297,7 +303,7 @@ def samplemotordatalimerick_get_datasets_for_train(data,
     Returns Sample Motor Data Limerick Dataset For Training Mode
     """
 
-    eval_mode = False   # Test set includes validation normals
+    eval_mode = False  # Test set includes validation normals
     label_as_signal = True
 
     signal_duration_in_sec = 0.25
@@ -335,7 +341,7 @@ def samplemotordatalimerick_get_datasets_for_eval_with_anomaly_label(data,
     Label is anomaly status
     """
 
-    eval_mode = True   # Test set includes validation normals
+    eval_mode = True  # Test set includes validation normals
     label_as_signal = False
 
     signal_duration_in_sec = 0.25
@@ -373,7 +379,7 @@ def samplemotordatalimerick_get_datasets_for_eval_with_signal(data,
     Label is signal
     """
 
-    eval_mode = True   # Test set includes anormal samples as well as validation normals
+    eval_mode = True  # Test set includes anormal samples as well as validation normals
     label_as_signal = True
 
     signal_duration_in_sec = 0.25

datasets/speechcom.py

@@ -1,9 +1,9 @@
 ###################################################################################################
 #
 # Copyright (C) 2019-2023 Maxim Integrated Products, Inc. All Rights Reserved.
+# Copyright (C) 2024 Analog Devices, Inc. All Rights Reserved.
 #
-# Maxim Integrated Products, Inc. Default Copyright Notice:
-# https://www.maximintegrated.com/en/aboutus/legal/copyrights.html
+# This software is proprietary to Analog Devices, Inc. and its licensors.
 #
 ###################################################################################################
 """
@@ -102,7 +102,7 @@ def __gen_datasets(self):
         print('Generating dataset from raw data samples.')
         with warnings.catch_warnings():
             warnings.simplefilter('error')
-            lst = os.listdir(self.raw_folder)
+            lst = sorted(os.listdir(self.raw_folder))
             labels = [d for d in lst if os.path.isdir(os.path.join(self.raw_folder, d)) and
                       d[0].isalpha()]
             train_images = []
@@ -113,8 +113,7 @@ def __gen_datasets(self):
             test_labels = []
             for i, label in enumerate(labels):
                 print(f'\tProcessing the label: {label}. {i+1} of {len(labels)}')
-                records = os.listdir(os.path.join(self.raw_folder, label))
-                records = sorted(records)
+                records = sorted(os.listdir(os.path.join(self.raw_folder, label)))
                 for record in records:
                     record_pth = os.path.join(self.raw_folder, label, record)
                     y, _ = librosa.load(record_pth, offset=0, sr=None)