From f2826dddf440a1ff5070ea7dcf56feb04a83da58 Mon Sep 17 00:00:00 2001 From: benedettoleto Date: Wed, 14 Aug 2024 17:37:07 -0700 Subject: [PATCH 01/10] refactor connection sparsity metric and update documentation connection sparsity can be calculated directly looking at the name of the parameters in the state dict --- neurobench/benchmarks/static_metrics.py | 91 ++----------------------- 1 file changed, 5 insertions(+), 86 deletions(-) diff --git a/neurobench/benchmarks/static_metrics.py b/neurobench/benchmarks/static_metrics.py index 920f07d..b551268 100644 --- a/neurobench/benchmarks/static_metrics.py +++ b/neurobench/benchmarks/static_metrics.py @@ -57,93 +57,12 @@ def connection_sparsity(model): float: Connection sparsity, rounded to 3 decimals. """ - def get_nr_zeros_weights(module): - """ - Get the number of zeros in a module's weights. - - Args: - module: A torch.nn.Module. - Returns: - int: Number of zeros in the module's weights. - - """ - children = list(module.children()) - regular_layers = ( - torch.nn.Linear, - torch.nn.Conv1d, - torch.nn.Conv2d, - torch.nn.Conv3d, - ) - recurr_layers = torch.nn.RNNBase - recurr_cells = torch.nn.RNNCellBase - if len(children) == 0: # it is a leaf - # print(module) - if isinstance(module, regular_layers): - count_zeros = torch.sum(module.weight == 0) - count_weights = module.weight.numel() - return count_zeros, count_weights - - elif isinstance(module, recurr_layers): - attribute_names = [] - for i in range(module.num_layers): - param_names = ["weight_ih_l{}{}", "weight_hh_l{}{}"] - if module.bias: - param_names += ["bias_ih_l{}{}", "bias_hh_l{}{}"] - if module.proj_size > 0: # it is lstm - param_names += ["weight_hr_l{}{}"] - - attribute_names += [x.format(i, "") for x in param_names] - if module.bidirectional: - suffix = "_reverse" - attribute_names += [x.format(i, suffix) for x in param_names] - - count_zeros = 0 - count_weights = 0 - for attr in attribute_names: - attr_val = getattr(module, attr) - count_zeros += torch.sum(attr_val == 0) - count_weights += attr_val.numel() - - return count_zeros, count_weights - - elif isinstance(module, recurr_cells): - attribute_names = ["weight_ih", "weight_hh"] - if module.bias: - attribute_names += ["bias_ih", "bias_hh"] - - count_zeros = 0 - count_weights = 0 - for attr in attribute_names: - attr_val = getattr(module, attr) - count_zeros += torch.sum(attr_val == 0) - count_weights += attr_val.numel() - - return count_zeros, count_weights - - elif isinstance(module, snn.SpikingNeuron): - return 0, 0 # it is a neuromorphic neuron layer - else: - # print('Module type: ', module, 'not found.') - return 0, 0 - - else: - count_zeros = 0 - count_weights = 0 - for child in children: - child_zeros, child_weights = get_nr_zeros_weights(child) - count_zeros += child_zeros - count_weights += child_weights - return count_zeros, count_weights - - # Pull the layers from the model's network - layers = model.__net__().children() - # For each layer, count where the weights are zero count_zeros = 0 count_weights = 0 - for module in layers: - zeros, weights = get_nr_zeros_weights(module) - count_zeros += zeros - count_weights += weights + for name, param in model.__net__().named_parameters(): + if "weight" in name: + count_zeros += param.numel() - torch.count_nonzero(param).item() + count_weights += param.numel() # Return the ratio of zeros to weights, rounded to 4 decimals - return round((count_zeros / count_weights).item(), 4) + return round((count_zeros / count_weights), 4) From b71c076a1d56161aa656073787c76ec1c9b29196 Mon Sep 17 00:00:00 2001 From: benedettoleto Date: Wed, 14 Aug 2024 17:39:23 -0700 Subject: [PATCH 02/10] update documentation --- neurobench/benchmarks/static_metrics.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/neurobench/benchmarks/static_metrics.py b/neurobench/benchmarks/static_metrics.py index b551268..85cff7a 100644 --- a/neurobench/benchmarks/static_metrics.py +++ b/neurobench/benchmarks/static_metrics.py @@ -42,8 +42,12 @@ def footprint(model): def connection_sparsity(model): - """Sparsity of model connections between layers. Based on number of zeros - in supported layers, other layers are not taken into account in the computation: + """Sparsity of model connections between layers. This function calculates the + sparsity + based on the number of zero-valued parameters that contribute to connections + between + layers. Only parameters that are involved in connection operations + are included in the calculation. Supported layers: Linear Conv1d, Conv2d, Conv3d From 8f454778d38e259601327e7cb64488a0913fe000 Mon Sep 17 00:00:00 2001 From: benedettoleto Date: Wed, 14 Aug 2024 17:39:55 -0700 Subject: [PATCH 03/10] update documentation --- neurobench/benchmarks/static_metrics.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/neurobench/benchmarks/static_metrics.py b/neurobench/benchmarks/static_metrics.py index 85cff7a..e242ebb 100644 --- a/neurobench/benchmarks/static_metrics.py +++ b/neurobench/benchmarks/static_metrics.py @@ -43,10 +43,8 @@ def footprint(model): def connection_sparsity(model): """Sparsity of model connections between layers. This function calculates the - sparsity - based on the number of zero-valued parameters that contribute to connections - between - layers. Only parameters that are involved in connection operations + sparsity based on the number of zero-valued parameters that contribute to connections + between layers. Only parameters that are involved in connection operations are included in the calculation. Supported layers: Linear From c32b061d2b52e67d3ab20463540d23ad1376eaf1 Mon Sep 17 00:00:00 2001 From: benedettoleto Date: Wed, 14 Aug 2024 17:40:20 -0700 Subject: [PATCH 04/10] update documentation --- neurobench/benchmarks/static_metrics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/neurobench/benchmarks/static_metrics.py b/neurobench/benchmarks/static_metrics.py index e242ebb..d45e0ed 100644 --- a/neurobench/benchmarks/static_metrics.py +++ b/neurobench/benchmarks/static_metrics.py @@ -42,10 +42,10 @@ def footprint(model): def connection_sparsity(model): - """Sparsity of model connections between layers. This function calculates the - sparsity based on the number of zero-valued parameters that contribute to connections - between layers. Only parameters that are involved in connection operations - are included in the calculation. + """Sparsity of model connections between layers. This function calculates the sparsity + based on the number of zero-valued parameters that contribute to connections between + layers. Only parameters that are involved in connection operations (e.g., matrix + multiplications) are included in the calculation. Supported layers: Linear Conv1d, Conv2d, Conv3d From 963aedf1088316fa54c01fde2ee9807fa435c1fc Mon Sep 17 00:00:00 2001 From: benedettoleto Date: Wed, 14 Aug 2024 17:43:30 -0700 Subject: [PATCH 05/10] update documentation --- neurobench/benchmarks/static_metrics.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/neurobench/benchmarks/static_metrics.py b/neurobench/benchmarks/static_metrics.py index d45e0ed..1976cbe 100644 --- a/neurobench/benchmarks/static_metrics.py +++ b/neurobench/benchmarks/static_metrics.py @@ -45,7 +45,7 @@ def connection_sparsity(model): """Sparsity of model connections between layers. This function calculates the sparsity based on the number of zero-valued parameters that contribute to connections between layers. Only parameters that are involved in connection operations (e.g., matrix - multiplications) are included in the calculation. + multiplications) should be included in the calculation. Supported layers: Linear Conv1d, Conv2d, Conv3d @@ -53,6 +53,10 @@ def connection_sparsity(model): LSTM, LSTMBase, LSTMCell GRU, GRUBase, GRUCell + These layers inherently have weight parameters that represent connections and + are automatically included in the sparsity calculation. + + Args: model: A NeuroBenchModel. Returns: From c47252e42ab978c8e0bba445e3aa21d7bc0cbe33 Mon Sep 17 00:00:00 2001 From: benedettoleto Date: Wed, 14 Aug 2024 17:43:58 -0700 Subject: [PATCH 06/10] update documentation --- neurobench/benchmarks/static_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurobench/benchmarks/static_metrics.py b/neurobench/benchmarks/static_metrics.py index 1976cbe..0d4c166 100644 --- a/neurobench/benchmarks/static_metrics.py +++ b/neurobench/benchmarks/static_metrics.py @@ -53,7 +53,7 @@ def connection_sparsity(model): LSTM, LSTMBase, LSTMCell GRU, GRUBase, GRUCell - These layers inherently have weight parameters that represent connections and + These layers inherently have 'weight' parameters that represent connections and are automatically included in the sparsity calculation. From 0b5fc0dcb4c8b942c068faf949ed8a6d0e045724 Mon Sep 17 00:00:00 2001 From: benedettoleto Date: Wed, 14 Aug 2024 17:44:48 -0700 Subject: [PATCH 07/10] update documentation --- neurobench/benchmarks/static_metrics.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/neurobench/benchmarks/static_metrics.py b/neurobench/benchmarks/static_metrics.py index 0d4c166..6d7d225 100644 --- a/neurobench/benchmarks/static_metrics.py +++ b/neurobench/benchmarks/static_metrics.py @@ -56,6 +56,10 @@ def connection_sparsity(model): These layers inherently have 'weight' parameters that represent connections and are automatically included in the sparsity calculation. + Custom connections: + Your model may contain layers which are involved in operations that implement a connection, such as a matrix multiplication (e.g., torch.matmul), but are not part of the above list. + To make sure this metric tracks these layers, ensure that the weight tensors are a nn.Parameter which has weight in its name, e.g. self.weight1 = torch.nn.Parameter(data1), self.weight2 = torch.nn.Parameter(data2). + Args: model: A NeuroBenchModel. From e613ddeb00f1626017c757141589e491e626f626 Mon Sep 17 00:00:00 2001 From: benedettoleto Date: Wed, 14 Aug 2024 18:23:46 -0700 Subject: [PATCH 08/10] update summary --- neurobench/benchmarks/static_metrics.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/neurobench/benchmarks/static_metrics.py b/neurobench/benchmarks/static_metrics.py index 6d7d225..be93a7a 100644 --- a/neurobench/benchmarks/static_metrics.py +++ b/neurobench/benchmarks/static_metrics.py @@ -46,12 +46,7 @@ def connection_sparsity(model): based on the number of zero-valued parameters that contribute to connections between layers. Only parameters that are involved in connection operations (e.g., matrix multiplications) should be included in the calculation. - Supported layers: - Linear - Conv1d, Conv2d, Conv3d - RNN, RNNBase, RNNCell - LSTM, LSTMBase, LSTMCell - GRU, GRUBase, GRUCell + Supported layers: Linear, Conv1d, Conv2d, Conv3d, RNN, RNNBase, RNNCell, LSTM, LSTMBase, LSTMCell, GRU, GRUBase, GRUCell These layers inherently have 'weight' parameters that represent connections and are automatically included in the sparsity calculation. From 8de45cee2720e12520f86b975712837f2e85ba50 Mon Sep 17 00:00:00 2001 From: benedettoleto Date: Wed, 14 Aug 2024 18:24:26 -0700 Subject: [PATCH 09/10] update summary --- neurobench/benchmarks/static_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurobench/benchmarks/static_metrics.py b/neurobench/benchmarks/static_metrics.py index be93a7a..2ddc20b 100644 --- a/neurobench/benchmarks/static_metrics.py +++ b/neurobench/benchmarks/static_metrics.py @@ -46,7 +46,7 @@ def connection_sparsity(model): based on the number of zero-valued parameters that contribute to connections between layers. Only parameters that are involved in connection operations (e.g., matrix multiplications) should be included in the calculation. - Supported layers: Linear, Conv1d, Conv2d, Conv3d, RNN, RNNBase, RNNCell, LSTM, LSTMBase, LSTMCell, GRU, GRUBase, GRUCell + Supported layers: Linear, Conv1d, Conv2d, Conv3d, RNN, RNNBase, RNNCell, LSTM, LSTMBase, LSTMCell, GRU, GRUBase, GRUCell. These layers inherently have 'weight' parameters that represent connections and are automatically included in the sparsity calculation. From 8062ad1ce530f7c9b75dc4e88f85001a1ebf6b19 Mon Sep 17 00:00:00 2001 From: benedettoleto Date: Wed, 14 Aug 2024 18:30:55 -0700 Subject: [PATCH 10/10] update rst documentation --- docs/metrics/static_metrics/connection_sparsity.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/metrics/static_metrics/connection_sparsity.rst b/docs/metrics/static_metrics/connection_sparsity.rst index a5a1601..da4aff7 100644 --- a/docs/metrics/static_metrics/connection_sparsity.rst +++ b/docs/metrics/static_metrics/connection_sparsity.rst @@ -16,4 +16,13 @@ The layers that are supported include: - LSTM, LSTMBase, LSTMCell - GRU, GRUBase, GRUCell +Custom connections: + Your model may contain layers which are involved in operations that implement a connection, such as a matrix multiplication (e.g., torch.matmul), but are not part of the above list. + +To ensure that your custom connection layers are tracked by metrics: + +1. **Define weights as `nn.Parameter`:** Any tensor used for matrix multiplication or custom operations should be defined as an `nn.Parameter`. +2. **Name with 'weight':** Include the word `weight` in the variable name (e.g., `self.weight1 = torch.nn.Parameter(..)`) so it is recognized as a weight parameter by the tracking system. + + We go through the network, extract instances of these layers, count the number of weights and count the number of zero weights. \ No newline at end of file