From b267c0b8c464d5d47ec32894a0cb8bcd16efc43e Mon Sep 17 00:00:00 2001 From: David Rotermund <54365609+davrot@users.noreply.github.com> Date: Wed, 15 Mar 2023 16:41:33 +0100 Subject: [PATCH] Add files via upload --- network/HDynamicLayer.py | 2 +- network/NNMFLayer.py | 4 +- network/build_network.py | 90 ++++++++++++++++++++++++++++++++ network/build_optimizer.py | 8 +-- network/load_previous_weights.py | 5 +- network/save_weight_and_bias.py | 5 +- 6 files changed, 107 insertions(+), 7 deletions(-) diff --git a/network/HDynamicLayer.py b/network/HDynamicLayer.py index 12c9641..4220f5f 100644 --- a/network/HDynamicLayer.py +++ b/network/HDynamicLayer.py @@ -383,7 +383,7 @@ class FunctionalSbS(torch.autograd.Function): grad_output /= last_grad_scale grad_output_scale = last_grad_scale.clone() - input /= input.sum(dim=1, keepdim=True, dtype=weights.dtype) + input /= input.sum(dim=1, keepdim=True, dtype=weights.dtype) + 1e-20 # ################################################# # User doesn't want us to calculate the gradients diff --git a/network/NNMFLayer.py b/network/NNMFLayer.py index 7b7d5e8..89e11ce 100644 --- a/network/NNMFLayer.py +++ b/network/NNMFLayer.py @@ -249,7 +249,9 @@ class NNMFLayer(torch.nn.Module): else: self.last_input_data = None - input_convolved = input_convolved / input_convolved.sum(dim=1, keepdim=True) + input_convolved = input_convolved / ( + input_convolved.sum(dim=1, keepdim=True) + 1e-20 + ) h = torch.tile( self._h_initial.unsqueeze(0).unsqueeze(-1).unsqueeze(-1), diff --git a/network/build_network.py b/network/build_network.py index c0482b2..3e9a06e 100644 --- a/network/build_network.py +++ b/network/build_network.py @@ -5,10 +5,12 @@ from network.calculate_output_size import calculate_output_size from network.Parameter import Config from network.SbSLayer import SbSLayer from network.NNMFLayer import NNMFLayer +from network.NNMFLayerSbSBP import NNMFLayerSbSBP from network.SplitOnOffLayer import SplitOnOffLayer from network.Conv2dApproximation import Conv2dApproximation from network.SbSReconstruction import SbSReconstruction from network.InputSpikeImage import InputSpikeImage +from network.PoissonLayer import PoissonLayer def build_network( @@ -476,6 +478,94 @@ def build_network( # Adding the x,y output dimensions input_size.append(network[-1]._output_size.tolist()) + # ############################################################# + # NNMF with SbS BP: + # ############################################################# + + elif ( + cfg.network_structure.layer_type[layer_id].upper().startswith("BP SBS NNMF") + is True + ): + + assert in_channels > 0 + assert out_channels > 0 + + number_of_iterations: int = -1 + if len(cfg.number_of_spikes) > layer_id: + number_of_iterations = cfg.number_of_spikes[layer_id] + elif len(cfg.number_of_spikes) == 1: + number_of_iterations = cfg.number_of_spikes[0] + + assert number_of_iterations > 0 + + logging.info( + ( + f"Layer: {layer_id} -> NNMF Layer (SbS BP) with {number_of_iterations} iterations " + ) + ) + + local_learning = False + if cfg.network_structure.layer_type[layer_id].upper().find("LOCAL") != -1: + local_learning = True + + output_layer = False + if layer_id == len(cfg.network_structure.layer_type) - 1: + output_layer = True + + network.append( + NNMFLayerSbSBP( + number_of_input_neurons=in_channels, + number_of_neurons=out_channels, + input_size=input_size[-1], + forward_kernel_size=kernel_size, + number_of_iterations=number_of_iterations, + epsilon_0=cfg.epsilon_0, + weight_noise_range=weight_noise_range, + strides=strides, + dilation=dilation, + padding=padding, + w_trainable=w_trainable, + device=device, + default_dtype=default_dtype, + layer_id=layer_id, + local_learning=local_learning, + output_layer=output_layer, + keep_last_grad_scale=cfg.learning_parameters.kepp_last_grad_scale, + disable_scale_grade=cfg.learning_parameters.disable_scale_grade, + ) + ) + # Adding the x,y output dimensions + input_size.append(network[-1]._output_size.tolist()) + + # ############################################################# + # Poisson layer: + # ############################################################# + + elif ( + cfg.network_structure.layer_type[layer_id].upper().startswith("POISSON") + is True + ): + + number_of_spikes: int = -1 + if len(cfg.number_of_spikes) > layer_id: + number_of_spikes = cfg.number_of_spikes[layer_id] + elif len(cfg.number_of_spikes) == 1: + number_of_spikes = cfg.number_of_spikes[0] + + assert number_of_spikes > 0 + + logging.info( + (f"Layer: {layer_id} -> Poisson Layer with {number_of_spikes} spikes ") + ) + + network.append( + PoissonLayer( + number_of_spikes=number_of_spikes, + ) + ) + # Adding the x,y output dimensions + input_size.append(input_size[-1]) + # ############################################################# # Failure becaue we didn't found the selection of layer # ############################################################# diff --git a/network/build_optimizer.py b/network/build_optimizer.py index ca2f1dd..ce8217d 100644 --- a/network/build_optimizer.py +++ b/network/build_optimizer.py @@ -3,6 +3,7 @@ import torch from network.Parameter import Config from network.SbSLayer import SbSLayer from network.NNMFLayer import NNMFLayer +from network.NNMFLayerSbSBP import NNMFLayerSbSBP from network.Conv2dApproximation import Conv2dApproximation from network.Adam import Adam @@ -28,9 +29,10 @@ def build_optimizer( parameter_list_weights.append(network[id]._weights) parameter_list_sbs.append(True) - if (isinstance(network[id], NNMFLayer) is True) and ( - network[id]._w_trainable is True - ): + if ( + (isinstance(network[id], NNMFLayer) is True) + or (isinstance(network[id], NNMFLayerSbSBP) is True) + ) and (network[id]._w_trainable is True): parameter_list_weights.append(network[id]._weights) parameter_list_sbs.append(True) diff --git a/network/load_previous_weights.py b/network/load_previous_weights.py index 999c3a2..8555340 100644 --- a/network/load_previous_weights.py +++ b/network/load_previous_weights.py @@ -5,6 +5,7 @@ import numpy as np from network.SbSLayer import SbSLayer from network.NNMFLayer import NNMFLayer +from network.NNMFLayerSbSBP import NNMFLayerSbSBP from network.SplitOnOffLayer import SplitOnOffLayer from network.Conv2dApproximation import Conv2dApproximation @@ -48,7 +49,9 @@ def load_previous_weights( ) logging.info(f"Weights file used for layer {id} : {file_to_load[0]}") - if isinstance(network[id], NNMFLayer) is True: + if (isinstance(network[id], NNMFLayer) is True) or ( + isinstance(network[id], NNMFLayerSbSBP) is True + ): filename_wilcard = os.path.join( overload_path, f"Weight_L{id}_*{post_fix}.npy" ) diff --git a/network/save_weight_and_bias.py b/network/save_weight_and_bias.py index 61f70a8..4405626 100644 --- a/network/save_weight_and_bias.py +++ b/network/save_weight_and_bias.py @@ -5,6 +5,7 @@ from network.Parameter import Config import numpy as np from network.SbSLayer import SbSLayer from network.NNMFLayer import NNMFLayer +from network.NNMFLayerSbSBP import NNMFLayerSbSBP from network.SplitOnOffLayer import SplitOnOffLayer from network.Conv2dApproximation import Conv2dApproximation @@ -43,7 +44,9 @@ def save_weight_and_bias( # Save the NNMF Weights # ################################################ - if isinstance(network[id], NNMFLayer) is True: + if (isinstance(network[id], NNMFLayer) is True) or ( + isinstance(network[id], NNMFLayerSbSBP) is True + ): if network[id]._w_trainable is True: np.save(