Add files via upload
This commit is contained in:
parent
d7a6b284ea
commit
b267c0b8c4
6 changed files with 107 additions and 7 deletions
|
@ -383,7 +383,7 @@ class FunctionalSbS(torch.autograd.Function):
|
||||||
grad_output /= last_grad_scale
|
grad_output /= last_grad_scale
|
||||||
grad_output_scale = last_grad_scale.clone()
|
grad_output_scale = last_grad_scale.clone()
|
||||||
|
|
||||||
input /= input.sum(dim=1, keepdim=True, dtype=weights.dtype)
|
input /= input.sum(dim=1, keepdim=True, dtype=weights.dtype) + 1e-20
|
||||||
|
|
||||||
# #################################################
|
# #################################################
|
||||||
# User doesn't want us to calculate the gradients
|
# User doesn't want us to calculate the gradients
|
||||||
|
|
|
@ -249,7 +249,9 @@ class NNMFLayer(torch.nn.Module):
|
||||||
else:
|
else:
|
||||||
self.last_input_data = None
|
self.last_input_data = None
|
||||||
|
|
||||||
input_convolved = input_convolved / input_convolved.sum(dim=1, keepdim=True)
|
input_convolved = input_convolved / (
|
||||||
|
input_convolved.sum(dim=1, keepdim=True) + 1e-20
|
||||||
|
)
|
||||||
|
|
||||||
h = torch.tile(
|
h = torch.tile(
|
||||||
self._h_initial.unsqueeze(0).unsqueeze(-1).unsqueeze(-1),
|
self._h_initial.unsqueeze(0).unsqueeze(-1).unsqueeze(-1),
|
||||||
|
|
|
@ -5,10 +5,12 @@ from network.calculate_output_size import calculate_output_size
|
||||||
from network.Parameter import Config
|
from network.Parameter import Config
|
||||||
from network.SbSLayer import SbSLayer
|
from network.SbSLayer import SbSLayer
|
||||||
from network.NNMFLayer import NNMFLayer
|
from network.NNMFLayer import NNMFLayer
|
||||||
|
from network.NNMFLayerSbSBP import NNMFLayerSbSBP
|
||||||
from network.SplitOnOffLayer import SplitOnOffLayer
|
from network.SplitOnOffLayer import SplitOnOffLayer
|
||||||
from network.Conv2dApproximation import Conv2dApproximation
|
from network.Conv2dApproximation import Conv2dApproximation
|
||||||
from network.SbSReconstruction import SbSReconstruction
|
from network.SbSReconstruction import SbSReconstruction
|
||||||
from network.InputSpikeImage import InputSpikeImage
|
from network.InputSpikeImage import InputSpikeImage
|
||||||
|
from network.PoissonLayer import PoissonLayer
|
||||||
|
|
||||||
|
|
||||||
def build_network(
|
def build_network(
|
||||||
|
@ -476,6 +478,94 @@ def build_network(
|
||||||
# Adding the x,y output dimensions
|
# Adding the x,y output dimensions
|
||||||
input_size.append(network[-1]._output_size.tolist())
|
input_size.append(network[-1]._output_size.tolist())
|
||||||
|
|
||||||
|
# #############################################################
|
||||||
|
# NNMF with SbS BP:
|
||||||
|
# #############################################################
|
||||||
|
|
||||||
|
elif (
|
||||||
|
cfg.network_structure.layer_type[layer_id].upper().startswith("BP SBS NNMF")
|
||||||
|
is True
|
||||||
|
):
|
||||||
|
|
||||||
|
assert in_channels > 0
|
||||||
|
assert out_channels > 0
|
||||||
|
|
||||||
|
number_of_iterations: int = -1
|
||||||
|
if len(cfg.number_of_spikes) > layer_id:
|
||||||
|
number_of_iterations = cfg.number_of_spikes[layer_id]
|
||||||
|
elif len(cfg.number_of_spikes) == 1:
|
||||||
|
number_of_iterations = cfg.number_of_spikes[0]
|
||||||
|
|
||||||
|
assert number_of_iterations > 0
|
||||||
|
|
||||||
|
logging.info(
|
||||||
|
(
|
||||||
|
f"Layer: {layer_id} -> NNMF Layer (SbS BP) with {number_of_iterations} iterations "
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
local_learning = False
|
||||||
|
if cfg.network_structure.layer_type[layer_id].upper().find("LOCAL") != -1:
|
||||||
|
local_learning = True
|
||||||
|
|
||||||
|
output_layer = False
|
||||||
|
if layer_id == len(cfg.network_structure.layer_type) - 1:
|
||||||
|
output_layer = True
|
||||||
|
|
||||||
|
network.append(
|
||||||
|
NNMFLayerSbSBP(
|
||||||
|
number_of_input_neurons=in_channels,
|
||||||
|
number_of_neurons=out_channels,
|
||||||
|
input_size=input_size[-1],
|
||||||
|
forward_kernel_size=kernel_size,
|
||||||
|
number_of_iterations=number_of_iterations,
|
||||||
|
epsilon_0=cfg.epsilon_0,
|
||||||
|
weight_noise_range=weight_noise_range,
|
||||||
|
strides=strides,
|
||||||
|
dilation=dilation,
|
||||||
|
padding=padding,
|
||||||
|
w_trainable=w_trainable,
|
||||||
|
device=device,
|
||||||
|
default_dtype=default_dtype,
|
||||||
|
layer_id=layer_id,
|
||||||
|
local_learning=local_learning,
|
||||||
|
output_layer=output_layer,
|
||||||
|
keep_last_grad_scale=cfg.learning_parameters.kepp_last_grad_scale,
|
||||||
|
disable_scale_grade=cfg.learning_parameters.disable_scale_grade,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# Adding the x,y output dimensions
|
||||||
|
input_size.append(network[-1]._output_size.tolist())
|
||||||
|
|
||||||
|
# #############################################################
|
||||||
|
# Poisson layer:
|
||||||
|
# #############################################################
|
||||||
|
|
||||||
|
elif (
|
||||||
|
cfg.network_structure.layer_type[layer_id].upper().startswith("POISSON")
|
||||||
|
is True
|
||||||
|
):
|
||||||
|
|
||||||
|
number_of_spikes: int = -1
|
||||||
|
if len(cfg.number_of_spikes) > layer_id:
|
||||||
|
number_of_spikes = cfg.number_of_spikes[layer_id]
|
||||||
|
elif len(cfg.number_of_spikes) == 1:
|
||||||
|
number_of_spikes = cfg.number_of_spikes[0]
|
||||||
|
|
||||||
|
assert number_of_spikes > 0
|
||||||
|
|
||||||
|
logging.info(
|
||||||
|
(f"Layer: {layer_id} -> Poisson Layer with {number_of_spikes} spikes ")
|
||||||
|
)
|
||||||
|
|
||||||
|
network.append(
|
||||||
|
PoissonLayer(
|
||||||
|
number_of_spikes=number_of_spikes,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# Adding the x,y output dimensions
|
||||||
|
input_size.append(input_size[-1])
|
||||||
|
|
||||||
# #############################################################
|
# #############################################################
|
||||||
# Failure becaue we didn't found the selection of layer
|
# Failure becaue we didn't found the selection of layer
|
||||||
# #############################################################
|
# #############################################################
|
||||||
|
|
|
@ -3,6 +3,7 @@ import torch
|
||||||
from network.Parameter import Config
|
from network.Parameter import Config
|
||||||
from network.SbSLayer import SbSLayer
|
from network.SbSLayer import SbSLayer
|
||||||
from network.NNMFLayer import NNMFLayer
|
from network.NNMFLayer import NNMFLayer
|
||||||
|
from network.NNMFLayerSbSBP import NNMFLayerSbSBP
|
||||||
|
|
||||||
from network.Conv2dApproximation import Conv2dApproximation
|
from network.Conv2dApproximation import Conv2dApproximation
|
||||||
from network.Adam import Adam
|
from network.Adam import Adam
|
||||||
|
@ -28,9 +29,10 @@ def build_optimizer(
|
||||||
parameter_list_weights.append(network[id]._weights)
|
parameter_list_weights.append(network[id]._weights)
|
||||||
parameter_list_sbs.append(True)
|
parameter_list_sbs.append(True)
|
||||||
|
|
||||||
if (isinstance(network[id], NNMFLayer) is True) and (
|
if (
|
||||||
network[id]._w_trainable is True
|
(isinstance(network[id], NNMFLayer) is True)
|
||||||
):
|
or (isinstance(network[id], NNMFLayerSbSBP) is True)
|
||||||
|
) and (network[id]._w_trainable is True):
|
||||||
parameter_list_weights.append(network[id]._weights)
|
parameter_list_weights.append(network[id]._weights)
|
||||||
parameter_list_sbs.append(True)
|
parameter_list_sbs.append(True)
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@ import numpy as np
|
||||||
|
|
||||||
from network.SbSLayer import SbSLayer
|
from network.SbSLayer import SbSLayer
|
||||||
from network.NNMFLayer import NNMFLayer
|
from network.NNMFLayer import NNMFLayer
|
||||||
|
from network.NNMFLayerSbSBP import NNMFLayerSbSBP
|
||||||
|
|
||||||
from network.SplitOnOffLayer import SplitOnOffLayer
|
from network.SplitOnOffLayer import SplitOnOffLayer
|
||||||
from network.Conv2dApproximation import Conv2dApproximation
|
from network.Conv2dApproximation import Conv2dApproximation
|
||||||
|
@ -48,7 +49,9 @@ def load_previous_weights(
|
||||||
)
|
)
|
||||||
logging.info(f"Weights file used for layer {id} : {file_to_load[0]}")
|
logging.info(f"Weights file used for layer {id} : {file_to_load[0]}")
|
||||||
|
|
||||||
if isinstance(network[id], NNMFLayer) is True:
|
if (isinstance(network[id], NNMFLayer) is True) or (
|
||||||
|
isinstance(network[id], NNMFLayerSbSBP) is True
|
||||||
|
):
|
||||||
filename_wilcard = os.path.join(
|
filename_wilcard = os.path.join(
|
||||||
overload_path, f"Weight_L{id}_*{post_fix}.npy"
|
overload_path, f"Weight_L{id}_*{post_fix}.npy"
|
||||||
)
|
)
|
||||||
|
|
|
@ -5,6 +5,7 @@ from network.Parameter import Config
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from network.SbSLayer import SbSLayer
|
from network.SbSLayer import SbSLayer
|
||||||
from network.NNMFLayer import NNMFLayer
|
from network.NNMFLayer import NNMFLayer
|
||||||
|
from network.NNMFLayerSbSBP import NNMFLayerSbSBP
|
||||||
from network.SplitOnOffLayer import SplitOnOffLayer
|
from network.SplitOnOffLayer import SplitOnOffLayer
|
||||||
from network.Conv2dApproximation import Conv2dApproximation
|
from network.Conv2dApproximation import Conv2dApproximation
|
||||||
|
|
||||||
|
@ -43,7 +44,9 @@ def save_weight_and_bias(
|
||||||
# Save the NNMF Weights
|
# Save the NNMF Weights
|
||||||
# ################################################
|
# ################################################
|
||||||
|
|
||||||
if isinstance(network[id], NNMFLayer) is True:
|
if (isinstance(network[id], NNMFLayer) is True) or (
|
||||||
|
isinstance(network[id], NNMFLayerSbSBP) is True
|
||||||
|
):
|
||||||
if network[id]._w_trainable is True:
|
if network[id]._w_trainable is True:
|
||||||
|
|
||||||
np.save(
|
np.save(
|
||||||
|
|
Loading…
Reference in a new issue