diff --git a/cluster.bash b/cluster.bash
new file mode 100644
index 0000000..52efa5c
--- /dev/null
+++ b/cluster.bash
@@ -0,0 +1,17 @@
+Directory="$(dirname "$(readlink -f "$0")")"
+Priority="-450"
+echo $Directory
+mkdir -p $Directory/logs/cluster
+project_name="MAHA"
+python="/home/mahbod/P3.11/bin/python3"
+
+for i in {1..3} 
+do
+    run="$python run_train.py -r $i"
+    echo "Running $run"
+    echo "hostname; nvidia-smi; cd $Directory ; $run" | qsub -o $Directory/logs/cluster -j y -p $Priority -q gp4u -N $project_name
+done
+
+# run="$python run_train.py -r 4"
+# echo "Running $run"
+# echo "hostname; nvidia-smi; cd $Directory ; $run" | qsub -o $Directory/logs/cluster -j y -p $Priority -q gp4u -N $project_name
\ No newline at end of file
diff --git a/config_data.json b/config_data.json
new file mode 100644
index 0000000..1af2760
--- /dev/null
+++ b/config_data.json
@@ -0,0 +1,13 @@
+{
+    // "CIFAR10", "FashionMNIST", "MNIST"
+    "dataset": "CIFAR10",
+    "batch_size_train": 50,
+    "batch_size_test": 50,
+    // Data augmentation 
+    "flip_p": 0.5,
+    "jitter_brightness": 0.5,
+    "jitter_contrast": 0.1,
+    "jitter_saturation": 0.1,
+    "jitter_hue": 0.15,
+    "da_auto_mode": false
+}
\ No newline at end of file
diff --git a/config_lr_parameter.json b/config_lr_parameter.json
new file mode 100644
index 0000000..c79d911
--- /dev/null
+++ b/config_lr_parameter.json
@@ -0,0 +1,11 @@
+{
+    "loss_mode": 0,
+    "loss_coeffs_mse": 0.5,
+    "loss_coeffs_kldiv": 1.0,
+    "lr_limit": 1e-9,
+    "lr_initial_neuron_a": 0.01,
+    "lr_initial_neuron_b": 0.001,
+    "lr_initial_norm": 0.001,
+    "lr_initial_batchnorm2d": 0.001,
+    "number_of_epoch": 500
+}
\ No newline at end of file
diff --git a/config_network.json b/config_network.json
new file mode 100644
index 0000000..50b02de
--- /dev/null
+++ b/config_network.json
@@ -0,0 +1,232 @@
+{
+    // Number of NNMF iterations
+    "iterations": 20,
+    "epsilon": "None",
+    "number_of_neurons_a": [
+        32,
+        64,
+        96,
+        10
+    ],
+    "number_of_neurons_b": [
+        -1,
+        -1,
+        -1,
+        -1
+    ],
+    "kernel_size_conv": [
+        [
+            5,
+            5
+        ],
+        [
+            5,
+            5
+        ],
+        [
+            -1,
+            -1
+        ],
+        [
+            1,
+            1
+        ]
+    ],
+    "stride_conv": [
+        [
+            1,
+            1
+        ],
+        [
+            1,
+            1
+        ],
+        [
+            1,
+            1
+        ],
+        [
+            1,
+            1
+        ]
+    ],
+    "padding_conv": [
+        [
+            0,
+            0
+        ],
+        [
+            0,
+            0
+        ],
+        [
+            0,
+            0
+        ],
+        [
+            0,
+            0
+        ]
+    ],
+    "dilation_conv": [
+        [
+            1,
+            1
+        ],
+        [
+            1,
+            1
+        ],
+        [
+            1,
+            1
+        ],
+        [
+            1,
+            1
+        ]
+    ],
+    "kernel_size_pool": [
+        [
+            2,
+            2
+        ],
+        [
+            2,
+            2
+        ],
+        [
+            -1,
+            -1
+        ],
+        [
+            -1,
+            -1
+        ]
+    ],
+    "stride_pool": [
+        [
+            2,
+            2
+        ],
+        [
+            2,
+            2
+        ],
+        [
+            -1,
+            -1
+        ],
+        [
+            -1,
+            -1
+        ]
+    ],
+    "padding_pool": [
+        [
+            0,
+            0
+        ],
+        [
+            0,
+            0
+        ],
+        [
+            0,
+            0
+        ],
+        [
+            0,
+            0
+        ]
+    ],
+    "dilation_pool": [
+        [
+            1,
+            1
+        ],
+        [
+            1,
+            1
+        ],
+        [
+            1,
+            1
+        ],
+        [
+            1,
+            1
+        ]
+    ],
+    // 0 == No pooling, 1 = Average Pooling, 2 = Max Pooling, 3 == Perceptron, 4 == NNMF
+    "type_of_pooling": [
+        1,
+        1,
+        1,
+        1
+    ],
+    "local_learning": [
+        false,
+        false,
+        false,
+        false
+    ],
+    "local_learning_kl": [
+        false,
+        false,
+        false,
+        false
+    ],
+    // NNMF:
+    "local_learning_pooling": [
+        false,
+        false,
+        false,
+        false
+    ],
+    // NNMF:
+    "local_learning_use_kl_pooling": [
+        false,
+        false,
+        false,
+        false
+    ],
+    // 1 == NNMF, 2 == Perceptron
+    "type_of_neuron_a": [
+        1,
+        1,
+        1,
+        1
+    ],
+    // 0 = None, 1 == NNMF, 2 == Perceptron
+    "type_of_neuron_b": [
+        3,
+        3,
+        3,
+        3
+    ],
+    "batch_norm_neuron_a": [
+        true,
+        true,
+        true,
+        true
+    ],
+    "batch_norm_neuron_b": [
+        true,
+        true,
+        true,
+        true
+    ],
+    "bias_norm_neuron_a": [
+        false,
+        false,
+        false,
+        false
+    ],
+    "bias_norm_neuron_b": [
+        true,
+        true,
+        true,
+        true
+    ]
+}
\ No newline at end of file
diff --git a/convert_log_to_numpy.py b/convert_log_to_numpy.py
new file mode 100644
index 0000000..05a5427
--- /dev/null
+++ b/convert_log_to_numpy.py
@@ -0,0 +1,31 @@
+import os
+import glob
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+from tensorboard.backend.event_processing import event_accumulator  # type: ignore
+import numpy as np
+
+
+def get_data(path: str = "log_cnn"):
+    acc = event_accumulator.EventAccumulator(path)
+    acc.Reload()
+
+    which_scalar = "Test Number Correct"
+    te = acc.Scalars(which_scalar)
+
+    np_temp = np.zeros((len(te), 2))
+
+    for id in range(0, len(te)):
+        np_temp[id, 0] = te[id].step
+        np_temp[id, 1] = te[id].value
+
+    print(np_temp[:, 1] / 100)
+    np_temp = np.nan_to_num(np_temp)
+    return np_temp
+
+
+for path in glob.glob("log_*"):
+    print(path)
+    data = get_data(path)
+    np.save("data_" + path + ".npy", data)
diff --git a/load_model.py b/load_model.py
new file mode 100644
index 0000000..33fe1ea
--- /dev/null
+++ b/load_model.py
@@ -0,0 +1,22 @@
+# %%
+import torch
+
+# load the state dict of the model
+state = torch.load("Models/Model_seed_4_30.pt")
+print(state)
+# %%
+from tools.make_network import make_network
+
+model, _, _ = make_network(
+    input_dim_x=28,
+    input_dim_y=28,
+    input_number_of_channel=3,
+    device="cuda",
+    config_network_filename="config_network.json",
+)
+model.load_state_dict(state)
+print(model)
+
+# %%
+(state['8.parametrizations.weight.original'] < 0).sum(), (model[8].weight < 0).sum()
+# %%
diff --git a/noise_holes.py b/noise_holes.py
new file mode 100644
index 0000000..8c6439b
--- /dev/null
+++ b/noise_holes.py
@@ -0,0 +1,102 @@
+import os
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+import argh
+
+import numpy as np
+import torch
+
+rand_seed: int = 21
+torch.manual_seed(rand_seed)
+torch.cuda.manual_seed(rand_seed)
+np.random.seed(rand_seed)
+
+from get_the_data_uniform import get_the_data
+
+
+def main(
+    dataset: str = "CIFAR10",  # "CIFAR10", "FashionMNIST", "MNIST"
+    only_print_network: bool = False,
+    model_name: str = "Model_iter20_lr_1.0000e-03_1.0000e-02_1.0000e-03_.pt",
+) -> None:
+
+    torch_device: torch.device = (
+        torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu")
+    )
+    torch.set_default_dtype(torch.float32)
+
+    # Some parameters
+    batch_size_test: int = 50  # 0
+
+    loss_mode: int = 0
+    loss_coeffs_mse: float = 0.5
+    loss_coeffs_kldiv: float = 1.0
+    print(
+        "loss_mode: ",
+        loss_mode,
+        "loss_coeffs_mse: ",
+        loss_coeffs_mse,
+        "loss_coeffs_kldiv: ",
+        loss_coeffs_kldiv,
+    )
+
+    if dataset == "MNIST" or dataset == "FashionMNIST":
+        input_dim_x: int = 24
+        input_dim_y: int = 24
+    else:
+        input_dim_x = 28
+        input_dim_y = 28
+
+    test_dataloader, test_processing_chain = get_the_data(
+        dataset,
+        batch_size_test,
+        torch_device,
+        input_dim_x,
+        input_dim_y,
+    )
+
+    network = torch.load(model_name)
+    network.to(device=torch_device)
+
+    print(network)
+
+    if only_print_network:
+        exit()
+
+    # Switch the network into evalution mode
+    network.eval()
+    number_of_noise_steps = 20
+    noise_scale = torch.arange(0, number_of_noise_steps + 1) / float(
+        number_of_noise_steps
+    )
+
+    results = torch.zeros_like(noise_scale)
+
+    with torch.no_grad():
+
+        for position in range(0, noise_scale.shape[0]):
+            test_correct: int = 0
+            test_number: int = 0
+            eta: float = noise_scale[position]
+            for image, target in test_dataloader:
+                noise = torch.rand_like(image) > eta
+
+                image = image * noise
+                image = image / (image.sum(dim=(1, 2, 3), keepdim=True) + 1e-20)
+                output = network(test_processing_chain(image))
+
+                test_correct += (output.argmax(dim=1) == target).sum().cpu().numpy()
+                test_number += target.shape[0]
+
+            perfomance_test_correct: float = 100.0 * test_correct / test_number
+            results[position] = perfomance_test_correct
+
+            print(f"{eta:.2f}: {perfomance_test_correct:.2f}%")
+
+    np.save("noise_holes_results.npy", results.cpu().numpy())
+    return
+
+
+if __name__ == "__main__":
+    argh.dispatch_command(main)
diff --git a/noise_holes_w_noise.py b/noise_holes_w_noise.py
new file mode 100644
index 0000000..5768e9b
--- /dev/null
+++ b/noise_holes_w_noise.py
@@ -0,0 +1,107 @@
+import os
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+import argh
+
+import numpy as np
+import torch
+
+rand_seed: int = 21
+torch.manual_seed(rand_seed)
+torch.cuda.manual_seed(rand_seed)
+np.random.seed(rand_seed)
+
+from get_the_data_uniform import get_the_data
+
+
+def main(
+    dataset: str = "CIFAR10",  # "CIFAR10", "FashionMNIST", "MNIST"
+    only_print_network: bool = False,
+    model_name: str = "Model_iter20_lr_1.0000e-03_1.0000e-02_1.0000e-03_.pt",
+) -> None:
+
+    torch_device: torch.device = (
+        torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu")
+    )
+    torch.set_default_dtype(torch.float32)
+
+    # Some parameters
+    batch_size_test: int = 50  # 0
+
+    loss_mode: int = 0
+    loss_coeffs_mse: float = 0.5
+    loss_coeffs_kldiv: float = 1.0
+    print(
+        "loss_mode: ",
+        loss_mode,
+        "loss_coeffs_mse: ",
+        loss_coeffs_mse,
+        "loss_coeffs_kldiv: ",
+        loss_coeffs_kldiv,
+    )
+
+    if dataset == "MNIST" or dataset == "FashionMNIST":
+        input_dim_x: int = 24
+        input_dim_y: int = 24
+    else:
+        input_dim_x = 28
+        input_dim_y = 28
+
+    test_dataloader, test_processing_chain = get_the_data(
+        dataset,
+        batch_size_test,
+        torch_device,
+        input_dim_x,
+        input_dim_y,
+    )
+
+    network = torch.load(model_name)
+    network.to(device=torch_device)
+
+    print(network)
+
+    if only_print_network:
+        exit()
+
+    # Switch the network into evalution mode
+    network.eval()
+    number_of_noise_steps = 20
+    noise_scale = torch.arange(0, number_of_noise_steps + 1) / float(
+        number_of_noise_steps
+    )
+
+    results = torch.zeros_like(noise_scale)
+
+    with torch.no_grad():
+
+        for position in range(0, noise_scale.shape[0]):
+            test_correct: int = 0
+            test_number: int = 0
+            eta: float = noise_scale[position]
+            for image, target in test_dataloader:
+                noise = torch.rand_like(image) > eta
+                noise_2 = torch.rand_like(image)
+                noise_2 = noise_2 / (noise_2.sum(dim=(1, 2, 3), keepdim=True) + 1e-20)
+
+                image = (
+                    image * noise.type(dtype=torch.float32)
+                    + (1.0 - noise.type(dtype=torch.float32)) * noise_2
+                )
+                image = image / (image.sum(dim=(1, 2, 3), keepdim=True) + 1e-20)
+                output = network(test_processing_chain(image))
+
+                test_correct += (output.argmax(dim=1) == target).sum().cpu().numpy()
+                test_number += target.shape[0]
+
+            perfomance_test_correct: float = 100.0 * test_correct / test_number
+            results[position] = perfomance_test_correct
+
+            print(f"{eta:.2f}: {perfomance_test_correct:.2f}%")
+
+    np.save("noise_holes_w_noise_results.npy", results.cpu().numpy())
+    return
+
+
+if __name__ == "__main__":
+    argh.dispatch_command(main)
diff --git a/noise_picture.py b/noise_picture.py
new file mode 100644
index 0000000..4522f0e
--- /dev/null
+++ b/noise_picture.py
@@ -0,0 +1,110 @@
+import os
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+import argh
+
+import numpy as np
+import torch
+
+rand_seed: int = 21
+torch.manual_seed(rand_seed)
+torch.cuda.manual_seed(rand_seed)
+np.random.seed(rand_seed)
+
+from get_the_data_picture import get_the_data
+
+
+def main(
+    dataset: str = "CIFAR10",  # "CIFAR10", "FashionMNIST", "MNIST"
+    only_print_network: bool = False,
+    model_name: str = "Model_iter20_lr_1.0000e-03_1.0000e-02_1.0000e-03_.pt",
+) -> None:
+
+    torch_device: torch.device = (
+        torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu")
+    )
+    torch.set_default_dtype(torch.float32)
+
+    # Some parameters
+    batch_size_test: int = 50  # 0
+
+    loss_mode: int = 0
+    loss_coeffs_mse: float = 0.5
+    loss_coeffs_kldiv: float = 1.0
+    print(
+        "loss_mode: ",
+        loss_mode,
+        "loss_coeffs_mse: ",
+        loss_coeffs_mse,
+        "loss_coeffs_kldiv: ",
+        loss_coeffs_kldiv,
+    )
+
+    if dataset == "MNIST" or dataset == "FashionMNIST":
+        input_dim_x: int = 24
+        input_dim_y: int = 24
+    else:
+        input_dim_x = 28
+        input_dim_y = 28
+
+    network = torch.load(model_name)
+    network.to(device=torch_device)
+
+    print(network)
+
+    if only_print_network:
+        exit()
+
+    # Switch the network into evalution mode
+    network.eval()
+    number_of_noise_steps = 20
+    noise_scale = (
+        0.5 * torch.arange(0, number_of_noise_steps + 1) / float(number_of_noise_steps)
+    )
+
+    results = torch.zeros_like(noise_scale)
+
+    with torch.no_grad():
+
+        for position in range(0, noise_scale.shape[0]):
+
+            train_dataloader, test_dataloader, test_processing_chain = get_the_data(
+                dataset,
+                batch_size_test,
+                batch_size_test,
+                torch_device,
+                input_dim_x,
+                input_dim_y,
+            )
+            train_dataloader_iter = iter(train_dataloader)
+            test_dataloader_iter = iter(test_dataloader)
+
+            test_correct: int = 0
+            test_number: int = 0
+            eta: float = noise_scale[position]
+            max_iters = len(test_dataloader)
+
+            for _ in range(0, max_iters):
+                (image, target) = next(test_dataloader_iter)
+                (noise, _) = next(train_dataloader_iter)
+                noise = noise / (noise.sum(dim=(1, 2, 3), keepdim=True) + 1e-20)
+                image = image / (image.sum(dim=(1, 2, 3), keepdim=True) + 1e-20)
+                output = network(
+                    test_processing_chain((1.0 - eta) * image + eta * noise)
+                )
+
+                test_correct += (output.argmax(dim=1) == target).sum().cpu().numpy()
+                test_number += target.shape[0]
+
+            perfomance_test_correct: float = 100.0 * test_correct / test_number
+            results[position] = perfomance_test_correct
+
+            print(f"{eta:.2f}: {perfomance_test_correct:.2f}%")
+
+    np.save("noise_picture_results.npy", results.cpu().numpy())
+    return
+
+
+if __name__ == "__main__":
+    argh.dispatch_command(main)
diff --git a/noise_run.sh b/noise_run.sh
new file mode 100644
index 0000000..b0edc9b
--- /dev/null
+++ b/noise_run.sh
@@ -0,0 +1,5 @@
+/data_1/davrot/P3.12/bin/python3 noise_uniform.py
+/data_1/davrot/P3.12/bin/python3 noise_picture.py
+/data_1/davrot/P3.12/bin/python3 noise_holes.py
+/data_1/davrot/P3.12/bin/python3 noise_holes_w_noise.py
+
diff --git a/noise_uniform.py b/noise_uniform.py
new file mode 100644
index 0000000..63af795
--- /dev/null
+++ b/noise_uniform.py
@@ -0,0 +1,103 @@
+import os
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+import argh
+
+import numpy as np
+import torch
+
+rand_seed: int = 21
+torch.manual_seed(rand_seed)
+torch.cuda.manual_seed(rand_seed)
+np.random.seed(rand_seed)
+
+from get_the_data_uniform import get_the_data
+
+
+def main(
+    dataset: str = "CIFAR10",  # "CIFAR10", "FashionMNIST", "MNIST"
+    only_print_network: bool = False,
+    model_name: str = "Model_iter20_lr_1.0000e-03_1.0000e-02_1.0000e-03_.pt",
+) -> None:
+
+    torch_device: torch.device = (
+        torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu")
+    )
+    torch.set_default_dtype(torch.float32)
+
+    # Some parameters
+    batch_size_test: int = 50  # 0
+
+    loss_mode: int = 0
+    loss_coeffs_mse: float = 0.5
+    loss_coeffs_kldiv: float = 1.0
+    print(
+        "loss_mode: ",
+        loss_mode,
+        "loss_coeffs_mse: ",
+        loss_coeffs_mse,
+        "loss_coeffs_kldiv: ",
+        loss_coeffs_kldiv,
+    )
+
+    if dataset == "MNIST" or dataset == "FashionMNIST":
+        input_dim_x: int = 24
+        input_dim_y: int = 24
+    else:
+        input_dim_x = 28
+        input_dim_y = 28
+
+    test_dataloader, test_processing_chain = get_the_data(
+        dataset,
+        batch_size_test,
+        torch_device,
+        input_dim_x,
+        input_dim_y,
+    )
+
+    network = torch.load(model_name)
+    network.to(device=torch_device)
+
+    print(network)
+
+    if only_print_network:
+        exit()
+
+    # Switch the network into evalution mode
+    network.eval()
+    number_of_noise_steps = 20
+    noise_scale = torch.arange(0, number_of_noise_steps + 1) / float(
+        number_of_noise_steps
+    )
+
+    results = torch.zeros_like(noise_scale)
+
+    with torch.no_grad():
+
+        for position in range(0, noise_scale.shape[0]):
+            test_correct: int = 0
+            test_number: int = 0
+            eta: float = noise_scale[position]
+            for image, target in test_dataloader:
+                noise = torch.rand_like(image)
+                noise = noise / (noise.sum(dim=(1, 2, 3), keepdim=True) + 1e-20)
+                image = image / (image.sum(dim=(1, 2, 3), keepdim=True) + 1e-20)
+                output = network(
+                    test_processing_chain((1.0 - eta) * image + eta * noise)
+                )
+
+                test_correct += (output.argmax(dim=1) == target).sum().cpu().numpy()
+                test_number += target.shape[0]
+
+            perfomance_test_correct: float = 100.0 * test_correct / test_number
+            results[position] = perfomance_test_correct
+
+            print(f"{eta:.2f}: {perfomance_test_correct:.2f}%")
+
+    np.save("noise_uniform_results.npy", results.cpu().numpy())
+    return
+
+
+if __name__ == "__main__":
+    argh.dispatch_command(main)
diff --git a/plot.py b/plot.py
new file mode 100644
index 0000000..ad22d33
--- /dev/null
+++ b/plot.py
@@ -0,0 +1,15 @@
+import numpy as np
+import matplotlib.pyplot as plt
+
+data = np.load("data_log.npy")
+plt.loglog(
+    data[:, 0],
+    100.0 * (1.0 - data[:, 1] / 10000.0),
+    "k",
+)
+
+plt.legend()
+plt.xlabel("Epoch")
+plt.ylabel("Error [%]")
+plt.title("CIFAR10")
+plt.show()
diff --git a/plot_test.py b/plot_test.py
new file mode 100644
index 0000000..3b62c04
--- /dev/null
+++ b/plot_test.py
@@ -0,0 +1,10 @@
+# %%
+import torch
+import matplotlib.pyplot as plt
+
+x = (torch.arange(0, 10000) / 1000) - 5.0
+beta = 10.0
+plt.plot(x, torch.nn.functional.relu(0.5 + 0.5 * torch.tanh(beta * x)))
+plt.show()
+
+# %%
diff --git a/run_test.py b/run_test.py
new file mode 100644
index 0000000..e9ad6b1
--- /dev/null
+++ b/run_test.py
@@ -0,0 +1,9 @@
+import os
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+import argh
+from tools.run_network_test import main
+
+if __name__ == "__main__":
+    argh.dispatch_command(main)
\ No newline at end of file
diff --git a/run_train.py b/run_train.py
new file mode 100644
index 0000000..8bf01bb
--- /dev/null
+++ b/run_train.py
@@ -0,0 +1,9 @@
+import os
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+
+import argh
+from tools.run_network_train import main
+
+if __name__ == "__main__":
+    argh.dispatch_command(main)
\ No newline at end of file