From e3d1fad2053769ccf49baf7f90eaf6b87fad79bb Mon Sep 17 00:00:00 2001
From: davrot <davrot@uni-bremen.de>
Date: Tue, 10 Dec 2024 12:47:31 +0100
Subject: [PATCH] =?UTF-8?q?Dateien=20nach=20=E2=80=9Etools=E2=80=9C=20hoch?=
 =?UTF-8?q?laden?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 tools/L1NormLayer.py      | 13 ++++++++
 tools/append_parameter.py |  8 +++++
 tools/data_loader.py      | 31 +++++++++++++++++++
 tools/loss_function.py    | 64 +++++++++++++++++++++++++++++++++++++++
 tools/make_optimize.py    | 32 ++++++++++++++++++++
 5 files changed, 148 insertions(+)
 create mode 100644 tools/L1NormLayer.py
 create mode 100644 tools/append_parameter.py
 create mode 100644 tools/data_loader.py
 create mode 100644 tools/loss_function.py
 create mode 100644 tools/make_optimize.py

diff --git a/tools/L1NormLayer.py b/tools/L1NormLayer.py
new file mode 100644
index 0000000..6816b3a
--- /dev/null
+++ b/tools/L1NormLayer.py
@@ -0,0 +1,13 @@
+import torch
+
+
+class L1NormLayer(torch.nn.Module):
+
+    epsilon: float
+
+    def __init__(self, epsilon: float = 10e-20) -> None:
+        super().__init__()
+        self.epsilon = epsilon
+
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        return input / (input.sum(dim=1, keepdim=True) + self.epsilon)
diff --git a/tools/append_parameter.py b/tools/append_parameter.py
new file mode 100644
index 0000000..b972e39
--- /dev/null
+++ b/tools/append_parameter.py
@@ -0,0 +1,8 @@
+import torch
+
+
+def append_parameter(
+    module: torch.nn.Module, parameter_list: list[torch.nn.parameter.Parameter]
+):
+    for netp in module.parameters():
+        parameter_list.append(netp)
diff --git a/tools/data_loader.py b/tools/data_loader.py
new file mode 100644
index 0000000..0a0d430
--- /dev/null
+++ b/tools/data_loader.py
@@ -0,0 +1,31 @@
+import torch
+
+
+def data_loader(
+    pattern: torch.Tensor,
+    labels: torch.Tensor,
+    worker_init_fn,
+    generator,
+    batch_size: int = 128,
+    shuffle: bool = True,
+    torch_device: torch.device = torch.device("cpu"),
+) -> torch.utils.data.dataloader.DataLoader:
+
+    assert pattern.ndim >= 3
+
+    pattern_storage: torch.Tensor = pattern.to(torch_device).type(torch.float32)
+    if pattern_storage.ndim == 3:
+        pattern_storage = pattern_storage.unsqueeze(1)
+    pattern_storage /= pattern_storage.max()
+
+    label_storage: torch.Tensor = labels.to(torch_device).type(torch.int64)
+
+    dataloader = torch.utils.data.DataLoader(
+        torch.utils.data.TensorDataset(pattern_storage, label_storage),
+        batch_size=batch_size,
+        shuffle=shuffle,
+        worker_init_fn=worker_init_fn,
+        generator=generator,
+    )
+
+    return dataloader
diff --git a/tools/loss_function.py b/tools/loss_function.py
new file mode 100644
index 0000000..e256840
--- /dev/null
+++ b/tools/loss_function.py
@@ -0,0 +1,64 @@
+import torch
+
+
+# loss_mode == 0: "normal" SbS loss function mixture
+# loss_mode == 1: cross_entropy
+def loss_function(
+    h: torch.Tensor,
+    labels: torch.Tensor,
+    loss_mode: int = 0,
+    number_of_output_neurons: int = 10,
+    loss_coeffs_mse: float = 0.0,
+    loss_coeffs_kldiv: float = 0.0,
+) -> torch.Tensor | None:
+
+    assert loss_mode >= 0
+    assert loss_mode <= 1
+
+    assert h.ndim == 2
+
+    if loss_mode == 0:
+
+        # Convert label into one hot
+        target_one_hot: torch.Tensor = torch.zeros(
+            (
+                labels.shape[0],
+                number_of_output_neurons,
+            ),
+            device=h.device,
+            dtype=h.dtype,
+        )
+
+        target_one_hot.scatter_(
+            1,
+            labels.to(h.device).unsqueeze(1),
+            torch.ones(
+                (labels.shape[0], 1),
+                device=h.device,
+                dtype=h.dtype,
+            ),
+        )
+
+        my_loss: torch.Tensor = ((h - target_one_hot) ** 2).sum(dim=0).mean(
+            dim=0
+        ) * loss_coeffs_mse
+
+        my_loss = (
+            my_loss
+            + (
+                (target_one_hot * torch.log((target_one_hot + 1e-20) / (h + 1e-20)))
+                .sum(dim=0)
+                .mean(dim=0)
+            )
+            * loss_coeffs_kldiv
+        )
+
+        my_loss = my_loss / (abs(loss_coeffs_kldiv) + abs(loss_coeffs_mse))
+
+        return my_loss
+
+    elif loss_mode == 1:
+        my_loss = torch.nn.functional.cross_entropy(h, labels.to(h.device))
+        return my_loss
+    else:
+        return None
diff --git a/tools/make_optimize.py b/tools/make_optimize.py
new file mode 100644
index 0000000..ab1a4e0
--- /dev/null
+++ b/tools/make_optimize.py
@@ -0,0 +1,32 @@
+import torch
+
+
+def make_optimize(
+    parameters: list[list[torch.nn.parameter.Parameter]],
+    lr_initial: list[float],
+    eps=1e-10,
+) -> tuple[
+    list[torch.optim.Adam | None],
+    list[torch.optim.lr_scheduler.ReduceLROnPlateau | None],
+]:
+    list_optimizer: list[torch.optim.Adam | None] = []
+    list_lr_scheduler: list[torch.optim.lr_scheduler.ReduceLROnPlateau | None] = []
+
+    assert len(parameters) == len(lr_initial)
+
+    for i in range(0, len(parameters)):
+        if len(parameters[i]) > 0:
+            list_optimizer.append(torch.optim.Adam(parameters[i], lr=lr_initial[i]))
+        else:
+            list_optimizer.append(None)
+
+    for i in range(0, len(list_optimizer)):
+        if list_optimizer[i] is not None:
+            pass
+            list_lr_scheduler.append(
+                torch.optim.lr_scheduler.ReduceLROnPlateau(list_optimizer[i], eps=eps)  # type: ignore
+            )
+        else:
+            list_lr_scheduler.append(None)
+
+    return (list_optimizer, list_lr_scheduler)