Add files via upload
This commit is contained in:
David Rotermund
GitHub
parent
34a688d546
commit
8505df62e3
2 changed files with 436 additions and 407 deletions
824
cnn_training.py
824
cnn_training.py
|
|
@ -1,400 +1,424 @@
|
||||||
import torch
|
import torch
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import datetime
|
import datetime
|
||||||
import argh
|
import argh
|
||||||
import time
|
import time
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
from jsmin import jsmin
|
import glob
|
||||||
|
from jsmin import jsmin
|
||||||
from functions.alicorn_data_loader import alicorn_data_loader
|
from natsort import natsorted
|
||||||
from functions.train import train
|
|
||||||
from functions.test import test
|
from functions.alicorn_data_loader import alicorn_data_loader
|
||||||
from functions.make_cnn import make_cnn
|
from functions.train import train
|
||||||
from functions.set_seed import set_seed
|
from functions.test import test
|
||||||
from functions.plot_intermediate import plot_intermediate
|
from functions.make_cnn import make_cnn
|
||||||
from functions.create_logger import create_logger
|
from functions.set_seed import set_seed
|
||||||
|
from functions.plot_intermediate import plot_intermediate
|
||||||
|
from functions.create_logger import create_logger
|
||||||
# to disable logging output from Tensorflow
|
|
||||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
|
|
||||||
from torch.utils.tensorboard import SummaryWriter
|
# to disable logging output from Tensorflow
|
||||||
|
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
|
||||||
|
from torch.utils.tensorboard import SummaryWriter
|
||||||
def main(
|
|
||||||
idx_conv_out_channels_list: int = 0,
|
|
||||||
idx_conv_kernel_sizes: int = 0,
|
def main(
|
||||||
idx_conv_stride_sizes: int = 0,
|
idx_conv_out_channels_list: int = 0,
|
||||||
seed_counter: int = 0,
|
idx_conv_kernel_sizes: int = 0,
|
||||||
) -> None:
|
idx_conv_stride_sizes: int = 0,
|
||||||
config_filenname = "config.json"
|
seed_counter: int = 0,
|
||||||
with open(config_filenname, "r") as file_handle:
|
) -> None:
|
||||||
config = json.loads(jsmin(file_handle.read()))
|
config_filenname = "config.json"
|
||||||
|
with open(config_filenname, "r") as file_handle:
|
||||||
logger = create_logger(
|
config = json.loads(jsmin(file_handle.read()))
|
||||||
save_logging_messages=bool(config["save_logging_messages"]),
|
|
||||||
display_logging_messages=bool(config["display_logging_messages"]),
|
logger = create_logger(
|
||||||
)
|
save_logging_messages=bool(config["save_logging_messages"]),
|
||||||
|
display_logging_messages=bool(config["display_logging_messages"]),
|
||||||
# network settings:
|
)
|
||||||
conv_out_channels_list: list[list[int]] = config["conv_out_channels_list"]
|
|
||||||
conv_kernel_sizes: list[list[int]] = config["conv_kernel_sizes"]
|
# network settings:
|
||||||
conv_stride_sizes: list[int] = config["conv_stride_sizes"]
|
conv_out_channels_list: list[list[int]] = config["conv_out_channels_list"]
|
||||||
|
conv_kernel_sizes: list[list[int]] = config["conv_kernel_sizes"]
|
||||||
num_pfinkel: list = np.arange(
|
conv_stride_sizes: list[int] = config["conv_stride_sizes"]
|
||||||
int(config["num_pfinkel_start"]),
|
|
||||||
int(config["num_pfinkel_stop"]),
|
num_pfinkel: list = np.arange(
|
||||||
int(config["num_pfinkel_step"]),
|
int(config["num_pfinkel_start"]),
|
||||||
).tolist()
|
int(config["num_pfinkel_stop"]),
|
||||||
|
int(config["num_pfinkel_step"]),
|
||||||
run_network(
|
).tolist()
|
||||||
out_channels=conv_out_channels_list[int(idx_conv_out_channels_list)],
|
|
||||||
kernel_size=conv_kernel_sizes[int(idx_conv_kernel_sizes)],
|
run_network(
|
||||||
stride=conv_stride_sizes[int(idx_conv_stride_sizes)],
|
out_channels=conv_out_channels_list[int(idx_conv_out_channels_list)],
|
||||||
activation_function=str(config["activation_function"]),
|
kernel_size=conv_kernel_sizes[int(idx_conv_kernel_sizes)],
|
||||||
train_first_layer=bool(config["train_first_layer"]),
|
stride=conv_stride_sizes[int(idx_conv_stride_sizes)],
|
||||||
seed_counter=seed_counter,
|
activation_function=str(config["activation_function"]),
|
||||||
minimum_learning_rate=float(config["minimum_learning_rate"]),
|
train_first_layer=bool(config["train_first_layer"]),
|
||||||
conv_0_kernel_size=int(config["conv_0_kernel_size"]),
|
seed_counter=seed_counter,
|
||||||
mp_1_kernel_size=int(config["mp_1_kernel_size"]),
|
minimum_learning_rate=float(config["minimum_learning_rate"]),
|
||||||
mp_1_stride=int(config["mp_1_stride"]),
|
conv_0_kernel_size=int(config["conv_0_kernel_size"]),
|
||||||
batch_size_train=int(config["batch_size_train"]),
|
mp_1_kernel_size=int(config["mp_1_kernel_size"]),
|
||||||
batch_size_test=int(config["batch_size_test"]),
|
mp_1_stride=int(config["mp_1_stride"]),
|
||||||
learning_rate=float(config["learning_rate"]),
|
batch_size_train=int(config["batch_size_train"]),
|
||||||
max_epochs=int(config["max_epochs"]),
|
batch_size_test=int(config["batch_size_test"]),
|
||||||
save_model=bool(config["save_model"]),
|
learning_rate=float(config["learning_rate"]),
|
||||||
stimuli_per_pfinkel=int(config["stimuli_per_pfinkel"]),
|
max_epochs=int(config["max_epochs"]),
|
||||||
num_pfinkel=num_pfinkel,
|
save_model=bool(config["save_model"]),
|
||||||
logger=logger,
|
stimuli_per_pfinkel=int(config["stimuli_per_pfinkel"]),
|
||||||
save_ever_x_epochs=int(config["save_ever_x_epochs"]),
|
num_pfinkel=num_pfinkel,
|
||||||
scheduler_patience=int(config["scheduler_patience"]),
|
logger=logger,
|
||||||
condition=str(config["condition"]),
|
save_ever_x_epochs=int(config["save_ever_x_epochs"]),
|
||||||
data_path=str(config["data_path"]),
|
scheduler_patience=int(config["scheduler_patience"]),
|
||||||
pooling_type=str(config["pooling_type"]),
|
condition=str(config["condition"]),
|
||||||
conv_0_enable_softmax=bool(config["conv_0_enable_softmax"]),
|
data_path=str(config["data_path"]),
|
||||||
scale_data=int(config["scale_data"]),
|
pooling_type=str(config["pooling_type"]),
|
||||||
use_scheduler=bool(config["use_scheduler"]),
|
conv_0_enable_softmax=bool(config["conv_0_enable_softmax"]),
|
||||||
use_adam=bool(config["use_adam"]),
|
scale_data=int(config["scale_data"]),
|
||||||
use_plot_intermediate=bool(config["use_plot_intermediate"]),
|
use_scheduler=bool(config["use_scheduler"]),
|
||||||
leak_relu_negative_slope=float(config["leak_relu_negative_slope"]),
|
use_adam=bool(config["use_adam"]),
|
||||||
scheduler_verbose=bool(config["scheduler_verbose"]),
|
use_plot_intermediate=bool(config["use_plot_intermediate"]),
|
||||||
scheduler_factor=float(config["scheduler_factor"]),
|
leak_relu_negative_slope=float(config["leak_relu_negative_slope"]),
|
||||||
precision_100_percent=int(config["precision_100_percent"]),
|
scheduler_verbose=bool(config["scheduler_verbose"]),
|
||||||
scheduler_threshold=float(config["scheduler_threshold"]),
|
scheduler_factor=float(config["scheduler_factor"]),
|
||||||
)
|
precision_100_percent=int(config["precision_100_percent"]),
|
||||||
|
scheduler_threshold=float(config["scheduler_threshold"]),
|
||||||
|
model_continue=bool(config["model_continue"]),
|
||||||
def run_network(
|
initial_model_path=str(config["initial_model_path"]),
|
||||||
out_channels: list[int],
|
tb_runs_path=str(config["tb_runs_path"]),
|
||||||
kernel_size: list[int],
|
trained_models_path=str(config["trained_models_path"]),
|
||||||
num_pfinkel: list,
|
performance_data_path=str(config["performance_data_path"]),
|
||||||
logger,
|
)
|
||||||
stride: int,
|
|
||||||
activation_function: str,
|
|
||||||
train_first_layer: bool,
|
def run_network(
|
||||||
seed_counter: int,
|
out_channels: list[int],
|
||||||
minimum_learning_rate: float,
|
kernel_size: list[int],
|
||||||
conv_0_kernel_size: int,
|
num_pfinkel: list,
|
||||||
mp_1_kernel_size: int,
|
logger,
|
||||||
mp_1_stride: int,
|
stride: int,
|
||||||
scheduler_patience: int,
|
activation_function: str,
|
||||||
batch_size_train: int,
|
train_first_layer: bool,
|
||||||
batch_size_test: int,
|
seed_counter: int,
|
||||||
learning_rate: float,
|
minimum_learning_rate: float,
|
||||||
max_epochs: int,
|
conv_0_kernel_size: int,
|
||||||
save_model: bool,
|
mp_1_kernel_size: int,
|
||||||
stimuli_per_pfinkel: int,
|
mp_1_stride: int,
|
||||||
save_ever_x_epochs: int,
|
scheduler_patience: int,
|
||||||
condition: str,
|
batch_size_train: int,
|
||||||
data_path: str,
|
batch_size_test: int,
|
||||||
pooling_type: str,
|
learning_rate: float,
|
||||||
conv_0_enable_softmax: bool,
|
max_epochs: int,
|
||||||
scale_data: float,
|
save_model: bool,
|
||||||
use_scheduler: bool,
|
stimuli_per_pfinkel: int,
|
||||||
use_adam: bool,
|
save_ever_x_epochs: int,
|
||||||
use_plot_intermediate: bool,
|
condition: str,
|
||||||
leak_relu_negative_slope: float,
|
data_path: str,
|
||||||
scheduler_verbose: bool,
|
pooling_type: str,
|
||||||
scheduler_factor: float,
|
conv_0_enable_softmax: bool,
|
||||||
precision_100_percent: int,
|
scale_data: float,
|
||||||
scheduler_threshold: float,
|
use_scheduler: bool,
|
||||||
) -> None:
|
use_adam: bool,
|
||||||
# define device:
|
use_plot_intermediate: bool,
|
||||||
device_str: str = "cuda:0" if torch.cuda.is_available() else "cpu"
|
leak_relu_negative_slope: float,
|
||||||
logger.info(f"Using {device_str} device")
|
scheduler_verbose: bool,
|
||||||
device: torch.device = torch.device(device_str)
|
scheduler_factor: float,
|
||||||
torch.set_default_dtype(torch.float32)
|
precision_100_percent: int,
|
||||||
|
scheduler_threshold: float,
|
||||||
# -------------------------------------------------------------------
|
model_continue: bool,
|
||||||
logger.info("-==- START -==-")
|
initial_model_path: str,
|
||||||
|
tb_runs_path: str,
|
||||||
train_accuracy: list[float] = []
|
trained_models_path: str,
|
||||||
train_losses: list[float] = []
|
performance_data_path: str,
|
||||||
train_loss: list[float] = []
|
) -> None:
|
||||||
test_accuracy: list[float] = []
|
# define device:
|
||||||
test_losses: list[float] = []
|
device_str: str = "cuda:0" if torch.cuda.is_available() else "cpu"
|
||||||
|
logger.info(f"Using {device_str} device")
|
||||||
# prepare data:
|
device: torch.device = torch.device(device_str)
|
||||||
|
torch.set_default_dtype(torch.float32)
|
||||||
logger.info(num_pfinkel)
|
|
||||||
logger.info(condition)
|
# -------------------------------------------------------------------
|
||||||
|
logger.info("-==- START -==-")
|
||||||
logger.info("Loading training data")
|
|
||||||
data_train = alicorn_data_loader(
|
train_accuracy: list[float] = []
|
||||||
num_pfinkel=num_pfinkel,
|
train_losses: list[float] = []
|
||||||
load_stimuli_per_pfinkel=stimuli_per_pfinkel,
|
train_loss: list[float] = []
|
||||||
condition=condition,
|
test_accuracy: list[float] = []
|
||||||
logger=logger,
|
test_losses: list[float] = []
|
||||||
data_path=data_path,
|
|
||||||
)
|
# prepare data:
|
||||||
|
|
||||||
logger.info("Loading test data")
|
logger.info(num_pfinkel)
|
||||||
data_test = alicorn_data_loader(
|
logger.info(condition)
|
||||||
num_pfinkel=num_pfinkel,
|
|
||||||
load_stimuli_per_pfinkel=stimuli_per_pfinkel,
|
logger.info("Loading training data")
|
||||||
condition=condition,
|
data_train = alicorn_data_loader(
|
||||||
logger=logger,
|
num_pfinkel=num_pfinkel,
|
||||||
data_path=data_path,
|
load_stimuli_per_pfinkel=stimuli_per_pfinkel,
|
||||||
)
|
condition=condition,
|
||||||
|
logger=logger,
|
||||||
logger.info("Loading done!")
|
data_path=data_path,
|
||||||
|
)
|
||||||
# data loader
|
|
||||||
loader_train = torch.utils.data.DataLoader(
|
logger.info("Loading test data")
|
||||||
data_train, shuffle=True, batch_size=batch_size_train
|
data_test = alicorn_data_loader(
|
||||||
)
|
num_pfinkel=num_pfinkel,
|
||||||
loader_test = torch.utils.data.DataLoader(
|
load_stimuli_per_pfinkel=stimuli_per_pfinkel,
|
||||||
data_test, shuffle=False, batch_size=batch_size_test
|
condition=condition,
|
||||||
)
|
logger=logger,
|
||||||
|
data_path=data_path,
|
||||||
previous_test_acc: float = -1
|
)
|
||||||
|
|
||||||
# set seed for reproducibility
|
logger.info("Loading done!")
|
||||||
set_seed(seed=int(seed_counter), logger=logger)
|
|
||||||
|
# data loader
|
||||||
# number conv layer:
|
loader_train = torch.utils.data.DataLoader(
|
||||||
if train_first_layer:
|
data_train, shuffle=True, batch_size=batch_size_train
|
||||||
num_conv_layers = len(out_channels)
|
)
|
||||||
else:
|
loader_test = torch.utils.data.DataLoader(
|
||||||
num_conv_layers = len(out_channels) if len(out_channels) >= 2 else 1
|
data_test, shuffle=False, batch_size=batch_size_test
|
||||||
|
)
|
||||||
# determine num conv layers
|
|
||||||
model_name = (
|
previous_test_acc: float = -1
|
||||||
f"ArghCNN__MPk3s2_numConvLayers{num_conv_layers}"
|
|
||||||
f"_outChannels{out_channels}_kernelSize{kernel_size}_"
|
# set seed for reproducibility
|
||||||
f"{activation_function}_stride{stride}_"
|
set_seed(seed=int(seed_counter), logger=logger)
|
||||||
f"trainFirstConvLayer{train_first_layer}_"
|
|
||||||
f"seed{seed_counter}_{condition}_MPk3s2"
|
# number conv layer:
|
||||||
)
|
if train_first_layer:
|
||||||
current = datetime.datetime.now().strftime("%d%m-%H%M")
|
num_conv_layers = len(out_channels)
|
||||||
|
else:
|
||||||
# new tb session
|
num_conv_layers = len(out_channels) if len(out_channels) >= 2 else 1
|
||||||
os.makedirs("tb_runs", exist_ok=True)
|
|
||||||
path: str = os.path.join("tb_runs", f"{model_name}")
|
# determine num conv layers
|
||||||
tb = SummaryWriter(path)
|
model_name = (
|
||||||
|
f"ArghCNN__MPk3s2_numConvLayers{num_conv_layers}"
|
||||||
# --------------------------------------------------------------------------
|
f"_outChannels{out_channels}_kernelSize{kernel_size}_"
|
||||||
|
f"{activation_function}_stride{stride}_"
|
||||||
# print network configuration:
|
f"trainFirstConvLayer{train_first_layer}_"
|
||||||
logger.info("----------------------------------------------------")
|
f"seed{seed_counter}_{condition}_MPk3s2"
|
||||||
logger.info(f"Number conv layers: {num_conv_layers}")
|
)
|
||||||
logger.info(f"Output channels: {out_channels}")
|
current = datetime.datetime.now().strftime("%d%m-%H%M")
|
||||||
logger.info(f"Kernel sizes: {kernel_size}")
|
|
||||||
logger.info(f"Stride: {stride}")
|
# new tb session
|
||||||
logger.info(f"Activation function: {activation_function}")
|
|
||||||
logger.info(f"Training conv 0: {train_first_layer}")
|
os.makedirs(tb_runs_path, exist_ok=True)
|
||||||
logger.info(f"Seed: {seed_counter}")
|
path: str = os.path.join(tb_runs_path, f"{model_name}")
|
||||||
logger.info(f"LR-scheduler patience: {scheduler_patience}")
|
tb = SummaryWriter(path)
|
||||||
logger.info(f"Pooling layer kernel: {mp_1_kernel_size}, stride: {mp_1_stride}")
|
|
||||||
|
# --------------------------------------------------------------------------
|
||||||
# define model:
|
|
||||||
model = make_cnn(
|
# print network configuration:
|
||||||
conv_out_channels_list=out_channels,
|
logger.info("----------------------------------------------------")
|
||||||
conv_kernel_size=kernel_size,
|
logger.info(f"Number conv layers: {num_conv_layers}")
|
||||||
conv_stride_size=stride,
|
logger.info(f"Output channels: {out_channels}")
|
||||||
conv_activation_function=activation_function,
|
logger.info(f"Kernel sizes: {kernel_size}")
|
||||||
train_conv_0=train_first_layer,
|
logger.info(f"Stride: {stride}")
|
||||||
conv_0_kernel_size=conv_0_kernel_size,
|
logger.info(f"Activation function: {activation_function}")
|
||||||
mp_1_kernel_size=mp_1_kernel_size,
|
logger.info(f"Training conv 0: {train_first_layer}")
|
||||||
mp_1_stride=mp_1_stride,
|
logger.info(f"Seed: {seed_counter}")
|
||||||
logger=logger,
|
logger.info(f"LR-scheduler patience: {scheduler_patience}")
|
||||||
pooling_type=pooling_type,
|
logger.info(f"Pooling layer kernel: {mp_1_kernel_size}, stride: {mp_1_stride}")
|
||||||
conv_0_enable_softmax=conv_0_enable_softmax,
|
|
||||||
l_relu_negative_slope=leak_relu_negative_slope,
|
# define model:
|
||||||
).to(device)
|
if model_continue:
|
||||||
|
filename_list: list = natsorted(
|
||||||
logger.info(model)
|
glob.glob(os.path.join(initial_model_path, str("*.pt")))
|
||||||
|
)
|
||||||
old_params: dict = {}
|
assert len(filename_list) > 0
|
||||||
for name, param in model.named_parameters():
|
model_filename: str = filename_list[-1]
|
||||||
old_params[name] = param.data.detach().cpu().clone()
|
logger.info(f"Load filename: {model_filename}")
|
||||||
|
else:
|
||||||
# pararmeters for training:
|
model = make_cnn(
|
||||||
param_list: list = []
|
conv_out_channels_list=out_channels,
|
||||||
|
conv_kernel_size=kernel_size,
|
||||||
for i in range(0, len(model)):
|
conv_stride_size=stride,
|
||||||
if (not train_first_layer) and (i == 0):
|
conv_activation_function=activation_function,
|
||||||
pass
|
train_conv_0=train_first_layer,
|
||||||
else:
|
conv_0_kernel_size=conv_0_kernel_size,
|
||||||
for name, param in model[i].named_parameters():
|
mp_1_kernel_size=mp_1_kernel_size,
|
||||||
logger.info(f"Learning parameter: layer: {i} name: {name}")
|
mp_1_stride=mp_1_stride,
|
||||||
param_list.append(param)
|
logger=logger,
|
||||||
|
pooling_type=pooling_type,
|
||||||
for name, param in model.named_parameters():
|
conv_0_enable_softmax=conv_0_enable_softmax,
|
||||||
assert (
|
l_relu_negative_slope=leak_relu_negative_slope,
|
||||||
torch.isfinite(param.data).sum().cpu()
|
).to(device)
|
||||||
== torch.tensor(param.data.size()).prod()
|
|
||||||
), name
|
model = torch.load(model_filename, map_location=device)
|
||||||
|
|
||||||
# optimizer and learning rate scheduler
|
logger.info(model)
|
||||||
if use_adam:
|
|
||||||
optimizer = torch.optim.Adam(param_list, lr=learning_rate)
|
old_params: dict = {}
|
||||||
else:
|
for name, param in model.named_parameters():
|
||||||
optimizer = torch.optim.SGD(param_list, lr=learning_rate) # type: ignore
|
old_params[name] = param.data.detach().cpu().clone()
|
||||||
|
|
||||||
if use_scheduler:
|
# pararmeters for training:
|
||||||
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
|
param_list: list = []
|
||||||
optimizer,
|
|
||||||
patience=scheduler_patience,
|
for i in range(0, len(model)):
|
||||||
eps=minimum_learning_rate / 10,
|
if (not train_first_layer) and (i == 0):
|
||||||
verbose=scheduler_verbose,
|
pass
|
||||||
factor=scheduler_factor,
|
else:
|
||||||
threshold=scheduler_threshold,
|
for name, param in model[i].named_parameters():
|
||||||
)
|
logger.info(f"Learning parameter: layer: {i} name: {name}")
|
||||||
|
param_list.append(param)
|
||||||
# training loop:
|
|
||||||
logger.info("-==- Data and network loader: Done -==-")
|
for name, param in model.named_parameters():
|
||||||
t_dis0 = time.perf_counter()
|
assert (
|
||||||
for epoch in range(1, max_epochs + 1):
|
torch.isfinite(param.data).sum().cpu()
|
||||||
# train
|
== torch.tensor(param.data.size()).prod()
|
||||||
logger.info("-==- Training... -==-")
|
), name
|
||||||
running_loss = train(
|
|
||||||
model=model,
|
# optimizer and learning rate scheduler
|
||||||
loader=loader_train,
|
if use_adam:
|
||||||
optimizer=optimizer,
|
optimizer = torch.optim.Adam(param_list, lr=learning_rate)
|
||||||
epoch=epoch,
|
else:
|
||||||
device=device,
|
optimizer = torch.optim.SGD(param_list, lr=learning_rate) # type: ignore
|
||||||
tb=tb,
|
|
||||||
test_acc=previous_test_acc,
|
if use_scheduler:
|
||||||
logger=logger,
|
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
|
||||||
train_accuracy=train_accuracy,
|
optimizer,
|
||||||
train_losses=train_losses,
|
patience=scheduler_patience,
|
||||||
train_loss=train_loss,
|
eps=minimum_learning_rate / 10,
|
||||||
scale_data=scale_data,
|
verbose=scheduler_verbose,
|
||||||
)
|
factor=scheduler_factor,
|
||||||
|
threshold=scheduler_threshold,
|
||||||
# logging:
|
)
|
||||||
logger.info("")
|
|
||||||
|
# training loop:
|
||||||
logger.info("Check for changes in the weights:")
|
logger.info("-==- Data and network loader: Done -==-")
|
||||||
for name, param in model.named_parameters():
|
t_dis0 = time.perf_counter()
|
||||||
if isinstance(old_params[name], torch.Tensor) and isinstance(
|
for epoch in range(1, max_epochs + 1):
|
||||||
param.data, torch.Tensor
|
# train
|
||||||
):
|
logger.info("-==- Training... -==-")
|
||||||
temp_torch = param.data.detach().cpu().clone()
|
running_loss = train(
|
||||||
if old_params[name].ndim == temp_torch.ndim:
|
model=model,
|
||||||
if old_params[name].size() == temp_torch.size():
|
loader=loader_train,
|
||||||
abs_diff = torch.abs(old_params[name] - temp_torch).max()
|
optimizer=optimizer,
|
||||||
logger.info(f"Parameter {name}: {abs_diff:.3e}")
|
epoch=epoch,
|
||||||
|
device=device,
|
||||||
old_params[name] = temp_torch
|
tb=tb,
|
||||||
|
test_acc=previous_test_acc,
|
||||||
logger.info("")
|
logger=logger,
|
||||||
|
train_accuracy=train_accuracy,
|
||||||
logger.info("-==- Testing... -==-")
|
train_losses=train_losses,
|
||||||
previous_test_acc = test( # type: ignore
|
train_loss=train_loss,
|
||||||
model=model,
|
scale_data=scale_data,
|
||||||
loader=loader_test,
|
)
|
||||||
device=device,
|
|
||||||
tb=tb,
|
# logging:
|
||||||
epoch=epoch,
|
logger.info("")
|
||||||
logger=logger,
|
|
||||||
test_accuracy=test_accuracy,
|
logger.info("Check for changes in the weights:")
|
||||||
test_losses=test_losses,
|
for name, param in model.named_parameters():
|
||||||
scale_data=scale_data,
|
if isinstance(old_params[name], torch.Tensor) and isinstance(
|
||||||
)
|
param.data, torch.Tensor
|
||||||
|
):
|
||||||
logger.info(f"Time required: {time.perf_counter()-t_dis0:.2e} sec")
|
temp_torch = param.data.detach().cpu().clone()
|
||||||
|
if old_params[name].ndim == temp_torch.ndim:
|
||||||
# save model after every 100th epoch:
|
if old_params[name].size() == temp_torch.size():
|
||||||
if save_model and (epoch % save_ever_x_epochs == 0):
|
abs_diff = torch.abs(old_params[name] - temp_torch).max()
|
||||||
pt_filename: str = f"{model_name}_{epoch}Epoch_{current}.pt"
|
logger.info(f"Parameter {name}: {abs_diff:.3e}")
|
||||||
logger.info("")
|
|
||||||
logger.info(f"Saved model: {pt_filename}")
|
old_params[name] = temp_torch
|
||||||
os.makedirs("trained_models", exist_ok=True)
|
|
||||||
torch.save(
|
logger.info("")
|
||||||
model,
|
|
||||||
os.path.join(
|
logger.info("-==- Testing... -==-")
|
||||||
"trained_models",
|
previous_test_acc = test( # type: ignore
|
||||||
pt_filename,
|
model=model,
|
||||||
),
|
loader=loader_test,
|
||||||
)
|
device=device,
|
||||||
|
tb=tb,
|
||||||
# check nan
|
epoch=epoch,
|
||||||
for name, param in model.named_parameters():
|
logger=logger,
|
||||||
assert (
|
test_accuracy=test_accuracy,
|
||||||
torch.isfinite(param.data).sum().cpu()
|
test_losses=test_losses,
|
||||||
== torch.tensor(param.data.size()).prod()
|
scale_data=scale_data,
|
||||||
), name
|
)
|
||||||
|
|
||||||
# update scheduler
|
logger.info(f"Time required: {time.perf_counter()-t_dis0:.2e} sec")
|
||||||
if use_scheduler:
|
|
||||||
if scheduler_verbose and isinstance(scheduler.best, float):
|
# save model after every 100th epoch:
|
||||||
logger.info(
|
if save_model and (epoch % save_ever_x_epochs == 0):
|
||||||
"Step LR scheduler: "
|
pt_filename: str = f"{model_name}_{epoch}Epoch_{current}.pt"
|
||||||
f"Loss: {running_loss:.2e} "
|
logger.info("")
|
||||||
f"Best: {scheduler.best:.2e} "
|
logger.info(f"Saved model: {pt_filename}")
|
||||||
f"Delta: {running_loss-scheduler.best:.2e} "
|
|
||||||
f"Threshold: {scheduler.threshold:.2e} "
|
os.makedirs(trained_models_path, exist_ok=True)
|
||||||
f"Number of bad epochs: {scheduler.num_bad_epochs} "
|
torch.save(
|
||||||
f"Patience: {scheduler.patience} "
|
model,
|
||||||
)
|
os.path.join(
|
||||||
scheduler.step(running_loss)
|
trained_models_path,
|
||||||
|
pt_filename,
|
||||||
# stop learning: lr too small
|
),
|
||||||
if optimizer.param_groups[0]["lr"] <= minimum_learning_rate:
|
)
|
||||||
logger.info("Learning rate is too small. Stop training.")
|
|
||||||
break
|
# check nan
|
||||||
|
for name, param in model.named_parameters():
|
||||||
# stop learning: done
|
assert (
|
||||||
if round(previous_test_acc, precision_100_percent) == 100.0:
|
torch.isfinite(param.data).sum().cpu()
|
||||||
logger.info("100% test performance reached. Stop training.")
|
== torch.tensor(param.data.size()).prod()
|
||||||
break
|
), name
|
||||||
|
|
||||||
if use_plot_intermediate:
|
# update scheduler
|
||||||
plot_intermediate(
|
if use_scheduler:
|
||||||
train_accuracy=train_accuracy,
|
if scheduler_verbose and isinstance(scheduler.best, float):
|
||||||
test_accuracy=test_accuracy,
|
logger.info(
|
||||||
train_losses=train_losses,
|
"Step LR scheduler: "
|
||||||
test_losses=test_losses,
|
f"Loss: {running_loss:.2e} "
|
||||||
save_name=model_name,
|
f"Best: {scheduler.best:.2e} "
|
||||||
)
|
f"Delta: {running_loss-scheduler.best:.2e} "
|
||||||
|
f"Threshold: {scheduler.threshold:.2e} "
|
||||||
os.makedirs("performance_data", exist_ok=True)
|
f"Number of bad epochs: {scheduler.num_bad_epochs} "
|
||||||
np.savez(
|
f"Patience: {scheduler.patience} "
|
||||||
os.path.join("performance_data", f"performances_{model_name}.npz"),
|
)
|
||||||
train_accuracy=np.array(train_accuracy),
|
scheduler.step(running_loss)
|
||||||
test_accuracy=np.array(test_accuracy),
|
|
||||||
train_losses=np.array(train_losses),
|
# stop learning: lr too small
|
||||||
test_losses=np.array(test_losses),
|
if optimizer.param_groups[0]["lr"] <= minimum_learning_rate:
|
||||||
)
|
logger.info("Learning rate is too small. Stop training.")
|
||||||
|
break
|
||||||
# end TB session:
|
|
||||||
tb.close()
|
# stop learning: done
|
||||||
|
if round(previous_test_acc, precision_100_percent) == 100.0:
|
||||||
# print model name:
|
logger.info("100% test performance reached. Stop training.")
|
||||||
logger.info("")
|
break
|
||||||
logger.info(f"Saved model: {model_name}_{epoch}Epoch_{current}")
|
|
||||||
if save_model:
|
if use_plot_intermediate:
|
||||||
os.makedirs("trained_models", exist_ok=True)
|
plot_intermediate(
|
||||||
torch.save(
|
train_accuracy=train_accuracy,
|
||||||
model,
|
test_accuracy=test_accuracy,
|
||||||
os.path.join(
|
train_losses=train_losses,
|
||||||
"trained_models",
|
test_losses=test_losses,
|
||||||
f"{model_name}_{epoch}Epoch_{current}.pt",
|
save_name=model_name,
|
||||||
),
|
)
|
||||||
)
|
|
||||||
|
os.makedirs(performance_data_path, exist_ok=True)
|
||||||
|
np.savez(
|
||||||
if __name__ == "__main__":
|
os.path.join(performance_data_path, f"performances_{model_name}.npz"),
|
||||||
argh.dispatch_command(main)
|
train_accuracy=np.array(train_accuracy),
|
||||||
|
test_accuracy=np.array(test_accuracy),
|
||||||
|
train_losses=np.array(train_losses),
|
||||||
|
test_losses=np.array(test_losses),
|
||||||
|
)
|
||||||
|
|
||||||
|
# end TB session:
|
||||||
|
tb.close()
|
||||||
|
|
||||||
|
# print model name:
|
||||||
|
logger.info("")
|
||||||
|
logger.info(f"Saved model: {model_name}_{epoch}Epoch_{current}")
|
||||||
|
if save_model:
|
||||||
|
os.makedirs(trained_models_path, exist_ok=True)
|
||||||
|
torch.save(
|
||||||
|
model,
|
||||||
|
os.path.join(
|
||||||
|
trained_models_path,
|
||||||
|
f"{model_name}_{epoch}Epoch_{current}.pt",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
argh.dispatch_command(main)
|
||||||
|
|
|
||||||
19
config.json
19
config.json
|
|
@ -1,10 +1,11 @@
|
||||||
{
|
{
|
||||||
"data_path": "/home/kk/Documents/Semester4/code/RenderStimuli/Output/",
|
"data_path": "/home/kk/Documents/Semester4/code/RenderStimuli/Output/",
|
||||||
|
"model_continue": false, // true, (false)
|
||||||
"save_logging_messages": true, // (true), false
|
"save_logging_messages": true, // (true), false
|
||||||
"display_logging_messages": true, // (true), false
|
"display_logging_messages": true, // (true), false
|
||||||
"batch_size_train": 250,
|
"batch_size_train": 250,
|
||||||
"batch_size_test": 500,
|
"batch_size_test": 500,
|
||||||
"max_epochs": 2000,
|
"max_epochs": 5000,
|
||||||
"save_model": true,
|
"save_model": true,
|
||||||
"conv_0_kernel_size": 11,
|
"conv_0_kernel_size": 11,
|
||||||
"mp_1_kernel_size": 3,
|
"mp_1_kernel_size": 3,
|
||||||
|
|
@ -22,14 +23,14 @@
|
||||||
// LR Scheduler ->
|
// LR Scheduler ->
|
||||||
"use_scheduler": true, // (true), false
|
"use_scheduler": true, // (true), false
|
||||||
"scheduler_verbose": true,
|
"scheduler_verbose": true,
|
||||||
"scheduler_factor": 0.1, //(0.1)
|
"scheduler_factor": 0.025, //(0.1)
|
||||||
"scheduler_patience": 10, // (10)
|
"scheduler_patience": 100, // (10)
|
||||||
"scheduler_threshold": 1e-5, // (1e-4)
|
"scheduler_threshold": 1e-5, // (1e-4)
|
||||||
"minimum_learning_rate": 1e-8,
|
"minimum_learning_rate": 1e-10,
|
||||||
"learning_rate": 0.0001,
|
"learning_rate": 1e-5,
|
||||||
// <- LR Scheduler
|
// <- LR Scheduler
|
||||||
"pooling_type": "max", // (max), average, none
|
"pooling_type": "max", // (max), average, none
|
||||||
"conv_0_enable_softmax": false, // true, (false)
|
"conv_0_enable_softmax": true, // true, (false)
|
||||||
"use_adam": true, // (true) => adam, false => SGD
|
"use_adam": true, // (true) => adam, false => SGD
|
||||||
"condition": "Coignless",
|
"condition": "Coignless",
|
||||||
"scale_data": 255.0, // (255.0)
|
"scale_data": 255.0, // (255.0)
|
||||||
|
|
@ -48,5 +49,9 @@
|
||||||
],
|
],
|
||||||
"conv_stride_sizes": [
|
"conv_stride_sizes": [
|
||||||
1
|
1
|
||||||
]
|
],
|
||||||
|
"initial_model_path": "initial_models",
|
||||||
|
"tb_runs_path": "tb_runs",
|
||||||
|
"trained_models_path": "trained_models",
|
||||||
|
"performance_data_path": "performance_data"
|
||||||
}
|
}
|
||||||
Loading…
Reference in a new issue