Skip to content

Feature & Bug fix: Logging overhaul #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fehret wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Feature & Bug fix: Logging overhaul #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c8135a9
Add a logging level to only show training progress
fehret Jun 22, 2026
0178855
Changed relevant logger calls in HPO as well
fehret Jun 23, 2026
c8092f1
Simplify logging to builtin types and a single command switch
fehret Jul 6, 2026
5915aa5
Added propagate back to ensure functionality of the feature, simplifi…
fehret Jul 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dpdl/callbacks/body_head_gradient.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,4 +143,4 @@ def _save_to_csv(self, file_path):
writer.writeheader()
writer.writerows(self.grad_history)

log.info(f'Gradient norms (body & head per class) saved to {file_path}')
log.debug(f'Gradient norms (body & head per class) saved to {file_path}')
4 changes: 2 additions & 2 deletions dpdl/callbacks/checkpoint.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,12 @@ def on_train_end(self, trainer, *args, **kwargs):

def save_checkpoint(self, trainer, checkpoint_path: str):
trainer.save_model(checkpoint_path)
log.info(f'Model checkpoint saved at {checkpoint_path}')
log.debug(f'Model checkpoint saved at {checkpoint_path}')

def save_metrics(self, metrics, metrics_path: str):
metrics = tensor_to_python_type(metrics)

with open(metrics_path, 'w') as fh:
json.dump(metrics, fh)

log.info(f'Model checkpoint metrics saved at {metrics_path}')
log.debug(f'Model checkpoint metrics saved at {metrics_path}')
2 changes: 1 addition & 1 deletion dpdl/callbacks/clipping_bias.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,4 +322,4 @@ def on_train_end(self, trainer, *args, **kwargs) -> None:
writer.writerow(header)
writer.writerows(self._rows)

log.info(f'Clipping MSE decomposition written to {out_path}')
log.debug(f'Clipping MSE decomposition written to {out_path}')
4 changes: 2 additions & 2 deletions dpdl/callbacks/cosine_similarity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def on_train_end(self, trainer, *args, **kwargs):

writer.writerow(row)

log.info(f'Cosine similarity data saved at {file_path}')
log.debug(f'Cosine similarity data saved at {file_path}')


class RecordPerClassCosineSimilarityCallback(Callback):
Expand Down Expand Up @@ -230,4 +230,4 @@ def _save_to_csv(self, file_path, history, header):
for record in history:
writer.writerow([record.get(col, None) for col in header])

log.info(f'Cosine Similarities data saved at {file_path}')
log.debug(f'Cosine Similarities data saved at {file_path}')
34 changes: 17 additions & 17 deletions dpdl/callbacks/debug.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,53 +10,53 @@ def __init__(self):
super().__init__()

def _is_global_zero(self):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] Calling _is_global_zero")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] Calling _is_global_zero")
super().__is_global_zero()

def on_train_start(self, trainer):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_start")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_start")

def on_train_end(self, trainer):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_end")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_end")

def on_train_epoch_start(self, trainer, epoch):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_epoch_start")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_epoch_start")

def on_train_epoch_end(self, trainer, epoch, epoch_loss):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_epoch_end")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_epoch_end")

def on_train_batch_start(self, trainer, batch_idx, batch):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_batch_start")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_batch_start")

def on_train_physical_batch_start(self, trainer, batch_idx, batch):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_physical_batch_start")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_physical_batch_start")

def on_train_batch_end(self, trainer, batch_idx, batch, loss):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_batch_end")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_batch_end")

def on_train_physical_batch_end(self, trainer, batch_idx, batch, loss):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_physical_batch_end")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_train_physical_batch_end")

def on_validation_epoch_start(self, trainer, epoch):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_validation_epoch_start")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_validation_epoch_start")

def on_validation_epoch_end(self, trainer, epoch, metrics):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_validation_epoch_end")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_validation_epoch_end")

def on_validation_batch_start(self, trainer, batch_idx, batch):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_validation_batch_start")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_validation_batch_start")

def on_validation_batch_end(self, trainer, batch_idx, batch, loss):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_validation_batch_end")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_validation_batch_end")

def on_test_epoch_start(self, trainer, epoch):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_test_epoch_start")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_test_epoch_start")

def on_test_epoch_end(self, trainer, epoch, metrics):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_test_epoch_end")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_test_epoch_end")

def on_test_batch_start(self, trainer, batch_idx, batch):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_test_batch_start")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_test_batch_start")

def on_test_batch_end(self, trainer, batch_idx, batch, loss):
log.info(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_test_batch_end")
log.debug(f"[DEBUG - RANK {torch.distributed.get_rank()}] on_test_batch_end")
2 changes: 1 addition & 1 deletion dpdl/callbacks/gradient_proportion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,4 +96,4 @@ def _save_to_csv(self, file_path, history):
for step, proportions in enumerate(history):
writer.writerow([step] + proportions)

log.info(f'Clipped proportions data saved at {file_path}')
log.debug(f'Clipped proportions data saved at {file_path}')
2 changes: 1 addition & 1 deletion dpdl/callbacks/gradient_stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,4 +112,4 @@ def _save_to_csv(self, file_name, grad_history):
writer.writeheader()
writer.writerows(grad_history)

log.info(f'{file_name} saved successfully at {file_path}.')
log.debug(f'{file_name} saved successfully at {file_path}.')
2 changes: 1 addition & 1 deletion dpdl/callbacks/per_class_accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,4 @@ def on_train_end(self, trainer, *args, **kwargs):
for i, accuracies in enumerate(self.per_class_accuracies_history):
writer.writerow([i] + accuracies)

log.info(f'Per-class accuracy data saved at {file_path}')
log.debug(f'Per-class accuracy data saved at {file_path}')
2 changes: 1 addition & 1 deletion dpdl/callbacks/record_accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,4 @@ def on_train_end(self, trainer):

writer.writerow([i + 1, train_acc, val_acc])

log.info(f'Per epoch accuracies written to {self.csv_path}')
log.debug(f'Per epoch accuracies written to {self.csv_path}')
4 changes: 2 additions & 2 deletions dpdl/callbacks/record_losses.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def on_train_end(self, trainer, *args, **kwargs):
writer.writeheader()
writer.writerows(self.train_losses)

log.info(f'Training losses (by step) saved to {train_loss_path}')
log.debug(f'Training losses (by step) saved to {train_loss_path}')


class RecordLossesByEpochCallback(Callback):
Expand Down Expand Up @@ -88,4 +88,4 @@ def on_train_end(self, trainer):
val_loss_val = self.val_losses[i] if i < len(self.val_losses) else ''
writer.writerow([i+1, train_loss_val, val_loss_val])

log.info('Training finished and all epoch losses have been logged to CSV.')
log.debug('Training finished and all epoch losses have been logged to CSV.')
2 changes: 1 addition & 1 deletion dpdl/callbacks/record_snr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,4 +146,4 @@ def on_train_end(self, trainer, *args, **kwargs):
writer.writerow(header)
writer.writerows(self._rows)

log.info("SNR log written to %s", out_path)
log.debug("SNR log written to %s", out_path)
34 changes: 21 additions & 13 deletions dpdl/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,14 @@ def cli(
rich_help_panel='Prediction options',
)
] = 'test',
quiet: Annotated[
bool,
typer.Option(
'--quiet',
help='Only show training metrics, suppressing other console output.',
rich_help_panel='Logging options',
)
] = False,
):

# Map from commands to functions
Expand Down Expand Up @@ -646,8 +654,8 @@ def cli(


def run_show_layers(config_manager: ConfigurationManager) -> None:
log.info(config_manager.configuration)
log.info('Showing model layers.')
log.debug(config_manager.configuration)
log.debug('Showing model layers.')

model, _, _ = ModelFactory.get_model(
config_manager.configuration,
Expand All @@ -663,9 +671,9 @@ def run_train(config_manager: ConfigurationManager) -> Optional[Path]:
rank_zero = torch.distributed.get_rank() == 0

if rank_zero:
log.info('Starting training.')
log.info(config_manager.hyperparams)
log.info(config_manager.configuration)
log.debug('Starting training.')
log.debug(config_manager.hyperparams)
log.debug(config_manager.configuration)

seed_everything(config_manager.configuration.seed)

Expand All @@ -678,7 +686,7 @@ def run_train(config_manager: ConfigurationManager) -> Optional[Path]:
# log final train accuracy if needed
if config_manager.configuration.record_final_train_accuracy:
if rank_zero:
log.info('Evaluating on train set..')
log.debug('Evaluating on train set..')

train_loss, train_metrics = trainer._evaluate('train', enable_callbacks=False)

Expand All @@ -687,7 +695,7 @@ def run_train(config_manager: ConfigurationManager) -> Optional[Path]:

# log test accuracy and run time, and save model if asked
if rank_zero:
log.info('Evaluating on test set..')
log.debug('Evaluating on test set..')
test_loss, test_metrics = trainer.test()

log_test_metrics(config_manager, test_metrics, test_loss)
Expand All @@ -713,9 +721,9 @@ def run_train(config_manager: ConfigurationManager) -> Optional[Path]:
config_manager.configuration.model_weights_path = str(save_path)

if rank_zero:
log.info(f'Saving model to "{save_path}"...')
log.debug(f'Saving model to "{save_path}"...')
trainer.save_model(save_path)
log.info('Saving model done.')
log.debug('Saving model done.')
saved_model_path = save_path

torch.distributed.barrier()
Expand All @@ -725,8 +733,8 @@ def run_train(config_manager: ConfigurationManager) -> Optional[Path]:

def run_optimize(config_manager: ConfigurationManager) -> None:
if torch.distributed.get_rank() == 0:
log.info('Starting hyperparameter optimization.')
log.info(config_manager.configuration)
log.debug('Starting hyperparameter optimization.')
log.debug(config_manager.configuration)

seed_everything(config_manager.configuration.seed)

Expand All @@ -741,8 +749,8 @@ def run_optimize(config_manager: ConfigurationManager) -> None:

def run_predict(config_manager: ConfigurationManager) -> None:
if torch.distributed.get_rank() == 0:
log.info('Starting prediction.')
log.info(config_manager.configuration)
log.debug('Starting prediction.')
log.debug(config_manager.configuration)

seed_everything(config_manager.configuration.seed)

Expand Down
4 changes: 2 additions & 2 deletions dpdl/configurationmanager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ def save_configuration(self, directory: pathlib.Path):
with open(directory / 'configuration.json', 'w') as fh:
fh.write(self.configuration.json())

log.info(f'Configuration saved to {directory}.')
log.debug(f'Configuration saved to {directory}.')

def save_hyperparameters(self, directory: pathlib.Path):
if torch.distributed.get_rank() == 0:
Expand All @@ -352,7 +352,7 @@ def save_hyperparameters(self, directory: pathlib.Path):
with open(directory / 'hyperparameters.json', 'w') as fh:
fh.write(self.hyperparams.json())

log.info(f'Hyperparameters saved to {directory}/.')
log.debug(f'Hyperparameters saved to {directory}/.')

def clone_with_overrides(self, **overrides) -> 'ConfigurationManager':
params = dict(self._cli_params)
Expand Down
Loading