ci/lint: simplification of used tools

Author: Borda

.actions/assistant.py

@@ -167,6 +167,7 @@ def _find_meta(folder: str) -> str:
 
         Args:
             folder: path to the folder with python script, meta and artefacts
+
         """
         files = glob.glob(os.path.join(folder, AssistantCLI._META_FILE_REGEX), flags=glob.BRACE)
         if len(files) == 1:
@@ -180,6 +181,7 @@ def _load_meta(folder: str, strict: bool = False) -> Optional[dict]:
         Args:
             folder: path to the folder with python script, meta and artefacts
             strict: raise error if meta is missing required fields
+
         """
         fpath = AssistantCLI._find_meta(folder)
         assert fpath, f"Missing meta file in folder: {folder}"
@@ -197,6 +199,7 @@ def _valid_conf_folder(folder: str) -> Tuple[str, str]:
 
         Args:
             folder: path to the folder with python script, meta and artefacts
+
         """
         meta_files = [os.path.join(folder, f".meta.{ext}") for ext in ("yml", "yaml")]
         meta_files = [pf for pf in meta_files if os.path.isfile(pf)]
@@ -217,6 +220,7 @@ def _valid_folder(folder: str, ext: str) -> Tuple[str, str, str]:
         Args:
             folder: path to the folder with python script, meta and artefacts
             ext: extension determining the stage - ".py" for python script nad ".ipynb" for notebook
+
         """
         files = glob.glob(os.path.join(folder, f"*{ext}"))
         if len(files) != 1:
@@ -231,6 +235,7 @@ def _valid_accelerator(folder: str) -> bool:
 
         Args:
             folder: path to the folder with python script, meta and artefacts
+
         """
         meta = AssistantCLI._load_meta(folder)
         meta_accels = [acc.lower() for acc in meta.get("accelerator", AssistantCLI._META_ACCEL_DEFAULT)]
@@ -243,6 +248,7 @@ def _parse_requirements(folder: str) -> Tuple[str, str]:
 
         Args:
             folder: path to the folder with python script, meta and artefacts
+
         """
         meta = AssistantCLI._load_meta(folder)
         reqs = meta.get("requirements", [])
@@ -268,6 +274,7 @@ def _bash_download_data(folder: str) -> List[str]:
 
         Args:
             folder: path to the folder with python script, meta and artefacts
+
         """
         meta = AssistantCLI._load_meta(folder)
         datasets = meta.get("datasets", {})
@@ -299,6 +306,7 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[
 
         Returns:
             string with nash script content
+
         """
         cmd = list(AssistantCLI._BASH_SCRIPT_BASE) + [f"# Rendering: {folder}"]
         if not AssistantCLI.DRY_RUN:
@@ -348,6 +356,7 @@ def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST) -> Optional[str]
 
         Returns:
             string with nash script content
+
         """
         cmd = list(AssistantCLI._BASH_SCRIPT_BASE) + [f"# Testing: {folder}"]
         cmd += AssistantCLI._bash_download_data(folder)
@@ -395,6 +404,7 @@ def convert_ipynb(folder: str) -> None:
 
         Args:
             folder: folder with python script
+
         """
         fpath, _, _ = AssistantCLI._valid_folder(folder, ext=".py")
         with open(fpath) as fopen:
@@ -426,6 +436,7 @@ def _replace_images(lines: list, local_dir: str) -> list:
         Args:
             lines: string lines from python script
             local_dir: relative path to the folder with script
+
         """
         md = os.linesep.join([ln.rstrip() for ln in lines])
         p_imgs = []
@@ -488,6 +499,7 @@ def group_folders(
         Example:
             $ python assistant.py group-folders ../target-diff.txt \
                 --fpath_actual_dirs "['../dirs-main.txt', '../dirs-publication.txt']"
+
         """
         with open(fpath_gitdiff) as fopen:
             changed = [ln.strip() for ln in fopen.readlines()]
@@ -534,6 +546,7 @@ def generate_matrix(fpath_change_folders: str, json_indent: Optional[int] = None
         Args:
             fpath_change_folders: output of previous ``group_folders``
             json_indent: makes the json more readable, recommendation is 4
+
         """
         with open(fpath_change_folders) as fopen:
             folders = [ln.strip() for ln in fopen.readlines()]
@@ -623,6 +636,7 @@ def copy_notebooks(
             path_docs_images: destination path to the images' location relative to ``docs_root``
             patterns: patterns to use when glob-ing notebooks
             ignore: ignore some specific notebooks even when the given string is in path
+
         """
         all_ipynb = []
         for pattern in patterns:
@@ -694,6 +708,7 @@ def update_env_details(folder: str, base_path: str = DIR_NOTEBOOKS) -> str:
         Args:
              folder: path to the folder
              base_path:
+
         """
         meta = AssistantCLI._load_meta(folder)
         # default is COU runtime

.pre-commit-config.yaml

@@ -22,13 +22,6 @@ repos:
       - id: check-docstring-first
       - id: detect-private-key
 
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
-    hooks:
-      - id: pyupgrade
-        args: ["--py38-plus"]
-        name: Upgrade code
-
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:
@@ -40,29 +33,17 @@ repos:
     rev: v1.7.5
     hooks:
       - id: docformatter
-        args:
-          - "--in-place"
-          - "--wrap-summaries=119"
-          - "--wrap-descriptions=120"
+        additional_dependencies: [tomli]
+        args: ["--in-place"]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v3.1.0
     hooks:
       - id: prettier
+        files: \.(json|yml|yaml|toml)
         # https://prettier.io/docs/en/options.html#print-width
         args: ["--print-width=120"]
 
-  - repo: https://github.com/psf/black
-    rev: 23.12.1
-    hooks:
-      - id: black
-        name: Format code
-
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
-    hooks:
-      - id: yesqa
-
   - repo: https://github.com/executablebooks/mdformat
     rev: 0.7.17
     hooks:
@@ -73,7 +54,12 @@ repos:
           - mdformat_frontmatter
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.9
+    rev: v0.3.5
     hooks:
+      # try to fix what is possible
       - id: ruff
         args: ["--fix"]
+      # perform formatting updates
+      - id: ruff-format
+      # validate if all is fine with preview mode
+      - id: ruff

.prettierignore

@@ -606,6 +606,7 @@ def __init__(self, size, std=0.1):
         Args:
             size: Number of data points we want to generate
             std: Standard deviation of the noise (see generate_continuous_xor function)
+
         """
         super().__init__()
         self.size = size

course_UvA-DL/01-introduction-to-pytorch/Introduction_to_PyTorch.py

@@ -218,6 +218,7 @@ def get_grads(act_fn, x):
 
     Returns:
         A tensor with the same size of x containing the gradients of act_fn at x.
+
     """
     x = x.clone().requires_grad_()  # Mark the input as tensor for which we want to store gradients
     out = act_fn(x)
@@ -287,6 +288,7 @@ def __init__(self, act_fn, input_size=784, num_classes=10, hidden_sizes=[512, 25
             input_size: Size of the input images in pixels
             num_classes: Number of classes we want to predict
             hidden_sizes: A list of integers specifying the hidden layer sizes in the NN
+
         """
         super().__init__()
 
@@ -338,6 +340,7 @@ def load_model(model_path, model_name, net=None):
         model_path: Path of the checkpoint directory
         model_name: Name of the model (str)
         net: (Optional) If given, the state dict is loaded into this model. Otherwise, a new model is created.
+
     """
     config_file, model_file = _get_config_file(model_path, model_name), _get_model_file(model_path, model_name)
     assert os.path.isfile(
@@ -363,6 +366,7 @@ def save_model(model, model_path, model_name):
         model: Network object to save parameters from
         model_path: Path of the checkpoint directory
         model_name: Name of the model (str)
+
     """
     config_dict = model.config
     os.makedirs(model_path, exist_ok=True)
@@ -438,6 +442,7 @@ def visualize_gradients(net, color="C0"):
     Args:
         net: Object of class BaseNetwork
         color: Color in which we want to visualize the histogram (for easier separation of activation functions)
+
     """
     net.eval()
     small_loader = data.DataLoader(train_set, batch_size=256, shuffle=False)
@@ -519,6 +524,7 @@ def train_model(net, model_name, max_epochs=50, patience=7, batch_size=256, over
         patience: If the performance on the validation set has not improved for #patience epochs, we stop training early
         batch_size: Size of batches used in training
         overwrite: Determines how to handle the case when there already exists a checkpoint. If True, it will be overwritten. Otherwise, we skip training.
+
     """
     file_exists = os.path.isfile(_get_model_file(CHECKPOINT_PATH, model_name))
     if file_exists and not overwrite:
@@ -591,6 +597,7 @@ def test_model(net, data_loader):
     Args:
         net: Trained model of type BaseNetwork
         data_loader: DataLoader object of the dataset to test on (validation or test)
+
     """
     net.eval()
     true_preds, count = 0.0, 0
@@ -717,6 +724,7 @@ def measure_number_dead_neurons(net):
 
     For each neuron, we create a boolean variable initially set to 1. If it has an activation unequals 0 at any time, we
     set this variable to 0. After running through the whole training set, only dead neurons will have a 1.
+
     """
     neurons_dead = [
         torch.ones(layer.weight.shape[0], device=device, dtype=torch.bool)

course_UvA-DL/02-activation-functions/Activation_Functions.py

@@ -158,6 +158,7 @@ def __init__(self, act_fn, input_size=784, num_classes=10, hidden_sizes=[512, 25
             input_size: Size of the input images in pixels
             num_classes: Number of classes we want to predict
             hidden_sizes: A list of integers specifying the hidden layer sizes in the NN
+
         """
         super().__init__()
 
@@ -618,6 +619,7 @@ def train_model(net, model_name, optim_func, max_epochs=50, batch_size=256, over
         patience: If the performance on the validation set has not improved for #patience epochs, we stop training early
         batch_size: Size of batches used in training
         overwrite: Determines how to handle the case when there already exists a checkpoint. If True, it will be overwritten. Otherwise, we skip training.
+
     """
     file_exists = os.path.isfile(_get_model_file(CHECKPOINT_PATH, model_name))
     if file_exists and not overwrite:
@@ -718,6 +720,7 @@ def test_model(net, data_loader):
     Args:
         net: Trained model of type BaseNetwork
         data_loader: DataLoader object of the dataset to test on (validation or test)
+
     """
     net.eval()
     true_preds, count = 0.0, 0

course_UvA-DL/03-initialization-and-optimization/Initialization_and_Optimization.py

@@ -224,6 +224,7 @@ def __init__(self, model_name, model_hparams, optimizer_name, optimizer_hparams)
             model_hparams: Hyperparameters for the model, as dictionary.
             optimizer_name: Name of the optimizer to use. Currently supported: Adam, SGD
             optimizer_hparams: Hyperparameters for the optimizer, as dictionary. This includes learning rate, weight decay, etc.
+
         """
         super().__init__()
         # Exports the hyperparameters to a YAML file, and create "self.hparams" namespace
@@ -343,6 +344,7 @@ def train_model(model_name, save_name=None, **kwargs):
     Args:
         model_name: Name of the model you want to run. Is used to look up the class in "model_dict"
         save_name (optional): If specified, this name will be used for creating the checkpoint and logging directory.
+
     """
     if save_name is None:
         save_name = model_name
@@ -426,6 +428,7 @@ def __init__(self, c_in, c_red: dict, c_out: dict, act_fn):
             c_red: Dictionary with keys "3x3" and "5x5" specifying the output of the dimensionality reducing 1x1 convolutions
             c_out: Dictionary with keys "1x1", "3x3", "5x5", and "max"
             act_fn: Activation class constructor (e.g. nn.ReLU)
+
         """
         super().__init__()
 
@@ -676,6 +679,7 @@ def __init__(self, c_in, act_fn, subsample=False, c_out=-1):
             act_fn: Activation class constructor (e.g. nn.ReLU)
             subsample - If True, we want to apply a stride inside the block and reduce the output shape by 2 in height and width
             c_out - Number of output features. Note that this is only relevant if subsample is True, as otherwise, c_out = c_in
+
         """
         super().__init__()
         if not subsample:
@@ -722,6 +726,7 @@ def __init__(self, c_in, act_fn, subsample=False, c_out=-1):
             act_fn - Activation class constructor (e.g. nn.ReLU)
             subsample - If True, we want to apply a stride inside the block and reduce the output shape by 2 in height and width
             c_out - Number of output features. Note that this is only relevant if subsample is True, as otherwise, c_out = c_in
+
         """
         super().__init__()
         if not subsample:
@@ -793,6 +798,7 @@ def __init__(
             c_hidden - List with the hidden dimensionalities in the different blocks. Usually multiplied by 2 the deeper we go.
             act_fn_name - Name of the activation function to use, looked up in "act_fn_by_name"
             block_name - Name of the ResNet block, looked up in "resnet_blocks_by_name"
+
         """
         super().__init__()
         assert block_name in resnet_blocks_by_name
@@ -962,6 +968,7 @@ def __init__(self, c_in, bn_size, growth_rate, act_fn):
             bn_size - Bottleneck size (factor of growth rate) for the output of the 1x1 convolution. Typically between 2 and 4.
             growth_rate - Number of output channels of the 3x3 convolution
             act_fn - Activation class constructor (e.g. nn.ReLU)
+
         """
         super().__init__()
         self.net = nn.Sequential(
@@ -995,6 +1002,7 @@ def __init__(self, c_in, num_layers, bn_size, growth_rate, act_fn):
             bn_size - Bottleneck size to use in the dense layers
             growth_rate - Growth rate to use in the dense layers
             act_fn - Activation function to use in the dense layers
+
         """
         super().__init__()
         layers = []

course_UvA-DL/04-inception-resnet-densenet/Inception_ResNet_DenseNet.py

@@ -470,6 +470,7 @@ def __init__(self, input_dim, num_heads, dim_feedforward, dropout=0.0):
             num_heads: Number of heads to use in the attention block
             dim_feedforward: Dimensionality of the hidden layer in the MLP
             dropout: Dropout probability to use in the dropout layers
+
         """
         super().__init__()
 
@@ -578,6 +579,7 @@ def __init__(self, d_model, max_len=5000):
         Args:
             d_model: Hidden dimensionality of the input.
             max_len: Maximum length of a sequence to expect.
+
         """
         super().__init__()
 
@@ -773,6 +775,7 @@ def __init__(
             max_iters: Number of maximum iterations the model is trained for. This is needed for the CosineWarmup scheduler
             dropout: Dropout to apply inside the model
             input_dropout: Dropout to apply on the input features
+
         """
         super().__init__()
         self.save_hyperparameters()
@@ -822,6 +825,7 @@ def get_attention_maps(self, x, mask=None, add_positional_encoding=True):
         """Function for extracting the attention matrices of the whole Transformer for a single batch.
 
         Input arguments same as the forward pass.
+
         """
         x = self.input_net(x)
         if add_positional_encoding:
@@ -1314,6 +1318,7 @@ def sample_img_set(self, anomaly_label):
         """Samples a new set of images, given the label of the anomaly.
 
         The sampled images come from a different class than anomaly_label
+
         """
         # Sample class from 0,...,num_classes-1 while skipping anomaly_label as class
         set_label = np.random.randint(self.num_labels - 1)