start exploration of dropout

Author: hyzhak

assignment2/TensorFlow.ipynb

@@ -0,0 +1,32 @@
+import numpy as np
+from cs231n.data_utils import load_CIFAR10
+
+
+def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=10000):
+    """
+    Load the CIFAR-10 dataset from disk and perform preprocessing to prepare
+    it for the two-layer neural net classifier. These are the same steps as
+    we used for the SVM, but condensed to a single function.
+    """
+    # Load the raw CIFAR-10 data
+    cifar10_dir = './cs231n/datasets/cifar-10-batches-py'
+    X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)
+
+    # Subsample the data
+    mask = range(num_training, num_training + num_validation)
+    X_val = X_train[mask]
+    y_val = y_train[mask]
+    mask = range(num_training)
+    X_train = X_train[mask]
+    y_train = y_train[mask]
+    mask = range(num_test)
+    X_test = X_test[mask]
+    y_test = y_test[mask]
+
+    # Normalize the data: subtract the mean image
+    mean_image = np.mean(X_train, axis=0)
+    X_train -= mean_image
+    X_val -= mean_image
+    X_test -= mean_image
+
+    return X_train, y_train, X_val, y_val, X_test, y_test

assignment2/cifar/__init__.py

@@ -0,0 +1,14 @@
+import numpy as np
+import tensorflow as tf
+
+
+def affine(idx, X, output_size):
+    input_size = np.prod(X.shape[1:])
+    W = tf.get_variable(f'W{idx}', shape=[input_size, output_size])
+    b = tf.get_variable(f'b{idx}', shape=[output_size])
+    if len(X.shape) > 3:
+        out = tf.reshape(X, [-1, input_size])
+    else:
+        out = X
+    out = tf.matmul(out, W) + b
+    return out, [W, b]

assignment2/cifar/loader.py

@@ -0,0 +1,18 @@
+import numpy as np
+import tensorflow as tf
+
+
+def cnn(idx, X, filters, kernel_size, is_training, strides=(1, 1, 1, 1), padding='SAME', use_batchnorm=True):
+    input_filter_size = X.shape[-1]
+    Wconv = tf.get_variable(f'Wconv{idx}', shape=[*kernel_size, input_filter_size, filters])
+    bconv = tf.get_variable(f'bconv{idx}', shape=[filters])
+    out = tf.nn.conv2d(X, filter=Wconv, strides=strides, padding=padding) + bconv
+
+    # ReLU Activation Layer
+    out = tf.nn.relu(out)
+
+    # Spatial Batch Normalization Layer (trainable parameters, with scale and centering)
+    # axis=3 channel axis
+    if use_batchnorm:
+        out = tf.layers.batch_normalization(out, axis=3, training=is_training)
+    return out, [Wconv, bconv]

assignment2/cifar10_exploration_dropout.ipynb

@@ -0,0 +1,45 @@
+#
+# links for inspiration
+# - https://medium.com/@awjuliani/visualizing-neural-network-layer-activation-tensorflow-tutorial-d45f8bf7bbc4
+# simple snippet to train and visualize cnn layers of simple model
+#
+
+import math
+import matplotlib.pyplot as plt
+from nn_visualization.models import get_model_name
+
+
+def draw_layers_of_models(models, **kwargs):
+    img_idx = 0
+    for m in models:
+        img_idx = draw_layers(m, img_idx, **kwargs)
+
+
+def draw_layers(m, img_idx):
+    if 'params' not in m:
+        return
+
+    print(f'Model {get_model_name(m)}')
+    for (layer_tf, layer) in m['params'].items():
+        draw_layer(layer, str(layer_tf.name), img_idx)
+        img_idx += layer.shape[3]
+
+    return img_idx
+
+
+def draw_layer(layer, title, img_idx):
+    filters = layer.shape[3]
+    n_columns = 6
+    n_rows = math.ceil(filters / n_columns) + 1
+
+    fig = plt.figure(figsize=(20, 20))
+    fig.suptitle(title, size=16)
+    for idx in range(filters):
+        ax = fig.add_subplot(n_rows, n_columns, idx + 1)
+        ax.set_title(f'Filter {idx}')
+        img = layer[:, :, :, idx]
+        img_min = img.min()
+        delta = img.max() - img_min
+        img = (img - img_min) / delta
+        plt.imshow(img, interpolation='nearest')
+    plt.show()

assignment2/nn/__init__.py

@@ -0,0 +1,34 @@
+# import tensorflow as tf
+# import numpy as np
+# import math
+# import time
+# import timeit
+import matplotlib.pyplot as plt
+from IPython.display import display, HTML
+
+from nn_visualization.models import get_model_name
+
+
+def show_graphs(models, ymax=5):
+    """
+    TODO:
+    - show 2 graphs: training and validation
+    - show accuracy
+    """
+
+    # plt.gcf().set_size_inches(15, 12)
+
+    # training dynamic
+    plt.grid(True)
+    plt.title('Training Dynamics')
+    plt.xlabel('minibatch iteration')
+    plt.ylabel('minibatch loss')
+    # plt.xlim(xmin=50)
+    plt.ylim(ymax=ymax)
+
+    for m in models:
+        name = get_model_name(m)
+        plt.plot(m['res']['training']['losses'], label=name)
+
+    plt.legend()
+    plt.show()