From ee9a0f14ecdb88b2d67c0ea2a3c9d57b21bfad42 Mon Sep 17 00:00:00 2001 From: Sasha Kolpakov Date: Tue, 31 Oct 2023 21:14:01 +0100 Subject: [PATCH 1/3] DenseNet on Cifar-10 ipynb notebook and py files --- cifar_10_DenseNet.ipynb | 4912 +++++++++++++++++++++++++++++++++++++++ cifar_10_DenseNet.py | 205 ++ 2 files changed, 5117 insertions(+) create mode 100644 cifar_10_DenseNet.ipynb create mode 100644 cifar_10_DenseNet.py diff --git a/cifar_10_DenseNet.ipynb b/cifar_10_DenseNet.ipynb new file mode 100644 index 00000000..afd59d7b --- /dev/null +++ b/cifar_10_DenseNet.ipynb @@ -0,0 +1,4912 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Cifar-10 test for image classification" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.insert(0, '../..')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "tags": [ + "imports" + ] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/sasha/jax-metal/lib/python3.8/site-packages/urllib3/__init__.py:34: NotOpenSSLWarning: urllib3 v2.0 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds\n", + "from tensorflow.keras.datasets import cifar10, cifar100\n", + "from tensorflow.keras.utils import to_categorical\n", + "from tensorflow.keras.models import Model\n", + "from tensorflow.keras.layers import Input, Resizing, Lambda, Flatten, Dense\n", + "import pandas as pd\n", + "import numpy as np\n", + "from cerebros.simplecerebrosrandomsearch.simple_cerebros_random_search\\\n", + " import SimpleCerebrosRandomSearch\n", + "import pendulum\n", + "from cerebros.units.units import DenseUnit\n", + "from cerebros.denseautomlstructuralcomponent.dense_automl_structural_component\\\n", + " import zero_7_exp_decay, zero_95_exp_decay, simple_sigmoid\n", + "from ast import literal_eval" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Download DenseNet121 with Imagenet weights (1000 classes)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "dnet = tf.keras.applications.densenet.DenseNet121(include_top=True,\n", + " weights='imagenet',\n", + " input_tensor=None,\n", + " input_shape=None,\n", + " pooling=None,\n", + " classes=1000,\n", + " classifier_activation='softmax'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"densenet121\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " input_4 (InputLayer) [(None, 224, 224, 3)] 0 [] \n", + " \n", + " zero_padding2d_2 (ZeroPadd (None, 230, 230, 3) 0 ['input_4[0][0]'] \n", + " ing2D) \n", + " \n", + " conv1/conv (Conv2D) (None, 112, 112, 64) 9408 ['zero_padding2d_2[0][0]'] \n", + " \n", + " conv1/bn (BatchNormalizati (None, 112, 112, 64) 256 ['conv1/conv[0][0]'] \n", + " on) \n", + " \n", + " conv1/relu (Activation) (None, 112, 112, 64) 0 ['conv1/bn[0][0]'] \n", + " \n", + " zero_padding2d_3 (ZeroPadd (None, 114, 114, 64) 0 ['conv1/relu[0][0]'] \n", + " ing2D) \n", + " \n", + " pool1 (MaxPooling2D) (None, 56, 56, 64) 0 ['zero_padding2d_3[0][0]'] \n", + " \n", + " conv2_block1_0_bn (BatchNo (None, 56, 56, 64) 256 ['pool1[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block1_0_relu (Activ (None, 56, 56, 64) 0 ['conv2_block1_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block1_1_conv (Conv2 (None, 56, 56, 128) 8192 ['conv2_block1_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block1_1_bn (BatchNo (None, 56, 56, 128) 512 ['conv2_block1_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block1_1_relu (Activ (None, 56, 56, 128) 0 ['conv2_block1_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block1_2_conv (Conv2 (None, 56, 56, 32) 36864 ['conv2_block1_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block1_concat (Conca (None, 56, 56, 96) 0 ['pool1[0][0]', \n", + " tenate) 'conv2_block1_2_conv[0][0]'] \n", + " \n", + " conv2_block2_0_bn (BatchNo (None, 56, 56, 96) 384 ['conv2_block1_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block2_0_relu (Activ (None, 56, 56, 96) 0 ['conv2_block2_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block2_1_conv (Conv2 (None, 56, 56, 128) 12288 ['conv2_block2_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block2_1_bn (BatchNo (None, 56, 56, 128) 512 ['conv2_block2_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block2_1_relu (Activ (None, 56, 56, 128) 0 ['conv2_block2_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block2_2_conv (Conv2 (None, 56, 56, 32) 36864 ['conv2_block2_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block2_concat (Conca (None, 56, 56, 128) 0 ['conv2_block1_concat[0][0]', \n", + " tenate) 'conv2_block2_2_conv[0][0]'] \n", + " \n", + " conv2_block3_0_bn (BatchNo (None, 56, 56, 128) 512 ['conv2_block2_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block3_0_relu (Activ (None, 56, 56, 128) 0 ['conv2_block3_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block3_1_conv (Conv2 (None, 56, 56, 128) 16384 ['conv2_block3_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block3_1_bn (BatchNo (None, 56, 56, 128) 512 ['conv2_block3_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block3_1_relu (Activ (None, 56, 56, 128) 0 ['conv2_block3_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block3_2_conv (Conv2 (None, 56, 56, 32) 36864 ['conv2_block3_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block3_concat (Conca (None, 56, 56, 160) 0 ['conv2_block2_concat[0][0]', \n", + " tenate) 'conv2_block3_2_conv[0][0]'] \n", + " \n", + " conv2_block4_0_bn (BatchNo (None, 56, 56, 160) 640 ['conv2_block3_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block4_0_relu (Activ (None, 56, 56, 160) 0 ['conv2_block4_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block4_1_conv (Conv2 (None, 56, 56, 128) 20480 ['conv2_block4_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block4_1_bn (BatchNo (None, 56, 56, 128) 512 ['conv2_block4_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block4_1_relu (Activ (None, 56, 56, 128) 0 ['conv2_block4_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block4_2_conv (Conv2 (None, 56, 56, 32) 36864 ['conv2_block4_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block4_concat (Conca (None, 56, 56, 192) 0 ['conv2_block3_concat[0][0]', \n", + " tenate) 'conv2_block4_2_conv[0][0]'] \n", + " \n", + " conv2_block5_0_bn (BatchNo (None, 56, 56, 192) 768 ['conv2_block4_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block5_0_relu (Activ (None, 56, 56, 192) 0 ['conv2_block5_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block5_1_conv (Conv2 (None, 56, 56, 128) 24576 ['conv2_block5_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block5_1_bn (BatchNo (None, 56, 56, 128) 512 ['conv2_block5_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block5_1_relu (Activ (None, 56, 56, 128) 0 ['conv2_block5_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block5_2_conv (Conv2 (None, 56, 56, 32) 36864 ['conv2_block5_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block5_concat (Conca (None, 56, 56, 224) 0 ['conv2_block4_concat[0][0]', \n", + " tenate) 'conv2_block5_2_conv[0][0]'] \n", + " \n", + " conv2_block6_0_bn (BatchNo (None, 56, 56, 224) 896 ['conv2_block5_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block6_0_relu (Activ (None, 56, 56, 224) 0 ['conv2_block6_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block6_1_conv (Conv2 (None, 56, 56, 128) 28672 ['conv2_block6_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block6_1_bn (BatchNo (None, 56, 56, 128) 512 ['conv2_block6_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv2_block6_1_relu (Activ (None, 56, 56, 128) 0 ['conv2_block6_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv2_block6_2_conv (Conv2 (None, 56, 56, 32) 36864 ['conv2_block6_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv2_block6_concat (Conca (None, 56, 56, 256) 0 ['conv2_block5_concat[0][0]', \n", + " tenate) 'conv2_block6_2_conv[0][0]'] \n", + " \n", + " pool2_bn (BatchNormalizati (None, 56, 56, 256) 1024 ['conv2_block6_concat[0][0]'] \n", + " on) \n", + " \n", + " pool2_relu (Activation) (None, 56, 56, 256) 0 ['pool2_bn[0][0]'] \n", + " \n", + " pool2_conv (Conv2D) (None, 56, 56, 128) 32768 ['pool2_relu[0][0]'] \n", + " \n", + " pool2_pool (AveragePooling (None, 28, 28, 128) 0 ['pool2_conv[0][0]'] \n", + " 2D) \n", + " \n", + " conv3_block1_0_bn (BatchNo (None, 28, 28, 128) 512 ['pool2_pool[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block1_0_relu (Activ (None, 28, 28, 128) 0 ['conv3_block1_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block1_1_conv (Conv2 (None, 28, 28, 128) 16384 ['conv3_block1_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block1_1_bn (BatchNo (None, 28, 28, 128) 512 ['conv3_block1_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block1_1_relu (Activ (None, 28, 28, 128) 0 ['conv3_block1_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block1_2_conv (Conv2 (None, 28, 28, 32) 36864 ['conv3_block1_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block1_concat (Conca (None, 28, 28, 160) 0 ['pool2_pool[0][0]', \n", + " tenate) 'conv3_block1_2_conv[0][0]'] \n", + " \n", + " conv3_block2_0_bn (BatchNo (None, 28, 28, 160) 640 ['conv3_block1_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block2_0_relu (Activ (None, 28, 28, 160) 0 ['conv3_block2_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block2_1_conv (Conv2 (None, 28, 28, 128) 20480 ['conv3_block2_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block2_1_bn (BatchNo (None, 28, 28, 128) 512 ['conv3_block2_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block2_1_relu (Activ (None, 28, 28, 128) 0 ['conv3_block2_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block2_2_conv (Conv2 (None, 28, 28, 32) 36864 ['conv3_block2_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block2_concat (Conca (None, 28, 28, 192) 0 ['conv3_block1_concat[0][0]', \n", + " tenate) 'conv3_block2_2_conv[0][0]'] \n", + " \n", + " conv3_block3_0_bn (BatchNo (None, 28, 28, 192) 768 ['conv3_block2_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block3_0_relu (Activ (None, 28, 28, 192) 0 ['conv3_block3_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block3_1_conv (Conv2 (None, 28, 28, 128) 24576 ['conv3_block3_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block3_1_bn (BatchNo (None, 28, 28, 128) 512 ['conv3_block3_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block3_1_relu (Activ (None, 28, 28, 128) 0 ['conv3_block3_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block3_2_conv (Conv2 (None, 28, 28, 32) 36864 ['conv3_block3_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block3_concat (Conca (None, 28, 28, 224) 0 ['conv3_block2_concat[0][0]', \n", + " tenate) 'conv3_block3_2_conv[0][0]'] \n", + " \n", + " conv3_block4_0_bn (BatchNo (None, 28, 28, 224) 896 ['conv3_block3_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block4_0_relu (Activ (None, 28, 28, 224) 0 ['conv3_block4_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block4_1_conv (Conv2 (None, 28, 28, 128) 28672 ['conv3_block4_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block4_1_bn (BatchNo (None, 28, 28, 128) 512 ['conv3_block4_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block4_1_relu (Activ (None, 28, 28, 128) 0 ['conv3_block4_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block4_2_conv (Conv2 (None, 28, 28, 32) 36864 ['conv3_block4_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block4_concat (Conca (None, 28, 28, 256) 0 ['conv3_block3_concat[0][0]', \n", + " tenate) 'conv3_block4_2_conv[0][0]'] \n", + " \n", + " conv3_block5_0_bn (BatchNo (None, 28, 28, 256) 1024 ['conv3_block4_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block5_0_relu (Activ (None, 28, 28, 256) 0 ['conv3_block5_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block5_1_conv (Conv2 (None, 28, 28, 128) 32768 ['conv3_block5_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block5_1_bn (BatchNo (None, 28, 28, 128) 512 ['conv3_block5_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block5_1_relu (Activ (None, 28, 28, 128) 0 ['conv3_block5_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block5_2_conv (Conv2 (None, 28, 28, 32) 36864 ['conv3_block5_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block5_concat (Conca (None, 28, 28, 288) 0 ['conv3_block4_concat[0][0]', \n", + " tenate) 'conv3_block5_2_conv[0][0]'] \n", + " \n", + " conv3_block6_0_bn (BatchNo (None, 28, 28, 288) 1152 ['conv3_block5_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block6_0_relu (Activ (None, 28, 28, 288) 0 ['conv3_block6_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block6_1_conv (Conv2 (None, 28, 28, 128) 36864 ['conv3_block6_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block6_1_bn (BatchNo (None, 28, 28, 128) 512 ['conv3_block6_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block6_1_relu (Activ (None, 28, 28, 128) 0 ['conv3_block6_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block6_2_conv (Conv2 (None, 28, 28, 32) 36864 ['conv3_block6_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block6_concat (Conca (None, 28, 28, 320) 0 ['conv3_block5_concat[0][0]', \n", + " tenate) 'conv3_block6_2_conv[0][0]'] \n", + " \n", + " conv3_block7_0_bn (BatchNo (None, 28, 28, 320) 1280 ['conv3_block6_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block7_0_relu (Activ (None, 28, 28, 320) 0 ['conv3_block7_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block7_1_conv (Conv2 (None, 28, 28, 128) 40960 ['conv3_block7_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block7_1_bn (BatchNo (None, 28, 28, 128) 512 ['conv3_block7_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block7_1_relu (Activ (None, 28, 28, 128) 0 ['conv3_block7_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block7_2_conv (Conv2 (None, 28, 28, 32) 36864 ['conv3_block7_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block7_concat (Conca (None, 28, 28, 352) 0 ['conv3_block6_concat[0][0]', \n", + " tenate) 'conv3_block7_2_conv[0][0]'] \n", + " \n", + " conv3_block8_0_bn (BatchNo (None, 28, 28, 352) 1408 ['conv3_block7_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block8_0_relu (Activ (None, 28, 28, 352) 0 ['conv3_block8_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block8_1_conv (Conv2 (None, 28, 28, 128) 45056 ['conv3_block8_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block8_1_bn (BatchNo (None, 28, 28, 128) 512 ['conv3_block8_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block8_1_relu (Activ (None, 28, 28, 128) 0 ['conv3_block8_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block8_2_conv (Conv2 (None, 28, 28, 32) 36864 ['conv3_block8_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block8_concat (Conca (None, 28, 28, 384) 0 ['conv3_block7_concat[0][0]', \n", + " tenate) 'conv3_block8_2_conv[0][0]'] \n", + " \n", + " conv3_block9_0_bn (BatchNo (None, 28, 28, 384) 1536 ['conv3_block8_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block9_0_relu (Activ (None, 28, 28, 384) 0 ['conv3_block9_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block9_1_conv (Conv2 (None, 28, 28, 128) 49152 ['conv3_block9_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block9_1_bn (BatchNo (None, 28, 28, 128) 512 ['conv3_block9_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv3_block9_1_relu (Activ (None, 28, 28, 128) 0 ['conv3_block9_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv3_block9_2_conv (Conv2 (None, 28, 28, 32) 36864 ['conv3_block9_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv3_block9_concat (Conca (None, 28, 28, 416) 0 ['conv3_block8_concat[0][0]', \n", + " tenate) 'conv3_block9_2_conv[0][0]'] \n", + " \n", + " conv3_block10_0_bn (BatchN (None, 28, 28, 416) 1664 ['conv3_block9_concat[0][0]'] \n", + " ormalization) \n", + " \n", + " conv3_block10_0_relu (Acti (None, 28, 28, 416) 0 ['conv3_block10_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv3_block10_1_conv (Conv (None, 28, 28, 128) 53248 ['conv3_block10_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv3_block10_1_bn (BatchN (None, 28, 28, 128) 512 ['conv3_block10_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv3_block10_1_relu (Acti (None, 28, 28, 128) 0 ['conv3_block10_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv3_block10_2_conv (Conv (None, 28, 28, 32) 36864 ['conv3_block10_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv3_block10_concat (Conc (None, 28, 28, 448) 0 ['conv3_block9_concat[0][0]', \n", + " atenate) 'conv3_block10_2_conv[0][0]']\n", + " \n", + " conv3_block11_0_bn (BatchN (None, 28, 28, 448) 1792 ['conv3_block10_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv3_block11_0_relu (Acti (None, 28, 28, 448) 0 ['conv3_block11_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv3_block11_1_conv (Conv (None, 28, 28, 128) 57344 ['conv3_block11_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv3_block11_1_bn (BatchN (None, 28, 28, 128) 512 ['conv3_block11_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv3_block11_1_relu (Acti (None, 28, 28, 128) 0 ['conv3_block11_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv3_block11_2_conv (Conv (None, 28, 28, 32) 36864 ['conv3_block11_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv3_block11_concat (Conc (None, 28, 28, 480) 0 ['conv3_block10_concat[0][0]',\n", + " atenate) 'conv3_block11_2_conv[0][0]']\n", + " \n", + " conv3_block12_0_bn (BatchN (None, 28, 28, 480) 1920 ['conv3_block11_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv3_block12_0_relu (Acti (None, 28, 28, 480) 0 ['conv3_block12_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv3_block12_1_conv (Conv (None, 28, 28, 128) 61440 ['conv3_block12_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv3_block12_1_bn (BatchN (None, 28, 28, 128) 512 ['conv3_block12_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv3_block12_1_relu (Acti (None, 28, 28, 128) 0 ['conv3_block12_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv3_block12_2_conv (Conv (None, 28, 28, 32) 36864 ['conv3_block12_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv3_block12_concat (Conc (None, 28, 28, 512) 0 ['conv3_block11_concat[0][0]',\n", + " atenate) 'conv3_block12_2_conv[0][0]']\n", + " \n", + " pool3_bn (BatchNormalizati (None, 28, 28, 512) 2048 ['conv3_block12_concat[0][0]']\n", + " on) \n", + " \n", + " pool3_relu (Activation) (None, 28, 28, 512) 0 ['pool3_bn[0][0]'] \n", + " \n", + " pool3_conv (Conv2D) (None, 28, 28, 256) 131072 ['pool3_relu[0][0]'] \n", + " \n", + " pool3_pool (AveragePooling (None, 14, 14, 256) 0 ['pool3_conv[0][0]'] \n", + " 2D) \n", + " \n", + " conv4_block1_0_bn (BatchNo (None, 14, 14, 256) 1024 ['pool3_pool[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block1_0_relu (Activ (None, 14, 14, 256) 0 ['conv4_block1_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block1_1_conv (Conv2 (None, 14, 14, 128) 32768 ['conv4_block1_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block1_1_bn (BatchNo (None, 14, 14, 128) 512 ['conv4_block1_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block1_1_relu (Activ (None, 14, 14, 128) 0 ['conv4_block1_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block1_2_conv (Conv2 (None, 14, 14, 32) 36864 ['conv4_block1_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block1_concat (Conca (None, 14, 14, 288) 0 ['pool3_pool[0][0]', \n", + " tenate) 'conv4_block1_2_conv[0][0]'] \n", + " \n", + " conv4_block2_0_bn (BatchNo (None, 14, 14, 288) 1152 ['conv4_block1_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block2_0_relu (Activ (None, 14, 14, 288) 0 ['conv4_block2_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block2_1_conv (Conv2 (None, 14, 14, 128) 36864 ['conv4_block2_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block2_1_bn (BatchNo (None, 14, 14, 128) 512 ['conv4_block2_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block2_1_relu (Activ (None, 14, 14, 128) 0 ['conv4_block2_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block2_2_conv (Conv2 (None, 14, 14, 32) 36864 ['conv4_block2_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block2_concat (Conca (None, 14, 14, 320) 0 ['conv4_block1_concat[0][0]', \n", + " tenate) 'conv4_block2_2_conv[0][0]'] \n", + " \n", + " conv4_block3_0_bn (BatchNo (None, 14, 14, 320) 1280 ['conv4_block2_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block3_0_relu (Activ (None, 14, 14, 320) 0 ['conv4_block3_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block3_1_conv (Conv2 (None, 14, 14, 128) 40960 ['conv4_block3_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block3_1_bn (BatchNo (None, 14, 14, 128) 512 ['conv4_block3_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block3_1_relu (Activ (None, 14, 14, 128) 0 ['conv4_block3_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block3_2_conv (Conv2 (None, 14, 14, 32) 36864 ['conv4_block3_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block3_concat (Conca (None, 14, 14, 352) 0 ['conv4_block2_concat[0][0]', \n", + " tenate) 'conv4_block3_2_conv[0][0]'] \n", + " \n", + " conv4_block4_0_bn (BatchNo (None, 14, 14, 352) 1408 ['conv4_block3_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block4_0_relu (Activ (None, 14, 14, 352) 0 ['conv4_block4_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block4_1_conv (Conv2 (None, 14, 14, 128) 45056 ['conv4_block4_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block4_1_bn (BatchNo (None, 14, 14, 128) 512 ['conv4_block4_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block4_1_relu (Activ (None, 14, 14, 128) 0 ['conv4_block4_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block4_2_conv (Conv2 (None, 14, 14, 32) 36864 ['conv4_block4_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block4_concat (Conca (None, 14, 14, 384) 0 ['conv4_block3_concat[0][0]', \n", + " tenate) 'conv4_block4_2_conv[0][0]'] \n", + " \n", + " conv4_block5_0_bn (BatchNo (None, 14, 14, 384) 1536 ['conv4_block4_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block5_0_relu (Activ (None, 14, 14, 384) 0 ['conv4_block5_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block5_1_conv (Conv2 (None, 14, 14, 128) 49152 ['conv4_block5_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block5_1_bn (BatchNo (None, 14, 14, 128) 512 ['conv4_block5_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block5_1_relu (Activ (None, 14, 14, 128) 0 ['conv4_block5_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block5_2_conv (Conv2 (None, 14, 14, 32) 36864 ['conv4_block5_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block5_concat (Conca (None, 14, 14, 416) 0 ['conv4_block4_concat[0][0]', \n", + " tenate) 'conv4_block5_2_conv[0][0]'] \n", + " \n", + " conv4_block6_0_bn (BatchNo (None, 14, 14, 416) 1664 ['conv4_block5_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block6_0_relu (Activ (None, 14, 14, 416) 0 ['conv4_block6_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block6_1_conv (Conv2 (None, 14, 14, 128) 53248 ['conv4_block6_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block6_1_bn (BatchNo (None, 14, 14, 128) 512 ['conv4_block6_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block6_1_relu (Activ (None, 14, 14, 128) 0 ['conv4_block6_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block6_2_conv (Conv2 (None, 14, 14, 32) 36864 ['conv4_block6_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block6_concat (Conca (None, 14, 14, 448) 0 ['conv4_block5_concat[0][0]', \n", + " tenate) 'conv4_block6_2_conv[0][0]'] \n", + " \n", + " conv4_block7_0_bn (BatchNo (None, 14, 14, 448) 1792 ['conv4_block6_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block7_0_relu (Activ (None, 14, 14, 448) 0 ['conv4_block7_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block7_1_conv (Conv2 (None, 14, 14, 128) 57344 ['conv4_block7_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block7_1_bn (BatchNo (None, 14, 14, 128) 512 ['conv4_block7_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block7_1_relu (Activ (None, 14, 14, 128) 0 ['conv4_block7_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block7_2_conv (Conv2 (None, 14, 14, 32) 36864 ['conv4_block7_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block7_concat (Conca (None, 14, 14, 480) 0 ['conv4_block6_concat[0][0]', \n", + " tenate) 'conv4_block7_2_conv[0][0]'] \n", + " \n", + " conv4_block8_0_bn (BatchNo (None, 14, 14, 480) 1920 ['conv4_block7_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block8_0_relu (Activ (None, 14, 14, 480) 0 ['conv4_block8_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block8_1_conv (Conv2 (None, 14, 14, 128) 61440 ['conv4_block8_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block8_1_bn (BatchNo (None, 14, 14, 128) 512 ['conv4_block8_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block8_1_relu (Activ (None, 14, 14, 128) 0 ['conv4_block8_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block8_2_conv (Conv2 (None, 14, 14, 32) 36864 ['conv4_block8_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block8_concat (Conca (None, 14, 14, 512) 0 ['conv4_block7_concat[0][0]', \n", + " tenate) 'conv4_block8_2_conv[0][0]'] \n", + " \n", + " conv4_block9_0_bn (BatchNo (None, 14, 14, 512) 2048 ['conv4_block8_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block9_0_relu (Activ (None, 14, 14, 512) 0 ['conv4_block9_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block9_1_conv (Conv2 (None, 14, 14, 128) 65536 ['conv4_block9_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block9_1_bn (BatchNo (None, 14, 14, 128) 512 ['conv4_block9_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv4_block9_1_relu (Activ (None, 14, 14, 128) 0 ['conv4_block9_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv4_block9_2_conv (Conv2 (None, 14, 14, 32) 36864 ['conv4_block9_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv4_block9_concat (Conca (None, 14, 14, 544) 0 ['conv4_block8_concat[0][0]', \n", + " tenate) 'conv4_block9_2_conv[0][0]'] \n", + " \n", + " conv4_block10_0_bn (BatchN (None, 14, 14, 544) 2176 ['conv4_block9_concat[0][0]'] \n", + " ormalization) \n", + " \n", + " conv4_block10_0_relu (Acti (None, 14, 14, 544) 0 ['conv4_block10_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block10_1_conv (Conv (None, 14, 14, 128) 69632 ['conv4_block10_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block10_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block10_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block10_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block10_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block10_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block10_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block10_concat (Conc (None, 14, 14, 576) 0 ['conv4_block9_concat[0][0]', \n", + " atenate) 'conv4_block10_2_conv[0][0]']\n", + " \n", + " conv4_block11_0_bn (BatchN (None, 14, 14, 576) 2304 ['conv4_block10_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block11_0_relu (Acti (None, 14, 14, 576) 0 ['conv4_block11_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block11_1_conv (Conv (None, 14, 14, 128) 73728 ['conv4_block11_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block11_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block11_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block11_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block11_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block11_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block11_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block11_concat (Conc (None, 14, 14, 608) 0 ['conv4_block10_concat[0][0]',\n", + " atenate) 'conv4_block11_2_conv[0][0]']\n", + " \n", + " conv4_block12_0_bn (BatchN (None, 14, 14, 608) 2432 ['conv4_block11_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block12_0_relu (Acti (None, 14, 14, 608) 0 ['conv4_block12_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block12_1_conv (Conv (None, 14, 14, 128) 77824 ['conv4_block12_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block12_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block12_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block12_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block12_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block12_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block12_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block12_concat (Conc (None, 14, 14, 640) 0 ['conv4_block11_concat[0][0]',\n", + " atenate) 'conv4_block12_2_conv[0][0]']\n", + " \n", + " conv4_block13_0_bn (BatchN (None, 14, 14, 640) 2560 ['conv4_block12_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block13_0_relu (Acti (None, 14, 14, 640) 0 ['conv4_block13_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block13_1_conv (Conv (None, 14, 14, 128) 81920 ['conv4_block13_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block13_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block13_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block13_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block13_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block13_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block13_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block13_concat (Conc (None, 14, 14, 672) 0 ['conv4_block12_concat[0][0]',\n", + " atenate) 'conv4_block13_2_conv[0][0]']\n", + " \n", + " conv4_block14_0_bn (BatchN (None, 14, 14, 672) 2688 ['conv4_block13_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block14_0_relu (Acti (None, 14, 14, 672) 0 ['conv4_block14_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block14_1_conv (Conv (None, 14, 14, 128) 86016 ['conv4_block14_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block14_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block14_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block14_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block14_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block14_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block14_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block14_concat (Conc (None, 14, 14, 704) 0 ['conv4_block13_concat[0][0]',\n", + " atenate) 'conv4_block14_2_conv[0][0]']\n", + " \n", + " conv4_block15_0_bn (BatchN (None, 14, 14, 704) 2816 ['conv4_block14_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block15_0_relu (Acti (None, 14, 14, 704) 0 ['conv4_block15_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block15_1_conv (Conv (None, 14, 14, 128) 90112 ['conv4_block15_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block15_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block15_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block15_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block15_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block15_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block15_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block15_concat (Conc (None, 14, 14, 736) 0 ['conv4_block14_concat[0][0]',\n", + " atenate) 'conv4_block15_2_conv[0][0]']\n", + " \n", + " conv4_block16_0_bn (BatchN (None, 14, 14, 736) 2944 ['conv4_block15_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block16_0_relu (Acti (None, 14, 14, 736) 0 ['conv4_block16_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block16_1_conv (Conv (None, 14, 14, 128) 94208 ['conv4_block16_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block16_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block16_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block16_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block16_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block16_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block16_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block16_concat (Conc (None, 14, 14, 768) 0 ['conv4_block15_concat[0][0]',\n", + " atenate) 'conv4_block16_2_conv[0][0]']\n", + " \n", + " conv4_block17_0_bn (BatchN (None, 14, 14, 768) 3072 ['conv4_block16_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block17_0_relu (Acti (None, 14, 14, 768) 0 ['conv4_block17_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block17_1_conv (Conv (None, 14, 14, 128) 98304 ['conv4_block17_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block17_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block17_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block17_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block17_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block17_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block17_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block17_concat (Conc (None, 14, 14, 800) 0 ['conv4_block16_concat[0][0]',\n", + " atenate) 'conv4_block17_2_conv[0][0]']\n", + " \n", + " conv4_block18_0_bn (BatchN (None, 14, 14, 800) 3200 ['conv4_block17_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block18_0_relu (Acti (None, 14, 14, 800) 0 ['conv4_block18_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block18_1_conv (Conv (None, 14, 14, 128) 102400 ['conv4_block18_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block18_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block18_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block18_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block18_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block18_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block18_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block18_concat (Conc (None, 14, 14, 832) 0 ['conv4_block17_concat[0][0]',\n", + " atenate) 'conv4_block18_2_conv[0][0]']\n", + " \n", + " conv4_block19_0_bn (BatchN (None, 14, 14, 832) 3328 ['conv4_block18_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block19_0_relu (Acti (None, 14, 14, 832) 0 ['conv4_block19_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block19_1_conv (Conv (None, 14, 14, 128) 106496 ['conv4_block19_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block19_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block19_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block19_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block19_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block19_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block19_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block19_concat (Conc (None, 14, 14, 864) 0 ['conv4_block18_concat[0][0]',\n", + " atenate) 'conv4_block19_2_conv[0][0]']\n", + " \n", + " conv4_block20_0_bn (BatchN (None, 14, 14, 864) 3456 ['conv4_block19_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block20_0_relu (Acti (None, 14, 14, 864) 0 ['conv4_block20_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block20_1_conv (Conv (None, 14, 14, 128) 110592 ['conv4_block20_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block20_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block20_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block20_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block20_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block20_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block20_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block20_concat (Conc (None, 14, 14, 896) 0 ['conv4_block19_concat[0][0]',\n", + " atenate) 'conv4_block20_2_conv[0][0]']\n", + " \n", + " conv4_block21_0_bn (BatchN (None, 14, 14, 896) 3584 ['conv4_block20_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block21_0_relu (Acti (None, 14, 14, 896) 0 ['conv4_block21_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block21_1_conv (Conv (None, 14, 14, 128) 114688 ['conv4_block21_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block21_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block21_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block21_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block21_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block21_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block21_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block21_concat (Conc (None, 14, 14, 928) 0 ['conv4_block20_concat[0][0]',\n", + " atenate) 'conv4_block21_2_conv[0][0]']\n", + " \n", + " conv4_block22_0_bn (BatchN (None, 14, 14, 928) 3712 ['conv4_block21_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block22_0_relu (Acti (None, 14, 14, 928) 0 ['conv4_block22_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block22_1_conv (Conv (None, 14, 14, 128) 118784 ['conv4_block22_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block22_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block22_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block22_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block22_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block22_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block22_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block22_concat (Conc (None, 14, 14, 960) 0 ['conv4_block21_concat[0][0]',\n", + " atenate) 'conv4_block22_2_conv[0][0]']\n", + " \n", + " conv4_block23_0_bn (BatchN (None, 14, 14, 960) 3840 ['conv4_block22_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block23_0_relu (Acti (None, 14, 14, 960) 0 ['conv4_block23_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block23_1_conv (Conv (None, 14, 14, 128) 122880 ['conv4_block23_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block23_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block23_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block23_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block23_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block23_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block23_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block23_concat (Conc (None, 14, 14, 992) 0 ['conv4_block22_concat[0][0]',\n", + " atenate) 'conv4_block23_2_conv[0][0]']\n", + " \n", + " conv4_block24_0_bn (BatchN (None, 14, 14, 992) 3968 ['conv4_block23_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block24_0_relu (Acti (None, 14, 14, 992) 0 ['conv4_block24_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block24_1_conv (Conv (None, 14, 14, 128) 126976 ['conv4_block24_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block24_1_bn (BatchN (None, 14, 14, 128) 512 ['conv4_block24_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv4_block24_1_relu (Acti (None, 14, 14, 128) 0 ['conv4_block24_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv4_block24_2_conv (Conv (None, 14, 14, 32) 36864 ['conv4_block24_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv4_block24_concat (Conc (None, 14, 14, 1024) 0 ['conv4_block23_concat[0][0]',\n", + " atenate) 'conv4_block24_2_conv[0][0]']\n", + " \n", + " pool4_bn (BatchNormalizati (None, 14, 14, 1024) 4096 ['conv4_block24_concat[0][0]']\n", + " on) \n", + " \n", + " pool4_relu (Activation) (None, 14, 14, 1024) 0 ['pool4_bn[0][0]'] \n", + " \n", + " pool4_conv (Conv2D) (None, 14, 14, 512) 524288 ['pool4_relu[0][0]'] \n", + " \n", + " pool4_pool (AveragePooling (None, 7, 7, 512) 0 ['pool4_conv[0][0]'] \n", + " 2D) \n", + " \n", + " conv5_block1_0_bn (BatchNo (None, 7, 7, 512) 2048 ['pool4_pool[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block1_0_relu (Activ (None, 7, 7, 512) 0 ['conv5_block1_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block1_1_conv (Conv2 (None, 7, 7, 128) 65536 ['conv5_block1_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block1_1_bn (BatchNo (None, 7, 7, 128) 512 ['conv5_block1_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block1_1_relu (Activ (None, 7, 7, 128) 0 ['conv5_block1_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block1_2_conv (Conv2 (None, 7, 7, 32) 36864 ['conv5_block1_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block1_concat (Conca (None, 7, 7, 544) 0 ['pool4_pool[0][0]', \n", + " tenate) 'conv5_block1_2_conv[0][0]'] \n", + " \n", + " conv5_block2_0_bn (BatchNo (None, 7, 7, 544) 2176 ['conv5_block1_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block2_0_relu (Activ (None, 7, 7, 544) 0 ['conv5_block2_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block2_1_conv (Conv2 (None, 7, 7, 128) 69632 ['conv5_block2_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block2_1_bn (BatchNo (None, 7, 7, 128) 512 ['conv5_block2_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block2_1_relu (Activ (None, 7, 7, 128) 0 ['conv5_block2_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block2_2_conv (Conv2 (None, 7, 7, 32) 36864 ['conv5_block2_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block2_concat (Conca (None, 7, 7, 576) 0 ['conv5_block1_concat[0][0]', \n", + " tenate) 'conv5_block2_2_conv[0][0]'] \n", + " \n", + " conv5_block3_0_bn (BatchNo (None, 7, 7, 576) 2304 ['conv5_block2_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block3_0_relu (Activ (None, 7, 7, 576) 0 ['conv5_block3_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block3_1_conv (Conv2 (None, 7, 7, 128) 73728 ['conv5_block3_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block3_1_bn (BatchNo (None, 7, 7, 128) 512 ['conv5_block3_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block3_1_relu (Activ (None, 7, 7, 128) 0 ['conv5_block3_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block3_2_conv (Conv2 (None, 7, 7, 32) 36864 ['conv5_block3_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block3_concat (Conca (None, 7, 7, 608) 0 ['conv5_block2_concat[0][0]', \n", + " tenate) 'conv5_block3_2_conv[0][0]'] \n", + " \n", + " conv5_block4_0_bn (BatchNo (None, 7, 7, 608) 2432 ['conv5_block3_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block4_0_relu (Activ (None, 7, 7, 608) 0 ['conv5_block4_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block4_1_conv (Conv2 (None, 7, 7, 128) 77824 ['conv5_block4_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block4_1_bn (BatchNo (None, 7, 7, 128) 512 ['conv5_block4_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block4_1_relu (Activ (None, 7, 7, 128) 0 ['conv5_block4_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block4_2_conv (Conv2 (None, 7, 7, 32) 36864 ['conv5_block4_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block4_concat (Conca (None, 7, 7, 640) 0 ['conv5_block3_concat[0][0]', \n", + " tenate) 'conv5_block4_2_conv[0][0]'] \n", + " \n", + " conv5_block5_0_bn (BatchNo (None, 7, 7, 640) 2560 ['conv5_block4_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block5_0_relu (Activ (None, 7, 7, 640) 0 ['conv5_block5_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block5_1_conv (Conv2 (None, 7, 7, 128) 81920 ['conv5_block5_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block5_1_bn (BatchNo (None, 7, 7, 128) 512 ['conv5_block5_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block5_1_relu (Activ (None, 7, 7, 128) 0 ['conv5_block5_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block5_2_conv (Conv2 (None, 7, 7, 32) 36864 ['conv5_block5_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block5_concat (Conca (None, 7, 7, 672) 0 ['conv5_block4_concat[0][0]', \n", + " tenate) 'conv5_block5_2_conv[0][0]'] \n", + " \n", + " conv5_block6_0_bn (BatchNo (None, 7, 7, 672) 2688 ['conv5_block5_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block6_0_relu (Activ (None, 7, 7, 672) 0 ['conv5_block6_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block6_1_conv (Conv2 (None, 7, 7, 128) 86016 ['conv5_block6_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block6_1_bn (BatchNo (None, 7, 7, 128) 512 ['conv5_block6_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block6_1_relu (Activ (None, 7, 7, 128) 0 ['conv5_block6_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block6_2_conv (Conv2 (None, 7, 7, 32) 36864 ['conv5_block6_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block6_concat (Conca (None, 7, 7, 704) 0 ['conv5_block5_concat[0][0]', \n", + " tenate) 'conv5_block6_2_conv[0][0]'] \n", + " \n", + " conv5_block7_0_bn (BatchNo (None, 7, 7, 704) 2816 ['conv5_block6_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block7_0_relu (Activ (None, 7, 7, 704) 0 ['conv5_block7_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block7_1_conv (Conv2 (None, 7, 7, 128) 90112 ['conv5_block7_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block7_1_bn (BatchNo (None, 7, 7, 128) 512 ['conv5_block7_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block7_1_relu (Activ (None, 7, 7, 128) 0 ['conv5_block7_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block7_2_conv (Conv2 (None, 7, 7, 32) 36864 ['conv5_block7_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block7_concat (Conca (None, 7, 7, 736) 0 ['conv5_block6_concat[0][0]', \n", + " tenate) 'conv5_block7_2_conv[0][0]'] \n", + " \n", + " conv5_block8_0_bn (BatchNo (None, 7, 7, 736) 2944 ['conv5_block7_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block8_0_relu (Activ (None, 7, 7, 736) 0 ['conv5_block8_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block8_1_conv (Conv2 (None, 7, 7, 128) 94208 ['conv5_block8_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block8_1_bn (BatchNo (None, 7, 7, 128) 512 ['conv5_block8_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block8_1_relu (Activ (None, 7, 7, 128) 0 ['conv5_block8_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block8_2_conv (Conv2 (None, 7, 7, 32) 36864 ['conv5_block8_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block8_concat (Conca (None, 7, 7, 768) 0 ['conv5_block7_concat[0][0]', \n", + " tenate) 'conv5_block8_2_conv[0][0]'] \n", + " \n", + " conv5_block9_0_bn (BatchNo (None, 7, 7, 768) 3072 ['conv5_block8_concat[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block9_0_relu (Activ (None, 7, 7, 768) 0 ['conv5_block9_0_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block9_1_conv (Conv2 (None, 7, 7, 128) 98304 ['conv5_block9_0_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block9_1_bn (BatchNo (None, 7, 7, 128) 512 ['conv5_block9_1_conv[0][0]'] \n", + " rmalization) \n", + " \n", + " conv5_block9_1_relu (Activ (None, 7, 7, 128) 0 ['conv5_block9_1_bn[0][0]'] \n", + " ation) \n", + " \n", + " conv5_block9_2_conv (Conv2 (None, 7, 7, 32) 36864 ['conv5_block9_1_relu[0][0]'] \n", + " D) \n", + " \n", + " conv5_block9_concat (Conca (None, 7, 7, 800) 0 ['conv5_block8_concat[0][0]', \n", + " tenate) 'conv5_block9_2_conv[0][0]'] \n", + " \n", + " conv5_block10_0_bn (BatchN (None, 7, 7, 800) 3200 ['conv5_block9_concat[0][0]'] \n", + " ormalization) \n", + " \n", + " conv5_block10_0_relu (Acti (None, 7, 7, 800) 0 ['conv5_block10_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block10_1_conv (Conv (None, 7, 7, 128) 102400 ['conv5_block10_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block10_1_bn (BatchN (None, 7, 7, 128) 512 ['conv5_block10_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block10_1_relu (Acti (None, 7, 7, 128) 0 ['conv5_block10_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block10_2_conv (Conv (None, 7, 7, 32) 36864 ['conv5_block10_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block10_concat (Conc (None, 7, 7, 832) 0 ['conv5_block9_concat[0][0]', \n", + " atenate) 'conv5_block10_2_conv[0][0]']\n", + " \n", + " conv5_block11_0_bn (BatchN (None, 7, 7, 832) 3328 ['conv5_block10_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block11_0_relu (Acti (None, 7, 7, 832) 0 ['conv5_block11_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block11_1_conv (Conv (None, 7, 7, 128) 106496 ['conv5_block11_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block11_1_bn (BatchN (None, 7, 7, 128) 512 ['conv5_block11_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block11_1_relu (Acti (None, 7, 7, 128) 0 ['conv5_block11_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block11_2_conv (Conv (None, 7, 7, 32) 36864 ['conv5_block11_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block11_concat (Conc (None, 7, 7, 864) 0 ['conv5_block10_concat[0][0]',\n", + " atenate) 'conv5_block11_2_conv[0][0]']\n", + " \n", + " conv5_block12_0_bn (BatchN (None, 7, 7, 864) 3456 ['conv5_block11_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block12_0_relu (Acti (None, 7, 7, 864) 0 ['conv5_block12_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block12_1_conv (Conv (None, 7, 7, 128) 110592 ['conv5_block12_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block12_1_bn (BatchN (None, 7, 7, 128) 512 ['conv5_block12_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block12_1_relu (Acti (None, 7, 7, 128) 0 ['conv5_block12_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block12_2_conv (Conv (None, 7, 7, 32) 36864 ['conv5_block12_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block12_concat (Conc (None, 7, 7, 896) 0 ['conv5_block11_concat[0][0]',\n", + " atenate) 'conv5_block12_2_conv[0][0]']\n", + " \n", + " conv5_block13_0_bn (BatchN (None, 7, 7, 896) 3584 ['conv5_block12_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block13_0_relu (Acti (None, 7, 7, 896) 0 ['conv5_block13_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block13_1_conv (Conv (None, 7, 7, 128) 114688 ['conv5_block13_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block13_1_bn (BatchN (None, 7, 7, 128) 512 ['conv5_block13_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block13_1_relu (Acti (None, 7, 7, 128) 0 ['conv5_block13_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block13_2_conv (Conv (None, 7, 7, 32) 36864 ['conv5_block13_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block13_concat (Conc (None, 7, 7, 928) 0 ['conv5_block12_concat[0][0]',\n", + " atenate) 'conv5_block13_2_conv[0][0]']\n", + " \n", + " conv5_block14_0_bn (BatchN (None, 7, 7, 928) 3712 ['conv5_block13_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block14_0_relu (Acti (None, 7, 7, 928) 0 ['conv5_block14_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block14_1_conv (Conv (None, 7, 7, 128) 118784 ['conv5_block14_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block14_1_bn (BatchN (None, 7, 7, 128) 512 ['conv5_block14_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block14_1_relu (Acti (None, 7, 7, 128) 0 ['conv5_block14_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block14_2_conv (Conv (None, 7, 7, 32) 36864 ['conv5_block14_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block14_concat (Conc (None, 7, 7, 960) 0 ['conv5_block13_concat[0][0]',\n", + " atenate) 'conv5_block14_2_conv[0][0]']\n", + " \n", + " conv5_block15_0_bn (BatchN (None, 7, 7, 960) 3840 ['conv5_block14_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block15_0_relu (Acti (None, 7, 7, 960) 0 ['conv5_block15_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block15_1_conv (Conv (None, 7, 7, 128) 122880 ['conv5_block15_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block15_1_bn (BatchN (None, 7, 7, 128) 512 ['conv5_block15_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block15_1_relu (Acti (None, 7, 7, 128) 0 ['conv5_block15_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block15_2_conv (Conv (None, 7, 7, 32) 36864 ['conv5_block15_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block15_concat (Conc (None, 7, 7, 992) 0 ['conv5_block14_concat[0][0]',\n", + " atenate) 'conv5_block15_2_conv[0][0]']\n", + " \n", + " conv5_block16_0_bn (BatchN (None, 7, 7, 992) 3968 ['conv5_block15_concat[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block16_0_relu (Acti (None, 7, 7, 992) 0 ['conv5_block16_0_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block16_1_conv (Conv (None, 7, 7, 128) 126976 ['conv5_block16_0_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block16_1_bn (BatchN (None, 7, 7, 128) 512 ['conv5_block16_1_conv[0][0]']\n", + " ormalization) \n", + " \n", + " conv5_block16_1_relu (Acti (None, 7, 7, 128) 0 ['conv5_block16_1_bn[0][0]'] \n", + " vation) \n", + " \n", + " conv5_block16_2_conv (Conv (None, 7, 7, 32) 36864 ['conv5_block16_1_relu[0][0]']\n", + " 2D) \n", + " \n", + " conv5_block16_concat (Conc (None, 7, 7, 1024) 0 ['conv5_block15_concat[0][0]',\n", + " atenate) 'conv5_block16_2_conv[0][0]']\n", + " \n", + " bn (BatchNormalization) (None, 7, 7, 1024) 4096 ['conv5_block16_concat[0][0]']\n", + " \n", + " relu (Activation) (None, 7, 7, 1024) 0 ['bn[0][0]'] \n", + " \n", + " avg_pool (GlobalAveragePoo (None, 1024) 0 ['relu[0][0]'] \n", + " ling2D) \n", + " \n", + " predictions (Dense) (None, 1000) 1025000 ['avg_pool[0][0]'] \n", + " \n", + "==================================================================================================\n", + "Total params: 8062504 (30.76 MB)\n", + "Trainable params: 7978856 (30.44 MB)\n", + "Non-trainable params: 83648 (326.75 KB)\n", + "__________________________________________________________________________________________________\n" + ] + } + ], + "source": [ + "dnet.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Make all layers untrainable except for the very last convolutional layer" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "for layer in dnet.layers:\n", + " layer.trainable = False\n", + "dnet.layers[-6].trainable = True" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Cifar-10 testing" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "(X_train, y_train), (X_test, y_test) = cifar10.load_data()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "y_train_cat = to_categorical(y_train, 1000)\n", + "y_test_cat = to_categorical(y_test, 1000)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Lambda layer for preprocessing" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "def preprocess(x):\n", + " x = tf.image.resize(x,size=(224,224),method='bicubic')\n", + " x = tf.keras.applications.densenet.preprocess_input(x)\n", + " return x" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Modify the model" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "input_shape = (32,32,3)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "input_layer = Input(shape=input_shape)\n", + "prep = Lambda(preprocess)(input_layer)\n", + "out = dnet(prep)\n", + "dnet_mod = Model(inputs=input_layer, outputs=out)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "dnet_mod.compile(optimizer='adam',\n", + " loss=tf.keras.losses.CategoricalCrossentropy(),\n", + " metrics=[tf.keras.metrics.TopKCategoricalAccuracy(k=1, name='top_1_categorical_accuracy')])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Try to fit it on Cifar-10 data and then evaluate (there is no hope this is gonna work ...)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1563/1563 [==============================] - 3842s 2s/step - loss: 5.9553 - top_1_categorical_accuracy: 0.1825\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dnet_mod.fit(X_train, y_train_cat)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "313/313 [==============================] - 648s 2s/step - loss: 4.8529 - top_1_categorical_accuracy: 0.2640\n" + ] + }, + { + "data": { + "text/plain": [ + "[4.8528923988342285, 0.2639999985694885]" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dnet_mod.evaluate(X_test, y_test_cat)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Try the same with adding a Cerebros \"add-on\" network" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "INPUT_SHAPES = [input_shape]\n", + "OUTPUT_SHAPES = [10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use 10k-15k random samples from Cifar-10 to speed up the process" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "num_samples = 15_000\n", + "rng = np.random.default_rng()\n", + "ind = rng.permutation(X_train.shape[0])[:num_samples]" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "training_x = [tf.constant(X_train[ind,:,:,:])]\n", + "y_train_cat = to_categorical(y_train[ind], 10)\n", + "train_labels = [tf.constant(y_train_cat)]" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "dnet = tf.keras.applications.densenet.DenseNet121(include_top=True,\n", + " weights='imagenet',\n", + " input_tensor=None,\n", + " input_shape=None,\n", + " pooling=None,\n", + " classes=1000,\n", + " classifier_activation='softmax'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "for layer in dnet.layers:\n", + " layer.trainable = False\n", + "dnet.layers[-6].trainable = True" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "dnet_io = Model(inputs=dnet.layers[0].input,\n", + " outputs=dnet.layers[-2].output)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "input_layer = Input(shape=input_shape)\n", + "prep = Lambda(preprocess)(input_layer)\n", + "out = Flatten()(dnet_io(prep))\n", + "base_mod = Model(inputs=input_layer, outputs=out)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [], + "source": [ + "activation = 'relu'\n", + "predecessor_level_connection_affinity_factor_first = 2.0\n", + "predecessor_level_connection_affinity_factor_main = 0.97\n", + "max_consecutive_lateral_connections = 5\n", + "p_lateral_connection = 0.97\n", + "num_lateral_connection_tries_per_unit = 2\n", + "learning_rate = 0.001\n", + "epochs = 10 # [1, 100]\n", + "batch_size = 20\n", + "maximum_levels = 4 # [3,7]\n", + "maximum_units_per_level = 7 # [2,10]\n", + "maximum_neurons_per_unit = 4 # [2,20]" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "# Final training task\n", + "TIME = pendulum.now(tz='America/New_York').__str__()[:16]\\\n", + " .replace('T', '_')\\\n", + " .replace(':', '_')\\\n", + " .replace('-', '_')\n", + "#\n", + "PROJECT_NAME = f'{TIME}_cerebros_auto_ml_test_cifar10_densenet'\n", + "#\n", + "meta_trial_number = 42\n", + "#\n", + "cerebros_automl = SimpleCerebrosRandomSearch(\n", + " unit_type=DenseUnit,\n", + " input_shapes=INPUT_SHAPES,\n", + " output_shapes=OUTPUT_SHAPES,\n", + " training_data=training_x,\n", + " labels=train_labels,\n", + " validation_split=0.2,\n", + " direction='maximize',\n", + " metric_to_rank_by=\"val_top_1_categorical_accuracy\",\n", + " minimum_levels=2,\n", + " maximum_levels=maximum_levels,\n", + " minimum_units_per_level=1,\n", + " maximum_units_per_level=maximum_units_per_level,\n", + " minimum_neurons_per_unit=1,\n", + " maximum_neurons_per_unit=maximum_neurons_per_unit,\n", + " activation=activation,\n", + " final_activation='softmax',\n", + " number_of_architecture_moities_to_try=3,\n", + " number_of_tries_per_architecture_moity=2,\n", + " minimum_skip_connection_depth=1,\n", + " maximum_skip_connection_depth=7,\n", + " predecessor_level_connection_affinity_factor_first=predecessor_level_connection_affinity_factor_first,\n", + " predecessor_level_connection_affinity_factor_first_rounding_rule='ceil',\n", + " predecessor_level_connection_affinity_factor_main=predecessor_level_connection_affinity_factor_main,\n", + " predecessor_level_connection_affinity_factor_main_rounding_rule='ceil',\n", + " predecessor_level_connection_affinity_factor_decay_main=zero_7_exp_decay,\n", + " seed=8675309,\n", + " max_consecutive_lateral_connections=max_consecutive_lateral_connections,\n", + " gate_after_n_lateral_connections=3,\n", + " gate_activation_function=simple_sigmoid,\n", + " p_lateral_connection=p_lateral_connection,\n", + " p_lateral_connection_decay=zero_95_exp_decay,\n", + " num_lateral_connection_tries_per_unit=num_lateral_connection_tries_per_unit,\n", + " learning_rate=learning_rate,\n", + " loss=tf.keras.losses.CategoricalCrossentropy(),\n", + " metrics=[tf.keras.metrics.TopKCategoricalAccuracy(\n", + " k=1, name='top_1_categorical_accuracy')\n", + " ],\n", + " epochs=epochs,\n", + " project_name=f\"{PROJECT_NAME}_meta_{meta_trial_number}\",\n", + " model_graphs='model_graphs',\n", + " batch_size=batch_size,\n", + " meta_trial_number=meta_trial_number,\n", + " base_models=[base_mod])" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'4': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'1': }, {'1': }, {'4': }]\n", + "2\n", + "Trying to create level 2\n", + "We think level 2's predecessors are: [0, 1]\n", + "k is: 3 value is: [{'2': }, {'2': }, {'1': }, {'2': }, {'2': }, {'4': }]\n", + "3\n", + "Trying to create level 3\n", + "We think level 3's predecessors are: [0, 1, 2]\n", + "k is: 4 value is: [{'10': }]\n", + "4\n", + "Trying to create Final level 4\n", + "Trying to create level 4\n", + "We think level final level 4's predecessors are: [0, 1, 2, 3]\n", + "levels:\n", + "[0, 1, 2, 3, 4]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'4': }\n", + "{'1': }\n", + "{'1': }\n", + "{'4': }\n", + "{'2': }\n", + "{'2': }\n", + "{'1': }\n", + "{'2': }\n", + "{'2': }\n", + "{'4': }\n", + "Debug: I am 4 selecting 3\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2, 3, 4]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2, 3, 4]\n", + "Setting levels_unmaterialized[2] level_number 2 to have first successor: levels_unmaterialized[:3], having level_numbers of [3, 4]\n", + "Setting levels_unmaterialized[3] level_number 3 to have first successor: levels_unmaterialized[:4], having level_numbers of [4]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'4': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "{'1': }\n", + "{'1': }\n", + "{'4': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "{'2': }\n", + "{'2': }\n", + "{'1': }\n", + "{'2': }\n", + "{'2': }\n", + "{'4': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_1 called\n", + "materialized network layers\n", + "[, , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_3 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_4 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_5 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000004_tr_0_FinalDenseUnit_0000000000000004_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , , , , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , ]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000004_tr_0_FinalDenseUnit_0000000000000004_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000004_tr_0_FinalDenseUnit_0000000000000004_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_3 (Functional) (None, 1024) 7037504 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 0 ['model_3[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_3[0][0]', \n", + " _0000000000000001_tr_0_Den 'model_3[0][0]', \n", + " seUnit_0000000000000001_tr 'model_3[0][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 16384 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 16388 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2056) 0 ['model_3[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000002_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000002_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_1_cat_ (Concatenate) 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2056) 8224 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 2057 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2058) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2058) 8232 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 8236 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2071) 0 ['model_3[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000003_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000003_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_1_cat_ (Concatenate) 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2056) 0 ['model_3[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000002_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000002_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_0_cat_ (Concatenate) 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2071) 8284 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2056) 8224 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 4144 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 2057 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2072) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2072) 8288 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 2073 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2066) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_2_dns_[0][0\n", + " _0_3_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2066) 8264 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_3_cat_[0][0\n", + " _0_3_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 4134 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_3_btn_[0][0\n", + " _0_3_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2073) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_3_dns_[0][0\n", + " _0_4_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2073) 8292 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_4_cat_[0][0\n", + " _0_4_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 4148 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_4_btn_[0][0\n", + " _0_4_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2074) 0 ['model_3[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000003_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000003_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_0_cat_ (Concatenate) 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2081) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_4_dns_[0][0\n", + " _0_5_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_4_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2074) 8296 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2081) 8324 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_5_cat_[0][0\n", + " _0_5_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 4150 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 8328 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_5_btn_[0][0\n", + " _0_5_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2107) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000004_tr_ 00000000003_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000003_tr_0_2_dns_[0][0\n", + " 0000004_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_5_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_4_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_5_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_5_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_4_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_5_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2107) 8428 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000004_tr_ 0000000000000004_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000004_tr_0_0\n", + " 0000004_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 21080 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000004_tr_ 0000000000000004_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000004_tr_0_0\n", + " 0000004_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 7213539 (27.52 MB)\n", + "Trainable params: 163279 (637.81 KB)\n", + "Non-trainable params: 7050260 (26.89 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 921s 2s/step - loss: 0.4709 - top_1_categorical_accuracy: 0.8411 - val_loss: 0.3004 - val_top_1_categorical_accuracy: 0.8960\n", + "Epoch 2/10\n", + "600/600 [==============================] - 877s 1s/step - loss: 0.3047 - top_1_categorical_accuracy: 0.8931 - val_loss: 0.2955 - val_top_1_categorical_accuracy: 0.9017\n", + "Epoch 3/10\n", + "600/600 [==============================] - 854s 1s/step - loss: 0.2633 - top_1_categorical_accuracy: 0.9092 - val_loss: 0.2940 - val_top_1_categorical_accuracy: 0.9040\n", + "Epoch 4/10\n", + "600/600 [==============================] - 850s 1s/step - loss: 0.2268 - top_1_categorical_accuracy: 0.9192 - val_loss: 0.3142 - val_top_1_categorical_accuracy: 0.8997\n", + "Epoch 5/10\n", + "600/600 [==============================] - 836s 1s/step - loss: 0.2008 - top_1_categorical_accuracy: 0.9310 - val_loss: 0.3326 - val_top_1_categorical_accuracy: 0.8980\n", + "Epoch 6/10\n", + "600/600 [==============================] - 853s 1s/step - loss: 0.1721 - top_1_categorical_accuracy: 0.9376 - val_loss: 0.3800 - val_top_1_categorical_accuracy: 0.8913\n", + "Epoch 7/10\n", + "600/600 [==============================] - 684s 1s/step - loss: 0.1505 - top_1_categorical_accuracy: 0.9465 - val_loss: 0.3464 - val_top_1_categorical_accuracy: 0.8987\n", + "Epoch 8/10\n", + "600/600 [==============================] - 1136s 2s/step - loss: 0.1307 - top_1_categorical_accuracy: 0.9494 - val_loss: 0.3665 - val_top_1_categorical_accuracy: 0.9000\n", + "Epoch 9/10\n", + "600/600 [==============================] - 896s 1s/step - loss: 0.1040 - top_1_categorical_accuracy: 0.9613 - val_loss: 0.4149 - val_top_1_categorical_accuracy: 0.8997\n", + "Epoch 10/10\n", + "600/600 [==============================] - 869s 1s/step - loss: 0.0921 - top_1_categorical_accuracy: 0.9670 - val_loss: 0.4557 - val_top_1_categorical_accuracy: 0.8927\n", + "this is neural_network_spec_file 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000000.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.470867 0.841083 0.300446 \n", + "1 0.304662 0.893083 0.295500 \n", + "2 0.263270 0.909167 0.293994 \n", + "3 0.226823 0.919167 0.314155 \n", + "4 0.200816 0.931000 0.332571 \n", + "5 0.172115 0.937583 0.379954 \n", + "6 0.150458 0.946500 0.346395 \n", + "7 0.130742 0.949417 0.366537 \n", + "8 0.104036 0.961333 0.414900 \n", + "9 0.092128 0.967000 0.455699 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.896000 0 0 \n", + "1 0.901667 0 0 \n", + "2 0.904000 0 0 \n", + "3 0.899667 0 0 \n", + "4 0.898000 0 0 \n", + "5 0.891333 0 0 \n", + "6 0.898667 0 0 \n", + "7 0.900000 0 0 \n", + "8 0.899667 0 0 \n", + "9 0.892667 0 0 \n", + "\n", + " model_name \n", + "0 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'4': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'1': }, {'1': }, {'4': }]\n", + "2\n", + "Trying to create level 2\n", + "We think level 2's predecessors are: [0, 1]\n", + "k is: 3 value is: [{'2': }, {'2': }, {'1': }, {'2': }, {'2': }, {'4': }]\n", + "3\n", + "Trying to create level 3\n", + "We think level 3's predecessors are: [0, 1, 2]\n", + "k is: 4 value is: [{'10': }]\n", + "4\n", + "Trying to create Final level 4\n", + "Trying to create level 4\n", + "We think level final level 4's predecessors are: [0, 1, 2, 3]\n", + "levels:\n", + "[0, 1, 2, 3, 4]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'4': }\n", + "{'1': }\n", + "{'1': }\n", + "{'4': }\n", + "{'2': }\n", + "{'2': }\n", + "{'1': }\n", + "{'2': }\n", + "{'2': }\n", + "{'4': }\n", + "Debug: I am 4 selecting 3\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2, 3, 4]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2, 3, 4]\n", + "Setting levels_unmaterialized[2] level_number 2 to have first successor: levels_unmaterialized[:3], having level_numbers of [3, 4]\n", + "Setting levels_unmaterialized[3] level_number 3 to have first successor: levels_unmaterialized[:4], having level_numbers of [4]\n", + "3\n", + "1\n", + "Debug: successor_connectivity_errors_2d []\n", + "Successor map:\n", + "{4: '0'}\n", + "I am: level #: 3 calling a correction to an error in:\n", + "error_0:[3 1]\n", + "I am: 4: My predecessors are [0, 1, 2, 3]\n", + "asigning unit level 3, unit: 1 to be the input of: level: 4 unit: 0\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'4': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "{'1': }\n", + "{'1': }\n", + "{'4': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000002_tr_0_DenseUnit_0000000000000002_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "{'2': }\n", + "{'2': }\n", + "{'1': }\n", + "{'2': }\n", + "{'2': }\n", + "{'4': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d [Array([3, 1], dtype=int32)]\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_1 called\n", + "materialized network layers\n", + "[, , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_3 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_4 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000003_tr_0_DenseUnit_0000000000000003_tr_0_5 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000004_tr_0_FinalDenseUnit_0000000000000004_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000004_tr_0_FinalDenseUnit_0000000000000004_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000004_tr_0_FinalDenseUnit_0000000000000004_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_3 (Functional) (None, 1024) 7037504 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 0 ['model_3[1][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_3[1][0]', \n", + " _0000000000000001_tr_0_Den 'model_3[1][0]', \n", + " seUnit_0000000000000001_tr 'model_3[1][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 16384 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 16388 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2056) 0 ['model_3[1][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000002_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000002_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_1_cat_ (Concatenate) 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2056) 8224 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 2057 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2058) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2056) 0 ['model_3[1][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000002_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000002_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_0_cat_ (Concatenate) 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2058) 8232 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2056) 8224 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 8236 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 2057 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000002_tr_0_Den 00000000002_tr_0_DenseUnit_000\n", + " seUnit_0000000000000002_tr 0000000000002_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2074) 0 ['model_3[1][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000003_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000003_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_1_cat_ (Concatenate) 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2074) 8296 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 4150 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2072) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2072) 8288 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 2073 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2071) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_2_dns_[0][0\n", + " _0_3_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2071) 8284 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_3_cat_[0][0\n", + " _0_3_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 4144 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_3_btn_[0][0\n", + " _0_3_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2069) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_3_dns_[0][0\n", + " _0_4_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2069) 8276 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_4_cat_[0][0\n", + " _0_4_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 4140 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_4_btn_[0][0\n", + " _0_4_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2078) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_4_dns_[0][0\n", + " _0_5_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_4_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2065) 0 ['model_3[1][0]', \n", + " 0000000nan_tr_0_DenseLevel 'NeuralNetworkFuture_00000000\n", + " _0000000000000003_tr_0_Den 00000nan_tr_0_DenseLevel_00000\n", + " seUnit_0000000000000003_tr 00000000001_tr_0_DenseUnit_000\n", + " _0_0_cat_ (Concatenate) 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2078) 8312 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_5_cat_[0][0\n", + " _0_5_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2065) 8260 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 8316 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_5_btn_[0][0\n", + " _0_5_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 4132 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000003_tr_0_Den 00000000003_tr_0_DenseUnit_000\n", + " seUnit_0000000000000003_tr 0000000000003_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2118) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000004_tr_ 00000000003_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000003_tr_0_3_dns_[0][0\n", + " 0000004_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_5_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_5_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_4_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_4_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_5_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_5_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_5_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000002_tr_0_DenseUnit_000\n", + " 0000000000002_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000003_tr_0_DenseUnit_000\n", + " 0000000000003_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2118) 8472 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000004_tr_ 0000000000000004_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000004_tr_0_0\n", + " 0000004_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 21190 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000004_tr_ 0000000000000004_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000004_tr_0_0\n", + " 0000004_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 7213639 (27.52 MB)\n", + "Trainable params: 163373 (638.18 KB)\n", + "Non-trainable params: 7050266 (26.89 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 909s 1s/step - loss: 0.3762 - top_1_categorical_accuracy: 0.8731 - val_loss: 0.3258 - val_top_1_categorical_accuracy: 0.8953\n", + "Epoch 2/10\n", + "600/600 [==============================] - 912s 2s/step - loss: 0.1597 - top_1_categorical_accuracy: 0.9438 - val_loss: 0.3654 - val_top_1_categorical_accuracy: 0.8937\n", + "Epoch 3/10\n", + "600/600 [==============================] - 886s 1s/step - loss: 0.1081 - top_1_categorical_accuracy: 0.9624 - val_loss: 0.4124 - val_top_1_categorical_accuracy: 0.8883\n", + "Epoch 4/10\n", + "600/600 [==============================] - 771s 1s/step - loss: 0.0989 - top_1_categorical_accuracy: 0.9650 - val_loss: 0.4400 - val_top_1_categorical_accuracy: 0.8823\n", + "Epoch 5/10\n", + "600/600 [==============================] - 762s 1s/step - loss: 0.0784 - top_1_categorical_accuracy: 0.9719 - val_loss: 0.4362 - val_top_1_categorical_accuracy: 0.8883\n", + "Epoch 6/10\n", + "600/600 [==============================] - 714s 1s/step - loss: 0.0663 - top_1_categorical_accuracy: 0.9763 - val_loss: 0.4725 - val_top_1_categorical_accuracy: 0.8913\n", + "Epoch 7/10\n", + "600/600 [==============================] - 697s 1s/step - loss: 0.0657 - top_1_categorical_accuracy: 0.9770 - val_loss: 0.5056 - val_top_1_categorical_accuracy: 0.8883\n", + "Epoch 8/10\n", + "600/600 [==============================] - 748s 1s/step - loss: 0.0538 - top_1_categorical_accuracy: 0.9809 - val_loss: 0.4913 - val_top_1_categorical_accuracy: 0.8847\n", + "Epoch 9/10\n", + "600/600 [==============================] - 751s 1s/step - loss: 0.0490 - top_1_categorical_accuracy: 0.9830 - val_loss: 0.5353 - val_top_1_categorical_accuracy: 0.8890\n", + "Epoch 10/10\n", + "600/600 [==============================] - 745s 1s/step - loss: 0.0520 - top_1_categorical_accuracy: 0.9820 - val_loss: 0.5248 - val_top_1_categorical_accuracy: 0.8910\n", + "this is neural_network_spec_file 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000001.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.376246 0.873067 0.325792 \n", + "1 0.159675 0.943750 0.365404 \n", + "2 0.108139 0.962417 0.412434 \n", + "3 0.098893 0.965000 0.440005 \n", + "4 0.078389 0.971917 0.436178 \n", + "5 0.066321 0.976333 0.472510 \n", + "6 0.065657 0.977000 0.505585 \n", + "7 0.053825 0.980917 0.491337 \n", + "8 0.049022 0.983000 0.535324 \n", + "9 0.051981 0.982000 0.524778 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.895333 0 1 \n", + "1 0.893667 0 1 \n", + "2 0.888333 0 1 \n", + "3 0.882333 0 1 \n", + "4 0.888333 0 1 \n", + "5 0.891333 0 1 \n", + "6 0.888333 0 1 \n", + "7 0.884667 0 1 \n", + "8 0.889000 0 1 \n", + "9 0.891000 0 1 \n", + "\n", + " model_name \n", + "0 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'3': }, {'4': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'10': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'3': }\n", + "{'4': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'3': }\n", + "{'4': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_3 (Functional) (None, 1024) 7037504 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 0 ['model_3[2][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_3[2][0]', \n", + " _0000000000000001_tr_0_Den 'model_3[2][0]', \n", + " seUnit_0000000000000001_tr 'model_3[2][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 0 ['model_3[2][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_3[2][0]', \n", + " _0000000000000001_tr_0_Den 'model_3[2][0]', \n", + " seUnit_0000000000000001_tr 'model_3[2][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 16384 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 16384 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 16388 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 12291 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2076) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_1_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[2][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[2][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2076) 8304 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 20770 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 7128025 (27.19 MB)\n", + "Trainable params: 106849 (417.38 KB)\n", + "Non-trainable params: 7021176 (26.78 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 739s 1s/step - loss: 0.3170 - top_1_categorical_accuracy: 0.8930 - val_loss: 0.3269 - val_top_1_categorical_accuracy: 0.8933\n", + "Epoch 2/10\n", + "600/600 [==============================] - 716s 1s/step - loss: 0.0922 - top_1_categorical_accuracy: 0.9688 - val_loss: 0.4104 - val_top_1_categorical_accuracy: 0.8800\n", + "Epoch 3/10\n", + "600/600 [==============================] - 721s 1s/step - loss: 0.0621 - top_1_categorical_accuracy: 0.9791 - val_loss: 0.4118 - val_top_1_categorical_accuracy: 0.8883\n", + "Epoch 4/10\n", + "600/600 [==============================] - 733s 1s/step - loss: 0.0451 - top_1_categorical_accuracy: 0.9852 - val_loss: 0.4812 - val_top_1_categorical_accuracy: 0.8783\n", + "Epoch 5/10\n", + "600/600 [==============================] - 631s 1s/step - loss: 0.0454 - top_1_categorical_accuracy: 0.9850 - val_loss: 0.5174 - val_top_1_categorical_accuracy: 0.8747\n", + "Epoch 6/10\n", + "600/600 [==============================] - 732s 1s/step - loss: 0.0411 - top_1_categorical_accuracy: 0.9861 - val_loss: 0.5825 - val_top_1_categorical_accuracy: 0.8757\n", + "Epoch 7/10\n", + "600/600 [==============================] - 741s 1s/step - loss: 0.0443 - top_1_categorical_accuracy: 0.9843 - val_loss: 0.5804 - val_top_1_categorical_accuracy: 0.8820\n", + "Epoch 8/10\n", + "600/600 [==============================] - 754s 1s/step - loss: 0.0365 - top_1_categorical_accuracy: 0.9874 - val_loss: 0.5288 - val_top_1_categorical_accuracy: 0.8923\n", + "Epoch 9/10\n", + "600/600 [==============================] - 725s 1s/step - loss: 0.0343 - top_1_categorical_accuracy: 0.9884 - val_loss: 0.5831 - val_top_1_categorical_accuracy: 0.8860\n", + "Epoch 10/10\n", + "600/600 [==============================] - 742s 1s/step - loss: 0.0405 - top_1_categorical_accuracy: 0.9856 - val_loss: 0.6483 - val_top_1_categorical_accuracy: 0.8780\n", + "this is neural_network_spec_file 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000000.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.317014 0.893000 0.326906 \n", + "1 0.092198 0.968750 0.410371 \n", + "2 0.062088 0.979083 0.411791 \n", + "3 0.045070 0.985250 0.481218 \n", + "4 0.045355 0.985000 0.517356 \n", + "5 0.041095 0.986083 0.582543 \n", + "6 0.044283 0.984250 0.580443 \n", + "7 0.036477 0.987417 0.528848 \n", + "8 0.034284 0.988417 0.583133 \n", + "9 0.040495 0.985583 0.648337 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.893333 0 0 \n", + "1 0.880000 0 0 \n", + "2 0.888333 0 0 \n", + "3 0.878333 0 0 \n", + "4 0.874667 0 0 \n", + "5 0.875667 0 0 \n", + "6 0.882000 0 0 \n", + "7 0.892333 0 0 \n", + "8 0.886000 0 0 \n", + "9 0.878000 0 0 \n", + "\n", + " model_name \n", + "0 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'3': }, {'4': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'10': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'3': }\n", + "{'4': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'3': }\n", + "{'4': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_3 (Functional) (None, 1024) 7037504 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 0 ['model_3[3][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_3[3][0]', \n", + " _0000000000000001_tr_0_Den 'model_3[3][0]', \n", + " seUnit_0000000000000001_tr 'model_3[3][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 0 ['model_3[3][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_3[3][0]', \n", + " _0000000000000001_tr_0_Den 'model_3[3][0]', \n", + " seUnit_0000000000000001_tr 'model_3[3][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 16384 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 16384 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 16388 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 12291 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2077) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_1_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_3[3][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[3][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2077) 8308 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 20780 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 7128039 (27.19 MB)\n", + "Trainable params: 106861 (417.43 KB)\n", + "Non-trainable params: 7021178 (26.78 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 750s 1s/step - loss: 0.2710 - top_1_categorical_accuracy: 0.9045 - val_loss: 0.3518 - val_top_1_categorical_accuracy: 0.8863\n", + "Epoch 2/10\n", + "600/600 [==============================] - 748s 1s/step - loss: 0.0559 - top_1_categorical_accuracy: 0.9846 - val_loss: 0.4296 - val_top_1_categorical_accuracy: 0.8857\n", + "Epoch 3/10\n", + "600/600 [==============================] - 738s 1s/step - loss: 0.0374 - top_1_categorical_accuracy: 0.9896 - val_loss: 0.4512 - val_top_1_categorical_accuracy: 0.8837\n", + "Epoch 4/10\n", + "600/600 [==============================] - 623s 1s/step - loss: 0.0313 - top_1_categorical_accuracy: 0.9903 - val_loss: 0.5007 - val_top_1_categorical_accuracy: 0.8823\n", + "Epoch 5/10\n", + "600/600 [==============================] - 726s 1s/step - loss: 0.0357 - top_1_categorical_accuracy: 0.9874 - val_loss: 0.5495 - val_top_1_categorical_accuracy: 0.8743\n", + "Epoch 6/10\n", + "600/600 [==============================] - 740s 1s/step - loss: 0.0307 - top_1_categorical_accuracy: 0.9901 - val_loss: 0.5481 - val_top_1_categorical_accuracy: 0.8807\n", + "Epoch 7/10\n", + "600/600 [==============================] - 738s 1s/step - loss: 0.0297 - top_1_categorical_accuracy: 0.9895 - val_loss: 0.6317 - val_top_1_categorical_accuracy: 0.8757\n", + "Epoch 8/10\n", + "600/600 [==============================] - 738s 1s/step - loss: 0.0250 - top_1_categorical_accuracy: 0.9907 - val_loss: 0.5992 - val_top_1_categorical_accuracy: 0.8823\n", + "Epoch 9/10\n", + "600/600 [==============================] - 728s 1s/step - loss: 0.0304 - top_1_categorical_accuracy: 0.9889 - val_loss: 0.6426 - val_top_1_categorical_accuracy: 0.8747\n", + "Epoch 10/10\n", + "600/600 [==============================] - 725s 1s/step - loss: 0.0235 - top_1_categorical_accuracy: 0.9928 - val_loss: 0.7473 - val_top_1_categorical_accuracy: 0.8737\n", + "this is neural_network_spec_file 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000001.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.271026 0.904533 0.351793 \n", + "1 0.055946 0.984583 0.429624 \n", + "2 0.037375 0.989583 0.451219 \n", + "3 0.031299 0.990333 0.500735 \n", + "4 0.035740 0.987417 0.549493 \n", + "5 0.030705 0.990083 0.548118 \n", + "6 0.029723 0.989500 0.631687 \n", + "7 0.024976 0.990667 0.599243 \n", + "8 0.030425 0.988917 0.642585 \n", + "9 0.023503 0.992833 0.747255 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.886333 0 1 \n", + "1 0.885667 0 1 \n", + "2 0.883667 0 1 \n", + "3 0.882333 0 1 \n", + "4 0.874333 0 1 \n", + "5 0.880667 0 1 \n", + "6 0.875667 0 1 \n", + "7 0.882333 0 1 \n", + "8 0.874667 0 1 \n", + "9 0.873667 0 1 \n", + "\n", + " model_name \n", + "0 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'1': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'10': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'1': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'1': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_3 (Functional) (None, 1024) 7037504 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 0 ['model_3[4][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_3[4][0]', \n", + " _0000000000000001_tr_0_Den 'model_3[4][0]', \n", + " seUnit_0000000000000001_tr 'model_3[4][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 16384 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 4097 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2052) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_0_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[4][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[4][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2052) 8208 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 20530 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 7086723 (27.03 MB)\n", + "Trainable params: 73787 (288.23 KB)\n", + "Non-trainable params: 7012936 (26.75 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 754s 1s/step - loss: 0.2649 - top_1_categorical_accuracy: 0.9061 - val_loss: 0.3346 - val_top_1_categorical_accuracy: 0.8840\n", + "Epoch 2/10\n", + "600/600 [==============================] - 733s 1s/step - loss: 0.0410 - top_1_categorical_accuracy: 0.9918 - val_loss: 0.3947 - val_top_1_categorical_accuracy: 0.8840\n", + "Epoch 3/10\n", + "600/600 [==============================] - 606s 1s/step - loss: 0.0272 - top_1_categorical_accuracy: 0.9941 - val_loss: 0.4384 - val_top_1_categorical_accuracy: 0.8830\n", + "Epoch 4/10\n", + "600/600 [==============================] - 732s 1s/step - loss: 0.0163 - top_1_categorical_accuracy: 0.9963 - val_loss: 0.4750 - val_top_1_categorical_accuracy: 0.8880\n", + "Epoch 5/10\n", + "600/600 [==============================] - 725s 1s/step - loss: 0.0221 - top_1_categorical_accuracy: 0.9932 - val_loss: 0.5780 - val_top_1_categorical_accuracy: 0.8783\n", + "Epoch 6/10\n", + "600/600 [==============================] - 720s 1s/step - loss: 0.0233 - top_1_categorical_accuracy: 0.9918 - val_loss: 0.5964 - val_top_1_categorical_accuracy: 0.8783\n", + "Epoch 7/10\n", + "600/600 [==============================] - 733s 1s/step - loss: 0.0255 - top_1_categorical_accuracy: 0.9915 - val_loss: 0.6104 - val_top_1_categorical_accuracy: 0.8820\n", + "Epoch 8/10\n", + "600/600 [==============================] - 741s 1s/step - loss: 0.0228 - top_1_categorical_accuracy: 0.9924 - val_loss: 0.6359 - val_top_1_categorical_accuracy: 0.8783\n", + "Epoch 9/10\n", + "600/600 [==============================] - 729s 1s/step - loss: 0.0222 - top_1_categorical_accuracy: 0.9918 - val_loss: 0.6596 - val_top_1_categorical_accuracy: 0.8753\n", + "Epoch 10/10\n", + "600/600 [==============================] - 730s 1s/step - loss: 0.0233 - top_1_categorical_accuracy: 0.9915 - val_loss: 0.6784 - val_top_1_categorical_accuracy: 0.8773\n", + "this is neural_network_spec_file 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000000.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.264904 0.906133 0.334645 \n", + "1 0.041018 0.991750 0.394694 \n", + "2 0.027192 0.994083 0.438355 \n", + "3 0.016322 0.996333 0.475004 \n", + "4 0.022072 0.993167 0.578026 \n", + "5 0.023345 0.991833 0.596374 \n", + "6 0.025543 0.991500 0.610430 \n", + "7 0.022781 0.992417 0.635906 \n", + "8 0.022238 0.991750 0.659581 \n", + "9 0.023292 0.991500 0.678422 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.884000 0 0 \n", + "1 0.884000 0 0 \n", + "2 0.883000 0 0 \n", + "3 0.888000 0 0 \n", + "4 0.878333 0 0 \n", + "5 0.878333 0 0 \n", + "6 0.882000 0 0 \n", + "7 0.878333 0 0 \n", + "8 0.875333 0 0 \n", + "9 0.877333 0 0 \n", + "\n", + " model_name \n", + "0 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'1': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'10': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'1': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'1': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_3 (Functional) (None, 1024) 7037504 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 0 ['model_3[5][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_3[5][0]', \n", + " _0000000000000001_tr_0_Den 'model_3[5][0]', \n", + " seUnit_0000000000000001_tr 'model_3[5][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 16384 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 4097 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2052) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_0_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[5][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[5][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2052) 8208 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 20530 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 7086723 (27.03 MB)\n", + "Trainable params: 73787 (288.23 KB)\n", + "Non-trainable params: 7012936 (26.75 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 729s 1s/step - loss: 0.2521 - top_1_categorical_accuracy: 0.9129 - val_loss: 0.3423 - val_top_1_categorical_accuracy: 0.8890\n", + "Epoch 2/10\n", + "600/600 [==============================] - 510s 849ms/step - loss: 0.0345 - top_1_categorical_accuracy: 0.9940 - val_loss: 0.3881 - val_top_1_categorical_accuracy: 0.8843\n", + "Epoch 3/10\n", + "600/600 [==============================] - 417s 696ms/step - loss: 0.0192 - top_1_categorical_accuracy: 0.9962 - val_loss: 0.4284 - val_top_1_categorical_accuracy: 0.8803\n", + "Epoch 4/10\n", + "600/600 [==============================] - 420s 700ms/step - loss: 0.0134 - top_1_categorical_accuracy: 0.9975 - val_loss: 0.4850 - val_top_1_categorical_accuracy: 0.8803\n", + "Epoch 5/10\n", + "600/600 [==============================] - 423s 704ms/step - loss: 0.0180 - top_1_categorical_accuracy: 0.9943 - val_loss: 0.5179 - val_top_1_categorical_accuracy: 0.8807\n", + "Epoch 6/10\n", + "600/600 [==============================] - 426s 709ms/step - loss: 0.0191 - top_1_categorical_accuracy: 0.9943 - val_loss: 0.5453 - val_top_1_categorical_accuracy: 0.8853\n", + "Epoch 7/10\n", + "600/600 [==============================] - 415s 692ms/step - loss: 0.0228 - top_1_categorical_accuracy: 0.9924 - val_loss: 0.5708 - val_top_1_categorical_accuracy: 0.8820\n", + "Epoch 8/10\n", + "600/600 [==============================] - 411s 685ms/step - loss: 0.0186 - top_1_categorical_accuracy: 0.9937 - val_loss: 0.6628 - val_top_1_categorical_accuracy: 0.8757\n", + "Epoch 9/10\n", + "600/600 [==============================] - 447s 746ms/step - loss: 0.0147 - top_1_categorical_accuracy: 0.9950 - val_loss: 0.6357 - val_top_1_categorical_accuracy: 0.8877\n", + "Epoch 10/10\n", + "600/600 [==============================] - 421s 702ms/step - loss: 0.0193 - top_1_categorical_accuracy: 0.9934 - val_loss: 0.7096 - val_top_1_categorical_accuracy: 0.8750\n", + "this is neural_network_spec_file 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000001.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.252051 0.912933 0.342295 \n", + "1 0.034482 0.994000 0.388051 \n", + "2 0.019183 0.996167 0.428428 \n", + "3 0.013406 0.997500 0.485034 \n", + "4 0.018032 0.994333 0.517872 \n", + "5 0.019061 0.994333 0.545307 \n", + "6 0.022755 0.992417 0.570794 \n", + "7 0.018599 0.993667 0.662809 \n", + "8 0.014659 0.995000 0.635707 \n", + "9 0.019295 0.993417 0.709606 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.889000 0 1 \n", + "1 0.884333 0 1 \n", + "2 0.880333 0 1 \n", + "3 0.880333 0 1 \n", + "4 0.880667 0 1 \n", + "5 0.885333 0 1 \n", + "6 0.882000 0 1 \n", + "7 0.875667 0 1 \n", + "8 0.887667 0 1 \n", + "9 0.875000 0 1 \n", + "\n", + " model_name \n", + "0 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_16_16_cerebros_auto_ml_test_cifar10... \n", + "Index(['loss', 'top_1_categorical_accuracy', 'val_loss',\n", + " 'val_top_1_categorical_accuracy', 'trial_number', 'subtrial_number',\n", + " 'model_name'],\n", + " dtype='object')\n", + "metric_to_rank_by is: 'val_top_1_categorical_accuracy'\n", + "Type of metric_to_rank_by is: \n", + "Best result this trial was: 0.9039999842643738\n", + "Type of best result: \n", + "Best model name: 2023_10_21_16_16_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000\n" + ] + } + ], + "source": [ + "%%time\n", + "result = cerebros_automl.run_random_search()" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best accuracy achieved is 0.9039999842643738\n", + "top-1 categorical accuracy\n", + "CPU times: user 147 µs, sys: 301 µs, total: 448 µs\n", + "Wall time: 2.49 ms\n" + ] + } + ], + "source": [ + "print(f'Best accuracy achieved is {result}')\n", + "print(f'top-1 categorical accuracy')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Evaluating the best model found" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [], + "source": [ + "best_model_found = cerebros_automl.get_best_model()" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [], + "source": [ + "#\n", + "eval_loss = tf.keras.losses.CategoricalCrossentropy()\n", + "#\n", + "eval_metrics =\\\n", + "[tf.keras.metrics.TopKCategoricalAccuracy(k=1,\\\n", + " name='eval_top_1_categorical_accuracy'),\n", + " tf.keras.metrics.TopKCategoricalAccuracy(k=5,\\\n", + " name='eval_top_5_categorical_accuracy')\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_3 (Functional) (None, 1024) 7037504 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 0 ['model_3[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_3[0][0]', \n", + " _0000000000000001_tr_0_Den 'model_3[0][0]', \n", + " seUnit_0000000000000001_tr 'model_3[0][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4096) 16384 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 4097 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2052) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_0_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_3[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2052) 8208 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 20530 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 7086723 (27.03 MB)\n", + "Trainable params: 73787 (288.23 KB)\n", + "Non-trainable params: 7012936 (26.75 MB)\n", + "__________________________________________________________________________________________________\n" + ] + } + ], + "source": [ + "best_model_found.compile(loss=eval_loss, metrics=eval_metrics)\n", + "best_model_found.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluating best model found ...\n", + "Loss | Top-1 accuracy | Top-5 accuracy\n", + "313/313 [==============================] - 552s 2s/step - loss: 0.7619 - eval_top_1_categorical_accuracy: 0.8688 - eval_top_5_categorical_accuracy: 0.9933\n" + ] + }, + { + "data": { + "text/plain": [ + "[0.761899471282959, 0.8687999844551086, 0.9933000206947327]" + ] + }, + "execution_count": 76, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"Evaluating best model found ...\")\n", + "print(\"Loss | Top-1 accuracy | Top-5 accuracy\")\n", + "y_test_cat = to_categorical(y_test, 10)\n", + "best_model_found.evaluate(X_test, y_test_cat)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "kubeflow_notebook": { + "autosnapshot": true, + "deploy_config": {}, + "docker_image": "", + "experiment_name": "image-classification-b", + "katib_metadata": { + "algorithm": { + "algorithmName": "bayesianoptimization", + "algorithmSettings": [ + { + "name": "random_state", + "value": "10" + }, + { + "name": "acq_optimizer", + "value": "auto" + }, + { + "name": "acq_func", + "value": "gp_hedge" + }, + { + "name": "base_estimator", + "value": "GP" + } + ] + }, + "maxFailedTrialCount": 10, + "maxTrialCount": 40, + "objective": { + "additionalMetricNames": [], + "goal": 1, + "objectiveMetricName": "val-top-1-categorical-accuracy", + "type": "maximize" + }, + "parallelTrialCount": 3, + "parameters": [ + { + "feasibleSpace": { + "list": [ + "relu", + "elu", + "gelu" + ] + }, + "name": "activation", + "parameterType": "categorical" + }, + { + "feasibleSpace": { + "max": "40", + "min": "1", + "step": "0.1" + }, + "name": "predecessor_level_connection_affinity_factor_first", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "65", + "min": "0.1", + "step": "0.1" + }, + "name": "predecessor_level_connection_affinity_factor_main", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "40", + "min": "2", + "step": "1" + }, + "name": "max_consecutive_lateral_connections", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "50", + "min": "0.1", + "step": "0.1" + }, + "name": "p_lateral_connection", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "40", + "min": "1", + "step": "1" + }, + "name": "num_lateral_connection_tries_per_unit", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "0.3", + "min": "0.00001", + "step": "0.00001" + }, + "name": "learning_rate", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "7", + "min": "1", + "step": "1" + }, + "name": "epochs", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "40", + "min": "5", + "step": "1" + }, + "name": "batch_size", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "8", + "min": "2" + }, + "name": "maximum_levels", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "10", + "min": "2", + "step": "1" + }, + "name": "maximum_units_per_level", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "10", + "min": "2", + "step": "1" + }, + "name": "maximum_neurons_per_unit", + "parameterType": "int" + } + ] + }, + "katib_run": true, + "pipeline_description": "Cifar 10 example", + "pipeline_name": "image-classification-b", + "snapshot_volumes": true, + "volume_access_mode": "rwm", + "volumes": [ + { + "annotations": [], + "mount_point": "/home/jovyan", + "name": "test-images-workspace-pmbn9", + "size": 5, + "size_type": "Gi", + "snapshot": false, + "type": "clone" + } + ] + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/cifar_10_DenseNet.py b/cifar_10_DenseNet.py new file mode 100644 index 00000000..dabe92f2 --- /dev/null +++ b/cifar_10_DenseNet.py @@ -0,0 +1,205 @@ +# Initialization + +import sys +sys.path.insert(0, '../..') + +import tensorflow as tf +import tensorflow_datasets as tfds +from tensorflow.keras.datasets import cifar10, cifar100 +from tensorflow.keras.utils import to_categorical +from tensorflow.keras.models import Model +from tensorflow.keras.layers import Input, Resizing, Lambda, Flatten, Dense +import pandas as pd +import numpy as np +from cerebros.simplecerebrosrandomsearch.simple_cerebros_random_search\ + import SimpleCerebrosRandomSearch +import pendulum +from cerebros.units.units import DenseUnit +from cerebros.denseautomlstructuralcomponent.dense_automl_structural_component\ + import zero_7_exp_decay, zero_95_exp_decay, simple_sigmoid +from ast import literal_eval + +# Download DenseNet121 with Imagenet weights (1000 classes) + +dnet = tf.keras.applications.densenet.DenseNet121(include_top=True, + weights='imagenet', + input_tensor=None, + input_shape=None, + pooling=None, + classes=1000, + classifier_activation='softmax' + ) + +dnet.summary() + +# Make all layers untrainable except for the very last convolutional layer + +for layer in dnet.layers: + layer.trainable = False +dnet.layers[-6].trainable = True + +# Cifar-10 testing + +(X_train, y_train), (X_test, y_test) = cifar10.load_data() + +y_train_cat = to_categorical(y_train, 1000) +y_test_cat = to_categorical(y_test, 1000) + +# Lambda layer for preprocessing + +def preprocess(x): + x = tf.image.resize(x,size=(224,224),method='bicubic') + x = tf.keras.applications.densenet.preprocess_input(x) + return x + +# Modify the model + +input_shape = (32,32,3) + +input_layer = Input(shape=input_shape) +prep = Lambda(preprocess)(input_layer) +out = dnet(prep) +dnet_mod = Model(inputs=input_layer, outputs=out) + +dnet_mod.compile(optimizer='adam', + loss=tf.keras.losses.CategoricalCrossentropy(), + metrics=[tf.keras.metrics.TopKCategoricalAccuracy(k=1, name='top_1_categorical_accuracy')]) + +# Try to fit it on Cifar-10 data and then evaluate (there is no hope this is gonna work ...) + +dnet_mod.fit(X_train, y_train_cat) + +dnet_mod.evaluate(X_test, y_test_cat) + +# Try the same with adding a Cerebros "add-on" network + +INPUT_SHAPES = [input_shape] +OUTPUT_SHAPES = [10] + +# Use 10k-15k random samples from Cifar-10 to speed up the process + +num_samples = 15_000 +rng = np.random.default_rng() +ind = rng.permutation(X_train.shape[0])[:num_samples] + +training_x = [tf.constant(X_train[ind,:,:,:])] +y_train_cat = to_categorical(y_train[ind], 10) +train_labels = [tf.constant(y_train_cat)] + +dnet = tf.keras.applications.densenet.DenseNet121(include_top=True, + weights='imagenet', + input_tensor=None, + input_shape=None, + pooling=None, + classes=1000, + classifier_activation='softmax' + ) + +for layer in dnet.layers: + layer.trainable = False +dnet.layers[-6].trainable = True + +dnet_io = Model(inputs=dnet.layers[0].input, + outputs=dnet.layers[-2].output) + +input_layer = Input(shape=input_shape) +prep = Lambda(preprocess)(input_layer) +out = Flatten()(dnet_io(prep)) +base_mod = Model(inputs=input_layer, outputs=out) + +activation = 'relu' +predecessor_level_connection_affinity_factor_first = 2.0 +predecessor_level_connection_affinity_factor_main = 0.97 +max_consecutive_lateral_connections = 5 +p_lateral_connection = 0.97 +num_lateral_connection_tries_per_unit = 2 +learning_rate = 0.001 +epochs = 10 # [1, 100] +batch_size = 20 +maximum_levels = 4 # [3,7] +maximum_units_per_level = 7 # [2,10] +maximum_neurons_per_unit = 4 # [2,20] + +# Final training task +TIME = pendulum.now(tz='America/New_York').__str__()[:16]\ + .replace('T', '_')\ + .replace(':', '_')\ + .replace('-', '_') +# +PROJECT_NAME = f'{TIME}_cerebros_auto_ml_test_cifar10_densenet' +# +meta_trial_number = 42 +# +cerebros_automl = SimpleCerebrosRandomSearch( + unit_type=DenseUnit, + input_shapes=INPUT_SHAPES, + output_shapes=OUTPUT_SHAPES, + training_data=training_x, + labels=train_labels, + validation_split=0.2, + direction='maximize', + metric_to_rank_by="val_top_1_categorical_accuracy", + minimum_levels=2, + maximum_levels=maximum_levels, + minimum_units_per_level=1, + maximum_units_per_level=maximum_units_per_level, + minimum_neurons_per_unit=1, + maximum_neurons_per_unit=maximum_neurons_per_unit, + activation=activation, + final_activation='softmax', + number_of_architecture_moities_to_try=3, + number_of_tries_per_architecture_moity=2, + minimum_skip_connection_depth=1, + maximum_skip_connection_depth=7, + predecessor_level_connection_affinity_factor_first=predecessor_level_connection_affinity_factor_first, + predecessor_level_connection_affinity_factor_first_rounding_rule='ceil', + predecessor_level_connection_affinity_factor_main=predecessor_level_connection_affinity_factor_main, + predecessor_level_connection_affinity_factor_main_rounding_rule='ceil', + predecessor_level_connection_affinity_factor_decay_main=zero_7_exp_decay, + seed=8675309, + max_consecutive_lateral_connections=max_consecutive_lateral_connections, + gate_after_n_lateral_connections=3, + gate_activation_function=simple_sigmoid, + p_lateral_connection=p_lateral_connection, + p_lateral_connection_decay=zero_95_exp_decay, + num_lateral_connection_tries_per_unit=num_lateral_connection_tries_per_unit, + learning_rate=learning_rate, + loss=tf.keras.losses.CategoricalCrossentropy(), + metrics=[tf.keras.metrics.TopKCategoricalAccuracy( + k=1, name='top_1_categorical_accuracy') + ], + epochs=epochs, + project_name=f"{PROJECT_NAME}_meta_{meta_trial_number}", + model_graphs='model_graphs', + batch_size=batch_size, + meta_trial_number=meta_trial_number, + base_models=[base_mod]) + +# Commented out IPython magic to ensure Python compatibility. +#%%time +result = cerebros_automl.run_random_search() + +print(f'Best accuracy achieved is {result}') +print(f'top-1 categorical accuracy') + +# Evaluating the best model found + +best_model_found = cerebros_automl.get_best_model() + +# +eval_loss = tf.keras.losses.CategoricalCrossentropy() +# +eval_metrics =\ +[tf.keras.metrics.TopKCategoricalAccuracy(k=1,\ + name='eval_top_1_categorical_accuracy'), + tf.keras.metrics.TopKCategoricalAccuracy(k=5,\ + name='eval_top_5_categorical_accuracy') +] + +best_model_found.compile(loss=eval_loss, metrics=eval_metrics) +best_model_found.summary() + +print("Evaluating best model found ...") +print("Loss | Top-1 accuracy | Top-5 accuracy") +y_test_cat = to_categorical(y_test, 10) +best_model_found.evaluate(X_test, y_test_cat) \ No newline at end of file From 32c2bb4fd22c4893b75fe4f529a399e01657162f Mon Sep 17 00:00:00 2001 From: Sasha Kolpakov Date: Tue, 31 Oct 2023 21:14:42 +0100 Subject: [PATCH 2/3] Update cifar_10_DenseNet.py --- cifar_10_DenseNet.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cifar_10_DenseNet.py b/cifar_10_DenseNet.py index dabe92f2..207eb925 100644 --- a/cifar_10_DenseNet.py +++ b/cifar_10_DenseNet.py @@ -1,8 +1,5 @@ # Initialization -import sys -sys.path.insert(0, '../..') - import tensorflow as tf import tensorflow_datasets as tfds from tensorflow.keras.datasets import cifar10, cifar100 @@ -202,4 +199,4 @@ def preprocess(x): print("Evaluating best model found ...") print("Loss | Top-1 accuracy | Top-5 accuracy") y_test_cat = to_categorical(y_test, 10) -best_model_found.evaluate(X_test, y_test_cat) \ No newline at end of file +best_model_found.evaluate(X_test, y_test_cat) From 0b61882f1c8e86b92398114319c197d23461d008 Mon Sep 17 00:00:00 2001 From: Sasha Kolpakov Date: Tue, 31 Oct 2023 21:16:30 +0100 Subject: [PATCH 3/3] Update cifar_10_DenseNet.ipynb --- cifar_10_DenseNet.ipynb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cifar_10_DenseNet.ipynb b/cifar_10_DenseNet.ipynb index afd59d7b..cefe0764 100644 --- a/cifar_10_DenseNet.ipynb +++ b/cifar_10_DenseNet.ipynb @@ -9,16 +9,6 @@ "# Cifar-10 test for image classification" ] }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "sys.path.insert(0, '../..')" - ] - }, { "cell_type": "code", "execution_count": 2,