update 4.1

Author: TrickyGo

ch4_DL_computation/.ipynb_checkpoints/4.1 build model-checkpoint.ipynb

@@ -1,6 +1,237 @@
 {
- "cells": [],
- "metadata": {},
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4.1.1 build model from block"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2.0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "import tensorflow as tf\n",
+    "import numpy as np\n",
+    "print(tf.__version__)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class MLP(tf.keras.Model):\n",
+    "    def __init__(self):\n",
+    "        super().__init__()\n",
+    "        self.flatten = tf.keras.layers.Flatten()    # Flatten层将除第一维（batch_size）以外的维度展平\n",
+    "        self.dense1 = tf.keras.layers.Dense(units=256, activation=tf.nn.relu)\n",
+    "        self.dense2 = tf.keras.layers.Dense(units=10)\n",
+    "\n",
+    "    def call(self, inputs):         \n",
+    "        x = self.flatten(inputs)   \n",
+    "        x = self.dense1(x)    \n",
+    "        output = self.dense2(x)     \n",
+    "        return output"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<tf.Tensor: id=62, shape=(2, 10), dtype=float32, numpy=\n",
+       "array([[ 0.25095996, -0.24893613, -0.45473456, -0.04311958, -0.2295383 ,\n",
+       "         0.27443463,  0.37376237, -0.14436285, -0.37055504, -0.00098359],\n",
+       "       [ 0.1769592 , -0.14420828, -0.3330693 , -0.14657606, -0.14307055,\n",
+       "         0.27779722,  0.33585754, -0.14960271, -0.38496172,  0.10907309]],\n",
+       "      dtype=float32)>"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "X = tf.random.uniform((2,20))\n",
+    "net = MLP()\n",
+    "net(X)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4.1.2 Sequential"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<tf.Tensor: id=117, shape=(2, 10), dtype=float32, numpy=\n",
+       "array([[ 0.18656988, -0.24001493, -0.22437742, -0.15284519, -0.10276742,\n",
+       "        -0.04557859,  0.49852332,  0.04370471, -0.26982784, -0.14277868],\n",
+       "       [-0.11792175, -0.28876868, -0.4724299 , -0.08530779, -0.17974694,\n",
+       "         0.07656129,  0.53052086,  0.13260414, -0.36211073, -0.08055006]],\n",
+       "      dtype=float32)>"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model = tf.keras.models.Sequential([\n",
+    "    tf.keras.layers.Flatten(),\n",
+    "    tf.keras.layers.Dense(256, activation=tf.nn.relu),\n",
+    "    tf.keras.layers.Dense(10),\n",
+    "])\n",
+    "\n",
+    "model(X)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4.1.3 build complex model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class FancyMLP(tf.keras.Model):\n",
+    "    def __init__(self):\n",
+    "        super().__init__()\n",
+    "        self.flatten = tf.keras.layers.Flatten()\n",
+    "        self.rand_weight = tf.constant(\n",
+    "            tf.random.uniform((20,20)))\n",
+    "        self.dense = tf.keras.layers.Dense(units=20, activation=tf.nn.relu)\n",
+    "\n",
+    "    def call(self, inputs):         \n",
+    "        x = self.flatten(inputs)   \n",
+    "        x = tf.nn.relu(tf.matmul(x, self.rand_weight) + 1)\n",
+    "        x = self.dense(x)    \n",
+    "        while tf.norm(x) > 1:\n",
+    "            x /= 2\n",
+    "        if tf.norm(x) < 0.8:\n",
+    "            x *= 10\n",
+    "        return tf.reduce_sum(x)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<tf.Tensor: id=229, shape=(), dtype=float32, numpy=28.175375>"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "net = FancyMLP()\n",
+    "net(X)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## debugging"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "TypeError",
+     "evalue": "add() takes 2 positional arguments but 4 were given",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[1;32m<ipython-input-7-ac9a7842702c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[0mnet\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSequential\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 15\u001b[1;33m \u001b[0mnet\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mNestMLP\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     16\u001b[0m \u001b[0mnet\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDense\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m20\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m \u001b[0mnet\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mFancyMLP\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32m<ipython-input-7-ac9a7842702c>\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m      5\u001b[0m         self.net.add(tf.keras.layers.Flatten(),\n\u001b[0;32m      6\u001b[0m             \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDense\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m64\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mactivation\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m             tf.keras.layers.Dense(32, activation=tf.nn.relu))\n\u001b[0m\u001b[0;32m      8\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdense\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDense\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0munits\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m16\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mactivation\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32mD:\\dev_tools\\Anaconda3_2019.07\\envs\\tf2.0\\lib\\site-packages\\tensorflow_core\\python\\training\\tracking\\base.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    455\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_self_setattr_tracking\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    456\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 457\u001b[1;33m       \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    458\u001b[0m     \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    459\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_self_setattr_tracking\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mprevious_value\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;31mTypeError\u001b[0m: add() takes 2 positional arguments but 4 were given"
+     ]
+    }
+   ],
+   "source": [
+    "class NestMLP(tf.keras.Model):\n",
+    "    def __init__(self):\n",
+    "        super().__init__()\n",
+    "        self.net = tf.keras.Sequential()\n",
+    "        self.net.add(tf.keras.layers.Flatten(),\n",
+    "            tf.keras.layers.Dense(64, activation=tf.nn.relu),\n",
+    "            tf.keras.layers.Dense(32, activation=tf.nn.relu))\n",
+    "        self.dense = tf.keras.layers.Dense(units=16, activation=tf.nn.relu)\n",
+    "\n",
+    "    \n",
+    "    def call(self, inputs):         \n",
+    "        return self.dense(self.net(inputs))\n",
+    "\n",
+    "net = tf.keras.Sequential()\n",
+    "net.add(NestMLP())\n",
+    "net.add(tf.keras.layers.Dense(20))\n",
+    "net.add(FancyMLP())\n",
+    "\n",
+    "net(X)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "tf2.0",
+   "language": "python",
+   "name": "tf2.0"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
  "nbformat": 4,
  "nbformat_minor": 2
 }

ch4_DL_computation/.ipynb_checkpoints/4.2 parameters' access, initialization and sharing-checkpoint.ipynb

@@ -0,0 +1,133 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2.0.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "import tensorflow as tf\n",
+    "import numpy as np\n",
+    "print(tf.__version__)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<tf.Tensor: id=62, shape=(2, 10), dtype=float32, numpy=\n",
+       "array([[ 0.47627547,  0.12908635, -0.04767011,  0.03762572, -0.22968994,\n",
+       "         0.07292195, -0.20956676,  0.24949205,  0.14892189, -0.14833519],\n",
+       "       [ 0.391208  ,  0.26524487, -0.05629939,  0.00716697, -0.06539508,\n",
+       "         0.17122187, -0.16350499,  0.5076758 ,  0.07857557, -0.05519376]],\n",
+       "      dtype=float32)>"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "net = tf.keras.models.Sequential()\n",
+    "net.add(tf.keras.layers.Flatten())\n",
+    "net.add(tf.keras.layers.Dense(256,activation=tf.nn.relu))\n",
+    "net.add(tf.keras.layers.Dense(10))\n",
+    "\n",
+    "X = tf.random.uniform((2,20))\n",
+    "Y = net(X)\n",
+    "Y"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 4.2.1 access model parameters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(<tf.Variable 'sequential/dense/kernel:0' shape=(20, 256) dtype=float32, numpy=\n",
+       " array([[-2.02483535e-02,  2.85128355e-02, -7.38415122e-03, ...,\n",
+       "          7.84173608e-03, -1.25107020e-02,  8.92712176e-02],\n",
+       "        [ 2.29488611e-02, -1.39583871e-01, -1.23225398e-01, ...,\n",
+       "          7.12196082e-02, -2.21055746e-03, -4.10574600e-02],\n",
+       "        [ 5.77151179e-02, -1.44912243e-01,  1.00331202e-01, ...,\n",
+       "         -1.19500108e-01, -7.26372823e-02, -1.00700825e-01],\n",
+       "        ...,\n",
+       "        [ 1.46352559e-01,  3.99160534e-02, -1.04723126e-01, ...,\n",
+       "          1.44056857e-01, -6.36567399e-02, -4.77666780e-02],\n",
+       "        [-2.45300457e-02, -7.79614747e-02, -6.78218603e-02, ...,\n",
+       "          1.62270069e-02, -1.57367885e-02, -7.33146816e-02],\n",
+       "        [-2.91189030e-02,  1.42401308e-01, -1.06634945e-02, ...,\n",
+       "         -7.19291270e-02, -2.29924917e-05,  1.06444657e-01]], dtype=float32)>,\n",
+       " tensorflow.python.ops.resource_variable_ops.ResourceVariable)"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "net.weights[0], type(net.weights[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## initialize params"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "tf2.0",
+   "language": "python",
+   "name": "tf2.0"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}