Three_model_training_method.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 第一种方法直接用keras的fit方法，以fashion mnist为例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import time\n",
    "import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "from functools import partial\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 配置超参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=64\n",
    "epochs=10\n",
    "regularizer=1e-3\n",
    "total_train_samples=60000\n",
    "total_test_samples=10000\n",
    "lr_decay_epochs=1\n",
    "output_folder=\"./model_output\"\n",
    "#用来保存模型以及我们需要的所有东西\n",
    "if not os.path.exists(output_folder):\n",
    "    os.makedirs(output_folder)\n",
    "save_format=\"hdf5\" #或saved_model\n",
    "if save_format==\"hdf5\":\n",
    "    save_path=os.path.join(output_folder,\"hdf5_models\")\n",
    "    if not os.path.exists(save_path):\n",
    "        os.makedirs(save_path)\n",
    "    save_path=os.path.join(output_folder,\"ckpt_{epoch:02d}_{val_accuracy:.3f}.hdf5\")\n",
    "elif save_format==\"saved_model\":\n",
    "    save_path=os.path.join(output_folder,\"saved_models\")\n",
    "    if not os.path.exists(save_path):\n",
    "        os.makedirs(save_path)\n",
    "    save_path=os.path.join(output_folder,\"ckpt_{epoch:02d}_{val_accuracy:.3f}.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 选择指定显卡及自动调用显存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All the available GPUs:\n",
      " [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n"
     ]
    }
   ],
   "source": [
    "physical_devices = tf.config.experimental.list_physical_devices('GPU')#列出所有可见显卡\n",
    "print(\"All the available GPUs:\\n\",physical_devices)\n",
    "if physical_devices:\n",
    "    gpu=physical_devices[0]#显示第一块显卡\n",
    "    tf.config.experimental.set_memory_growth(gpu, True)#根据需要自动增长显存\n",
    "    tf.config.experimental.set_visible_devices(gpu, 'GPU')#只选择第一块"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fashion_mnist=tf.keras.datasets.fashion_mnist\n",
    "(train_x,train_y),(test_x,test_y)=fashion_mnist.load_data()\n",
    "\n",
    "train_x,test_x = train_x[...,np.newaxis]/255.0,test_x[...,np.newaxis]/255.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用tf.data来准备训练集和测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = tf.data.Dataset.from_tensor_slices((train_x,train_y))\n",
    "test_ds = tf.data.Dataset.from_tensor_slices((test_x,test_y))\n",
    " \n",
    "train_ds=train_ds.shuffle(buffer_size=batch_size*10).batch(batch_size).prefetch(buffer_size = tf.data.experimental.AUTOTUNE).repeat()\n",
    "test_ds = test_ds.batch(batch_size).prefetch(buffer_size = tf.data.experimental.AUTOTUNE).repeat()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 准备模型定义"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "l2 = tf.keras.regularizers.l2(regularizer)#定义模型正则化方法\n",
    "ini = tf.keras.initializers.he_normal()#定义参数初始化方法\n",
    "conv2d = partial(tf.keras.layers.Conv2D,activation='relu',padding='same',kernel_regularizer=l2,bias_regularizer=l2)\n",
    "fc = partial(tf.keras.layers.Dense,activation='relu',kernel_regularizer=l2,bias_regularizer=l2)\n",
    "maxpool=tf.keras.layers.MaxPooling2D\n",
    "dropout=tf.keras.layers.Dropout\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 开始定义模型,用functional方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_input = tf.keras.layers.Input(shape=(28,28,1))\n",
    "x = conv2d(128,(5,5))(x_input)\n",
    "x = maxpool((2,2))(x)\n",
    "x = conv2d(256,(5,5))(x)\n",
    "x = maxpool((2,2))(x)\n",
    "x = tf.keras.layers.Flatten()(x)\n",
    "x = fc(128)(x)\n",
    "x_output=fc(10,activation=None)(x)\n",
    "model = tf.keras.models.Model(inputs=x_input,outputs=x_output)                \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 打印模型结构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model architure:\n",
      "\n",
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         [(None, 28, 28, 1)]       0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 28, 28, 128)       3328      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, 14, 14, 256)       819456    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2 (None, 7, 7, 256)         0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 12544)             0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 128)               1605760   \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 2,429,834\n",
      "Trainable params: 2,429,834\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(\"The model architure:\\n\")\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义优化算法和损失函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#学习率变化设置，使用指数衰减\n",
    "train_steps_per_epoch=int(total_train_samples//batch_size)\n",
    "initial_learning_rate=0.01\n",
    "# lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate,\n",
    "#                                                              decay_steps=1*train_steps_per_epoch,\n",
    "#                                                             decay_rate=0.96,\n",
    "#                                                             staircase=True)#initial_learning_rate*0.96**(step/decay_steps)\n",
    "#优化算法\n",
    "optimizer = tf.keras.optimizers.SGD(learning_rate=initial_learning_rate,momentum=0.95)\n",
    "#损失函数\n",
    "loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "#评价指标\n",
    "#metrics=['accuracy','sparse_categorical_crossentropy']#第二个会返回交叉熵的结果，用loss减去该值就会得到正则化的值\n",
    "metrics=[tf.keras.metrics.Accuracy(),tf.keras.metrics.SparseCategoricalCrossentropy()]#这两种定义方式都可以\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 编译模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=optimizer,loss=loss,metrics=metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 定义callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 要有模型checkpoints,我们先用hdf5格式，只有一个文件简单易用，如果要用saved_model，也可以加载后再转存成saved_model,过程简单\n",
    "#我们做两种试验,保存不同的模型结构\n",
    "ckpt = tf.keras.callbacks.ModelCheckpoint(save_path,monitor='val_accuracy',verbose=1,\n",
    "                                         save_best_only=False,save_weights_only=False,\n",
    "                                         save_frequency=1)#参数具体函意，查看官方文档\n",
    "#当模型训练不符合我们要求时停止训练\n",
    "earlystop=tf.keras.callbacks.EarlyStopping(monitor='val_accuracy',min_delta = 0.00001,patience=5)#连续5个epoch验证集精度没有提高0.001%停\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 自定义学习率衰减并打印出来"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LearningRateExponentialDecay(tf.keras.optimizers.schedules.ExponentialDecay):\n",
    "    def __init__(self,initial_learning_rate,decay_steps,decay_rate,staircase=False,name=None):\n",
    "        super().__init__(initial_learning_rate,decay_steps,decay_rate,staircase=False,name=None)\n",
    "    #在原来\n",
    "    def plot(self,epochs,title=\"Learning Rate Schedule\"):\n",
    "        #计算学习率随epoch的变化值\n",
    "        lrs = [self(i) for i in epochs]\n",
    "        plt.figure()\n",
    "        plt.plot(epochs,lrs)\n",
    "        plt.title(title)\n",
    "        plt.xlable(\"Epoch #\")\n",
    "        plt.ylable(\"Learning Rate\")\n",
    "    \n",
    "lr_schedule = LearningRateExponentialDecay(initial_learning_rate,\n",
    "                                                             decay_steps=lr_decay_epochs*train_steps_per_epoch,\n",
    "                                                            decay_rate=0.96,\n",
    "                                                            staircase=True)#initial_learning_rate*0.96**(step/decay_steps)\n",
    "lr = tf.keras.callbacks.LearningRateScheduler(lr_schedule)\n",
    "#把学习率在每个epoch结速的时候打印\n",
    "class PrintLR(tf.keras.callbacks.Callback):\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        print('\\nLearning rate for epoch {} is {}'.format(epoch + 1,\n",
    "                                                      model.optimizer.lr.numpy()))\n",
    "print_lr = PrintLR()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#还要加入tensorboard的使用,这种方法记录的内容有限\n",
    "log_dir= os.path.join(output_folder,'logs_{}'.format(datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")))\n",
    "tensorboard = tf.keras.callbacks.TensorBoard(log_dir=log_dir)\n",
    "\n",
    "# class Tensorboard(tf.keras.callbacks.TensorBoard):\n",
    "#     def on_epoch_end(self,epoch,logs=None)\n",
    "#         super().on_epoch_end(epoch,logs=None)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "datetime.datetime.now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on class LearningRateExponentialDecay in module __main__:\n",
      "\n",
      "class LearningRateExponentialDecay(tensorflow.python.keras.optimizer_v2.learning_rate_schedule.ExponentialDecay)\n",
      " |  A LearningRateSchedule that uses an exponential decay schedule.\n",
      " |  \n",
      " |  Method resolution order:\n",
      " |      LearningRateExponentialDecay\n",
      " |      tensorflow.python.keras.optimizer_v2.learning_rate_schedule.ExponentialDecay\n",
      " |      tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule\n",
      " |      builtins.object\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  plot(self, epochs, title='Learning Rate Schedule')\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from tensorflow.python.keras.optimizer_v2.learning_rate_schedule.ExponentialDecay:\n",
      " |  \n",
      " |  __call__(self, step)\n",
      " |      Call self as a function.\n",
      " |  \n",
      " |  __init__(self, initial_learning_rate, decay_steps, decay_rate, staircase=False, name=None)\n",
      " |      Applies exponential decay to the learning rate.\n",
      " |      \n",
      " |      When training a model, it is often recommended to lower the learning rate as\n",
      " |      the training progresses. This schedule applies an exponential decay function\n",
      " |      to an optimizer step, given a provided initial learning rate.\n",
      " |      \n",
      " |      The schedule a 1-arg callable that produces a decayed learning\n",
      " |      rate when passed the current optimizer step. This can be useful for changing\n",
      " |      the learning rate value across different invocations of optimizer functions.\n",
      " |      It is computed as:\n",
      " |      \n",
      " |      ```python\n",
      " |      def decayed_learning_rate(step):\n",
      " |        return initial_learning_rate * decay_rate ^ (step / decay_steps)\n",
      " |      ```\n",
      " |      \n",
      " |      If the argument `staircase` is `True`, then `step / decay_steps` is\n",
      " |      an integer division and the decayed learning rate follows a\n",
      " |      staircase function.\n",
      " |      \n",
      " |      You can pass this schedule directly into a `tf.keras.optimizers.Optimizer`\n",
      " |      as the learning rate.\n",
      " |      Example: When fitting a Keras model, decay every 100000 steps with a base\n",
      " |      of 0.96:\n",
      " |      \n",
      " |      ```python\n",
      " |      initial_learning_rate = 0.1\n",
      " |      lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n",
      " |          initial_learning_rate,\n",
      " |          decay_steps=100000,\n",
      " |          decay_rate=0.96,\n",
      " |          staircase=True)\n",
      " |      \n",
      " |      model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=lr_schedule),\n",
      " |                    loss='sparse_categorical_crossentropy',\n",
      " |                    metrics=['accuracy'])\n",
      " |      \n",
      " |      model.fit(data, labels, epochs=5)\n",
      " |      ```\n",
      " |      \n",
      " |      The learning rate schedule is also serializable and deserializable using\n",
      " |      `tf.keras.optimizers.schedules.serialize` and\n",
      " |      `tf.keras.optimizers.schedules.deserialize`.\n",
      " |      \n",
      " |      Args:\n",
      " |        initial_learning_rate: A scalar `float32` or `float64` `Tensor` or a\n",
      " |          Python number.  The initial learning rate.\n",
      " |        decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.\n",
      " |          Must be positive.  See the decay computation above.\n",
      " |        decay_rate: A scalar `float32` or `float64` `Tensor` or a\n",
      " |          Python number.  The decay rate.\n",
      " |        staircase: Boolean.  If `True` decay the learning rate at discrete\n",
      " |          intervals\n",
      " |        name: String.  Optional name of the operation.  Defaults to\n",
      " |          'ExponentialDecay'.\n",
      " |      \n",
      " |      Returns:\n",
      " |        A 1-arg callable learning rate schedule that takes the current optimizer\n",
      " |        step and outputs the decayed learning rate, a scalar `Tensor` of the same\n",
      " |        type as `initial_learning_rate`.\n",
      " |  \n",
      " |  get_config(self)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Class methods inherited from tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule:\n",
      " |  \n",
      " |  from_config(config) from builtins.type\n",
      " |      Instantiates a `LearningRateSchedule` from its config.\n",
      " |      \n",
      " |      Args:\n",
      " |          config: Output of `get_config()`.\n",
      " |      \n",
      " |      Returns:\n",
      " |          A `LearningRateSchedule` instance.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from tensorflow.python.keras.optimizer_v2.learning_rate_schedule.LearningRateSchedule:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(LearningRateExponentialDecay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on class ExponentialDecay in module tensorflow.python.keras.optimizer_v2.learning_rate_schedule:\n",
      "\n",
      "class ExponentialDecay(LearningRateSchedule)\n",
      " |  A LearningRateSchedule that uses an exponential decay schedule.\n",
      " |  \n",
      " |  Method resolution order:\n",
      " |      ExponentialDecay\n",
      " |      LearningRateSchedule\n",
      " |      builtins.object\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __call__(self, step)\n",
      " |      Call self as a function.\n",
      " |  \n",
      " |  __init__(self, initial_learning_rate, decay_steps, decay_rate, staircase=False, name=None)\n",
      " |      Applies exponential decay to the learning rate.\n",
      " |      \n",
      " |      When training a model, it is often recommended to lower the learning rate as\n",
      " |      the training progresses. This schedule applies an exponential decay function\n",
      " |      to an optimizer step, given a provided initial learning rate.\n",
      " |      \n",
      " |      The schedule a 1-arg callable that produces a decayed learning\n",
      " |      rate when passed the current optimizer step. This can be useful for changing\n",
      " |      the learning rate value across different invocations of optimizer functions.\n",
      " |      It is computed as:\n",
      " |      \n",
      " |      ```python\n",
      " |      def decayed_learning_rate(step):\n",
      " |        return initial_learning_rate * decay_rate ^ (step / decay_steps)\n",
      " |      ```\n",
      " |      \n",
      " |      If the argument `staircase` is `True`, then `step / decay_steps` is\n",
      " |      an integer division and the decayed learning rate follows a\n",
      " |      staircase function.\n",
      " |      \n",
      " |      You can pass this schedule directly into a `tf.keras.optimizers.Optimizer`\n",
      " |      as the learning rate.\n",
      " |      Example: When fitting a Keras model, decay every 100000 steps with a base\n",
      " |      of 0.96:\n",
      " |      \n",
      " |      ```python\n",
      " |      initial_learning_rate = 0.1\n",
      " |      lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n",
      " |          initial_learning_rate,\n",
      " |          decay_steps=100000,\n",
      " |          decay_rate=0.96,\n",
      " |          staircase=True)\n",
      " |      \n",
      " |      model.compile(optimizer=tf.keras.optimizers.SGD(learning_rate=lr_schedule),\n",
      " |                    loss='sparse_categorical_crossentropy',\n",
      " |                    metrics=['accuracy'])\n",
      " |      \n",
      " |      model.fit(data, labels, epochs=5)\n",
      " |      ```\n",
      " |      \n",
      " |      The learning rate schedule is also serializable and deserializable using\n",
      " |      `tf.keras.optimizers.schedules.serialize` and\n",
      " |      `tf.keras.optimizers.schedules.deserialize`.\n",
      " |      \n",
      " |      Args:\n",
      " |        initial_learning_rate: A scalar `float32` or `float64` `Tensor` or a\n",
      " |          Python number.  The initial learning rate.\n",
      " |        decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.\n",
      " |          Must be positive.  See the decay computation above.\n",
      " |        decay_rate: A scalar `float32` or `float64` `Tensor` or a\n",
      " |          Python number.  The decay rate.\n",
      " |        staircase: Boolean.  If `True` decay the learning rate at discrete\n",
      " |          intervals\n",
      " |        name: String.  Optional name of the operation.  Defaults to\n",
      " |          'ExponentialDecay'.\n",
      " |      \n",
      " |      Returns:\n",
      " |        A 1-arg callable learning rate schedule that takes the current optimizer\n",
      " |        step and outputs the decayed learning rate, a scalar `Tensor` of the same\n",
      " |        type as `initial_learning_rate`.\n",
      " |  \n",
      " |  get_config(self)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Class methods inherited from LearningRateSchedule:\n",
      " |  \n",
      " |  from_config(config) from builtins.type\n",
      " |      Instantiates a `LearningRateSchedule` from its config.\n",
      " |      \n",
      " |      Args:\n",
      " |          config: Output of `get_config()`.\n",
      " |      \n",
      " |      Returns:\n",
      " |          A `LearningRateSchedule` instance.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from LearningRateSchedule:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(tf.keras.optimizers.schedules.ExponentialDecay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=605, shape=(), dtype=float32, numpy=9.260999>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.math.pow(2.1,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf2",
   "language": "python",
   "name": "tf2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}