11_过拟合与学习率衰减.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "11_过拟合与学习率衰减.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mengcius/pytorch-learn/blob/master/11_%E8%BF%87%E6%8B%9F%E5%90%88%E4%B8%8E%E5%AD%A6%E4%B9%A0%E7%8E%87%E8%A1%B0%E5%87%8F.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xh7xdSB-1Qrh",
        "colab_type": "text"
      },
      "source": [
        "## 11_过拟合与学习率衰减"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BEVVyoS_1VHi",
        "colab_type": "text"
      },
      "source": [
        "### 划分train-val-test集来训练\n",
        " 可将原来的train集划分为train集和val集，test集不变。\n",
        " \n",
        " train集用来训练，val集用来挑选模型参数，test仅仅用来评价。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pdBgUU101OIr",
        "colab_type": "code",
        "outputId": "c39d2641-5f4b-4c78-d6f8-2c132663d946",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "import  torch\n",
        "import  torch.nn as nn\n",
        "import  torch.nn.functional as F\n",
        "import  torch.optim as optim\n",
        "from    torchvision import datasets, transforms\n",
        "\n",
        "batch_size=200\n",
        "learning_rate=0.01\n",
        "epochs=10\n",
        "\n",
        "# 加载数据，划分train、val、test集\n",
        "train_db = datasets.MNIST('mnist_data', train=True, download=True,\n",
        "            transform=transforms.Compose([\n",
        "                transforms.ToTensor(),\n",
        "                transforms.Normalize((0.1307,), (0.3081,))]))     \n",
        "# train_loader = torch.utils.data.DataLoader(train_db, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "test_db = datasets.MNIST('mnist_data', train=False, download=True, #train=False，不训练\n",
        "            transform=transforms.Compose([\n",
        "                transforms.ToTensor(),\n",
        "                transforms.Normalize((0.1307,), (0.3081,))]))\n",
        "test_loader = torch.utils.data.DataLoader(test_db, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "print('train:', len(train_db), 'test:', len(test_db))\n",
        "train_db, val_db = torch.utils.data.random_split(train_db, [50000, 10000]) #将原train集随机划分为train和val集\n",
        "print('train_db:', len(train_db), 'val_db:', len(val_db))\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(train_db, batch_size=batch_size, shuffle=True)\n",
        "val_loader = torch.utils.data.DataLoader(val_db, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "\n",
        "# 模型构建\n",
        "class MLP(nn.Module):\n",
        "\n",
        "    def __init__(self):\n",
        "        super(MLP, self).__init__()\n",
        "\n",
        "        self.model = nn.Sequential(\n",
        "            nn.Linear(784, 200),\n",
        "            nn.LeakyReLU(inplace=True),\n",
        "            nn.Linear(200, 200),\n",
        "            nn.LeakyReLU(inplace=True),\n",
        "            nn.Linear(200, 10),\n",
        "            nn.LeakyReLU(inplace=True),\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.model(x)\n",
        "\n",
        "        return x\n",
        "\n",
        "\n",
        "# train集训练\n",
        "device = torch.device('cuda:0')\n",
        "net = MLP().to(device)\n",
        "optimizer = optim.SGD(net.parameters(), lr=learning_rate)\n",
        "# optimizer = optim.SGD(net.parameters(), lr=learning_rate, weight_decay=0.01) #weight_decay就是开L2正则化，设置lambda参数\n",
        "criteon = nn.CrossEntropyLoss().to(device)\n",
        "\n",
        "for epoch in range(epochs):\n",
        "\n",
        "    for batch_idx, (data, target) in enumerate(train_loader):\n",
        "        data = data.view(-1, 28*28)\n",
        "        data, target = data.to(device), target.cuda()\n",
        "\n",
        "        logits = net(data)\n",
        "        loss = criteon(logits, target)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        # print(w1.grad.norm(), w2.grad.norm())\n",
        "        optimizer.step()\n",
        "\n",
        "        if batch_idx % 100 == 0:\n",
        "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
        "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
        "                       100. * batch_idx / len(train_loader), loss.item()))\n",
        "\n",
        "\n",
        "    # val集验证，以挑选出最优的模型\n",
        "    test_loss = 0\n",
        "    correct = 0\n",
        "    for data, target in val_loader:\n",
        "        data = data.view(-1, 28 * 28)\n",
        "        data, target = data.to(device), target.cuda()\n",
        "        logits = net(data)\n",
        "        test_loss += criteon(logits, target).item()\n",
        "\n",
        "        pred = logits.data.max(1)[1]\n",
        "        correct += pred.eq(target.data).sum()\n",
        "\n",
        "    test_loss /= len(val_loader.dataset)\n",
        "    print('\\nVAL set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
        "        test_loss, correct, len(val_loader.dataset),\n",
        "        100. * correct / len(val_loader.dataset)))\n",
        "\n",
        "\n",
        "# test集测试评价\n",
        "#...#这里可加载从val中找到的最佳的模型，这里默认用最后一个\n",
        "test_loss = 0\n",
        "correct = 0\n",
        "for data, target in test_loader:\n",
        "    data = data.view(-1, 28 * 28)\n",
        "    data, target = data.to(device), target.cuda()\n",
        "    logits = net(data)\n",
        "    test_loss += criteon(logits, target).item()\n",
        "\n",
        "    pred = logits.data.max(1)[1]\n",
        "    correct += pred.eq(target.data).sum()\n",
        "\n",
        "test_loss /= len(test_loader.dataset)\n",
        "print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
        "    test_loss, correct, len(test_loader.dataset),\n",
        "    100. * correct / len(test_loader.dataset)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train: 60000 test: 10000\n",
            "train_db: 50000 val_db: 10000\n",
            "Train Epoch: 0 [0/50000 (0%)]\tLoss: 2.305666\n",
            "Train Epoch: 0 [20000/50000 (40%)]\tLoss: 2.178528\n",
            "Train Epoch: 0 [40000/50000 (80%)]\tLoss: 1.900783\n",
            "\n",
            "VAL set: Average loss: 0.0079, Accuracy: 6807/10000 (68%)\n",
            "\n",
            "Train Epoch: 1 [0/50000 (0%)]\tLoss: 1.550711\n",
            "Train Epoch: 1 [20000/50000 (40%)]\tLoss: 1.173832\n",
            "Train Epoch: 1 [40000/50000 (80%)]\tLoss: 1.074587\n",
            "\n",
            "VAL set: Average loss: 0.0049, Accuracy: 7105/10000 (71%)\n",
            "\n",
            "Train Epoch: 2 [0/50000 (0%)]\tLoss: 0.958005\n",
            "Train Epoch: 2 [20000/50000 (40%)]\tLoss: 0.765293\n",
            "Train Epoch: 2 [40000/50000 (80%)]\tLoss: 0.739508\n",
            "\n",
            "VAL set: Average loss: 0.0033, Accuracy: 7978/10000 (79%)\n",
            "\n",
            "Train Epoch: 3 [0/50000 (0%)]\tLoss: 0.666098\n",
            "Train Epoch: 3 [20000/50000 (40%)]\tLoss: 0.674212\n",
            "Train Epoch: 3 [40000/50000 (80%)]\tLoss: 0.412203\n",
            "\n",
            "VAL set: Average loss: 0.0019, Accuracy: 8927/10000 (89%)\n",
            "\n",
            "Train Epoch: 4 [0/50000 (0%)]\tLoss: 0.466578\n",
            "Train Epoch: 4 [20000/50000 (40%)]\tLoss: 0.313460\n",
            "Train Epoch: 4 [40000/50000 (80%)]\tLoss: 0.434926\n",
            "\n",
            "VAL set: Average loss: 0.0017, Accuracy: 9044/10000 (90%)\n",
            "\n",
            "Train Epoch: 5 [0/50000 (0%)]\tLoss: 0.336351\n",
            "Train Epoch: 5 [20000/50000 (40%)]\tLoss: 0.286367\n",
            "Train Epoch: 5 [40000/50000 (80%)]\tLoss: 0.299649\n",
            "\n",
            "VAL set: Average loss: 0.0015, Accuracy: 9124/10000 (91%)\n",
            "\n",
            "Train Epoch: 6 [0/50000 (0%)]\tLoss: 0.287438\n",
            "Train Epoch: 6 [20000/50000 (40%)]\tLoss: 0.292483\n",
            "Train Epoch: 6 [40000/50000 (80%)]\tLoss: 0.280009\n",
            "\n",
            "VAL set: Average loss: 0.0014, Accuracy: 9169/10000 (91%)\n",
            "\n",
            "Train Epoch: 7 [0/50000 (0%)]\tLoss: 0.321975\n",
            "Train Epoch: 7 [20000/50000 (40%)]\tLoss: 0.316500\n",
            "Train Epoch: 7 [40000/50000 (80%)]\tLoss: 0.168006\n",
            "\n",
            "VAL set: Average loss: 0.0014, Accuracy: 9216/10000 (92%)\n",
            "\n",
            "Train Epoch: 8 [0/50000 (0%)]\tLoss: 0.251309\n",
            "Train Epoch: 8 [20000/50000 (40%)]\tLoss: 0.281243\n",
            "Train Epoch: 8 [40000/50000 (80%)]\tLoss: 0.209324\n",
            "\n",
            "VAL set: Average loss: 0.0013, Accuracy: 9243/10000 (92%)\n",
            "\n",
            "Train Epoch: 9 [0/50000 (0%)]\tLoss: 0.173660\n",
            "Train Epoch: 9 [20000/50000 (40%)]\tLoss: 0.261065\n",
            "Train Epoch: 9 [40000/50000 (80%)]\tLoss: 0.264052\n",
            "\n",
            "VAL set: Average loss: 0.0012, Accuracy: 9291/10000 (92%)\n",
            "\n",
            "\n",
            "Test set: Average loss: 0.0011, Accuracy: 9347/10000 (93%)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hvm2LmY2h8eu",
        "colab_type": "text"
      },
      "source": [
        "### 正则化\n",
        "如果没有过拟合的话，设置正则化会使性能急剧下降，因为表达能力还不够。如果过拟合后使用了它，则train不变test性能会上升一些。\n",
        "\n",
        "不加正则化：Test set: Average loss: 0.0011, Accuracy: 9347/10000 (93%)\n",
        "\n",
        "加正则化：Test set: Average loss: 0.0013, Accuracy: 9268/10000 (92%)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0XFTjAGch7zN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = optim.SGD(net.parameters(), lr=learning_rate, weight_decay=0.01) #weight_decay就是开L2正则化，设置lambda参数"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lsC8hKkVoIM3",
        "colab_type": "text"
      },
      "source": [
        "### Dropout"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B4GsdP44oH5U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net_dropped = torch.nn.Sequential(\n",
        "    nn.Linear(784, 200),\n",
        "    nn.Dropout(0.5), #Dropout层：在两层之间丢弃断掉50%的连接，不断掉之前是直连。这不是在层里面断\n",
        "    nn.LeakyReLU(inplace=True),\n",
        "    nn.Linear(200, 200),\n",
        "    nn.Dropout(1), #断掉100%。但在TF相反是保留100%\n",
        "    nn.LeakyReLU(inplace=True),\n",
        "    nn.Linear(200, 10),\n",
        "    # nn.LeakyReLU(inplace=True),\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jbkgUfmGp__S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for epoch in range(epochs):\n",
        "    # train\n",
        "    net_dropped.train()\n",
        "    for batch_idx,(data,target) in enumerate(train_loader):\n",
        "        ...\n",
        "        net_dropped.eval() #在test/val时所有连接都要连上，不能Dropout，切换到eval里去\n",
        "        test_loss=0\n",
        "        correct=0\n",
        "        for data,target in test_loader\n",
        "            ..."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QsVFaBpCknsq",
        "colab_type": "text"
      },
      "source": [
        "### 动量与学习率衰减\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UOAFeoiq6K-j",
        "colab_type": "text"
      },
      "source": [
        "动量\n",
        "\n",
        "学习率衰减监听法"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G_H3dnQei7g5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer=torch.optim.SGD(model.parameters(),args.lr,\n",
        "             momentum=arg.momentum, #加入动量，传入动量的belta参数\n",
        "             weight_decay=args.weight_decay) #L2正则化\n",
        "scheduler=ReduceLROnPlateau(optim,'min') #监听法：min是要监听连续几次loss如果没有下降就减少学习率\n",
        "\n",
        "for epoch in xrange(args.start_epoch,arg.epochs):\n",
        "    train(train_loader,model,criterion,optimizer,epoch)\n",
        "    result_avg,loss_val=validate(val_loader,model,criterion,epoch)\n",
        "    scheduler.step(loss_val) #每调用一次就监听一次loss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "08CLJ_ey6FL1",
        "colab_type": "text"
      },
      "source": [
        "学习率衰减步进法"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r6_jTO2hkorO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "scheduler=StepLR(optimizer,step_size=30,gamma=0.1) #步进法：每30个epoch学习率降低0.1倍\n",
        "for epoch in range(100):\n",
        "    scheduler.step() #调度学习率衰减方法\n",
        "    train(...)\n",
        "    validate(...)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}