CNNLec10_6_ResNet_for_cifar10.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CNNLec10-6_ResNet_for_cifar10.ipynb",
      "provenance": [],
      "mount_file_id": "16ACf0ZRJb0o6_o5iMw4_mBqaaDsQN5NW",
      "authorship_tag": "ABX9TyPf5KTwPPq++NT/93o1Li4P",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "2317039d5800482dbb19c246529d7dca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5c1145cc37114cb6b5bbab6f90435344",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dd350f20014f44c5908ec6a9cddf1096",
              "IPY_MODEL_95d2b476e57a43639005568f2fd8b752"
            ]
          }
        },
        "5c1145cc37114cb6b5bbab6f90435344": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd350f20014f44c5908ec6a9cddf1096": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9dba5d996bc34bd8bee27661fac9e33f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_01a9f6a5dd3249249c5664c10ac716ce"
          }
        },
        "95d2b476e57a43639005568f2fd8b752": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f1f7409512e44bf6bac0d26d0f76306c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170500096/? [00:30&lt;00:00, 13481123.03it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_14fa1043175f44839d029db995aa955a"
          }
        },
        "9dba5d996bc34bd8bee27661fac9e33f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "01a9f6a5dd3249249c5664c10ac716ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f1f7409512e44bf6bac0d26d0f76306c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "14fa1043175f44839d029db995aa955a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6fdaa31801874bed84a89a0a0bb6d885": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_173e80657bb84405828b0e5969ba39fe",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_11a280fbb0b4469bb9eab65c1dac80d8",
              "IPY_MODEL_a72ff39784c34086ac864c624e595a2f"
            ]
          }
        },
        "173e80657bb84405828b0e5969ba39fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "11a280fbb0b4469bb9eab65c1dac80d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d77c5f526859482a9154f3015d621e46",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b89cf8e7cfcd469d943f00c0adea26e1"
          }
        },
        "a72ff39784c34086ac864c624e595a2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7b516b8e6f7b4de0a36c7b0d9f358c52",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170500096/? [00:32&lt;00:00, 13363279.27it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a0dcdd6d64994322a721bfdbe1505b98"
          }
        },
        "d77c5f526859482a9154f3015d621e46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b89cf8e7cfcd469d943f00c0adea26e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7b516b8e6f7b4de0a36c7b0d9f358c52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a0dcdd6d64994322a721bfdbe1505b98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/khodid/2020_SAI_MONING2/blob/master/CNNLec10_6_ResNet_for_cifar10.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xq96FM5weROj",
        "colab_type": "text"
      },
      "source": [
        "# Resnet을 cifar에\n",
        "\n",
        "적용해서 학습을 진행해보자.\n",
        "\n",
        "colab에서 visdom을 쓰기는 어려운 것 같으니\n",
        "\n",
        "일단은 visdom 관련 코드는 지우고 진행하겠다.\n",
        "\n",
        "anaconda와 jupyter notebook을 예전에 받아놓은 적이 있어서 이걸 쓰면 되긴 하는데.. PyTorch 환경에서는 돌려본 적이 없어서 세팅할 것이 많다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fgx8Vc3PdhuN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "import torch.optim as optim"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_lAoyYV3I9A3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "outputId": "5e90b1f6-b0c7-4f6c-fc18-4eea786f5fb6"
      },
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nQ3UxfeYwtQU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "torch.manual_seed(777)\n",
        "if device =='cuda':\n",
        "    torch.cuda.manual_seed_all(777)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cV_TxgIHxLBN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 172,
          "referenced_widgets": [
            "2317039d5800482dbb19c246529d7dca",
            "5c1145cc37114cb6b5bbab6f90435344",
            "dd350f20014f44c5908ec6a9cddf1096",
            "95d2b476e57a43639005568f2fd8b752",
            "9dba5d996bc34bd8bee27661fac9e33f",
            "01a9f6a5dd3249249c5664c10ac716ce",
            "f1f7409512e44bf6bac0d26d0f76306c",
            "14fa1043175f44839d029db995aa955a"
          ]
        },
        "outputId": "a8d244fb-7460-4a65-8af8-366b753a8d44"
      },
      "source": [
        "transform = transforms.Compose([\n",
        "    transforms.ToTensor()\n",
        "])\n",
        "\n",
        "# 테스트용 데이터셋 가져오기\n",
        "trainset = torchvision.datasets.CIFAR10(root='./data/cifar10', train=True, download=True, transform=transform)\n",
        "\n",
        "train_data = trainset.data # 강의 github에 있던 train_data 부분이 현재는 그냥 'data'라는 이름인 것 같다.\n",
        "print(train_data.shape)\n",
        "\n",
        "# 계산하는 방법\n",
        "train_data_mean = train_data.mean( axis=(0,1,2) )  # 각 축의 평균\n",
        "train_data_std = train_data.std( axis=(0,1,2) )    # 각 축의 표준 편차\n",
        "\n",
        "\n",
        "print(train_data_mean)\n",
        "print(train_data_std)\n",
        "\n",
        "train_data_mean = train_data_mean / 255                    # 평균을 255로 나눔\n",
        "train_data_std = train_data_std / 255                      # 표준편차를 255로 나눔\n",
        "\n",
        "print(train_data_mean)\n",
        "print(train_data_std)\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar10/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2317039d5800482dbb19c246529d7dca",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/cifar10/cifar-10-python.tar.gz to ./data/cifar10\n",
            "(50000, 32, 32, 3)\n",
            "[125.30691805 122.95039414 113.86538318]\n",
            "[62.99321928 62.08870764 66.70489964]\n",
            "[0.49139968 0.48215841 0.44653091]\n",
            "[0.24703223 0.24348513 0.26158784]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "59eXsmyzxL_8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102,
          "referenced_widgets": [
            "6fdaa31801874bed84a89a0a0bb6d885",
            "173e80657bb84405828b0e5969ba39fe",
            "11a280fbb0b4469bb9eab65c1dac80d8",
            "a72ff39784c34086ac864c624e595a2f",
            "d77c5f526859482a9154f3015d621e46",
            "b89cf8e7cfcd469d943f00c0adea26e1",
            "7b516b8e6f7b4de0a36c7b0d9f358c52",
            "a0dcdd6d64994322a721bfdbe1505b98"
          ]
        },
        "outputId": "5f12ef35-a87f-4d8d-8811-9aa119dd384a"
      },
      "source": [
        "transform_train = transforms.Compose([\n",
        "    transforms.RandomCrop(32, padding=4),     # padding=4로 둘러준 후에 랜덤하게 32만큼 뜯어오겠다는 뜻. \n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(train_data_mean, train_data_std)\n",
        "])\n",
        "\n",
        "transform_test = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(train_data_mean, train_data_std)\n",
        "])\n",
        "\n",
        "trainset = torchvision.datasets.CIFAR10(root='./cifar10', train=True,\n",
        "                                        download=True, transform=transform_train)\n",
        "trainloader = torch.utils.data.DataLoader(trainset, batch_size=256,\n",
        "                                          shuffle=True, num_workers=0)\n",
        "\n",
        "testset = torchvision.datasets.CIFAR10(root='./cifar10', train=False,\n",
        "                                       download=True, transform=transform_test)\n",
        "\n",
        "testloader = torch.utils.data.DataLoader(testset, batch_size=256,\n",
        "                                         shuffle=False, num_workers=0)\n",
        "\n",
        "classes = ('plane', 'car', 'bird', 'cat',\n",
        "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar10/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6fdaa31801874bed84a89a0a0bb6d885",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./cifar10/cifar-10-python.tar.gz to ./cifar10\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v0KUFnFo2ggv",
        "colab_type": "text"
      },
      "source": [
        "위 까지가 데이터를 받아오는 과정이었고,\n",
        "\n",
        "이 다음 resnet부터가 주목할 부분이다.\n",
        "\n",
        "일단 colab 환경에서 resnet.py를 불러오는 거 자체에서 좀 고민을 했는데... 강의 일시정시 시켜놓고 한참 알아보고 겨우 해결하니깐 정지시킨 부분 바로 5초 뒤에 \n",
        "```python \n",
        "import torchvision.models.restnet as resnet\n",
        "```\n",
        "을 알려준다... ^^.\n",
        "\n",
        "암튼 알아보는 과정에서 resnet에 대한 분석 설명이 잘 되어있는 [문서](https://colab.research.google.com/github/d2l-ai/d2l-en-colab/blob/master/chapter_convolutional-modern/resnet.ipynb#scrollTo=cD3nWXgM7D_C)를 하나 건졌는데, 이걸 작성한 [dive into deep github](https://github.com/d2l-ai/d2l-en)에 유용한 자료가 꽤 있다. 외국의 모딥 같은 사이트인 것 같은데...\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9FDVxEQjxNj5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torchvision.models.resnet as resnet"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PGGAtHesxOvU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# resnet을 'as resnet' 없이 그대로 torchvision에서 import할 경우엔 생략해도 되지 않나...? 라고 생각했는데 아니었음\n",
        "conv1x1=resnet.conv1x1\n",
        "Bottleneck = resnet.Bottleneck\n",
        "BasicBlock= resnet.BasicBlock"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EjW4O-OxxTrg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 모딥 강의에서는 원래의 선언에서 변형을 가했다.\n",
        "# cifar에서 사용하는 것은 resnet이 목표하던 224x224보다 훨씬 작은 32x32크기의 이미지이기 때문.\n",
        "# 때문에 padding도 줄였고, maxpooling도 빼고, 레이어도 줄였다.\n",
        "# size도 32x32에 맞춰서 바꾸었다.\n",
        "\n",
        "class ResNet(nn.Module):\n",
        "\n",
        "    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False):\n",
        "        super(ResNet, self).__init__()\n",
        "        self.inplanes = 16\n",
        "        self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1,\n",
        "                               bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(16)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        #self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
        "        \n",
        "        self.layer1 = self._make_layer(block, 16, layers[0], stride=1)\n",
        "        self.layer2 = self._make_layer(block, 32, layers[1], stride=1)\n",
        "        self.layer3 = self._make_layer(block, 64, layers[2], stride=2)\n",
        "        self.layer4 = self._make_layer(block, 128, layers[3], stride=2)\n",
        "        \n",
        "        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n",
        "        self.fc = nn.Linear(128 * block.expansion, num_classes)\n",
        "\n",
        "        for m in self.modules():\n",
        "            if isinstance(m, nn.Conv2d):\n",
        "                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n",
        "            elif isinstance(m, nn.BatchNorm2d):\n",
        "                nn.init.constant_(m.weight, 1)\n",
        "                nn.init.constant_(m.bias, 0)\n",
        "\n",
        "        # Zero-initialize the last BN in each residual branch,\n",
        "        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n",
        "        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n",
        "        if zero_init_residual:\n",
        "            for m in self.modules():\n",
        "                if isinstance(m, Bottleneck):\n",
        "                    nn.init.constant_(m.bn3.weight, 0)\n",
        "                elif isinstance(m, BasicBlock):\n",
        "                    nn.init.constant_(m.bn2.weight, 0)\n",
        "\n",
        "    def _make_layer(self, block, planes, blocks, stride=1):\n",
        "        downsample = None\n",
        "        if stride != 1 or self.inplanes != planes * block.expansion:\n",
        "            downsample = nn.Sequential(\n",
        "                conv1x1(self.inplanes, planes * block.expansion, stride),\n",
        "                nn.BatchNorm2d(planes * block.expansion),\n",
        "            )\n",
        "\n",
        "        layers = []\n",
        "        layers.append(block(self.inplanes, planes, stride, downsample))\n",
        "        self.inplanes = planes * block.expansion\n",
        "        for _ in range(1, blocks):\n",
        "            layers.append(block(self.inplanes, planes))\n",
        "\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.conv1(x)\n",
        "        #x.shape =[1, 16, 32,32]\n",
        "        x = self.bn1(x)\n",
        "        x = self.relu(x)\n",
        "        #x = self.maxpool(x)\n",
        "\n",
        "        x = self.layer1(x)\n",
        "        #x.shape =[1, 128, 32,32]\n",
        "        x = self.layer2(x)\n",
        "        #x.shape =[1, 256, 32,32]\n",
        "        x = self.layer3(x)\n",
        "        #x.shape =[1, 512, 16,16]\n",
        "        x = self.layer4(x)\n",
        "        #x.shape =[1, 1024, 8,8]\n",
        "        \n",
        "        x = self.avgpool(x)\n",
        "        x = x.view(x.size(0), -1)\n",
        "        x = self.fc(x)\n",
        "\n",
        "        return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BQzxXP9LxUOi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "resnet50 = ResNet(resnet.Bottleneck, [3, 4, 6, 3], 10, True).to(device) \n",
        "#1(conv1) + 9(layer1) + 12(layer2) + 18(layer3) + 9(layer4) +1(fc)= ResNet50"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_NJK4f2YxVnf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "38f028bd-bf97-4ed2-e6d6-34a53fd1e632"
      },
      "source": [
        "resnet50"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ResNet(\n",
              "  (conv1): Conv2d(3, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "  (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "  (relu): ReLU(inplace=True)\n",
              "  (layer1): Sequential(\n",
              "    (0): Bottleneck(\n",
              "      (conv1): Conv2d(16, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(16, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(16, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): Bottleneck(\n",
              "      (conv1): Conv2d(64, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(16, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (2): Bottleneck(\n",
              "      (conv1): Conv2d(64, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(16, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (layer2): Sequential(\n",
              "    (0): Bottleneck(\n",
              "      (conv1): Conv2d(64, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): Bottleneck(\n",
              "      (conv1): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (2): Bottleneck(\n",
              "      (conv1): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (3): Bottleneck(\n",
              "      (conv1): Conv2d(128, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(32, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (layer3): Sequential(\n",
              "    (0): Bottleneck(\n",
              "      (conv1): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): Bottleneck(\n",
              "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (2): Bottleneck(\n",
              "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (3): Bottleneck(\n",
              "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (4): Bottleneck(\n",
              "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (5): Bottleneck(\n",
              "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (layer4): Sequential(\n",
              "    (0): Bottleneck(\n",
              "      (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): Bottleneck(\n",
              "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (2): Bottleneck(\n",
              "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
              "  (fc): Linear(in_features=512, out_features=10, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bfOsjCQsxXWW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "c6cd8eb4-963b-41ce-9914-ae8efd091c6c"
      },
      "source": [
        "# 모델 테스트\n",
        "a=torch.Tensor(1,3,32,32).to(device)\n",
        "out = resnet50(a)\n",
        "print(out)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]], device='cuda:0',\n",
            "       grad_fn=<AddmmBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wJ9-Kyc9xY3D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "criterion = nn.CrossEntropyLoss().to(device)\n",
        "optimizer = torch.optim.Adam(resnet50.parameters(), lr = 0.001)\n",
        "lr_sche = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZZJmSOECAs8Y",
        "colab_type": "text"
      },
      "source": [
        "Plot 선언하는 부분은 colab의 한계로 못 넣었음."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SX5QxyYTxeJt",
        "colab_type": "text"
      },
      "source": [
        "**define acc_check function**\n",
        "\n",
        "정확도를 체크하는 부분을 복붙해서 약간 수정한 코드라고 한다.\n",
        "\n",
        "save를 하면 model이라는 폴더에 정확도와 에폭을 저장하겠다는 의미로."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a8bBJ0S0xaSs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def acc_check(net, test_set, epoch, save=1):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    with torch.no_grad():\n",
        "        for data in test_set:\n",
        "            images, labels = data\n",
        "            images = images.to(device)\n",
        "            labels = labels.to(device)\n",
        "            outputs = net(images)\n",
        "\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "    \n",
        "    acc = (100 * correct / total)\n",
        "    print('Accuracy of the network on the 10000 test images: %d %%' % acc)\n",
        "    if save:\n",
        "        torch.save(net.state_dict(), \"/content/gdrive/My Drive/Colab Notebooks/model_epoch_{}_acc_{}.pth\".format(epoch, int(acc)))\n",
        "    return acc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "njy5aq2KhwwD",
        "colab_type": "text"
      },
      "source": [
        "**Training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SqmLzIAoxoEn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 809
        },
        "outputId": "69e9eaba-61f8-4d41-a0f6-240360a99b8a"
      },
      "source": [
        "print(len(trainloader))\n",
        "# 에폭은 줄이고 돌리자...\n",
        "epochs = 30\n",
        "\n",
        "for epoch in range(epochs):  # loop over the dataset multiple times\n",
        "    print(epoch)\n",
        "    running_loss = 0.0\n",
        "    for i, data in enumerate(trainloader, 0):\n",
        "        # get the inputs\n",
        "        inputs, labels = data\n",
        "        inputs = inputs.to(device)\n",
        "        labels = labels.to(device)\n",
        "\n",
        "        # zero the parameter gradients\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # forward + backward + optimize\n",
        "        outputs = resnet50(inputs)\n",
        "        # print(\"finished resnet50\")\n",
        "        loss = criterion(outputs, labels)\n",
        "        # print(\"finished loss\")\n",
        "        loss.backward()\n",
        "        # print(\"finished backward\")\n",
        "        optimizer.step()\n",
        "        # print(\"finished opt\")\n",
        "        # 너무 오래 걸려서 돌아가는 건지 오류가 난 건지 알 수가 없어서...\n",
        "        # 한 번 출력하게 해본건데 이미지 하나 처리하는데 30초는 넘게 걸리는 것 같다.\n",
        "        # 실습... 마칠 수 있을까...\n",
        "\n",
        "        lr_sche.step() \n",
        "        # 모딥 github의 코드를 그대로 돌리니깐 \n",
        "        #optimizer.step()보다 lr_sche.step()을 앞에 두지 말라는 경고가 떠서 위치를 옮겼음.\n",
        "\n",
        "        # print statistics\n",
        "        running_loss += loss.item()\n",
        "        if i % 30 == 29:    # print every 30 mini-batches\n",
        "            #value_tracker(loss_plt, torch.Tensor([running_loss/30]), torch.Tensor([i + epoch*len(trainloader) ]))\n",
        "            print('[%d, %5d] loss: %.3f' %\n",
        "                  (epoch + 1, i + 1, running_loss / 30))\n",
        "            running_loss = 0.0\n",
        "    \n",
        "    #Check Accuracy\n",
        "    acc = acc_check(resnet50, testloader, epoch, save=1)\n",
        "    #value_tracker(acc_plt, torch.Tensor([acc]), torch.Tensor([epoch]))\n",
        "    \n",
        "\n",
        "print('Finished Training')"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[14,   150] loss: 1.784\n",
            "[14,   180] loss: 1.794\n",
            "Accuracy of the network on the 10000 test images: 34 %\n",
            "14\n",
            "[15,    30] loss: 1.786\n",
            "[15,    60] loss: 1.772\n",
            "[15,    90] loss: 1.784\n",
            "[15,   120] loss: 1.777\n",
            "[15,   150] loss: 1.780\n",
            "[15,   180] loss: 1.779\n",
            "Accuracy of the network on the 10000 test images: 34 %\n",
            "15\n",
            "[16,    30] loss: 1.785\n",
            "[16,    60] loss: 1.783\n",
            "[16,    90] loss: 1.776\n",
            "[16,   120] loss: 1.781\n",
            "[16,   150] loss: 1.778\n",
            "[16,   180] loss: 1.774\n",
            "Accuracy of the network on the 10000 test images: 34 %\n",
            "16\n",
            "[17,    30] loss: 1.785\n",
            "[17,    60] loss: 1.769\n",
            "[17,    90] loss: 1.777\n",
            "[17,   120] loss: 1.786\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-25-daa356371eeb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0;31m# print(\"finished backward\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m         \u001b[0;31m# print(\"finished opt\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0;31m# 너무 오래 걸려서 돌아가는 건지 오류가 난 건지 알 수가 없어서...\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     65\u001b[0m                 \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step_count\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m                 \u001b[0mwrapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m             \u001b[0;31m# Note that the returned function here is no longer a bound method,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    105\u001b[0m                     \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmax_exp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m                     \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m                 \u001b[0mstep_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DLua5kutRLgL",
        "colab_type": "text"
      },
      "source": [
        "진행이 하도 안 돼서 colab 설정도 gpu로 바꾸어 주고\n",
        "\n",
        "SGD가 잘 안 먹히는 것 같아서 optimizer도 Adam으로 바꾸었는데\n",
        "\n",
        "학습상태가 많이 안 좋다...\n",
        "\n",
        "CNN 공부를 이어가려면 구글에서 TPU나 GPU 클라우드를 월정액으로 써야 하나...?\n",
        "\n",
        "10 epoch 진행하는데 25분 넘게 걸림. 근데 학습도 거의 안 되는 걸 봐선 뭔가 잘못 설정한게 있나 싶다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cYkg0pfSxq-N",
        "colab_type": "text"
      },
      "source": [
        "**Model Accuracy Testing**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2fLE2C4sxo2C",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2526e845-7705-4261-a2f2-5cdb27f24e6d"
      },
      "source": [
        "correct = 0\n",
        "total = 0\n",
        "\n",
        "with torch.no_grad():\n",
        "    for data in testloader:\n",
        "        images, labels = data\n",
        "        images = images.to(device)\n",
        "        labels = labels.to(device)\n",
        "        outputs = resnet50(images)\n",
        "        \n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        \n",
        "        total += labels.size(0)\n",
        "        \n",
        "        correct += (predicted == labels).sum().item()\n",
        "\n",
        "print('Accuracy of the network on the 10000 test images: %d %%' % (\n",
        "    100 * correct / total))"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy of the network on the 10000 test images: 34 %\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}