Kaggle_oxford_102_flower_pytorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Kaggle_oxford-102-flower-pytorch.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aysunakarsu/kaggle/blob/master/Kaggle_oxford_102_flower_pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "HCqeJbPqjRQ7",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "This notebook demonstrates how to test a flower classifier and produces a submission.csv file for submitting to the kaggle competition \n",
        "https://www.kaggle.com/c/oxford-102-flower-pytorch. The notebook is directly accessible on google colab. The classifier is trained elsewhere, in this notebook it will be loaded from a saved checkpoint on google drive. The kaggle competition uses the same datasets as in Udacity Facebook Pytorch Challenge final project.  The data originally comes from http://www.robots.ox.ac.uk/~vgg/data/flowers/102/. There are 102 flower categories, and the test dataset contains 819 test images. For this competition the labels of the test dataset are not provided."
      ]
    },
    {
      "metadata": {
        "id": "bU93pbeB_cAh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "!pip install -q http://download.pytorch.org/whl/{accelerator}/torch-0.4.1-{platform}-linux_x86_64.whl torchvision\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AaE16HpaWaI9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install --no-cache-dir -I pillow\n",
        "%matplotlib inline\n",
        "import time\n",
        "import os\n",
        "import json\n",
        "import copy\n",
        "from PIL import Image\n",
        "from collections import OrderedDict\n",
        "import torch\n",
        "from torch import nn, optim\n",
        "from torch.optim import lr_scheduler\n",
        "from torch.autograd import Variable\n",
        "from torchvision import datasets, models, transforms\n",
        "from google.colab import files\n",
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "owayRNYDXFYZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import PIL\n",
        "print(PIL.PILLOW_VERSION)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wHwPjrCI4kNP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install -q kaggle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-GN16sVW4mdN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!mkdir -p ~/.kaggle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yM7S7HMR4o50",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!cp kaggle.json ~/.kaggle/\n",
        "!chmod 600 /root/.kaggle/kaggle.json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "B6P9cQ0w4r9J",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!kaggle competitions download -c oxford-102-flower-pytorch"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "h5kb7TDhN_tZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!unzip flower_data.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "e9p_b_pGDv37",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!mkdir ./flower_data/test/testdir"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "DlLXUnHqD66p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!cp ./flower_data/test/*.jpg ./flower_data/test/testdir/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AVttx-3o4x7a",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_on_gpu = torch.cuda.is_available()\n",
        "if not train_on_gpu:\n",
        "    print('Not your lucky day! Training on CPU ...')\n",
        "else:\n",
        "    print('Yay! Training on GPU ...')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "DzIZ_O16_1zy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "s8aNsMEdAP3e",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install -U -q PyDrive\n",
        "!git clone https://gist.github.com/dc7e60aa487430ea704a8cb3f2c5d6a6.git /tmp/colab_util_repo\n",
        "!mv /tmp/colab_util_repo/colab_util.py colab_util.py \n",
        "!rm -r /tmp/colab_util_repo"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "K-ooLlfgATyQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "62hXJSyGABZl",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def load_model(filepath):\n",
        "    checkpoint = torch.load(filepath)\n",
        "    model = models.densenet161(pretrained=False)\n",
        "    for param in model.parameters():\n",
        "        param.requires_grad = False\n",
        "\n",
        "    # Put the classifier on the pretrained network\n",
        "    model.classifier = checkpoint['classifier']\n",
        "    model.load_state_dict(checkpoint['state_dict'], strict=False)\n",
        "    model.eval() \n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ejrjWDYkAHiv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model = load_model('/content/drive/My Drive/densenet161_checkpoint.pth')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8vgoCJThDuCI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class ImageFolderWithPaths(datasets.ImageFolder):\n",
        "    \"\"\"Custom dataset that includes image file paths. Extends\n",
        "    torchvision.datasets.ImageFolder\n",
        "    \"\"\"\n",
        "\n",
        "    # override the __getitem__ method. this is the method dataloader calls\n",
        "    def __getitem__(self, index):\n",
        "        # this is what ImageFolder normally returns \n",
        "        original_tuple = super(ImageFolderWithPaths, self).__getitem__(index)\n",
        "        # the image file path\n",
        "        path = self.imgs[index][0]\n",
        "        # make a new tuple that includes original and the path\n",
        "        tuple_with_path = (original_tuple + (path,))\n",
        "        return tuple_with_path"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "13mSzoplKdPS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "data_dir = './flower_data'\n",
        "data_transforms = {\n",
        "    'valid': transforms.Compose([\n",
        "        transforms.Resize(256),\n",
        "        transforms.CenterCrop(224),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], \n",
        "                             [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "     'test': transforms.Compose([\n",
        "        transforms.Resize(256),\n",
        "        transforms.CenterCrop(224),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], \n",
        "                             [0.229, 0.224, 0.225])\n",
        "    ])\n",
        "    \n",
        "}\n",
        "\n",
        "batch_size = 32\n",
        "\n",
        "image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),\n",
        "                                          data_transforms[x])\n",
        "                  for x in ['valid']}\n",
        "\n",
        "image_datasets['test'] =  ImageFolderWithPaths(os.path.join(data_dir, 'test'),\n",
        "                                          data_transforms['test'])                                   \n",
        "                                              \n",
        "dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=4) for x in ['valid']}\n",
        "\n",
        "dataloaders['test'] = torch.utils.data.DataLoader(image_datasets['test'], batch_size=batch_size, shuffle=False, num_workers=0)\n",
        "\n",
        "\n",
        "dataset_sizes = {x: len(image_datasets[x]) for x in ['valid','test']}\n",
        "\n",
        "print(\"Datasets load has finished:\")\n",
        "print(\"\\tNumber of validation images:{}\".format(dataset_sizes['valid']))\n",
        "print(\"\\tNumber of test images:{}\".format(dataset_sizes['test']))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "O0fZXm63Jk4C",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def kaggle_prep(k_model,k_device, k_idx_to_class, k_dataloaders, phase):\n",
        "    predictions = []\n",
        "    image_names = []\n",
        "    \n",
        "    # Requires use of test_dataloader as it has paths component\n",
        "    \n",
        "    k_model.eval()\n",
        "    k_model.to(device)   \n",
        "\n",
        "    with torch.no_grad():\n",
        "        for data, target, paths in k_dataloaders[phase]:\n",
        "            \n",
        "            data, target = data.to(device), target.to(device)\n",
        "            output = model(data)\n",
        "            _, pred = torch.max(output, 1)\n",
        "            for idx in range(len(target.data)):            \n",
        "                image_names.append(paths[idx].split('/')[-1])\n",
        "                predictions.append(idx_to_class[pred[idx].item()])\n",
        "             \n",
        "   \n",
        "    return  np.vstack((image_names, predictions))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aauQWNhMJX-E",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        ""
      ]
    },
    {
      "metadata": {
        "id": "EnNdkXkCQ9_q",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "idx_to_class = {v: k for k, v in image_datasets['valid'].class_to_idx.items()}\n",
        "print(idx_to_class)\n",
        "kaggle_test_images = kaggle_prep(model, device,idx_to_class, dataloaders,'test')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "v42_QuNtNr27",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "df = pd.DataFrame({'file_name': kaggle_test_images[0], 'id': kaggle_test_images[1]})\n",
        "print (df)\n",
        "df.to_csv('submission_02.csv' ,index=False)\n",
        "df.describe()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}