ML2022Spring_HW8.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "bDk9r2YOcDc9",
        "Oi12tJMYWi0Q",
        "DCgNXSsEWuY7",
        "HNe7QU7n7cqh",
        "6X6fkGPnYyaF",
        "1EbfwRREhA7c",
        "vrJ9bScg9AgO",
        "XKNUImqUhIeq"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/leibohan/-/blob/master/ML2022Spring_HW8.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YiVfKn-6tXz8"
      },
      "source": [
        "# **Homework 8 - Anomaly Detection**\n",
        "\n",
        "If there are any questions, please contact mlta-2022spring-ta@googlegroups.com\n",
        "\n",
        "Slide:    [Link]()　Kaggle: [Link](https://www.kaggle.com/c/ml2022spring-hw8)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bDk9r2YOcDc9"
      },
      "source": [
        "# Set up the environment\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Oi12tJMYWi0Q"
      },
      "source": [
        "## Package installation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7LexxyPWWjJB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3a733a84-fca3-4e7c-fb9b-bfd5f890114d"
      },
      "source": [
        "# Training progress bar\n",
        "!pip install -q qqdm"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Building wheel for qqdm (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DCgNXSsEWuY7"
      },
      "source": [
        "## Downloading data"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://github.com/MachineLearningHW/HW8_Dataset/releases/download/v1.0.0/data.zip"
      ],
      "metadata": {
        "id": "SCLJtgF2BLSK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "54d462c4-2121-46a9-a966-1ca9b01e7b61"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-05-03 12:14:20--  https://github.com/MachineLearningHW/HW8_Dataset/releases/download/v1.0.0/data.zip\n",
            "Resolving github.com (github.com)... 140.82.112.3\n",
            "Connecting to github.com (github.com)|140.82.112.3|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/481862611/86ebe7c2-7087-49a9-9985-301851008195?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20220503%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20220503T121421Z&X-Amz-Expires=300&X-Amz-Signature=efae9d64de9ef8635e0a433b08109c15b5b2c7435484b7b80cd9a9ac012c61df&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=481862611&response-content-disposition=attachment%3B%20filename%3Ddata.zip&response-content-type=application%2Foctet-stream [following]\n",
            "--2022-05-03 12:14:21--  https://objects.githubusercontent.com/github-production-release-asset-2e65be/481862611/86ebe7c2-7087-49a9-9985-301851008195?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20220503%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20220503T121421Z&X-Amz-Expires=300&X-Amz-Signature=efae9d64de9ef8635e0a433b08109c15b5b2c7435484b7b80cd9a9ac012c61df&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=481862611&response-content-disposition=attachment%3B%20filename%3Ddata.zip&response-content-type=application%2Foctet-stream\n",
            "Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.111.133, 185.199.108.133, 185.199.109.133, ...\n",
            "Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1227511338 (1.1G) [application/octet-stream]\n",
            "Saving to: ‘data.zip’\n",
            "\n",
            "data.zip            100%[===================>]   1.14G  9.30MB/s    in 44s     \n",
            "\n",
            "2022-05-03 12:15:05 (26.7 MB/s) - ‘data.zip’ saved [1227511338/1227511338]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0K5kmlkuWzhJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c12176a4-f513-4ed3-c351-c82ef26e8072"
      },
      "source": [
        "!unzip data.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  data.zip\n",
            "   creating: data/\n",
            "  inflating: data/trainingset.npy    \n",
            "  inflating: data/testingset.npy     \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HNe7QU7n7cqh"
      },
      "source": [
        "# Import packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jk3qFK_a7k8P"
      },
      "source": [
        "import random\n",
        "import numpy as np\n",
        "import torch\n",
        "from torch import nn\n",
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset\n",
        "import torchvision.transforms as transforms\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "import torchvision.models as models\n",
        "from torch.optim import Adam, AdamW\n",
        "from qqdm import qqdm, format_str\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6X6fkGPnYyaF"
      },
      "source": [
        "# Loading data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k7Wd4yiUYzAm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2c2fa7bf-1c0a-4090-ac16-627c1fe5ca5c"
      },
      "source": [
        "\n",
        "train = np.load('data/trainingset.npy', allow_pickle=True)\n",
        "test = np.load('data/testingset.npy', allow_pickle=True)\n",
        "\n",
        "print(train.shape)\n",
        "print(test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(100000, 64, 64, 3)\n",
            "(19636, 64, 64, 3)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_flpmj6OYIa6"
      },
      "source": [
        "## Random seed\n",
        "Set the random seed to a certain value for reproducibility."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gb-dgXQYYI2Q"
      },
      "source": [
        "def same_seeds(seed):\n",
        "    random.seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    if torch.cuda.is_available():\n",
        "        torch.cuda.manual_seed(seed)\n",
        "        torch.cuda.manual_seed_all(seed)\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "\n",
        "same_seeds(48763)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zR9zC0_Df-CR"
      },
      "source": [
        "# Autoencoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1EbfwRREhA7c"
      },
      "source": [
        "# Models & loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wi8ds1fugCkR"
      },
      "source": [
        "class fcn_autoencoder(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(fcn_autoencoder, self).__init__()\n",
        "        self.encoder = nn.Sequential(\n",
        "            nn.Linear(64 * 64 * 3, 128),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(128, 64),\n",
        "            nn.ReLU(), \n",
        "            nn.Linear(64, 12), \n",
        "            nn.ReLU(), \n",
        "            nn.Linear(12, 3)\n",
        "        )\n",
        "        \n",
        "        self.decoder = nn.Sequential(\n",
        "            nn.Linear(3, 12),\n",
        "            nn.ReLU(), \n",
        "            nn.Linear(12, 64),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(64, 128),\n",
        "            nn.ReLU(), \n",
        "            nn.Linear(128, 64 * 64 * 3), \n",
        "            nn.Tanh()\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.encoder(x)\n",
        "        x = self.decoder(x)\n",
        "        return x\n",
        "\n",
        "\n",
        "class conv_autoencoder(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(conv_autoencoder, self).__init__()\n",
        "        self.encoder = nn.Sequential(\n",
        "            nn.Conv2d(3, 12, 4, stride=2, padding=1),         \n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(12, 24, 4, stride=2, padding=1),        \n",
        "            nn.ReLU(),\n",
        "\t\t\t      nn.Conv2d(24, 48, 4, stride=2, padding=1),         \n",
        "            nn.ReLU(),\n",
        "        )\n",
        "        self.decoder = nn.Sequential(\n",
        "\t\t\t      nn.ConvTranspose2d(48, 24, 4, stride=2, padding=1),\n",
        "            nn.ReLU(),\n",
        "\t\t\t      nn.ConvTranspose2d(24, 12, 4, stride=2, padding=1), \n",
        "            nn.ReLU(),\n",
        "            nn.ConvTranspose2d(12, 3, 4, stride=2, padding=1),\n",
        "            nn.Tanh(),\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.encoder(x)\n",
        "        x = self.decoder(x)\n",
        "        return x\n",
        "\n",
        "\n",
        "class VAE(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(VAE, self).__init__()\n",
        "        self.encoder = nn.Sequential(\n",
        "            nn.Conv2d(3, 12, 4, stride=2, padding=1),            \n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(12, 24, 4, stride=2, padding=1),    \n",
        "            nn.ReLU(),\n",
        "        )\n",
        "        self.enc_out_1 = nn.Sequential(\n",
        "            nn.Conv2d(24, 48, 4, stride=2, padding=1),  \n",
        "            nn.ReLU(),\n",
        "        )\n",
        "        self.enc_out_2 = nn.Sequential(\n",
        "            nn.Conv2d(24, 48, 4, stride=2, padding=1),\n",
        "            nn.ReLU(),\n",
        "        )\n",
        "        self.decoder = nn.Sequential(\n",
        "\t\t\t      nn.ConvTranspose2d(48, 24, 4, stride=2, padding=1), \n",
        "            nn.ReLU(),\n",
        "\t\t\t      nn.ConvTranspose2d(24, 12, 4, stride=2, padding=1), \n",
        "            nn.ReLU(),\n",
        "            nn.ConvTranspose2d(12, 3, 4, stride=2, padding=1), \n",
        "            nn.Tanh(),\n",
        "        )\n",
        "\n",
        "    def encode(self, x):\n",
        "        h1 = self.encoder(x)\n",
        "        return self.enc_out_1(h1), self.enc_out_2(h1)\n",
        "\n",
        "    def reparametrize(self, mu, logvar):\n",
        "        std = logvar.mul(0.5).exp_()\n",
        "        if torch.cuda.is_available():\n",
        "            eps = torch.cuda.FloatTensor(std.size()).normal_()\n",
        "        else:\n",
        "            eps = torch.FloatTensor(std.size()).normal_()\n",
        "        eps = Variable(eps)\n",
        "        return eps.mul(std).add_(mu)\n",
        "\n",
        "    def decode(self, z):\n",
        "        return self.decoder(z)\n",
        "\n",
        "    def forward(self, x):\n",
        "        mu, logvar = self.encode(x)\n",
        "        z = self.reparametrize(mu, logvar)\n",
        "        return self.decode(z), mu, logvar\n",
        "\n",
        "\n",
        "def loss_vae(recon_x, x, mu, logvar, criterion):\n",
        "    \"\"\"\n",
        "    recon_x: generating images\n",
        "    x: origin images\n",
        "    mu: latent mean\n",
        "    logvar: latent log variance\n",
        "    \"\"\"\n",
        "    mse = criterion(recon_x, x)\n",
        "    KLD_element = mu.pow(2).add_(logvar.exp()).mul_(-1).add_(1).add_(logvar)\n",
        "    KLD = torch.sum(KLD_element).mul_(-0.5)\n",
        "    return mse + KLD"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vrJ9bScg9AgO"
      },
      "source": [
        "# Dataset module\n",
        "\n",
        "Module for obtaining and processing data. The transform function here normalizes image's pixels from [0, 255] to [-1.0, 1.0].\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "33fWhE-h9LPq"
      },
      "source": [
        "class CustomTensorDataset(TensorDataset):\n",
        "    \"\"\"TensorDataset with support of transforms.\n",
        "    \"\"\"\n",
        "    def __init__(self, tensors):\n",
        "        self.tensors = tensors\n",
        "        if tensors.shape[-1] == 3:\n",
        "            self.tensors = tensors.permute(0, 3, 1, 2)\n",
        "        \n",
        "        self.transform = transforms.Compose([\n",
        "          transforms.Lambda(lambda x: x.to(torch.float32)),\n",
        "          transforms.Lambda(lambda x: 2. * x/255. - 1.),\n",
        "        ])\n",
        "        \n",
        "    def __getitem__(self, index):\n",
        "        x = self.tensors[index]\n",
        "        \n",
        "        if self.transform:\n",
        "            # mapping images to [-1.0, 1.0]\n",
        "            x = self.transform(x)\n",
        "\n",
        "        return x\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.tensors)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XKNUImqUhIeq"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7ebAJdjFmS08"
      },
      "source": [
        "## Configuration\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "in7yLfmqtZTk"
      },
      "source": [
        "# Training hyperparameters\n",
        "num_epochs = 50\n",
        "batch_size = 2000\n",
        "learning_rate = 1e-3\n",
        "\n",
        "# Build training dataloader\n",
        "x = torch.from_numpy(train)\n",
        "train_dataset = CustomTensorDataset(x)\n",
        "\n",
        "train_sampler = RandomSampler(train_dataset)\n",
        "train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "# Model\n",
        "model_type = 'vae'   # selecting a model type from {'cnn', 'fcn', 'vae', 'resnet'}\n",
        "model_classes = {'fcn': fcn_autoencoder(), 'cnn': conv_autoencoder(), 'vae': VAE()}\n",
        "model = model_classes[model_type].cuda()\n",
        "\n",
        "# Loss and optimizer\n",
        "criterion = nn.MSELoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wyooN-JPm8sS"
      },
      "source": [
        "## Training loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JoW1UrrxgI_U"
      },
      "source": [
        "\n",
        "best_loss = np.inf\n",
        "model.train()\n",
        "\n",
        "qqdm_train = qqdm(range(num_epochs), desc=format_str('bold', 'Description'))\n",
        "for epoch in qqdm_train:\n",
        "    tot_loss = list()\n",
        "    for data in train_dataloader:\n",
        "\n",
        "        # ===================loading=====================\n",
        "        img = data.float().cuda()\n",
        "        if model_type in ['fcn']:\n",
        "            img = img.view(img.shape[0], -1)\n",
        "\n",
        "        # ===================forward=====================\n",
        "        output = model(img)\n",
        "        if model_type in ['vae']:\n",
        "            loss = loss_vae(output[0], img, output[1], output[2], criterion)\n",
        "        else:\n",
        "            loss = criterion(output, img)\n",
        "\n",
        "        tot_loss.append(loss.item())\n",
        "        # ===================backward====================\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "    # ===================save_best====================\n",
        "    mean_loss = np.mean(tot_loss)\n",
        "    if mean_loss < best_loss:\n",
        "        best_loss = mean_loss\n",
        "        torch.save(model, 'best_model_{}.pt'.format(model_type))\n",
        "    # ===================log========================\n",
        "    qqdm_train.set_infos({\n",
        "        'epoch': f'{epoch + 1:.0f}/{num_epochs:.0f}',\n",
        "        'loss': f'{mean_loss:.4f}',\n",
        "    })\n",
        "    # ===================save_last========================\n",
        "    torch.save(model, 'last_model_{}.pt'.format(model_type))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wk0UxFuchLzR"
      },
      "source": [
        "# Inference\n",
        "Model is loaded and generates its anomaly score predictions."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "evgMW3OwoGqD"
      },
      "source": [
        "## Initialize\n",
        "- dataloader\n",
        "- model\n",
        "- prediction file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_MBnXAswoKmq"
      },
      "source": [
        "eval_batch_size = 200\n",
        "\n",
        "# build testing dataloader\n",
        "data = torch.tensor(test, dtype=torch.float32)\n",
        "test_dataset = CustomTensorDataset(data)\n",
        "test_sampler = SequentialSampler(test_dataset)\n",
        "test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=eval_batch_size, num_workers=1)\n",
        "eval_loss = nn.MSELoss(reduction='none')\n",
        "\n",
        "# load trained model\n",
        "checkpoint_path = f'last_model_{model_type}.pt'\n",
        "model = torch.load(checkpoint_path)\n",
        "model.eval()\n",
        "\n",
        "# prediction file \n",
        "out_file = 'prediction.csv'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "anomality = list()\n",
        "with torch.no_grad():\n",
        "  for i, data in enumerate(test_dataloader):\n",
        "    img = data.float().cuda()\n",
        "    if model_type in ['fcn']:\n",
        "      img = img.view(img.shape[0], -1)\n",
        "    output = model(img)\n",
        "    if model_type in ['vae']:\n",
        "      output = output[0]\n",
        "    if model_type in ['fcn']:\n",
        "        loss = eval_loss(output, img).sum(-1)\n",
        "    else:\n",
        "        loss = eval_loss(output, img).sum([1, 2, 3])\n",
        "    anomality.append(loss)\n",
        "anomality = torch.cat(anomality, axis=0)\n",
        "anomality = torch.sqrt(anomality).reshape(len(test), 1).cpu().numpy()\n",
        "\n",
        "df = pd.DataFrame(anomality, columns=['score'])\n",
        "df.to_csv(out_file, index_label = 'ID')"
      ],
      "metadata": {
        "id": "_1IxCX2iCW6V"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}