ipynb/003.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare zero-shot split \n",
    "Based on the paper: Bansal, Ankan, et al. \"Zero-shot object detection.\" Proceedings of the European Conference on Computer Vision (ECCV). 2018."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from maskrcnn_benchmark.config import cfg\n",
    "from maskrcnn_benchmark.modeling.language_backbone.transformers import BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/annotations/instances_train2017.json', 'r') as fin:\n",
    "    coco_train_anno_all = json.load(fin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/annotations/instances_train2017.json', 'r') as fin:\n",
    "    coco_train_anno_seen = json.load(fin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/annotations/instances_train2017.json', 'r') as fin:\n",
    "    coco_train_anno_unseen = json.load(fin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/annotations/instances_val2017.json', 'r') as fin:\n",
    "    coco_val_anno_all = json.load(fin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/annotations/instances_val2017.json', 'r') as fin:\n",
    "    coco_val_anno_seen = json.load(fin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/annotations/instances_val2017.json', 'r') as fin:\n",
    "    coco_val_anno_unseen = json.load(fin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/zero-shot/mscoco_seen_classes.json', 'r') as fin:\n",
    "    labels_seen = json.load(fin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/zero-shot/mscoco_unseen_classes.json', 'r') as fin:\n",
    "    labels_unseen = json.load(fin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49, 17)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(labels_seen), len(labels_unseen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_all = [item['name'] for item in coco_val_anno_all['categories']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'background'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(labels_seen) - set(labels_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(labels_unseen) - set(labels_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_id_to_split = {}\n",
    "class_name_to_split = {}\n",
    "for item in coco_val_anno_all['categories']:\n",
    "    if item['name'] in labels_seen:\n",
    "        class_id_to_split[item['id']] = 'seen'\n",
    "        class_name_to_split[item['name']] = 'seen'\n",
    "    elif item['name'] in labels_unseen:\n",
    "        class_id_to_split[item['id']] = 'unseen'\n",
    "        class_name_to_split[item['name']] = 'unseen'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_name_to_glove = {}\n",
    "with open('../datasets/coco/zero-shot/glove.6B.300d.txt', 'r') as fin:\n",
    "    for row in fin:\n",
    "        row_tk = row.split()\n",
    "        if row_tk[0] in class_name_to_split:\n",
    "            class_name_to_glove[row_tk[0]] = [float(num) for num in row_tk[1:]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert = BERT(cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = bert.to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "giraffe not found\n",
      "handbag not found\n",
      "frisbee not found\n",
      "skis not found\n",
      "snowboard not found\n",
      "skateboard not found\n",
      "surfboard not found\n",
      "broccoli not found\n",
      "donut not found\n",
      "toaster not found\n",
      "toothbrush not found\n"
     ]
    }
   ],
   "source": [
    "class_name_to_bertemb = {}\n",
    "for c in class_name_to_split:\n",
    "    if c not in bert.tokenizer.vocab:\n",
    "        print(f'{c} not found')\n",
    "        continue\n",
    "    cid = bert.tokenizer.vocab[c]\n",
    "    class_name_to_bertemb[c] = bert.embeddings[cid]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_list = list(class_name_to_split.keys())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_class_list = bert(class_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = (1 - encoded_class_list['special_tokens_mask']).to(torch.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 3., 1., 1., 2., 1., 1., 3., 2., 2., 1., 2., 2., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 1., 1., 3., 1., 1., 2., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "        1., 1., 1., 2., 1., 1., 1., 1., 1., 1., 2.], device='cuda:0')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask.sum(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = (encoded_class_list['input_embeddings'] * mask[:, :, None]).sum(1) / mask.sum(1)[:, None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = embeddings.cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(65, 768)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_name_to_bertemb = {}\n",
    "for c, emb in zip(class_list, embeddings.tolist()):\n",
    "    class_name_to_bertemb[c] = emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(65, 65, 65)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(class_name_to_bertemb), len(class_name_to_glove), len(class_name_to_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_annotation(anno_dict, split_name_list):\n",
    "    filtered_categories = []\n",
    "    for item in anno_dict['categories']:\n",
    "        if class_id_to_split.get(item['id']) in split_name_list:\n",
    "            item['embedding'] = {}\n",
    "            item['embedding']['GloVE'] = class_name_to_glove[item['name']]\n",
    "            item['embedding']['BertEmb'] = class_name_to_bertemb[item['name']]\n",
    "            item['split'] = class_id_to_split.get(item['id'])\n",
    "            filtered_categories.append(item)\n",
    "    anno_dict['categories'] = filtered_categories\n",
    "    \n",
    "    filtered_images = []\n",
    "    filtered_annotations = []\n",
    "    useful_image_ids = set()\n",
    "    for item in anno_dict['annotations']:\n",
    "        if class_id_to_split.get(item['category_id']) in split_name_list:\n",
    "            filtered_annotations.append(item)\n",
    "            useful_image_ids.add(item['image_id'])\n",
    "    for item in anno_dict['images']:\n",
    "        if item['id'] in useful_image_ids:\n",
    "            filtered_images.append(item)\n",
    "    anno_dict['annotations'] = filtered_annotations\n",
    "    anno_dict['images'] = filtered_images    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_annotation(coco_train_anno_seen, ['seen'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_annotation(coco_train_anno_unseen, ['unseen'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_annotation(coco_train_anno_all, ['seen', 'unseen'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_annotation(coco_val_anno_seen, ['seen'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_annotation(coco_val_anno_unseen, ['unseen'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_annotation(coco_val_anno_all, ['seen', 'unseen'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48, 17, 65)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(coco_val_anno_seen['categories']), len(coco_val_anno_unseen['categories']), len(coco_val_anno_all['categories'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/zero-shot/instances_train2017_seen_2.json', 'w') as fout:\n",
    "    json.dump(coco_train_anno_seen, fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/zero-shot/instances_train2017_unseen_2.json', 'w') as fout:\n",
    "    json.dump(coco_train_anno_unseen, fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/zero-shot/instances_train2017_all_2.json', 'w') as fout:\n",
    "    json.dump(coco_train_anno_all, fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/zero-shot/instances_val2017_seen_2.json', 'w') as fout:\n",
    "    json.dump(coco_val_anno_seen, fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/zero-shot/instances_val2017_unseen_2.json', 'w') as fout:\n",
    "    json.dump(coco_val_anno_unseen, fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../datasets/coco/zero-shot/instances_val2017_all_2.json', 'w') as fout:\n",
    "    json.dump(coco_val_anno_all, fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "name": "pytorch-gpu.1-4.m46",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/pytorch-gpu.1-4:m46"
  },
  "kernelspec": {
   "display_name": "maskrcnn2",
   "language": "python",
   "name": "maskrcnn2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}