OccludedFaceDetection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Licensed under the EUPL."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DatasetModifier\n",
    "This will modify the [CFP](http://www.cfpw.io/) dataset in order to verify which part of the face is most important for face detection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from facerec.faceextractor import FaceExtractor\n",
    "from FaceAnalysis import FaceAnalysis\n",
    "\n",
    "from detectionalgorithms.MyMTCNN import MyMTCNN\n",
    "from detectionalgorithms.MyRetinaFace import MyRetinaFace\n",
    "from detectionalgorithms.MyDLIB import MyDLIB\n",
    "\n",
    "from datasets.CFPfrontal import CFPFrontal\n",
    "from datasets.CFPprofile import CFPProfile\n",
    "from datasets.ImageFolder import ImageFolder\n",
    "from datasets.afdb import AFDB\n",
    "\n",
    "\n",
    "from pathlib import Path\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import cv2\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instantiate the three face detection algorithms which are analyzed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "extractor = FaceExtractor()\n",
    "mt = MyMTCNN()\n",
    "rf = MyRetinaFace()\n",
    "dl = MyDLIB()\n",
    "\n",
    "path=\"/home/philipp/Downloads/\" # Path which contains 'cfp-dataset'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyze(path):\n",
    "    dataset = ImageFolder(path)\n",
    "    \n",
    "    print(\"=== {} ===\".format(path))\n",
    "    analysis = FaceAnalysis(mt,dataset)\n",
    "    analysis.calcAcc(copy_file_to_downloads_folder=False)\n",
    "    print(\"= MTCNN =\")\n",
    "    print(analysis)\n",
    "\n",
    "    analysis = FaceAnalysis(rf,dataset)\n",
    "    analysis.calcAcc(copy_file_to_downloads_folder=False)\n",
    "    print(\"= RETINAFACE =\")\n",
    "    print(analysis)\n",
    "\n",
    "    analysis = FaceAnalysis(dl,dataset)\n",
    "    analysis.calcAcc(copy_file_to_downloads_folder=False)\n",
    "    print(\"= DLIB =\")\n",
    "    print(analysis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"cfp-no-eyes-perc\"\n",
    "original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "for person_nr in tqdm(os.listdir(original_path)):\n",
    "    frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "    Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "    for img_nr in os.listdir(frontal_path):\n",
    "        img_path = os.path.join(frontal_path, img_nr)\n",
    "        img = cv2.imread(img_path)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "        if len(landmarks) == 1:\n",
    "            left_eye = landmarks[0][\"keypoints\"][\"left_eye\"][1]\n",
    "            right_eye = landmarks[0][\"keypoints\"][\"right_eye\"][1]\n",
    "\n",
    "            min_eye = min(left_eye,right_eye)\n",
    "            max_eye = max(left_eye,right_eye)\n",
    "            img[int(min_eye-img.shape[0]/10):int(max_eye+img.shape[0]/10), :, :] = 0\n",
    "            img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "            new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "            img_to_print.save(new_path_img)\n",
    "        else:\n",
    "            pass\n",
    "            #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "            \n",
    "print_results(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"cfp-no-eyes-fixed\"\n",
    "original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "for person_nr in tqdm(os.listdir(original_path)):\n",
    "    frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "    Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "    for img_nr in os.listdir(frontal_path):\n",
    "        img_path = os.path.join(frontal_path, img_nr)\n",
    "        img = cv2.imread(img_path)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "        if len(landmarks) == 1:\n",
    "            left_eye = landmarks[0][\"keypoints\"][\"left_eye\"][1]\n",
    "            right_eye = landmarks[0][\"keypoints\"][\"right_eye\"][1]\n",
    "\n",
    "            min_eye = min(left_eye,right_eye)\n",
    "            max_eye = max(left_eye,right_eye)\n",
    "            img[int(min_eye-5):int(max_eye+5), :, :] = 0\n",
    "            \n",
    "            \n",
    "            img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "            new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "            img_to_print.save(new_path_img)\n",
    "        else:\n",
    "            pass\n",
    "            #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "            \n",
    "print_results(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"cfp-no-eyes-rectangle\"\n",
    "original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "for person_nr in tqdm(os.listdir(original_path)):\n",
    "    frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "    Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "    for img_nr in os.listdir(frontal_path):\n",
    "        img_path = os.path.join(frontal_path, img_nr)\n",
    "        img = cv2.imread(img_path)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "        if len(landmarks) == 1:\n",
    "            left=landmarks[0][\"keypoints\"][\"left_eye\"]\n",
    "            right=landmarks[0][\"keypoints\"][\"right_eye\"]\n",
    "\n",
    "            width=int(abs(right[0]-left[0])/10)\n",
    "            img[left[1]-width:left[1]+width, left[0]-width*3:left[0]+width*3, :] = 0\n",
    "            img[right[1]-width:right[1]+width, right[0]-width*3:right[0]+width*3, :] = 0\n",
    "            \n",
    "            \n",
    "            img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "            new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "            img_to_print.save(new_path_img)\n",
    "        else:\n",
    "            pass\n",
    "            #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "            \n",
    "print_results(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"cfp-top-half-only\"\n",
    "original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "for person_nr in tqdm(os.listdir(original_path)):\n",
    "    frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "    Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "    for img_nr in os.listdir(frontal_path):\n",
    "        img_path = os.path.join(frontal_path, img_nr)\n",
    "        img = cv2.imread(img_path)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "        if len(landmarks) == 1:\n",
    "            landmarks = landmarks[0]\n",
    "            nose = landmarks[\"keypoints\"][\"nose\"]\n",
    "            img[int(nose[1]+nose[1]/20):, :, :] = 0\n",
    "            \n",
    "            \n",
    "            img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "            new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "            img_to_print.save(new_path_img)\n",
    "        else:\n",
    "            pass\n",
    "            #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "            \n",
    "print_results(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"cfp-top-half-only-mask\"\n",
    "original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "for person_nr in tqdm(os.listdir(original_path)):\n",
    "    frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "    Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "    for img_nr in os.listdir(frontal_path):\n",
    "        img_path = os.path.join(frontal_path, img_nr)\n",
    "        img = cv2.imread(img_path)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "        if len(landmarks) == 1:\n",
    "            left_eye = landmarks[0][\"keypoints\"][\"left_eye\"][1]\n",
    "            right_eye = landmarks[0][\"keypoints\"][\"right_eye\"][1]\n",
    "            min_eye = max(left_eye,right_eye)\n",
    "            nose = landmarks[0][\"keypoints\"][\"nose\"][1]\n",
    "            middle = int((nose+min_eye)/2)\n",
    "            img[middle:, :, :] = 0\n",
    "            \n",
    "            \n",
    "            img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "            new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "            img_to_print.save(new_path_img)\n",
    "        else:\n",
    "            pass\n",
    "            #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "            \n",
    "print_results(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"cfp-no-top\"\n",
    "original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "for person_nr in tqdm(os.listdir(original_path)):\n",
    "    frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "    Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "    for img_nr in os.listdir(frontal_path):\n",
    "        img_path = os.path.join(frontal_path, img_nr)\n",
    "        img = cv2.imread(img_path)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "        if len(landmarks) == 1:\n",
    "            keypoints = landmarks[0][\"keypoints\"]\n",
    "            offset = 25\n",
    "            eye = min(keypoints[\"left_eye\"][1]-offset, keypoints[\"right_eye\"][1]-offset)\n",
    "            img[:int(eye), :, :] = 0\n",
    "            \n",
    "            \n",
    "            img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "            new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "            img_to_print.save(new_path_img)\n",
    "        else:\n",
    "            pass\n",
    "            #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "            \n",
    "print_results(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "for i in range(10):\n",
    "    name = \"cfp-random-black-bars-{}\".format(i)\n",
    "    original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "    new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "    for person_nr in tqdm(os.listdir(original_path)):\n",
    "        frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "        Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "        for img_nr in os.listdir(frontal_path):\n",
    "            img_path = os.path.join(frontal_path, img_nr)\n",
    "            img = cv2.imread(img_path)\n",
    "            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "            landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "            if len(landmarks) == 1:\n",
    "                height = int(img.shape[0]/10)\n",
    "                start_y = random.randint(0,int(img.shape[0]-height))\n",
    "                img[start_y-height:start_y+height, :, :] = 0\n",
    "\n",
    "\n",
    "                img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "                new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "                img_to_print.save(new_path_img)\n",
    "            else:\n",
    "                pass\n",
    "                #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "\n",
    "    print_results(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"cfp-top-half-only-more\"\n",
    "original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "for person_nr in tqdm(os.listdir(original_path)):\n",
    "    frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "    Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "    for img_nr in os.listdir(frontal_path):\n",
    "        img_path = os.path.join(frontal_path, img_nr)\n",
    "        img = cv2.imread(img_path)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "        if len(landmarks) == 1:\n",
    "            landmarks = landmarks[0]\n",
    "            nose = landmarks[\"keypoints\"][\"nose\"]\n",
    "            img[int(nose[1]+nose[1]/10):, :, :] = 0\n",
    "            \n",
    "            \n",
    "            img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "            new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "            img_to_print.save(new_path_img)\n",
    "        else:\n",
    "            pass\n",
    "            #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "            \n",
    "print_results(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"cfp-vert-bar-larger\"\n",
    "original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "for person_nr in tqdm(os.listdir(original_path)):\n",
    "    frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "    Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "    for img_nr in os.listdir(frontal_path):\n",
    "        img_path = os.path.join(frontal_path, img_nr)\n",
    "        img = cv2.imread(img_path)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "        if len(landmarks) == 1:\n",
    "            landmarks = landmarks[0]\n",
    "            nose = landmarks[\"keypoints\"][\"nose\"]\n",
    "            img[:, int(nose[0]+nose[0]/10):, :] = 0\n",
    "            \n",
    "            \n",
    "            img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "            new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "            img_to_print.save(new_path_img)\n",
    "        else:\n",
    "            pass\n",
    "            #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "            \n",
    "print_results(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"cfp-vert-bar-larger-small\"\n",
    "original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "for person_nr in tqdm(os.listdir(original_path)):\n",
    "    frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "    Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "    for img_nr in os.listdir(frontal_path):\n",
    "        img_path = os.path.join(frontal_path, img_nr)\n",
    "        img = cv2.imread(img_path)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "        if len(landmarks) == 1:\n",
    "            landmarks = landmarks[0]\n",
    "            nose = landmarks[\"keypoints\"][\"nose\"]\n",
    "            img[:, int(nose[0]+nose[0]/5):, :] = 0\n",
    "            \n",
    "            \n",
    "            img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "            new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "            img_to_print.save(new_path_img)\n",
    "        else:\n",
    "            pass\n",
    "            #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "            \n",
    "print_results(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"cfp-vert-bars\"\n",
    "original_path=os.path.join(path,\"cfp-dataset/Data/Images\")\n",
    "new_path=os.path.join(path,name+\"/Data/Images\")\n",
    "for person_nr in tqdm(os.listdir(original_path)):\n",
    "    frontal_path = os.path.join(original_path, person_nr, \"frontal\")\n",
    "    Path(os.path.join(new_path, person_nr, \"frontal\")).mkdir(parents=True, exist_ok=True)\n",
    "    for img_nr in os.listdir(frontal_path):\n",
    "        img_path = os.path.join(frontal_path, img_nr)\n",
    "        img = cv2.imread(img_path)\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        landmarks = extractor.get_landmarks_from_path(img_path)\n",
    "        if len(landmarks) == 1:\n",
    "            img_path = os.path.join(frontal_path, img_nr)\n",
    "            img = cv2.imread(img_path)\n",
    "            img[:, int(img.shape[0]*2/3):, :] = 0\n",
    "            \n",
    "            \n",
    "            img_to_print = Image.fromarray(img.astype(np.uint8))\n",
    "            new_path_img = os.path.join(new_path, person_nr, \"frontal\", img_nr)\n",
    "            img_to_print.save(new_path_img)\n",
    "        else:\n",
    "            pass\n",
    "            #print(\"{} has {} faces\".format(img_path, len(landmarks)))\n",
    "            \n",
    "print_results(name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}