step_5_camera.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7a1536cc-91cd-45fb-8205-b2346eaec2f7",
   "metadata": {},
   "source": [
    "# Embedded ML Lab - Challenge - Demonstration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a0ee8260-14f8-47c4-a988-308dfc93c8d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.camera import CameraDisplay\n",
    "from utils.yolo import nms, filter_boxes\n",
    "from utils.viz import display_result\n",
    "import torch\n",
    "import time\n",
    "import cv2\n",
    "import onnxruntime as ort\n",
    "from models.pruned_my_tinyyolo2 import PrunedMyTinyYoloV2\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bd6f272b-f2d8-4f27-ad15-09778deab698",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing camera...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1883e9b4a0384521b971d8e12a8784e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Image(value=b'\\xff\\xd8\\xff\\xe0\\x00\\x10JFIF\\x00\\x01\\x01\\x00\\x00\\x01\\x00\\x01\\x00\\x00\\xff\\xdb\\x00C\\x00\\x02\\x01\\x0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Camera released\n"
     ]
    }
   ],
   "source": [
    "def dummy(image):\n",
    "    return image\n",
    "cam = CameraDisplay(dummy)\n",
    "cam.start()\n",
    "cam.stop()\n",
    "cam.release()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a406ed87-7666-45e6-a339-4cc736b3d917",
   "metadata": {},
   "outputs": [],
   "source": [
    "ort_sess = ort.InferenceSession('onnx_models/9x05x55.onnx')\n",
    "#ort_sess = ort.InferenceSession('onnx/pruned3.onnx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e5898cfa-32b3-4c25-abe3-c501857d197d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:58<00:00, 11.69s/it]\n"
     ]
    }
   ],
   "source": [
    "# execute network, because first times always take longer\n",
    "torch_input = torch.randn(1, 3, 320, 320)\n",
    "for i in tqdm.tqdm(enumerate(range(5)), total=5):\n",
    "    output = torch.from_numpy(ort_sess.run(None, {'input': torch_input.numpy()})[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f273cb05-1fa3-45cf-b637-66a6e22dbea0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a callback function (your detection pipeline)\n",
    "# Make sure to first load all your pipeline code and only at the end init the camera\n",
    "def callback(image):\n",
    "    global now\n",
    "    global counter\n",
    "    global times\n",
    "    global fpss\n",
    "    \n",
    "    # execute section a few times before adding next one, otherwise a run would take longer and crash the cam\n",
    "    counter += 1\n",
    "    \n",
    "    if counter < 10:\n",
    "        return image\n",
    "    \n",
    "    if counter == 10:\n",
    "        print(\"Begin Cropping\")\n",
    "    \n",
    "    fps = int(1/(time.time() - now))\n",
    "    now = time.time()\n",
    "    image = image[0:320,0:320, :]\n",
    "        \n",
    "    \n",
    "    if counter < 20:\n",
    "        return image\n",
    "    \n",
    "    if counter == 20:\n",
    "        print(\"Begin Conversion\")\n",
    "    \n",
    "    # convert image to torch\n",
    "    # from 320 x 320 x 3 to 1 x 3 x 320 x 320\n",
    "    torch_image2 = torch.from_numpy(image)\n",
    "    torch_image = torch.zeros([1, 3, 320, 320])\n",
    "    #torch_image = torch.zeros([1, 3, int(320 / downscale), int(320 / downscale)])\n",
    "    \n",
    "    # from BGR to RGB and from uint8 to float\n",
    "    for i in range(3):\n",
    "        torch_image[0, 2-i, :, :] = torch_image2[:, :, i] / 256\n",
    "    \n",
    "    \n",
    "    if counter < 30:\n",
    "        return image\n",
    "    \n",
    "    if counter == 30:\n",
    "        print(\"Begin NN\")\n",
    "    \n",
    "    '''if downscale != 1:\n",
    "        for i in range(torch_image.size()[2]):\n",
    "            torch_image[:, :, i, i] = torch.mean(torch_image3[:, :, downscale*i:downscale*i+down_add, downscale*i:downscale*i+down_add])\n",
    "    else:\n",
    "        torch_image = torch_image3'''\n",
    "    \n",
    "    # calculate result\n",
    "    #input is a 1 x 3 x 320 x 320 image\n",
    "    #torch_image = torch_image.to(torch.device(\"cuda\"))\n",
    "    output = torch.from_numpy(ort_sess.run(None, {'input': torch_image.numpy()})[0])\n",
    "    #output = net(torch_image)\n",
    "    #output = output.cpu()\n",
    "    \n",
    "    \n",
    "    \n",
    "    if counter < 40:\n",
    "        return image\n",
    "    \n",
    "    if counter == 40:\n",
    "        print(\"Begin Filter\")\n",
    "    \n",
    "    #output is a 32 x 125 x 10 x 10 tensor\n",
    "    #filter boxes based on confidence score (class_score*confidence)\n",
    "    output = filter_boxes(output, 0.4)\n",
    "    #filter boxes based on overlap\n",
    "    output = nms(output, 0)\n",
    "    \n",
    "    \n",
    "    if counter < 50:\n",
    "        return image\n",
    "    \n",
    "    if counter == 50:\n",
    "        print(\"Begin Drawing\")\n",
    "    \n",
    "    # draw result on camera image\n",
    "    for out1 in output:\n",
    "        for out in out1:\n",
    "            #convert relative to absolute width\n",
    "            w = int(out[2] * 320)\n",
    "            h = int(out[3] * 320)\n",
    "            # convert middle point to upper left corner\n",
    "            x = int(out[0] * 320 - int(w/2))\n",
    "            y = int(out[1] * 320 - int(h/2))\n",
    "            # draw\n",
    "            cv2.rectangle(image, (x, y), (x + w, y + h), (255, 0, 0), 2)\n",
    "            cv2.putText(image, f\"{int(out[4]*100)}\", (x, y-5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 2, cv2.LINE_AA)\n",
    "            \n",
    "    cv2.putText(image, f\"fps={fps}\", (2, 25), cv2.FONT_HERSHEY_SIMPLEX, 1,\n",
    "                (100, 255, 0), 2, cv2.LINE_AA)\n",
    "    \n",
    "    # prints current frame with output\n",
    "    #display_result(torch_image, output, torch.zeros([1,10,6]), file_path='yolo_prediction.png')\n",
    "    \n",
    "    \n",
    "    if counter > 60 and counter <= 200:\n",
    "        fpss.append(fps)\n",
    "    \n",
    "    if counter == 200:\n",
    "        print(\"avg fps: \", sum(fpss) / 140)\n",
    "    \n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3862254a-5f10-4d50-a109-fae94009e508",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing camera...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57597187ac3546d7b3401d15694e5c2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Image(value=b'\\xff\\xd8\\xff\\xe0\\x00\\x10JFIF\\x00\\x01\\x01\\x00\\x00\\x01\\x00\\x01\\x00\\x00\\xff\\xdb\\x00C\\x00\\x02\\x01\\x0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize the camera with the callback\n",
    "cam = CameraDisplay(callback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ef4f134f-6d8e-4e28-b9cd-ea47247856c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Begin Cropping\n",
      "Begin Conversion\n",
      "Begin NN\n",
      "Begin Filter\n",
      "Begin Drawing\n",
      "avg fps:  14.05\n"
     ]
    }
   ],
   "source": [
    "# The camera stream can be started with cam.start()\n",
    "# The callback gets asynchronously called (can be stopped with cam.stop())\n",
    "counter = 0\n",
    "times = [[], [], [], [], []]\n",
    "fpss = []\n",
    "now = time.time()\n",
    "cam.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f9baf3b9-a78c-4709-a1df-4fabd57366a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Camera released\n"
     ]
    }
   ],
   "source": [
    "# The camera should always be stopped and released for a new camera is instantiated (calling CameraDisplay(callback) again)\n",
    "cam.stop()\n",
    "cam.release()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cc9012d-e89e-461c-8e17-c93746d07c43",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}