step_5_camera_debug.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7a1536cc-91cd-45fb-8205-b2346eaec2f7",
   "metadata": {},
   "source": [
    "# Embedded ML Lab - Challenge (Camera example)\n",
    "\n",
    "This is an example notebook for the camera usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a0ee8260-14f8-47c4-a988-308dfc93c8d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.camera import CameraDisplay\n",
    "from utils.yolo import nms, filter_boxes\n",
    "from utils.viz import display_result\n",
    "import torch\n",
    "import time\n",
    "import cv2\n",
    "import onnxruntime as ort\n",
    "from models.pruned_my_tinyyolo2 import PrunedMyTinyYoloV2\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd6f272b-f2d8-4f27-ad15-09778deab698",
   "metadata": {},
   "outputs": [],
   "source": [
    "def dummy(image):\n",
    "    return image\n",
    "cam = CameraDisplay(dummy)\n",
    "cam.start()\n",
    "cam.stop()\n",
    "cam.release()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1611da4e-7053-4d07-8370-e33ae8bc028c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_to_string(time):\n",
    "    s = f\"{round(time * 1000, 2)}\"\n",
    "    while (len(s) < 7):\n",
    "        s = \"0\" + s\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "a406ed87-7666-45e6-a339-4cc736b3d917",
   "metadata": {},
   "outputs": [],
   "source": [
    "ort_sess = ort.InferenceSession('onnx_models/9x05x55.onnx')\n",
    "#ort_sess = ort.InferenceSession('onnx/pruned3.onnx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "e5898cfa-32b3-4c25-abe3-c501857d197d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:00<00:00,  6.33it/s]\n"
     ]
    }
   ],
   "source": [
    "torch_input = torch.randn(1, 3, 320, 320)\n",
    "for i in tqdm.tqdm(enumerate(range(5)), total=5):\n",
    "    output = torch.from_numpy(ort_sess.run(None, {'input': torch_input.numpy()})[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "f273cb05-1fa3-45cf-b637-66a6e22dbea0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Camera released\n",
      "Camera released\n"
     ]
    }
   ],
   "source": [
    "# Define a callback function (your detection pipeline)\n",
    "# Make sure to first load all your pipeline code and only at the end init the camera\n",
    "def callback(image):\n",
    "    global now\n",
    "    global counter\n",
    "    global times\n",
    "    \n",
    "    counter += 1\n",
    "    \n",
    "    if counter < 50:\n",
    "        return image\n",
    "    \n",
    "    if counter == 50:\n",
    "        print(\"Begin Cropping\")\n",
    "    \n",
    "    doPrint = False\n",
    "    fps = f\"{int(1/(time.time() - now))}\"\n",
    "    now = time.time()\n",
    "    t = time.time()\n",
    "    image = image[0:320,0:320, :]\n",
    "    \n",
    "    d = time.time() - t\n",
    "    if doPrint:\n",
    "        print(\"FPS: \", d)\n",
    "    t = time.time()\n",
    "    times[0].append(d)\n",
    "    if (len(times[0]) > 10):\n",
    "        times[0].pop(0)\n",
    "    \n",
    "    \n",
    "    if counter <100:\n",
    "        return image\n",
    "    \n",
    "    if counter == 100:\n",
    "        print(\"Begin Conversion\")\n",
    "    \n",
    "    # convert image to torch\n",
    "    # from 320 x 320 x 3 to 1 x 3 x 320 x 320\n",
    "    torch_image2 = torch.from_numpy(image)\n",
    "    torch_image = torch.zeros([1, 3, 320, 320])\n",
    "    #torch_image = torch.zeros([1, 3, int(320 / downscale), int(320 / downscale)])\n",
    "    \n",
    "    # from BGR to RGB and from uint8 to float\n",
    "    for i in range(3):\n",
    "        torch_image[0, 2-i, :, :] = torch_image2[:, :, i] / 256\n",
    "    \n",
    "    d = time.time() - t\n",
    "    if doPrint:\n",
    "        print(\"Convert: \", d)\n",
    "    t = time.time()\n",
    "    times[1].append(d)\n",
    "    if (len(times[1]) > 10):\n",
    "        times[1].pop(0)\n",
    "    \n",
    "    \n",
    "    if counter < 150:\n",
    "        return image\n",
    "    \n",
    "    if counter == 150:\n",
    "        print(\"Begin NN\")\n",
    "    \n",
    "    '''if downscale != 1:\n",
    "        for i in range(torch_image.size()[2]):\n",
    "            torch_image[:, :, i, i] = torch.mean(torch_image3[:, :, downscale*i:downscale*i+down_add, downscale*i:downscale*i+down_add])\n",
    "    else:\n",
    "        torch_image = torch_image3'''\n",
    "    \n",
    "    # calculate result\n",
    "    #input is a 1 x 3 x 320 x 320 image\n",
    "    #torch_image = torch_image.to(torch.device(\"cuda\"))\n",
    "    output = torch.from_numpy(ort_sess.run(None, {'input': torch_image.numpy()})[0])\n",
    "    #output = net(torch_image)\n",
    "    #output = output.cpu()\n",
    "    \n",
    "    d = time.time() - t\n",
    "    if doPrint:\n",
    "        print(\"NN: \", d)\n",
    "    t = time.time()\n",
    "    \n",
    "    times[2].append(d)\n",
    "    if (len(times[2]) > 10):\n",
    "        times[2].pop(2)\n",
    "    \n",
    "    \n",
    "    if counter < 200:\n",
    "        return image\n",
    "    \n",
    "    if counter == 200:\n",
    "        print(\"Begin Filter\")\n",
    "    \n",
    "    #output is a 32 x 125 x 10 x 10 tensor\n",
    "    #filter boxes based on confidence score (class_score*confidence)\n",
    "    output = filter_boxes(output, 0.4)\n",
    "    #filter boxes based on overlap\n",
    "    output = nms(output, 0)\n",
    "    \n",
    "    d = time.time() - t\n",
    "    if doPrint:\n",
    "        print(\"filter: \", d)\n",
    "    t = time.time()\n",
    "    times[3].append(d)\n",
    "    if (len(times[3]) > 10):\n",
    "        times[3].pop(0)\n",
    "    \n",
    "    \n",
    "    if counter < 250:\n",
    "        return image\n",
    "    \n",
    "    if counter == 250:\n",
    "        print(\"Begin Drawing\")\n",
    "    \n",
    "    # draw result on camera image\n",
    "    for out1 in output:\n",
    "        for out in out1:\n",
    "            #convert relative to absolute width\n",
    "            w = int(out[2] * 320)\n",
    "            h = int(out[3] * 320)\n",
    "            # convert middle point to upper left corner\n",
    "            x = int(out[0] * 320 - int(w/2))\n",
    "            y = int(out[1] * 320 - int(h/2))\n",
    "            # draw\n",
    "            cv2.rectangle(image, (x, y), (x + w, y + h), (255, 0, 0), 2)\n",
    "            cv2.putText(image, f\"{int(out[4]*100)}\", (x, y-5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 2, cv2.LINE_AA)\n",
    "    \n",
    "    cv2.putText(image, \"fps=\"+fps, (2, 25), cv2.FONT_HERSHEY_SIMPLEX, 1,\n",
    "                (100, 255, 0), 2, cv2.LINE_AA)\n",
    "    \n",
    "    # prints current frame with output\n",
    "    #display_result(torch_image, output, torch.zeros([1,10,6]), file_path='yolo_prediction.png')\n",
    "    \n",
    "    d = time.time() - t\n",
    "    if doPrint:\n",
    "        print(\"draw: \", d)\n",
    "    times[4].append(d)\n",
    "    if (len(times[4]) > 10):\n",
    "        times[4].pop(0)\n",
    "    \n",
    "    avg_times = [sum(x) / len(x) for x in times]\n",
    "    print(f\"FPS: {time_to_string(avg_times[0])}, Convert: {time_to_string(avg_times[1])}, NN: {time_to_string(avg_times[2])}, Filter: {time_to_string(avg_times[3])}, Draw: {time_to_string(avg_times[4])},\", end=\"\\r\")\n",
    "    \n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "3862254a-5f10-4d50-a109-fae94009e508",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing camera...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3c088935be0450ab076d9b70bd7acf5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Image(value=b'\\xff\\xd8\\xff\\xe0\\x00\\x10JFIF\\x00\\x01\\x01\\x00\\x00\\x01\\x00\\x01\\x00\\x00\\xff\\xdb\\x00C\\x00\\x02\\x01\\x0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize the camera with the callback\n",
    "cam = CameraDisplay(callback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "ef4f134f-6d8e-4e28-b9cd-ea47247856c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Begin Cropping\n",
      "Begin Conversion\n",
      "Begin NN\n",
      "Begin Filter\n",
      "Begin Drawing\n",
      "FPS: 0000.69, Convert: 0001.97, NN: 0048.53, Filter: 0004.33, Draw: 00000.7,\r"
     ]
    }
   ],
   "source": [
    "# The camera stream can be started with cam.start()\n",
    "# The callback gets asynchronously called (can be stopped with cam.stop())\n",
    "counter = 0\n",
    "times = [[], [], [], [], []]\n",
    "now = time.time()\n",
    "cam.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "f9baf3b9-a78c-4709-a1df-4fabd57366a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Camera releasedonvert: 0002.73, NN: 0032.59, Filter: 0003.24, Draw: 0000.62,\n"
     ]
    }
   ],
   "source": [
    "# The camera should always be stopped and released for a new camera is instantiated (calling CameraDisplay(callback) again)\n",
    "cam.stop()\n",
    "cam.release()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cc9012d-e89e-461c-8e17-c93746d07c43",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}