Added the final working version of the realtime demo

for FAZE.

.gitmodules

@@ -1,3 +1,12 @@
 [submodule "preprocess"]
 	path = preprocess
 	url = https://github.com/swook/faze_preprocess
+[submodule "demo/ext/eos"]
+	path = demo/ext/eos
+	url = https://github.com/patrikhuber/eos
+[submodule "demo/ext/HRNet-Facial-Landmark-Detection"]
+	path = demo/ext/HRNet-Facial-Landmark-Detection
+	url = https://github.com/HRNet/HRNet-Facial-Landmark-Detection
+[submodule "demo/ext/mtcnn-pytorch"]
+	path = demo/ext/mtcnn-pytorch
+	url = https://github.com/TropComplique/mtcnn-pytorch

demo/KalmanFilter1D.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+# --------------------------------------------------------
+# Copyright (C) 2020 NVIDIA Corporation. All rights reserved.
+# NVIDIA Source Code License (1-Way Commercial)
+# Code written by Pavlo Molchanov, Shalini De Mello.
+# --------------------------------------------------------
+
+import numpy as np
+
+class Kalman1D(object):
+
+    def __init__(self, R=0.001**2, sz=100):
+        self.Q = 1e-5 # process variance
+        # allocate space for arrays
+        self.xhat = np.zeros(sz, dtype=complex)      # a posteri estimate of x
+        self.P = np.zeros(sz, dtype=complex)         # a posteri error estimate
+        self.xhatminus = np.zeros(sz, dtype=complex) # a priori estimate of x
+        self.Pminus = np.zeros(sz, dtype=complex)    # a priori error estimate
+        self.K = np.zeros(sz, dtype=complex)         # gain or blending factor
+        self.R = R # estimate of measurement variance, change to see effect
+        self.sz = sz
+        # intial guesses
+        self.xhat[0] = 0.0
+        self.P[0] = 1.0
+        self.k = 1
+
+    def update(self, val):
+        k = self.k % self.sz
+        km = (self.k-1) % self.sz
+        self.xhatminus[k] = self.xhat[km]
+        self.Pminus[k] = self.P[km] + self.Q
+
+        # measurement update
+        self.K[k] = self.Pminus[k]/( self.Pminus[k]+self.R )
+        self.xhat[k] = self.xhatminus[k]+self.K[k]*(val-self.xhatminus[k])
+        self.P[k] = (1-self.K[k])*self.Pminus[k]
+        self.k = self.k + 1
+        return self.xhat[k]

demo/camera.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+
+# --------------------------------------------------------
+# Copyright (C) 2020 NVIDIA Corporation. All rights reserved.
+# NVIDIA Source Code License (1-Way Commercial)
+# Code written by Shalini De Mello, Seonwook Park.
+# --------------------------------------------------------
+
+import cv2
+import numpy as np
+import pickle
+
+def cam_calibrate(cam_idx, cap, cam_calib):
+
+    # termination criteria
+    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
+
+    # prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
+    pts = np.zeros((6 * 9, 3), np.float32)
+    pts[:, :2] = np.mgrid[0:9, 0:6].T.reshape(-1, 2)
+
+    # capture calibration frames
+    obj_points = []  # 3d point in real world space
+    img_points = []  # 2d points in image plane.
+    frames = []
+    while True:
+        ret, frame = cap.read()
+
+        if ret:
+            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+            ret, corners = cv2.findChessboardCorners(gray, (9, 6), None)
+            if ret:
+                cv2.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
+                # Draw and display the corners
+                frame_copy = frame.copy()
+                cv2.drawChessboardCorners(frame_copy, (9, 6), corners, ret)
+                cv2.imshow('points', frame_copy)
+
+                # s to save, c to continue, q to quit
+                if cv2.waitKey(0) & 0xFF == ord('s'):
+                    img_points.append(corners)
+                    obj_points.append(pts)
+                    frames.append(frame)
+                elif cv2.waitKey(0) & 0xFF == ord('n'):
+                    continue
+                elif cv2.waitKey(0) & 0xFF == ord('q'):
+                    cv2.destroyAllWindows()
+                    break
+
+    # compute calibration matrices
+    ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(obj_points, img_points, frames[0].shape[0:2], None, None)
+
+    # check
+    error = 0.0
+    for i in range(len(frames)):
+        proj_imgpoints, _ = cv2.projectPoints(obj_points[i], rvecs[i], tvecs[i], mtx, dist)
+        error += (cv2.norm(img_points[i], proj_imgpoints, cv2.NORM_L2) / len(proj_imgpoints))
+    print("Camera calibrated successfully, total re-projection error: %f" % (error / len(frames)))
+
+    cam_calib['mtx'] = mtx
+    cam_calib['dist'] = dist
+    print("Camera parameters:")
+    print(cam_calib)
+
+    pickle.dump(cam_calib, open("calib_cam%d.pkl" % (cam_idx), "wb"))

demo/ext/mtcnn-pytorch/.gitignore

@@ -0,0 +1,3 @@
+.ipynb_checkpoints
+__pycache__
+

demo/ext/mtcnn-pytorch/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Dan Antoshchenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

demo/ext/mtcnn-pytorch/README.md

@@ -0,0 +1,26 @@
+# MTCNN
+
+`pytorch` implementation of **inference stage** of face detection algorithm described in  
+[Joint Face Detection and Alignment using Multi-task Cascaded Convolutional Networks](https://arxiv.org/abs/1604.02878).
+
+## Example
+![example of a face detection](images/example.png)
+
+## How to use it
+Just download the repository and then do this
+```python
+from src import detect_faces
+from PIL import Image
+
+image = Image.open('image.jpg')
+bounding_boxes, landmarks = detect_faces(image)
+```
+For examples see `test_on_images.ipynb`.
+
+## Requirements
+* pytorch 0.2
+* Pillow, numpy
+
+## Credit
+This implementation is heavily inspired by:
+* [pangyupo/mxnet_mtcnn_face_detection](https://github.com/pangyupo/mxnet_mtcnn_face_detection)  

demo/ext/mtcnn-pytorch/caffe_models/det1.prototxt

@@ -0,0 +1,177 @@
+name: "PNet"
+input: "data"
+input_dim: 1
+input_dim: 3
+input_dim: 12
+input_dim: 12
+
+layer {
+  name: "conv1"
+  type: "Convolution"
+  bottom: "data"
+  top: "conv1"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 10
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "PReLU1"
+  type: "PReLU"
+  bottom: "conv1"
+  top: "conv1"
+}
+layer {
+  name: "pool1"
+  type: "Pooling"
+  bottom: "conv1"
+  top: "pool1"
+  pooling_param {
+    pool: MAX
+    kernel_size: 2
+    stride: 2
+  }
+}
+
+layer {
+  name: "conv2"
+  type: "Convolution"
+  bottom: "pool1"
+  top: "conv2"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 16
+    kernel_size: 3
+    stride: 1
+     weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "PReLU2"
+  type: "PReLU"
+  bottom: "conv2"
+  top: "conv2"
+}
+
+layer {
+  name: "conv3"
+  type: "Convolution"
+  bottom: "conv2"
+  top: "conv3"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 32
+    kernel_size: 3
+    stride: 1
+     weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+	  type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "PReLU3"
+  type: "PReLU"
+  bottom: "conv3"
+  top: "conv3"
+}
+
+
+layer {
+  name: "conv4-1"
+  type: "Convolution"
+  bottom: "conv3"
+  top: "conv4-1"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 2
+    kernel_size: 1
+    stride: 1
+     weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+
+layer {
+  name: "conv4-2"
+  type: "Convolution"
+  bottom: "conv3"
+  top: "conv4-2"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 4
+    kernel_size: 1
+    stride: 1
+     weight_filler {
+      type: "xavier"
+	}
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "prob1"
+  type: "Softmax"
+  bottom: "conv4-1"
+  top: "prob1"
+}