Merge branch 'demo'

.gitmodules

@@ -1,3 +1,12 @@
 [submodule "preprocess"]
 	path = preprocess
 	url = https://github.com/swook/faze_preprocess
+[submodule "demo/ext/eos"]
+	path = demo/ext/eos
+	url = https://github.com/patrikhuber/eos
+[submodule "demo/ext/HRNet-Facial-Landmark-Detection"]
+	path = demo/ext/HRNet-Facial-Landmark-Detection
+	url = https://github.com/HRNet/HRNet-Facial-Landmark-Detection
+[submodule "demo/ext/mtcnn-pytorch"]
+	path = demo/ext/mtcnn-pytorch
+	url = https://github.com/TropComplique/mtcnn-pytorch

demo/KalmanFilter1D.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+# --------------------------------------------------------
+# Copyright (C) 2020 NVIDIA Corporation. All rights reserved.
+# NVIDIA Source Code License (1-Way Commercial)
+# Code written by Pavlo Molchanov, Shalini De Mello.
+# --------------------------------------------------------
+
+import numpy as np
+
+class Kalman1D(object):
+
+    def __init__(self, R=0.001**2, sz=100):
+        self.Q = 1e-5 # process variance
+        # allocate space for arrays
+        self.xhat = np.zeros(sz, dtype=complex)      # a posteri estimate of x
+        self.P = np.zeros(sz, dtype=complex)         # a posteri error estimate
+        self.xhatminus = np.zeros(sz, dtype=complex) # a priori estimate of x
+        self.Pminus = np.zeros(sz, dtype=complex)    # a priori error estimate
+        self.K = np.zeros(sz, dtype=complex)         # gain or blending factor
+        self.R = R # estimate of measurement variance, change to see effect
+        self.sz = sz
+        # intial guesses
+        self.xhat[0] = 0.0
+        self.P[0] = 1.0
+        self.k = 1
+
+    def update(self, val):
+        k = self.k % self.sz
+        km = (self.k-1) % self.sz
+        self.xhatminus[k] = self.xhat[km]
+        self.Pminus[k] = self.P[km] + self.Q
+
+        # measurement update
+        self.K[k] = self.Pminus[k]/( self.Pminus[k]+self.R )
+        self.xhat[k] = self.xhatminus[k]+self.K[k]*(val-self.xhatminus[k])
+        self.P[k] = (1-self.K[k])*self.Pminus[k]
+        self.k = self.k + 1
+        return self.xhat[k]

demo/README.md

@@ -0,0 +1,76 @@
+## Setup
+
+### 1. Setup
+
+This codebase should run on most standard Linux systems. It is tested with Ubuntu 16.04, pytorch v1.3.1, cuda v10.1, python v3.5.2.
+
+a. This demo uses two external submodules: [EOS](https://pypi.org/project/eos-py/) and
+   [HRNET](https://github.com/HRNet/HRNet-Facial-Landmark-Detection) for face and facial landmarks detection, respectively.
+
+If you have already cloned this (`few_shot_gaze`) repository without pulling the submodules, please run:
+
+    git submodule update --init --recursive
+
+Also, download the pre-trained `HR18-WFLW.pth` model for HRNet from [here](https://1drv.ms/u/s!AiWjZ1LamlxzdTsr_9QZCwJsn5U)
+   and place it inside the folder:
+
+    mkdir demo/ext/HRNet-Facial-Landmark-Detection/hrnetv2_pretrained
+
+*Please note* that the Python Pip dependencies for the live demo (found under `/demo`) are different to the training/evaluation code of the network. You must install the additional dependencies. This is described in the next step.
+
+b. Create a Python virtual environment:
+
+    cd demo
+    python3 -m venv venv
+    source venv/bin/activate
+    pip install -r requirements.txt
+    sudo apt-get install -y software-properties-common
+    sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+    sudo apt update
+    sudo apt install g++-7 -y
+    CC=`which gcc-7` CXX=`which g++-7` pip3 install eos-py
+
+### 2. Camera and Monitor calibration
+  a. Calibrate your camera:
+
+    python calibrate_camera.py
+
+   This should generate a file named `calib_cam<id>.pkl` inside the `demo` folder.
+
+   b. Calibrate your monitor's orientation and the position of its upper-left corner w.r.t. to the
+   camera using the [Mirror-based Calibration](https://computer-vision.github.io/takahashi2012cvpr/) routine and
+   update the methods `camera_to_monitor` and `monitor_to_camera` in `monitor.py` for your system appropriately.
+
+   We recommend using the in-built camera in laptops or attaching an external webcam **rigidly** to your monitor.
+   If you move your webcam relative to the monitor you will have to calibrate it again.
+
+### 3. Download pre-trained models for FAZE from [here](https://ait.ethz.ch/projects/2019/faze/downloads/demo_weights.zip).
+    cd demo
+    wget https://ait.ethz.ch/projects/2019/faze/downloads/demo_weights.zip
+    unzip demo_weights.zip
+
+   These are slightly updated models that perform better than the originals ones documented in the published ICCV 2019
+   paper.
+
+### 4. Run demo
+    python run_demo.py
+
+This will collect user calibration data (9-point by default) and fine-tune the gaze network with it. The calibration
+targets are the letter 'E' shown on a 3x3 grid on the screen in any of the 4 orientations: up, down, left or right.
+The user must press the corresponding arrow key to advance to the next calibration target, otherwise another randomly
+oriented target will be shown again at the same screen location. After calibration, the updated gaze network will be
+used to continuously compute the user's on-screen point-of-regard and shown on the display.
+
+### Best practices:
+
+* A user should always look directly at the targets when pressing the arrow
+keys and not at the keyboard to record accurate calibration data.
+
+* For best results, experiment with the contrast, brightness and sharpness settings of your webcam .
+    * see top of `run_demo.py`
+
+* For best results, experiment with the learning rate and number of training steps used for fine-tuning.
+    * Adjust the `lr` argument of `fine_tune` as called from `run_demo.py`.
+
+* To change the delay/smoothing of the estimated on-screen point-of-regard modify the Kalman filter settings
+in `frame_processor.py`.

demo/ext/mtcnn-pytorch/.gitignore

@@ -0,0 +1,3 @@
+.ipynb_checkpoints
+__pycache__
+

demo/ext/mtcnn-pytorch/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Dan Antoshchenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

demo/ext/mtcnn-pytorch/README.md

@@ -0,0 +1,26 @@
+# MTCNN
+
+`pytorch` implementation of **inference stage** of face detection algorithm described in  
+[Joint Face Detection and Alignment using Multi-task Cascaded Convolutional Networks](https://arxiv.org/abs/1604.02878).
+
+## Example
+![example of a face detection](images/example.png)
+
+## How to use it
+Just download the repository and then do this
+```python
+from src import detect_faces
+from PIL import Image
+
+image = Image.open('image.jpg')
+bounding_boxes, landmarks = detect_faces(image)
+```
+For examples see `test_on_images.ipynb`.
+
+## Requirements
+* pytorch 0.2
+* Pillow, numpy
+
+## Credit
+This implementation is heavily inspired by:
+* [pangyupo/mxnet_mtcnn_face_detection](https://github.com/pangyupo/mxnet_mtcnn_face_detection)  

demo/ext/mtcnn-pytorch/caffe_models/det1.prototxt

@@ -0,0 +1,177 @@
+name: "PNet"
+input: "data"
+input_dim: 1
+input_dim: 3
+input_dim: 12
+input_dim: 12
+
+layer {
+  name: "conv1"
+  type: "Convolution"
+  bottom: "data"
+  top: "conv1"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 10
+    kernel_size: 3
+    stride: 1
+    weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "PReLU1"
+  type: "PReLU"
+  bottom: "conv1"
+  top: "conv1"
+}
+layer {
+  name: "pool1"
+  type: "Pooling"
+  bottom: "conv1"
+  top: "pool1"
+  pooling_param {
+    pool: MAX
+    kernel_size: 2
+    stride: 2
+  }
+}
+
+layer {
+  name: "conv2"
+  type: "Convolution"
+  bottom: "pool1"
+  top: "conv2"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 16
+    kernel_size: 3
+    stride: 1
+     weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "PReLU2"
+  type: "PReLU"
+  bottom: "conv2"
+  top: "conv2"
+}
+
+layer {
+  name: "conv3"
+  type: "Convolution"
+  bottom: "conv2"
+  top: "conv3"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 32
+    kernel_size: 3
+    stride: 1
+     weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+	  type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "PReLU3"
+  type: "PReLU"
+  bottom: "conv3"
+  top: "conv3"
+}
+
+
+layer {
+  name: "conv4-1"
+  type: "Convolution"
+  bottom: "conv3"
+  top: "conv4-1"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 2
+    kernel_size: 1
+    stride: 1
+     weight_filler {
+      type: "xavier"
+    }
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+
+layer {
+  name: "conv4-2"
+  type: "Convolution"
+  bottom: "conv3"
+  top: "conv4-2"
+  param {
+    lr_mult: 1
+    decay_mult: 1
+  }
+  param {
+    lr_mult: 2
+    decay_mult: 0
+  }
+  convolution_param {
+    num_output: 4
+    kernel_size: 1
+    stride: 1
+     weight_filler {
+      type: "xavier"
+	}
+    bias_filler {
+      type: "constant"
+      value: 0
+    }
+  }
+}
+layer {
+  name: "prob1"
+  type: "Softmax"
+  bottom: "conv4-1"
+  top: "prob1"
+}