Merge pull request #26 from hicsail/facetracklog

Face tracking info bug fix

src/controlApis/mouseCommands.ts

@@ -2,6 +2,7 @@ import { Button, down, left, right, up } from "@nut-tree/nut-js";
 import { LPFStream } from "../utils/filters";
 import { configuration } from "../config/config";
 
+const fs = require('fs/promises');
 const { mouse, straightTo } = require("@nut-tree/nut-js");
 
 const lPFStreamX = new LPFStream(
@@ -13,6 +14,17 @@ const lPFStreamY = new LPFStream(
   configuration.smoothingFactor
 );
 
+type TRACKING_INFO = {
+  face: string, 
+  frameSize: string,
+  pose: string,
+  pos: string,
+  gesture: string,
+  face_confidence: string,
+  gesture_confidences: string
+}
+
+
 /**
  * adjust the position ratios based on configuration.mouseMovementScaleFactor
  * helps move mouse across screen with only tiny movements
@@ -120,17 +132,16 @@ async function moveMouse(requestBody: {
     y: number;
   };
 
-  const content = "Movement to " + requestBody.x + " " + requestBody.y + "\n";
-  const fs = require('fs/promises');
+  const movementContent = "Movement to " + requestBody.x + " " + requestBody.y + "\n";
 
-  async function example() {
+  async function writeLogs() {
   try {
-    await fs.appendFile('movement_log.txt', content);
+    await fs.appendFile('movement_log.txt', movementContent);
   } catch (err) {
     console.log(err);
   }
 }
-example();
+  writeLogs();
   if (configuration.trackingMode == "position") {
     newPosition = await moveByRatioCoordinates({
       x: requestBody.x,
@@ -197,7 +208,6 @@ async function moveByYawAndPitch(yaw: number, pitch: number) {
 function click(direction: "left" | "right") {
   const content = direction + " click \n";
 
-  const fs = require('fs/promises');
 
   async function example() {
   try {
@@ -233,5 +243,14 @@ const demoMove = async () => {
   await mouse.move(down(500));
 };
 
-export { moveByRatioCoordinates, click, doubleClick, moveMouse, demoMove };
+const writeTrackingInfo = async (trackingInfo: TRACKING_INFO) => {
+  try {
+    const content = trackingInfo.face + " " + trackingInfo.frameSize + " " + trackingInfo.pose + " " + trackingInfo.pos + " " + trackingInfo.gesture + " " + trackingInfo.face_confidence + " " + trackingInfo.gesture_confidences + "\n";
+    await fs.appendFile('tracking_log.txt', content);
+  } catch (err) {
+    console.log(err);
+  }
+}
+
+export { moveByRatioCoordinates, click, doubleClick, moveMouse, demoMove, writeTrackingInfo };
 

src/index.ts

@@ -16,6 +16,7 @@ import {
 import { IPC_FUNCTION_KEYS } from "./constants/ipcFunctionKeys";
 import { startServer } from "./server/server";
 import * as path from "path";
+import { create } from "@mui/material/styles/createTransitions";
 const log = require("electron-log");
 const isDev = require("electron-is-dev"); //whether elctron is running in prod or dev mode
 
@@ -255,7 +256,7 @@ app.whenReady().then(async () => {
     }
   }
   createFile("movement_log.txt");
-
+  createFile("tracking_log.txt");
 
 });
 

src/pyTracker/src/api/requests.py

@@ -6,6 +6,7 @@
 MOUSE_MOVEMENT_PATH = "mouse/moveto"
 MOUSE_ACTION_PATH = "mouse/action"
 SETTINGS_PATH = "mouse/settings"
+TRACKING_LOG_PATH = "mouse/tracking_log"
 
 class MOUSE_ACTIONS:
     LEFT_CLICK = {"action": "leftClick"}

src/pyTracker/src/main.py

@@ -1,14 +1,11 @@
 from api.requests import getLatestAppSettingsFromServer
-from videoProcessing.track2Command import convertFaceTrackingToMouseMovement
+from videoProcessing.track2Command import convertFaceTrackingToMouseMovement, sendTrackingInfo
 from videoProcessing.ssdFaceTrack import getFrameSize, trackFace
 from videoProcessing.trackerState import trackerState
-from utils.utils import writeFaceTrackingLogToFile
 import cv2
 import os
 import sys
 
-f = open("trackingLog.txt", "w")
-
 if __name__ == "__main__":
 
 
@@ -17,7 +14,7 @@
     count = 0
     while True:
         face, pose, pos, guesture, face_confidence, gesture_confidences = trackFace(trackerState)
-        writeFaceTrackingLogToFile(face, frameSize, pose, pos, guesture, face_confidence, gesture_confidences)
+        sendTrackingInfo(face, frameSize, pose, pos, guesture, face_confidence, gesture_confidences)
         convertFaceTrackingToMouseMovement(face, frameSize, pose, pos, guesture)
 
 

src/pyTracker/src/videoProcessing/track2Command.py

@@ -3,7 +3,7 @@
 """ 
 from collections import deque
 import itertools
-from api.requests import MOUSE_ACTION_PATH, MOUSE_ACTIONS, MOUSE_MOVEMENT_PATH, sendRequest
+from api.requests import MOUSE_ACTION_PATH, MOUSE_ACTIONS, MOUSE_MOVEMENT_PATH, TRACKING_LOG_PATH, sendRequest
 from videoProcessing.trackerState import trackerState
 from videoProcessing.config import config
 
@@ -55,7 +55,18 @@ def convertFaceTrackingToMouseMovement(face, frameSize, pose, pos, gesture):
             if (doubleClick == "mouth" and gesture[0]) or (doubleClick == "eyebrow-raise" and gesture[1]):
                 sendRequest(MOUSE_ACTION_PATH, MOUSE_ACTIONS.DOUBLE_CLICK)
 
+def sendTrackingInfo(face, frameSize, pose, pos, guesture, face_confidence, gesture_confidences):
+    data = {
+        "face": str(face),
+        "frameSize": str(frameSize),
+        "pose": str(pose),
+        "pos": str(pos),
+        "gesture": str(guesture),
+        "face_confidence": str(face_confidence),
+        "gesture_confidences": str(gesture_confidences)
+    }
 
+    sendRequest(TRACKING_LOG_PATH, data)
 
 """
 sends a left click command to server if face has been hovering around the same position for a while

src/server/routes/mouseCommands.routes.ts

@@ -1,14 +1,15 @@
 import { Router } from "express";
 import { mainWindow } from "../../index";
 import { configuration, TRACKING_STATUS } from "../../config/config";
-import { click, doubleClick, moveMouse } from "../../controlApis/mouseCommands";
+import { click, doubleClick, moveMouse, writeTrackingInfo} from "../../controlApis/mouseCommands";
 import { IPC_FUNCTION_KEYS } from "../../constants/ipcFunctionKeys";
 
 let router = Router();
 const { handleUnknownError } = require("../utils");
 const MOVEMENT_PATH = "/moveto";
 const ACTION_PATH = "/action";
 const SETTINGS_PATH = "/settings";
+const LOG_PATH = "/tracking_log";
 
 // get current app configuration
 router.route(SETTINGS_PATH).get(async (req, res) => {
@@ -70,6 +71,21 @@ router.route(ACTION_PATH).post(async (req: any, res: any) => {
   }
 });
 
+router.route(LOG_PATH).post(async (req: any, res: any) => {
+  try {
+    const log = req.body;
+
+    if (log) {
+      writeTrackingInfo(log);
+    }
+    res.send({ status: "ok" });
+  } catch (error) {
+    handleUnknownError(res, error);
+  }
+}
+
+);
+
 /**
  * detects if app is shutting down and responds with shutdown status
  * @param httpResponse