diff --git a/src/controlApis/mouseCommands.ts b/src/controlApis/mouseCommands.ts index da4f972..6252d40 100644 --- a/src/controlApis/mouseCommands.ts +++ b/src/controlApis/mouseCommands.ts @@ -2,6 +2,7 @@ import { Button, down, left, right, up } from "@nut-tree/nut-js"; import { LPFStream } from "../utils/filters"; import { configuration } from "../config/config"; +const fs = require('fs/promises'); const { mouse, straightTo } = require("@nut-tree/nut-js"); const lPFStreamX = new LPFStream( @@ -13,6 +14,17 @@ const lPFStreamY = new LPFStream( configuration.smoothingFactor ); +type TRACKING_INFO = { + face: string, + frameSize: string, + pose: string, + pos: string, + gesture: string, + face_confidence: string, + gesture_confidences: string +} + + /** * adjust the position ratios based on configuration.mouseMovementScaleFactor * helps move mouse across screen with only tiny movements @@ -120,17 +132,16 @@ async function moveMouse(requestBody: { y: number; }; - const content = "Movement to " + requestBody.x + " " + requestBody.y + "\n"; - const fs = require('fs/promises'); + const movementContent = "Movement to " + requestBody.x + " " + requestBody.y + "\n"; - async function example() { + async function writeLogs() { try { - await fs.appendFile('movement_log.txt', content); + await fs.appendFile('movement_log.txt', movementContent); } catch (err) { console.log(err); } } -example(); + writeLogs(); if (configuration.trackingMode == "position") { newPosition = await moveByRatioCoordinates({ x: requestBody.x, @@ -197,7 +208,6 @@ async function moveByYawAndPitch(yaw: number, pitch: number) { function click(direction: "left" | "right") { const content = direction + " click \n"; - const fs = require('fs/promises'); async function example() { try { @@ -233,5 +243,14 @@ const demoMove = async () => { await mouse.move(down(500)); }; -export { moveByRatioCoordinates, click, doubleClick, moveMouse, demoMove }; +const writeTrackingInfo = async (trackingInfo: TRACKING_INFO) => { + try { + const content = trackingInfo.face + " " + trackingInfo.frameSize + " " + trackingInfo.pose + " " + trackingInfo.pos + " " + trackingInfo.gesture + " " + trackingInfo.face_confidence + " " + trackingInfo.gesture_confidences + "\n"; + await fs.appendFile('tracking_log.txt', content); + } catch (err) { + console.log(err); + } +} + +export { moveByRatioCoordinates, click, doubleClick, moveMouse, demoMove, writeTrackingInfo }; diff --git a/src/index.ts b/src/index.ts index da5617a..500fc21 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,7 @@ import { import { IPC_FUNCTION_KEYS } from "./constants/ipcFunctionKeys"; import { startServer } from "./server/server"; import * as path from "path"; +import { create } from "@mui/material/styles/createTransitions"; const log = require("electron-log"); const isDev = require("electron-is-dev"); //whether elctron is running in prod or dev mode @@ -255,7 +256,7 @@ app.whenReady().then(async () => { } } createFile("movement_log.txt"); - + createFile("tracking_log.txt"); }); diff --git a/src/pyTracker/src/api/requests.py b/src/pyTracker/src/api/requests.py index c66a421..f1cf414 100644 --- a/src/pyTracker/src/api/requests.py +++ b/src/pyTracker/src/api/requests.py @@ -6,6 +6,7 @@ MOUSE_MOVEMENT_PATH = "mouse/moveto" MOUSE_ACTION_PATH = "mouse/action" SETTINGS_PATH = "mouse/settings" +TRACKING_LOG_PATH = "mouse/tracking_log" class MOUSE_ACTIONS: LEFT_CLICK = {"action": "leftClick"} diff --git a/src/pyTracker/src/main.py b/src/pyTracker/src/main.py index c20a964..7749ee0 100644 --- a/src/pyTracker/src/main.py +++ b/src/pyTracker/src/main.py @@ -1,14 +1,11 @@ from api.requests import getLatestAppSettingsFromServer -from videoProcessing.track2Command import convertFaceTrackingToMouseMovement +from videoProcessing.track2Command import convertFaceTrackingToMouseMovement, sendTrackingInfo from videoProcessing.ssdFaceTrack import getFrameSize, trackFace from videoProcessing.trackerState import trackerState -from utils.utils import writeFaceTrackingLogToFile import cv2 import os import sys -f = open("trackingLog.txt", "w") - if __name__ == "__main__": @@ -17,7 +14,7 @@ count = 0 while True: face, pose, pos, guesture, face_confidence, gesture_confidences = trackFace(trackerState) - writeFaceTrackingLogToFile(face, frameSize, pose, pos, guesture, face_confidence, gesture_confidences) + sendTrackingInfo(face, frameSize, pose, pos, guesture, face_confidence, gesture_confidences) convertFaceTrackingToMouseMovement(face, frameSize, pose, pos, guesture) diff --git a/src/pyTracker/src/videoProcessing/track2Command.py b/src/pyTracker/src/videoProcessing/track2Command.py index 38e58ef..cd02c12 100644 --- a/src/pyTracker/src/videoProcessing/track2Command.py +++ b/src/pyTracker/src/videoProcessing/track2Command.py @@ -3,7 +3,7 @@ """ from collections import deque import itertools -from api.requests import MOUSE_ACTION_PATH, MOUSE_ACTIONS, MOUSE_MOVEMENT_PATH, sendRequest +from api.requests import MOUSE_ACTION_PATH, MOUSE_ACTIONS, MOUSE_MOVEMENT_PATH, TRACKING_LOG_PATH, sendRequest from videoProcessing.trackerState import trackerState from videoProcessing.config import config @@ -55,7 +55,18 @@ def convertFaceTrackingToMouseMovement(face, frameSize, pose, pos, gesture): if (doubleClick == "mouth" and gesture[0]) or (doubleClick == "eyebrow-raise" and gesture[1]): sendRequest(MOUSE_ACTION_PATH, MOUSE_ACTIONS.DOUBLE_CLICK) +def sendTrackingInfo(face, frameSize, pose, pos, guesture, face_confidence, gesture_confidences): + data = { + "face": str(face), + "frameSize": str(frameSize), + "pose": str(pose), + "pos": str(pos), + "gesture": str(guesture), + "face_confidence": str(face_confidence), + "gesture_confidences": str(gesture_confidences) + } + sendRequest(TRACKING_LOG_PATH, data) """ sends a left click command to server if face has been hovering around the same position for a while diff --git a/src/server/routes/mouseCommands.routes.ts b/src/server/routes/mouseCommands.routes.ts index 5ddd7cd..e382f01 100644 --- a/src/server/routes/mouseCommands.routes.ts +++ b/src/server/routes/mouseCommands.routes.ts @@ -1,7 +1,7 @@ import { Router } from "express"; import { mainWindow } from "../../index"; import { configuration, TRACKING_STATUS } from "../../config/config"; -import { click, doubleClick, moveMouse } from "../../controlApis/mouseCommands"; +import { click, doubleClick, moveMouse, writeTrackingInfo} from "../../controlApis/mouseCommands"; import { IPC_FUNCTION_KEYS } from "../../constants/ipcFunctionKeys"; let router = Router(); @@ -9,6 +9,7 @@ const { handleUnknownError } = require("../utils"); const MOVEMENT_PATH = "/moveto"; const ACTION_PATH = "/action"; const SETTINGS_PATH = "/settings"; +const LOG_PATH = "/tracking_log"; // get current app configuration router.route(SETTINGS_PATH).get(async (req, res) => { @@ -70,6 +71,21 @@ router.route(ACTION_PATH).post(async (req: any, res: any) => { } }); +router.route(LOG_PATH).post(async (req: any, res: any) => { + try { + const log = req.body; + + if (log) { + writeTrackingInfo(log); + } + res.send({ status: "ok" }); + } catch (error) { + handleUnknownError(res, error); + } +} + +); + /** * detects if app is shutting down and responds with shutdown status * @param httpResponse