Merge pull request #29 from hicsail/customdwelltime

Customdwelltime

src/config/config.ts

@@ -44,7 +44,7 @@ var configuration = {
   doubleClickGesture: "none", // valid values => "dwell", "mouth", "eyebrow-raise" and "none"
 
   // Dwell time hovering time 
-  dwellTime: 1000, // in milliseconds
+  dwellTime: 0.833, // in seconds
 
   // Thresholds for the gestures
   mouthGestureThreshold: 0.4,

src/index.ts

@@ -58,7 +58,7 @@ const createWindow = (): void => {
 };
 
 const turnOffTracking = () => {
-  console.log("Turned off tracking!");
+  //console.log("Turned off tracking!");
   configuration.trackingStatus = TRACKING_STATUS.OFF;
   mainWindow.webContents.send(
     IPC_FUNCTION_KEYS.HANDLE_CONFIGURATION_UPDATE,

src/pyTracker/.spec

@@ -0,0 +1,50 @@
+# -*- mode: python ; coding: utf-8 -*-
+
+
+block_cipher = None
+
+
+a = Analysis(
+    ['main.spec'],
+    pathex=[],
+    binaries=[],
+    datas=[],
+    hiddenimports=[],
+    hookspath=[],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    win_no_prefer_redirects=False,
+    win_private_assemblies=False,
+    cipher=block_cipher,
+    noarchive=False,
+)
+pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name='',
+)

src/pyTracker/src/api/requests.py

@@ -47,8 +47,7 @@ def getLatestAppSettingsFromServer(trackerState):
         trackerState.setScaleFactorValues(config['mouseMovementScaleFactor'], config['mouseMovementScaleFactorY'])
         trackerState.updateGestures(config['leftClickGesture'], config['rightClickGesture'], config['doubleClickGesture'])
         trackerState.updateGestureThresholds(config['mouthGestureThreshold'], config['eyebrowGestureThreshold'])
-        print("Mouth Gesture Threshold: ", config['mouthGestureThreshold'])
-        print("Eyebrow Gesture Threshold: ", config['eyebrowGestureThreshold'])
+        trackerState.updateDwellTime(config['dwellTime'])
 
 # detect shutdown command from server and end python process immediately
 def processShutDownCommand(response):

src/pyTracker/src/main.py

@@ -15,8 +15,7 @@
     while True:
         face, pose, pos, guesture, face_confidence, gesture_confidences = trackFace(trackerState)
         sendTrackingInfo(face, frameSize, pose, pos, guesture, face_confidence, gesture_confidences)
-        convertFaceTrackingToMouseMovement(face, frameSize, pose, pos, guesture)
-
+        convertFaceTrackingToMouseMovement(face, frameSize, pose, pos, guesture, trackerState)
 
         # get config every now and then
         if count % 20 == 0:
@@ -27,7 +26,7 @@
         if k == 27:
             break
 
-        # Do the if statement if the OS is Windows
+        # On Windows, close both windows if the close button is pressed
         if sys.platform == 'win32':
             if cv2.getWindowProperty('Face Tracker', cv2.WND_PROP_VISIBLE) < 1:        
                 break

src/pyTracker/src/videoProcessing/ssdFaceTrack.py

@@ -98,7 +98,7 @@ def getFrameSize():
 		return (config.FRAME_WIDTH,config.FRAME_HEIGHT)
 	frame = imutils.resize(frame, width=config.FRAME_WIDTH, height=config.FRAME_HEIGHT)
 	(h, w) = frame.shape[:2]
-	print("Webcam size (h, w)",(h, w) )
+	#print("Webcam size (h, w)",(h, w) )
 	return (w,h)
 
 from math import cos, sin, hypot
@@ -250,10 +250,6 @@ def detect_eyebrows_raised(left_eyebrow, right_eyebrow, left_eye, right_eye):
 	relative_left = abs(left_eyebrow[2][1] - left_eye[-1][1]) / dist.euclidean(left_eyebrow[0], left_eyebrow[-1])
 	relative_right = abs(right_eyebrow[2][1] - right_eye[-1][1]) / dist.euclidean(right_eyebrow[0], right_eyebrow[-1])
 
-	# print('-------------------')
-	# print("left", relative_left)
-	# print("right", relative_right)
-
 	return min(relative_left, relative_right) > EYEBROW_DIST_THRESHOLD
 
 

src/pyTracker/src/videoProcessing/track2Command.py

@@ -22,7 +22,7 @@ def getCenterOfFace(face):
 frameSize: (w, h) 
     width and height of the frame in which the face was tracked
 """
-def convertFaceTrackingToMouseMovement(face, frameSize, pose, pos, gesture):
+def convertFaceTrackingToMouseMovement(face, frameSize, pose, pos, gesture, state):
     w, h = frameSize
     if pos and len(pos) > 0:
         newMousePosition = { "x" : pos[0] / w, "y" : pos[1] / h }
@@ -38,13 +38,14 @@ def convertFaceTrackingToMouseMovement(face, frameSize, pose, pos, gesture):
             rightClick = trackerState.rightClickGesture
             doubleClick = trackerState.doubleClickGesture
 
+            frames = trackerState.dwellTime
             # Handle left click
             if leftClick == "dwell" : 
-                detectHoverToClickGesture(MOUSE_ACTIONS.LEFT_CLICK)
+                detectHoverToClickGesture(MOUSE_ACTIONS.LEFT_CLICK, frames)
             elif rightClick == "dwell":
-                detectHoverToClickGesture(MOUSE_ACTIONS.RIGHT_CLICK)
+                detectHoverToClickGesture(MOUSE_ACTIONS.RIGHT_CLICK, frames)
             elif doubleClick == "dwell":
-                detectHoverToClickGesture(MOUSE_ACTIONS.DOUBLE_CLICK)
+                detectHoverToClickGesture(MOUSE_ACTIONS.DOUBLE_CLICK, frames)
 
             elif (leftClick == "mouth" and gesture[0]) or (leftClick == "eyebrow-raise" and gesture[1]): 
                 sendRequest(MOUSE_ACTION_PATH, MOUSE_ACTIONS.LEFT_CLICK)
@@ -71,10 +72,10 @@ def sendTrackingInfo(face, frameSize, pose, pos, guesture, face_confidence, gest
 """
 sends a left click command to server if face has been hovering around the same position for a while
 """
-def detectHoverToClickGesture(command):
-    if len(trackerState.trackedPositions) >= config.HOVER_TO_CLICK_MIN_POINTS: 
+def detectHoverToClickGesture(command, frames):
+    if len(trackerState.trackedPositions) >= frames: 
         # get last HOVER_TO_CLICK_MIN_POINTS tracked positions.
-        startIndex = len(trackerState.trackedPositions) - config.HOVER_TO_CLICK_MIN_POINTS
+        startIndex = len(trackerState.trackedPositions) - frames
         endIndex = len(trackerState.trackedPositions)
         cluster = deque(itertools.islice(trackerState.trackedPositions, startIndex, endIndex))# a deque can't be sliced with [x:x]
         # get distance between furthest two points in cluster 
@@ -84,4 +85,5 @@ def detectHoverToClickGesture(command):
         if longestDistanceX < config.HOVER_TO_CLICK_DISTANCE_THRESHOLD and longestDistanceY < config.HOVER_TO_CLICK_DISTANCE_THRESHOLD:
             # clear past positions to prevent unintentional consecutive clicks
             trackerState.trackedPositions.clear()
+            #print("frames: ", frames)
             sendRequest(MOUSE_ACTION_PATH, command)

src/pyTracker/src/videoProcessing/trackerState.py

@@ -20,6 +20,8 @@ def __init__(self):
         self.mouthGestureThreshold = 0.4
         self.eyebrowGestureThreshold = 0.4
 
+        self.dwellTime = 25 # in frames
+
     def updateTracked(confidence):
         trackedReq += 1
         trackedProb += confidence
@@ -46,4 +48,9 @@ def updateGestureThresholds(self, mouthGestureThreshold, eyebrowGestureThreshold
         self.mouthGestureThreshold = mouthGestureThreshold
         self.eyebrowGestureThreshold = eyebrowGestureThreshold
 
+    def updateDwellTime(self, dwellTime):
+        # input: dwellTime - in seconds
+        frames = int((30*dwellTime) // 1)
+        self.dwellTime = frames
+
 trackerState = TrackerState()

src/screens/Settings.tsx

@@ -43,6 +43,7 @@ const Settings: React.FC<Props> = (props) => {
   const [doubleClickGesture, setDoubleClickGesture] = useState(appConfig.doubleClickGesture);
   const [mouthThreshold, setMouthThreshold] = useState(appConfig.mouthGestureThreshold);
   const [eyebrowThreshold, setEyebrowThreshold] = useState(appConfig.eyebrowGestureThreshold);
+  const [dwellTime, setDwellTime] = useState(appConfig.dwellTime);
 
   console.log(TAG, "appConfig", appConfig);
 
@@ -128,7 +129,6 @@ const Settings: React.FC<Props> = (props) => {
         // create alert to user
         return;
     }
-    console.log(event);
     setLeftClickGesture(event.target.value as string);
 
 
@@ -137,7 +137,6 @@ const Settings: React.FC<Props> = (props) => {
     await window.electronAPI.updateConfiguration(newAppConfig);
     await setAppConfig(newAppConfig);
 
-    console.log("leftClickGesture after", leftClickGesture)
 
   };
 
@@ -187,6 +186,16 @@ const Settings: React.FC<Props> = (props) => {
     }
   };
 
+  const handleDwellTimeChange = async (_:Event, newValue: number | number[]) => {
+    if (typeof newValue == "number") {
+      setDwellTime(newValue);
+      console.log("dwellTime", newValue);
+      const newAppConfig = { ...appConfig, dwellTime: newValue};
+      window.electronAPI.updateConfiguration(newAppConfig);
+      await setAppConfig(newAppConfig);
+    }
+  }
+
   return (
     <Stack minWidth={350} width={"100%"}>
       {!appConfig.monoTrackingMode && (
@@ -288,9 +297,22 @@ const Settings: React.FC<Props> = (props) => {
               Thresholds
             </Typography>
 
-            <Stack direction='row' justifyContent={"space-between"} spacing={10}>
-
+            <Stack direction='row' justifyContent={"space-between"} spacing={10} >
 
+              <Stack>
+                <Typography gutterBottom>Dwell Time</Typography>
+                <StyledSlider
+                  aria-label="eyebrow threshold slider"
+                  value={dwellTime}
+                  onChangeCommitted={handleDwellTimeChange}
+                  getAriaValueText={(value: string) => value.toString()}
+                  marks
+                  step={0.1}
+                  min={0.3}
+                  max={1.7}
+                  valueLabelDisplay="auto"
+                />
+              </Stack>
               <Stack>
                 <Typography gutterBottom>Mouth threshold</Typography>
                 <StyledSlider
@@ -303,6 +325,7 @@ const Settings: React.FC<Props> = (props) => {
                   min={0.1}
                   max={0.9}
                   valueLabelDisplay="auto"
+                  style={{flexGrow: 2}}
                 />
               </Stack>
               <Stack>