fix video save

vpalmisano · Oct 11, 2024 · 26f817f · 26f817f
1 parent 96b5a25
commit 26f817f
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 20 deletions.
diff --git a/scripts/save-tracks.js b/scripts/save-tracks.js
@@ -74,12 +74,9 @@ const saveFileWorkerFn = () => {
     const ws = await wsClient(url)
     websockets.set(id, ws)
     if (kind === 'video') {
-      writeIvfHeader(ws, width, height, frameRate, 1, 'MJPG')
-
-      const canvas = new OffscreenCanvas(width, height)
-      const ctx = canvas.getContext('2d')
       const header = new ArrayBuffer(12)
       const view = new DataView(header)
+      let headerWritten = false
       let startTimestamp = -1
       let lastPts = -1
       const writableStream = new WritableStream(
@@ -101,29 +98,48 @@ const saveFileWorkerFn = () => {
               frame.close()
               return
             }
-            const bitmap = await createImageBitmap(frame)
+            const bitmap = await createImageBitmap(
+              frame,
+              x,
+              y,
+              Math.min(width, codedWidth),
+              Math.min(height, codedHeight),
+            )
+            frame.close()
+            const canvas = new OffscreenCanvas(bitmap.width, bitmap.height)
+            const ctx = canvas.getContext('bitmaprenderer')
+            ctx.transferFromImageBitmap(bitmap)
+            bitmap.close()
             try {
-              ctx.drawImage(bitmap, x, y, width, height, 0, 0, width, height)
               const blob = await canvas.convertToBlob({
                 type: 'image/jpeg',
                 quality,
               })
               const data = await blob.arrayBuffer()
-              /* log(
-                `writer ${data.byteLength} bytes timestamp=${
-                  videoFrame.timestamp / 1000000
-                } pts=${pts}`,
-              ) */
+              if (!headerWritten) {
+                headerWritten = true
+                log(
+                  `saveTrack ${url} writeIvfHeader ${canvas.width}x${canvas.height}@${frameRate}`,
+                )
+                writeIvfHeader(
+                  ws,
+                  canvas.width,
+                  canvas.height,
+                  frameRate,
+                  1,
+                  'MJPG',
+                )
+              }
               view.setUint32(0, data.byteLength, true)
               view.setBigUint64(4, BigInt(pts), true)
-              ws.send(header)
-              ws.send(data)
+              const buf = new Uint8Array(header.byteLength + data.byteLength)
+              buf.set(new Uint8Array(header), 0)
+              buf.set(new Uint8Array(data), header.byteLength)
+              ws.send(buf)
               lastPts = pts
             } catch (err) {
               log(`saveMediaTrack ${url} error=${err.message}`)
             }
-            frame.close()
-            bitmap.close()
           },
           close() {
             log(`saveTrack ${url} close`)

diff --git a/src/config.ts b/src/config.ts
@@ -251,7 +251,7 @@ chromium process.`,
   },
   deviceScaleFactor: {
     doc: `The browser device scale factor.`,
-    format: 'nat',
+    format: 'float',
     default: 1,
     env: 'DEVICE_SCALE_FACTOR',
     arg: 'device-scale-factor',

diff --git a/src/vmaf.ts b/src/vmaf.ts
@@ -414,12 +414,23 @@ export async function runVmaf(
   } = await parseIvf(degradedPath, false)
   const refTextHeight = Math.round(Math.round(refHeight / 18) * 1.2)
   const degTextHeight = Math.round(Math.round(degHeight / 18) * 1.2)
+
   if (!crop.ref) crop.ref = {}
   crop.ref.top = (crop.ref.top || 0) + refTextHeight
   crop.ref.bottom = (crop.ref.bottom || 0) + refTextHeight
+
   if (!crop.deg) crop.deg = {}
   crop.deg.top = (crop.deg.top || 0) + degTextHeight
   crop.deg.bottom = (crop.deg.bottom || 0) + degTextHeight
+
+  const refAspectRatio = refWidth / refHeight
+  const degAspectRatio = degWidth / degHeight
+  if (refAspectRatio > degAspectRatio) {
+    const diff = (refWidth - refHeight * degAspectRatio) / 2
+    crop.ref.left = (crop.ref.left || 0) + diff
+    crop.ref.right = (crop.ref.right || 0) + diff
+  }
+
   const width = Math.min(
     refWidth - (crop.ref.left || 0) - (crop.ref.right || 0),
     degWidth - (crop.deg.left || 0) - (crop.deg.right || 0),
@@ -444,11 +455,12 @@ export async function runVmaf(
       commonDegFrames.push(degFrame)
     }
   }
-  log.debug(
-    `common frames ref: ${commonRefFrames.length}/${refFrames.size} deg: ${commonDegFrames.length}/${degFrames.size}`,
-  )
   referencePath = await filterIvfFrames(referencePath, commonRefFrames)
   degradedPath = await filterIvfFrames(degradedPath, commonDegFrames)
+  log.debug(
+    `common frames: ${commonRefFrames.length} ref: ${refFrames.size} deg: ${degFrames.size}`,
+    { width, height, ...crop },
+  )
 
   const ffmpegCmd = `ffmpeg -loglevel warning -y -threads ${cpus} \
 -i ${degradedPath} \
@@ -643,7 +655,7 @@ export async function calculateVmafScore(
           vmafReferencePath,
           degradedPath,
           vmafPreview,
-          crop,
+          { ...crop },
         )
         ret.push(metrics)
       } catch (err) {