Remove MSE support (#98)

.vscode/settings.json

@@ -8,7 +8,7 @@
 	],
 	"eslint.run": "onSave",
 	"editor.codeActionsOnSave": {
-		"source.fixAll": true
+		"source.fixAll": "explicit"
 	},
 	"editor.formatOnSave": true, // Tell VSCode to format files on save
 	"editor.defaultFormatter": "esbenp.prettier-vscode",

lib/playback/backend.ts

@@ -1,12 +1,102 @@
-import { Catalog } from "../media/catalog"
-import { GroupHeader } from "../transport/objects"
+/// <reference types="vite/client" />
+
+import * as Message from "./worker/message"
+import { Context } from "./context"
 
-// TODO make an interface for backends
+import MediaWorker from "./worker?worker"
+import { RingShared } from "../common/ring"
+import { Catalog, isAudioTrack } from "../media/catalog"
+import { GroupHeader } from "../transport/objects"
 
-export interface Config {
+export interface PlayerConfig {
+	canvas: OffscreenCanvas
 	catalog: Catalog
 }
 
+// This is a non-standard way of importing worklet/workers.
+// Unfortunately, it's the only option because of a Vite bug: https://github.com/vitejs/vite/issues/11823
+
+// Responsible for sending messages to the worker and worklet.
+export default class Backend {
+	// General worker
+	#worker: Worker
+
+	// The audio context, which must be created on the main thread.
+	#context?: Context
+
+	constructor(config: PlayerConfig) {
+		// TODO does this block the main thread? If so, make this async
+		// @ts-expect-error: The Vite typing is wrong https://github.com/vitejs/vite/blob/22bd67d70a1390daae19ca33d7de162140d533d6/packages/vite/client.d.ts#L182
+		this.#worker = new MediaWorker({ format: "es" })
+		this.#worker.addEventListener("message", this.on.bind(this))
+
+		let sampleRate: number | undefined
+		let channels: number | undefined
+
+		for (const track of config.catalog.tracks) {
+			if (isAudioTrack(track)) {
+				if (sampleRate && track.sample_rate !== sampleRate) {
+					throw new Error(`TODO multiple audio tracks with different sample rates`)
+				}
+
+				sampleRate = track.sample_rate
+				channels = Math.max(track.channel_count, channels ?? 0)
+			}
+		}
+
+		const msg: Message.Config = {}
+
+		// Only configure audio is we have an audio track
+		if (sampleRate && channels) {
+			msg.audio = {
+				channels: channels,
+				sampleRate: sampleRate,
+				ring: new RingShared(2, sampleRate / 20), // 50ms
+			}
+
+			this.#context = new Context(msg.audio)
+		}
+
+		// TODO only send the canvas if we have a video track
+		msg.video = {
+			canvas: config.canvas,
+		}
+
+		this.send({ config: msg }, msg.video.canvas)
+	}
+
+	// TODO initialize context now since the user clicked
+	play() {}
+
+	init(init: Init) {
+		this.send({ init })
+	}
+
+	segment(segment: Segment) {
+		this.send({ segment }, segment.stream)
+	}
+
+	async close() {
+		this.#worker.terminate()
+		await this.#context?.close()
+	}
+
+	// Enforce we're sending valid types to the worker
+	private send(msg: Message.ToWorker, ...transfer: Transferable[]) {
+		//console.log("sent message from main to worker", msg)
+		this.#worker.postMessage(msg, transfer)
+	}
+
+	private on(e: MessageEvent) {
+		const msg = e.data as Message.FromWorker
+
+		// Don't print the verbose timeline message.
+		if (!msg.timeline) {
+			//console.log("received message from worker to main", msg)
+		}
+	}
+}
+
 export interface Init {
 	name: string // name of the init track
 	data: Uint8Array

lib/playback/webcodecs/context.ts → lib/playback/context.ts

@@ -1,6 +1,6 @@
 /// <reference types="vite/client" />
 
-import * as Message from "./message"
+import * as Message from "./worker/message"
 
 // This is a non-standard way of importing worklet/workers.
 // Unfortunately, it's the only option because of a Vite bug: https://github.com/vitejs/vite/issues/11823

lib/playback/index.ts

@@ -1,12 +1,11 @@
-import * as Message from "./webcodecs/message"
+import * as Message from "./worker/message"
 
 import { Connection } from "../transport/connection"
-import { Catalog, isAudioTrack, isMp4Track, Mp4Track } from "../media/catalog"
+import { Catalog, isMp4Track, Mp4Track } from "../media/catalog"
 import { asError } from "../common/error"
 
-// We support two different playback implementations:
-import Webcodecs from "./webcodecs"
-import MSE from "./mse"
+import Backend from "./backend"
+
 import { Client } from "../transport/client"
 import { GroupReader } from "../transport/objects"
 
@@ -17,12 +16,12 @@ export interface PlayerConfig {
 	url: string
 	namespace: string
 	fingerprint?: string // URL to fetch TLS certificate fingerprint
-	element: HTMLCanvasElement | HTMLVideoElement
+	canvas: HTMLCanvasElement
 }
 
 // This class must be created on the main thread due to AudioContext.
 export class Player {
-	#backend: Webcodecs | MSE
+	#backend: Backend
 
 	// A periodically updated timeline
 	//#timeline = new Watch<Timeline | undefined>(undefined)
@@ -36,7 +35,7 @@ export class Player {
 	#close!: () => void
 	#abort!: (err: Error) => void
 
-	private constructor(connection: Connection, catalog: Catalog, backend: Webcodecs | MSE) {
+	private constructor(connection: Connection, catalog: Catalog, backend: Backend) {
 		this.#connection = connection
 		this.#catalog = catalog
 		this.#backend = backend
@@ -57,14 +56,8 @@ export class Player {
 		const catalog = new Catalog(config.namespace)
 		await catalog.fetch(connection)
 
-		let backend
-
-		if (config.element instanceof HTMLCanvasElement) {
-			const element = config.element.transferControlToOffscreen()
-			backend = new Webcodecs({ element, catalog })
-		} else {
-			backend = new MSE({ element: config.element })
-		}
+		const canvas = config.canvas.transferControlToOffscreen()
+		const backend = new Backend({ canvas, catalog })
 
 		return new Player(connection, catalog, backend)
 	}
@@ -78,11 +71,6 @@ export class Player {
 				throw new Error(`expected CMAF track`)
 			}
 
-			if (isAudioTrack(track) && this.#backend instanceof MSE) {
-				// TODO temporary hack to disable audio in MSE
-				continue
-			}
-
 			inits.add(track.init_track)
 			tracks.push(track)
 		}
@@ -173,8 +161,8 @@ export class Player {
 	}
 	*/
 
-	async play() {
-		await this.#backend.play()
+	play() {
+		this.#backend.play()
 	}
 
 	/*