component(audio): add beep on region-in and region-out

* add beep on region-in and region-out

* add 2 different beep for region in / out events

... and a raw checkbox to enable/disable beep

* update checkbox style

* update changelog

* bump audio component version to 0.10.0

CHANGELOG.md

@@ -13,8 +13,9 @@ See `HoverOptions` documentation for more details about available options
 from gryannote_audio import Player
 player = Player(video="video.mp4")
 ```
-
 A video file can also be upload directly from the interface when using `AudioLabeling` in interactive mode.
+- beep on annotation in/out, to check alignment between audio and annotation. This feature can be enabled directly
+from `AudioLabeling`'s interface.
 
 ### Fixes
 

gryannote/audio/frontend/player/AudioPlayer.svelte

@@ -13,8 +13,9 @@
 	import { Empty } from "@gradio/atoms";
 	import { resolve_wasm_src } from "@gradio/wasm/svelte";
 	import AnnotatedAudioData from "../shared/AnnotatedAudioData";
-	import { createEventDispatcher } from "svelte";
 	import { renderLineWaveform } from "../shared/utils";
+	import { createBeep } from "../shared/utils"
+	import { createEventDispatcher, onMount } from "svelte";
 
 	export let label: string;
 	export let i18n: I18nFormatter;
@@ -43,6 +44,11 @@
 
 	let regionsControl: RegionsControl;
 
+	const audioContext = new AudioContext();
+	const gainNodeIn = audioContext.createGain();
+	const gainNodeOut = audioContext.createGain();
+	const beepDuration = 0.05;
+
 	const dispatch = createEventDispatcher<{
 		stop: undefined;
 		play: undefined;
@@ -70,6 +76,27 @@
 		});
 	}
 
+	/**
+	 * Play beep for `duration` seconds
+	 * @param gainNode gain mapped to the bepp tp play
+	 * @param duration beep duration, in seconds
+	 */
+	function playBeep(gainNode: GainNode, duration: number): void {
+		const now = audioContext.currentTime
+		gainNode.gain.setValueAtTime(1, now);
+		gainNode.gain.setValueAtTime(0, now + duration);
+	}
+
+	function onRegionInOut(event: "region-in" | "region-out"): void {
+		const checkbox = document.getElementById("beep-checkbox") as HTMLInputElement;
+		const beepOn = checkbox.checked;
+		const gainNode = (event === "region-in") ? gainNodeIn : gainNodeOut;
+
+		if(waveform.isPlaying() && beepOn){
+			playBeep(gainNode, beepDuration);
+		}
+	}
+
 	/**
 	 * Adjust position of the cursor on the waveform
 	 * @param key shortcut name. Indicates direction: forward or backward.
@@ -172,6 +199,20 @@
 	$: url = value.file_data?.url;
 	$: url && load_audio(url);
 
+	onMount(() => {
+		gainNodeIn.connect(audioContext.destination);
+		gainNodeIn.gain.setValueAtTime(0, audioContext.currentTime);
+		const beepIn = createBeep(audioContext, {"type": "sine", "freq": 1000});
+		beepIn.connect(gainNodeIn);
+		beepIn.start();
+
+		gainNodeOut.connect(audioContext.destination);
+		gainNodeOut.gain.setValueAtTime(0, audioContext.currentTime);
+		const beepOut = createBeep(audioContext, {"type": "sine", "freq": 500});
+		beepOut.connect(gainNodeOut);
+		beepOut.start();
+	});
+
 </script>
 
 {#if value === null}
@@ -215,6 +256,10 @@
 						on:stop={() => dispatch("stop")}
 						on:pause={() => dispatch("pause")}
 					/>
+					<label>
+						<input type="checkbox" id="beep-checkbox">
+						beep on annotation in/out
+					</label>
 				</div>
 				<div class="regions-controls">
 					<RegionsControl
@@ -235,8 +280,10 @@
 							}
 							dispatch("edit", e.detail)
 						}}
+						on:edit={(e) => dispatch("edit", e.detail)}
+						on:region-in={(region) => onRegionInOut("region-in")}
+						on:region-out={(region) => onRegionInOut("region-out")}
 					/>
-
 				</div>
 			</div>
 			{#if value}
@@ -270,6 +317,25 @@
 {/if}
 
 <style>
+	label {
+		font-family: var(--font);
+		font-size: var(--text-md);
+	}
+
+	input[type="checkbox"] {
+		appearance: none;
+		border-color: var(--neutral-400);
+		border-radius: 20%;
+		margin-left: 0.3em;
+		transform: translateY(-0.075em);
+	}
+
+	input[type="checkbox"]:checked {
+		background-color: var(--color-accent);
+		border-color: var(--color-accent);
+		border-radius: 20%;
+	}
+
 	.commands {
 		display: flex;
 		justify-content: space-between;

gryannote/audio/frontend/player/Caption.svelte

@@ -256,7 +256,6 @@
 .caption-container {
     display: flex;
     justify-content: center;
-    margin-top: 1em;
 }
 
 div:global(.caption-container button){

gryannote/audio/frontend/player/RegionsControl.svelte

@@ -36,6 +36,8 @@
 
     const dispatch = createEventDispatcher<{
 		edit: typeof value;
+		"region-in": Region;
+		"region-out": Region;
 	}>();
 
 	/**
@@ -485,16 +487,22 @@
 			wsRegions = waveform.registerPlugin(RegionsPlugin.create());
 			if(interactive){
 				// add region-clicked event listener
-				wsRegions?.on("region-clicked", (region, e) => {
+				wsRegions.on("region-clicked", (region, e) => {
 					switch(mode){
 						case "remove": removeRegion(region); break;
 						case "split": splitRegion(region, region.start + (region.end - region.start) / 2); break;
 						default: setActiveRegion(region); region.play();
 					}
 				});
-				wsRegions?.on("region-updated", (region) => {
+				wsRegions.on("region-updated", (region) => {
 					onRegionUpdate(region);
 				});
+				wsRegions.on("region-in", (region: Region) => {
+					dispatch("region-in", region);
+				});
+				wsRegions.on("region-out", (region: Region) => {
+					dispatch("region-out", region);
+				});
 			}
 		}
 	});

gryannote/audio/frontend/shared/utils.ts

@@ -60,6 +60,14 @@ export function audio_loaded(
 	}
 }
 
+export function createBeep(audioContext: AudioContext, options: {"type": OscillatorType, "freq": number}){
+	const beep = audioContext.createOscillator();
+	beep.type = options.type;
+	beep.frequency.value = options.freq;
+
+	return beep
+}
+
 export const skip_audio = (waveform: WaveSurfer, amount: number): void => {
 	if (!waveform) return;
 	waveform.skip(amount);

gryannote/audio/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "gryannote_audio"
-version = "0.9.5"
+version = "0.10.0"
 description = "audio component with speaker annotations"
 readme = "README.md"
 license = "MIT"