Skip to content

Commit

Permalink
component(audio): drag to add new annotation on waveform
Browse files Browse the repository at this point in the history 
* drag to add region instead of dbl clicking

* fix color of added region

* fix regions splitting

* modify the way dragged region is colored

* fix regions color when initialized from annotations

* fix unexpected label attribution on region split

* update to @gryannote/wavesurfer.js 7.8.12

* bump audio component version to 0.11.0

* update changelogs

* disable drag in non-interactive mode
  • Loading branch information
@clement-pages
clement-pages authored Dec 3, 2024
1 parent e090830 commit c1e2027
Show file tree
Hide file tree
Showing 7 changed files with 116 additions and 106 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ from `AudioLabeling`'s interface.
- minimap's waveform is now colored according to segments added on the player.
- improve behavior of region's button (remove and trim button). Now these buttons will keep focus while the user is in removing or trimming mode, respectively. Also, it is now possible to remove or trim several regions in a row, without having to click
again on the corresponding button.
- add a region by dragging on an empty space of the waveform, instead of double clicking. This allows to set a region with custom end bound.

## 0.3.0

Expand Down
8 changes: 4 additions & 4 deletions gryannote/audio/frontend/package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion gryannote/audio/frontend/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"@gradio/upload": "0.8.2",
"@gradio/utils": "0.3.0",
"@gradio/wasm": "0.10.0",
"@gryannote/wavesurfer.js": "^7.8.11",
"@gryannote/wavesurfer.js": "^7.8.12",
"@types/wavesurfer.js": "^6.0.10",
"extendable-media-recorder": "^9.0.0",
"extendable-media-recorder-wav-encoder": "^7.0.76",
Expand Down
13 changes: 11 additions & 2 deletions gryannote/audio/frontend/player/AudioPlayer.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import type { I18nFormatter } from "@gradio/utils";
import { Music,} from "@gradio/icons";
import WaveSurfer from "@gryannote/wavesurfer.js";
import RegionsPlugin from "@gryannote/wavesurfer.js/dist/plugins/regions";
import GamepadPlugin from "@gryannote/wavesurfer.js/dist/plugins/gamepad.js";
import MiniMapPlugin from "@gryannote/wavesurfer.js/dist/plugins/minimap.js";
import TimelinePlugin from '@gryannote/wavesurfer.js/dist/plugins/timeline.js';
Expand Down Expand Up @@ -31,6 +32,8 @@
export let mode: string = "";
let container: HTMLDivElement;
let wsRegions: RegionsPlugin;
let wsGamepad: GamepadPlugin;
let wsTimeline: TimelinePlugin;
let wsHover: HoverPlugin;
Expand Down Expand Up @@ -141,7 +144,12 @@
}
);
$: waveform?.on("ready", () => {
$: waveform?.on("init", () => {
if(!wsRegions){
wsRegions = waveform.registerPlugin(RegionsPlugin.create());
if(interactive) wsRegions.enableDragSelection({});
}
if(!wsGamepad){
wsGamepad = waveform.registerPlugin(GamepadPlugin.create());
}
Expand Down Expand Up @@ -270,6 +278,7 @@
{waveform}
{caption}
{interactive}
{wsRegions}
{wsGamepad}
{i18n}
{value}
Expand All @@ -293,7 +302,7 @@
{interactive}
{wsGamepad}
on:select={(e) => {
regionsControl.setRegionLabel(e.detail)
regionsControl.setRegionLabel(e.detail);
}}
on:name_update={(e) => {
regionsControl.getRegions().forEach(region => {
Expand Down
58 changes: 29 additions & 29 deletions gryannote/audio/frontend/player/Caption.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,35 @@
return label
}
/**
* Set the label mapped to specified shortcut as the active one
* @param shortcut shortcut mapped to the label to activate. If shortcut does not correspond
* to any existing label, a new one will be created and assigned to this shortcut. If not value
* is specified, will deselect active label, if any.
*/
export function setActiveLabel(shortcut?: string): void {
let label = null;
// retrieve label and create it if not found
if(shortcut !== undefined){
label = getLabel("shortcut", shortcut, true);
}
// reset active label
if(activeLabel){
document.getElementById(activeLabel.shortcut).classList.remove("active-button");
document.getElementById(activeLabel.shortcut).blur();
}
// update active label
activeLabel = label;
if(activeLabel){
document.getElementById(activeLabel.shortcut).classList.add("active-button");
document.getElementById(activeLabel.shortcut).focus();
dispatch("select", activeLabel);
}
}
/**
* Update the User Interface of the specified label.
* @param label label to update
Expand Down Expand Up @@ -166,35 +195,6 @@
updateLabelUI(label);
}
/**
* Set the label mapped to specified shortcut as the active one
* @param shortcut shortcut mapped to the label to activate. If shortcut does not correspond
* to any existing label, a new one will be created and assigned to this shortcut. If not value
* is specified, will deselect active label, if any.
*/
function setActiveLabel(shortcut?: string): void {
let label = null;
// retrieve label and create it if not found
if(shortcut !== undefined){
label = getLabel("shortcut", shortcut, true);
}
// reset active label
if(activeLabel){
document.getElementById(activeLabel.shortcut).classList.remove("active-button");
document.getElementById(activeLabel.shortcut).blur();
}
// update active label
activeLabel = label;
if(activeLabel){
document.getElementById(activeLabel.shortcut).classList.add("active-button");
document.getElementById(activeLabel.shortcut).focus();
dispatch("select", activeLabel);
}
}
function onGamepadAxeValueUpdated(event: ButtonEvent): void {
let activeLabelIdx = labels.findIndex((label) => label.shortcut === activeLabel?.shortcut);
switch(event.idx){
Expand Down
138 changes: 69 additions & 69 deletions gryannote/audio/frontend/player/RegionsControl.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
export let adjustTimeCursorPosition: (s:string, b:boolean) => void;
export let i18n: I18nFormatter;
export let waveform: WaveSurfer;
export let wsRegions: RegionsPlugin;
export let wsGamepad: GamepadPlugin;
export let value: null | AnnotatedAudioData = null;
export let interactive = true;
Expand All @@ -22,8 +23,6 @@
let container: HTMLDivElement;
let wsRegions: RegionsPlugin;
let leftRegionHandle: HTMLDivElement | null = null;
let rightRegionHandle: HTMLDivElement | null = null;
let activeHandle = "";
Expand Down Expand Up @@ -174,23 +173,20 @@
};
/**
* Add a region onto waveform given its parameters and speaker label
* @param options region's params (start, end, color)
* @param label region's label
*
* @returns the added region
* Add a region on the waveform.
* @param start start bound of the region to add, in seconds
* @param end end bound of the region to add, in seconds
* @param label label associated to the region to add
*/
function addRegion(options: RegionParams, label: string): Region {
let region = wsRegions.addRegion(options);
const annotation = {start: region.start, end: region.end, speaker: label, color: region.color};
regionsMap.set(region.id, annotation);
// if this is the first region added on the waveform
if(!initialAnnotations){
initialAnnotations = [annotation];
}
updateAnnotations();
function addRegion(start: number, end: number, label?: Label): Region {
label = label || caption.getActiveLabel() || caption.getDefaultLabel();
let region = wsRegions.addRegion({
start: start,
end: end,
color: label.color,
drag: interactive,
resize: interactive,
});
return region;
}
Expand All @@ -205,12 +201,13 @@
throw new RangeError("split time out of region bounds");
}
const label = regionsMap.get(region.id).speaker;
const speaker = regionsMap.get(region.id).speaker;
const {start, id, ...rightRegionOpt} = region
const regionRight = addRegion({
start: splitTime,
...rightRegionOpt,
}, label);
const regionRight = addRegion(
splitTime,
rightRegionOpt.end,
caption.getLabel("name", speaker),
);
const {end, ...leftRegionOpt} = region
region.setOptions({end: splitTime, ...leftRegionOpt});
Expand Down Expand Up @@ -274,17 +271,16 @@
annotation.start === currentAnnotation.start &&
annotation.end === currentAnnotation.end &&
annotation.speaker === currentAnnotation.speaker
))
));
annotations.forEach(annotation => {
let label = caption.getLabel("name", annotation.speaker, true)
addRegion({
start: annotation.start,
end: annotation.end,
color: label.color,
drag: interactive,
resize: interactive,
}, annotation.speaker);
let label = caption.getLabel("name", annotation.speaker, true);
caption.setActiveLabel(label.shortcut);
addRegion(
annotation.start,
annotation.end,
label,
);
});
}
Expand Down Expand Up @@ -351,22 +347,23 @@
}
/**
* Handle add region event. The new added region becomes the active one.
* Do nothing if the component is not in interactive mode.
* @param relativeY mouse y-coordinate relative to waveform start
* Handle the region creation event. The new added region becomes the active one.
* @param region the created region
*/
function onRegionAdd(relativeY: number): void{
if(!interactive) return;
function onRegionCreated(region: Region): void{
// map an annotation object to current region
const label = caption.getActiveLabel() || caption.getDefaultLabel();
const annotation = {start: region.start, end: region.end, speaker: label.name};
regionsMap.set(region.id, annotation);
updateAnnotations();
let region = addRegion({
start: relativeY - 1.0,
end: relativeY + 1.0,
color: label.color,
drag: true,
resize: true,
}, label.name);
region.color = label.color;
region.setOptions(region);
// if this is the first region added on the waveform
if(!initialAnnotations){
initialAnnotations = [annotation];
}
// set region as active one
setActiveRegion(region);
Expand Down Expand Up @@ -435,7 +432,7 @@
function onGamepadButtonPressed(event: ButtonEvent): void {
switch(event.idx){
case 0: setActiveRegion(null); break;
case 1: onRegionAdd(waveform.getCurrentTime()); break;
case 1: addRegion(waveform.getCurrentTime() -1, waveform.getCurrentTime() + 1); break;
case 2: onRegionRemove("Delete", false); break;
case 3: onRegionSplit(waveform.getCurrentTime()); break;
case 4: selectRegion("backward"); break;
Expand Down Expand Up @@ -483,32 +480,34 @@
});
}
if(!wsRegions){
wsRegions = waveform.registerPlugin(RegionsPlugin.create());
if(interactive){
// add region-clicked event listener
wsRegions.on("region-clicked", (region, e) => {
switch(mode){
case "remove": removeRegion(region); break;
case "split": splitRegion(region, region.start + (region.end - region.start) / 2); break;
default: setActiveRegion(region); region.play();
}
});
wsRegions.on("region-updated", (region) => {
onRegionUpdate(region);
});
wsRegions.on("region-in", (region: Region) => {
dispatch("region-in", region);
});
wsRegions.on("region-out", (region: Region) => {
dispatch("region-out", region);
});
}
if(wsRegions){
// add region-clicked event listener
wsRegions.on("region-clicked", (region, e) => {
if(!interactive) return;
switch(mode){
case "remove": removeRegion(region); break;
case "split": splitRegion(region, region.start + (region.end - region.start) / 2); break;
default: setActiveRegion(region); region.play();
}
});
wsRegions.on("region-created", (region: Region) => {
onRegionCreated(region);
});
wsRegions.on("region-updated", (region) => {
onRegionUpdate(region);
});
wsRegions.on("region-in", (region: Region) => {
dispatch("region-in", region);
});
wsRegions.on("region-out", (region: Region) => {
dispatch("region-out", region);
});
}
});
$: waveform.on("dblclick", () => {
onRegionAdd(waveform.getCurrentTime());
window.alert("To add a new annotation, please drag on an empty space of the waveform.");
});
$: if (activeRegion) {
Expand Down Expand Up @@ -551,7 +550,8 @@
if(e.shiftKey){
onRegionSplit(waveform.getCurrentTime());
} else {
onRegionAdd(waveform.getCurrentTime());
const currentTime = waveform.getCurrentTime();
addRegion(currentTime -1, currentTime + 1);
}
break;
default: //do nothing
Expand Down
2 changes: 1 addition & 1 deletion gryannote/audio/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ build-backend = "hatchling.build"

[project]
name = "gryannote_audio"
version = "0.10.0"
version = "0.11.0"
description = "audio component with speaker annotations"
readme = "README.md"
license = "MIT"
Expand Down

0 comments on commit c1e2027

Please sign in to comment.