From 389106c175aac2c84cfab017feed899d707bb579 Mon Sep 17 00:00:00 2001 From: Ian Date: Mon, 30 Oct 2023 16:19:30 -0700 Subject: [PATCH 1/8] web --- .github/workflows/web-demos.yml | 6 +- .github/workflows/web.yml | 2 +- binding/web/package.json | 4 +- binding/web/src/index.ts | 2 + binding/web/src/koala.ts | 401 ++++++++++++++++++------ binding/web/src/koala_errors.ts | 252 +++++++++++++++ binding/web/src/koala_worker.ts | 163 +++++++--- binding/web/src/koala_worker_handler.ts | 41 ++- binding/web/src/types.ts | 23 +- binding/web/test/koala.test.ts | 333 ++++++++++---------- demo/web/package.json | 4 +- demo/web/yarn.lock | 6 +- 12 files changed, 890 insertions(+), 347 deletions(-) create mode 100644 binding/web/src/koala_errors.ts diff --git a/.github/workflows/web-demos.yml b/.github/workflows/web-demos.yml index 6e4fa73..9764178 100644 --- a/.github/workflows/web-demos.yml +++ b/.github/workflows/web-demos.yml @@ -25,7 +25,7 @@ jobs: strategy: matrix: - node-version: [14.x, 16.x, 18.x, 20.x] + node-version: [16.x, 18.x, 20.x] steps: - uses: actions/checkout@v3 @@ -35,6 +35,10 @@ jobs: with: node-version: ${{ matrix.node-version }} + - name: Build Web SDK + run: yarn && yarn copywasm && yarn copyppn && yarn build + working-directory: binding/web + - name: Pre-build dependencies run: npm install yarn diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index ddecc08..4ae734d 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -31,7 +31,7 @@ jobs: strategy: matrix: - node-version: [14.x, 16.x, 18.x, 20.x] + node-version: [16.x, 18.x, 20.x] steps: - uses: actions/checkout@v3 diff --git a/binding/web/package.json b/binding/web/package.json index 246ad7e..276a072 100644 --- a/binding/web/package.json +++ b/binding/web/package.json @@ -3,7 +3,7 @@ "description": "Koala Noise Suppression engine for web browsers (via WebAssembly)", "author": "Picovoice Inc", "license": "Apache-2.0", - "version": "1.0.3", + "version": "2.0.0", "keywords": [ "koala", "web", @@ -67,6 +67,6 @@ "wasm-feature-detect": "^1.5.0" }, "engines": { - "node": ">=14" + "node": ">=16" } } diff --git a/binding/web/src/index.ts b/binding/web/src/index.ts index d759296..666a409 100644 --- a/binding/web/src/index.ts +++ b/binding/web/src/index.ts @@ -1,5 +1,6 @@ import { Koala } from './koala'; import { KoalaWorker } from './koala_worker'; +import * as KoalaErrors from './koala_errors'; import { KoalaModel, @@ -25,6 +26,7 @@ KoalaWorker.setWasmSimd(koalaWasmSimd); export { Koala, + KoalaErrors, KoalaModel, KoalaOptions, KoalaWorker, diff --git a/binding/web/src/koala.ts b/binding/web/src/koala.ts index 718bf45..930b996 100644 --- a/binding/web/src/koala.ts +++ b/binding/web/src/koala.ts @@ -15,30 +15,50 @@ import { Mutex } from 'async-mutex'; import { aligned_alloc_type, - pv_free_type, - buildWasm, arrayBufferToStringAtIndex, + buildWasm, isAccessKeyValid, loadModel, - PvError + pv_free_type, + PvError, } from '@picovoice/web-utils'; import { simd } from 'wasm-feature-detect'; -import { KoalaModel, KoalaOptions } from './types'; +import { KoalaModel, KoalaOptions, PvStatus } from './types'; + +import * as KoalaErrors from './koala_errors'; +import { pvStatusToException } from './koala_errors'; /** * WebAssembly function types */ -type pv_koala_init_type = (accessKey: number, modelPath: number, object: number) => Promise; -type pv_koala_delay_sample_type = (object: number, delaySample: number) => Promise; -type pv_koala_process_type = (object: number, pcm: number, enhancedPcm: number) => Promise; +type pv_koala_init_type = ( + accessKey: number, + modelPath: number, + object: number +) => Promise; +type pv_koala_delay_sample_type = ( + object: number, + delaySample: number +) => Promise; +type pv_koala_process_type = ( + object: number, + pcm: number, + enhancedPcm: number +) => Promise; type pv_koala_reset_type = (object: number) => Promise; type pv_koala_delete_type = (object: number) => Promise; -type pv_status_to_string_type = (status: number) => Promise +type pv_status_to_string_type = (status: number) => Promise; type pv_koala_frame_length_type = () => Promise; type pv_sample_rate_type = () => Promise; type pv_koala_version_type = () => Promise; +type pv_set_sdk_type = (sdk: number) => Promise; +type pv_get_error_stack_type = ( + messageStack: number, + messageStackDepth: number +) => Promise; +type pv_free_error_stack_type = (messageStack: number) => Promise; /** * JavaScript/WebAssembly Binding for Koala @@ -48,12 +68,16 @@ type KoalaWasmOutput = { memory: WebAssembly.Memory; pvFree: pv_free_type; objectAddress: number; + messageStackAddressAddressAddress: number; + messageStackDepthAddress: number; pvKoalaDelete: pv_koala_delete_type; pvKoalaProcess: pv_koala_process_type; pvKoalaReset: pv_koala_reset_type; pvStatusToString: pv_status_to_string_type; + pvGetErrorStack: pv_get_error_stack_type; + pvFreeErrorStack: pv_free_error_stack_type; delaySample: number; - frameLength: number + frameLength: number; sampleRate: number; version: string; inputBufferAddress: number; @@ -68,6 +92,8 @@ export class Koala { private readonly _pvKoalaProcess: pv_koala_process_type; private readonly _pvKoalaReset: pv_koala_reset_type; private readonly _pvStatusToString: pv_status_to_string_type; + private readonly _pvGetErrorStack: pv_get_error_stack_type; + private readonly _pvFreeErrorStack: pv_free_error_stack_type; private _wasmMemory: WebAssembly.Memory | undefined; private readonly _pvFree: pv_free_type; @@ -76,6 +102,8 @@ export class Koala { private readonly _objectAddress: number; private readonly _inputBufferAddress: number; private readonly _outputBufferAddress: number; + private readonly _messageStackAddressAddressAddress: number; + private readonly _messageStackDepthAddress: number; private static _delaySample: number; private static _frameLength: number; @@ -83,18 +111,21 @@ export class Koala { private static _version: string; private static _wasm: string; private static _wasmSimd: string; + private static _sdk: string = 'web'; private static _koalaMutex = new Mutex(); private readonly _processCallback: (enhancedPcm: Int16Array) => void; - private readonly _processErrorCallback?: (error: string) => void; + private readonly _processErrorCallback?: ( + error: KoalaErrors.KoalaError + ) => void; private readonly _pvError: PvError; private constructor( handleWasm: KoalaWasmOutput, processCallback: (enhancedPcm: Int16Array) => void, - processErrorCallback?: (error: string) => void, + processErrorCallback?: (error: KoalaErrors.KoalaError) => void ) { Koala._delaySample = handleWasm.delaySample; Koala._frameLength = handleWasm.frameLength; @@ -105,12 +136,17 @@ export class Koala { this._pvKoalaProcess = handleWasm.pvKoalaProcess; this._pvKoalaReset = handleWasm.pvKoalaReset; this._pvStatusToString = handleWasm.pvStatusToString; + this._pvGetErrorStack = handleWasm.pvGetErrorStack; + this._pvFreeErrorStack = handleWasm.pvFreeErrorStack; this._wasmMemory = handleWasm.memory; this._pvFree = handleWasm.pvFree; this._objectAddress = handleWasm.objectAddress; this._inputBufferAddress = handleWasm.inputBufferAddress; this._outputBufferAddress = handleWasm.outputBufferAddress; + this._messageStackAddressAddressAddress = + handleWasm.messageStackDepthAddress; + this._messageStackDepthAddress = handleWasm.messageStackDepthAddress; this._pvError = handleWasm.pvError; this._processMutex = new Mutex(); @@ -119,6 +155,10 @@ export class Koala { this._processErrorCallback = processErrorCallback; } + public static setSdk(sdk: string): void { + Koala._sdk = sdk; + } + /** * Delay in samples. If the input and output of consecutive calls to `.process()` are viewed as two contiguous * streams of audio data, this delay specifies the time shift between the input and output stream. @@ -195,36 +235,37 @@ export class Koala { accessKey: string, processCallback: (enhancedPcm: Int16Array) => void, model: KoalaModel, - options: KoalaOptions = {}, + options: KoalaOptions = {} ): Promise { - const customWritePath = (model.customWritePath) ? model.customWritePath : 'koala_model'; + const customWritePath = model.customWritePath + ? model.customWritePath + : 'koala_model'; const modelPath = await loadModel({ ...model, customWritePath }); - return Koala._init( - accessKey, - processCallback, - modelPath, - options - ); + return Koala._init(accessKey, processCallback, modelPath, options); } public static async _init( accessKey: string, processCallback: (enhancedPcm: Int16Array) => void, modelPath: string, - options: KoalaOptions = {}, + options: KoalaOptions = {} ): Promise { const { processErrorCallback } = options; if (!isAccessKeyValid(accessKey)) { - throw new Error('Invalid AccessKey'); + throw new KoalaErrors.KoalaInvalidArgumentError('Invalid AccessKey'); } return new Promise((resolve, reject) => { Koala._koalaMutex .runExclusive(async () => { const isSimd = await simd(); - const wasmOutput = await Koala.initWasm(accessKey.trim(), (isSimd) ? this._wasmSimd : this._wasm, modelPath, options); + const wasmOutput = await Koala.initWasm( + accessKey.trim(), + isSimd ? this._wasmSimd : this._wasm, + modelPath + ); return new Koala(wasmOutput, processCallback, processErrorCallback); }) .then((result: Koala) => { @@ -247,9 +288,11 @@ export class Koala { */ public async process(pcm: Int16Array): Promise { if (!(pcm instanceof Int16Array)) { - const error = new Error('The argument \'pcm\' must be provided as an Int16Array'); + const error = new KoalaErrors.KoalaInvalidArgumentError( + "The argument 'pcm' must be provided as an Int16Array" + ); if (this._processErrorCallback) { - this._processErrorCallback(error.toString()); + this._processErrorCallback(error); } else { // eslint-disable-next-line no-console console.error(error); @@ -257,10 +300,11 @@ export class Koala { } if (pcm.length !== this.frameLength) { - const error = new Error(`Koala process requires frames of length ${this.frameLength}. + const error = + new KoalaErrors.KoalaInvalidArgumentError(`Koala process requires frames of length ${this.frameLength}. Received frame of size ${pcm.length}.`); if (this._processErrorCallback) { - this._processErrorCallback(error.toString()); + this._processErrorCallback(error); } else { // eslint-disable-next-line no-console console.error(error); @@ -270,13 +314,15 @@ export class Koala { this._processMutex .runExclusive(async () => { if (this._wasmMemory === undefined) { - throw new Error('Attempted to call Koala process after release.'); + throw new KoalaErrors.KoalaInvalidStateError( + 'Attempted to call Koala process after release.' + ); } const memoryBuffer = new Int16Array(this._wasmMemory.buffer); memoryBuffer.set( pcm, - this._inputBufferAddress / Int16Array.BYTES_PER_ELEMENT, + this._inputBufferAddress / Int16Array.BYTES_PER_ELEMENT ); const status = await this._pvKoalaProcess( @@ -285,29 +331,44 @@ export class Koala { this._outputBufferAddress ); - const memoryBufferUint8 = new Uint8Array(this._wasmMemory.buffer); - - if (status !== PV_STATUS_SUCCESS) { - const msg = `process failed with status ${arrayBufferToStringAtIndex( - memoryBufferUint8, - await this._pvStatusToString(status), - )}`; + if (status !== PvStatus.SUCCESS) { + const memoryBufferUint8 = new Uint8Array(this._wasmMemory.buffer); + const memoryBufferView = new DataView(this._wasmMemory.buffer); + const messageStack = await Koala.getMessageStack( + this._pvGetErrorStack, + this._pvFreeErrorStack, + this._messageStackAddressAddressAddress, + this._messageStackDepthAddress, + memoryBufferView, + memoryBufferUint8 + ); - throw new Error( - `${msg}\nDetails: ${this._pvError.getErrorString()}` + const error = pvStatusToException( + status, + 'Process failed', + messageStack ); + if (this._processErrorCallback) { + this._processErrorCallback(error); + } else { + // eslint-disable-next-line no-console + console.error(error); + } } const output = memoryBuffer.slice( this._outputBufferAddress / Int16Array.BYTES_PER_ELEMENT, - (this._outputBufferAddress / Int16Array.BYTES_PER_ELEMENT) + this.frameLength + this._outputBufferAddress / Int16Array.BYTES_PER_ELEMENT + + this.frameLength ); this._processCallback(output); }) .catch((error: any) => { if (this._processErrorCallback) { - this._processErrorCallback(error.toString()); + this._processErrorCallback( + pvStatusToException(PvStatus.RUNTIME_ERROR, error.toString()) + ); } else { // eslint-disable-next-line no-console console.error(error); @@ -320,33 +381,51 @@ export class Koala { * Call this function in between calls to `process` that do not provide consecutive frames of audio. */ public async reset(): Promise { - return new Promise((resolve, reject) => { - this._processMutex - .runExclusive(async () => { - if (this._wasmMemory === undefined) { - throw new Error('Attempted to call Koala reset after release.'); - } - - const status = await this._pvKoalaReset(this._objectAddress); + this._processMutex + .runExclusive(async () => { + if (this._wasmMemory === undefined) { + throw new KoalaErrors.KoalaInvalidStateError( + 'Attempted to call Koala reset after release.' + ); + } - if (status !== PV_STATUS_SUCCESS) { - const memoryBuffer = new Uint8Array(this._wasmMemory.buffer); - const msg = `process failed with status ${arrayBufferToStringAtIndex( - memoryBuffer, - await this._pvStatusToString(status), - )}`; + const status = await this._pvKoalaReset(this._objectAddress); + + if (status !== PvStatus.SUCCESS) { + const memoryBufferUint8 = new Uint8Array(this._wasmMemory.buffer); + const memoryBufferView = new DataView(this._wasmMemory.buffer); + const messageStack = await Koala.getMessageStack( + this._pvGetErrorStack, + this._pvFreeErrorStack, + this._messageStackAddressAddressAddress, + this._messageStackDepthAddress, + memoryBufferView, + memoryBufferUint8 + ); - throw new Error( - `${msg}\nDetails: ${this._pvError.getErrorString()}` - ); + const error = pvStatusToException( + status, + 'Reset failed', + messageStack + ); + if (this._processErrorCallback) { + this._processErrorCallback(error); + } else { + // eslint-disable-next-line no-console + console.error(error); } - - resolve(); - }) - .catch((error: any) => { - reject(error); - }); - }); + } + }) + .catch((error: any) => { + if (this._processErrorCallback) { + this._processErrorCallback( + pvStatusToException(PvStatus.RUNTIME_ERROR, error.toString()) + ); + } else { + // eslint-disable-next-line no-console + console.error(error); + } + }); } /** @@ -371,11 +450,16 @@ export class Koala { } } - private static async initWasm(accessKey: string, wasmBase64: string, modelPath: string, _: KoalaOptions): Promise { + private static async initWasm( + accessKey: string, + wasmBase64: string, + modelPath: string + ): Promise { // A WebAssembly page has a constant size of 64KiB. -> 1MiB ~= 16 pages const memory = new WebAssembly.Memory({ initial: 300 }); const memoryBufferUint8 = new Uint8Array(memory.buffer); + const memoryBufferView = new DataView(memory.buffer); const pvError = new PvError(); @@ -384,30 +468,42 @@ export class Koala { const aligned_alloc = exports.aligned_alloc as aligned_alloc_type; const pv_free = exports.pv_free as pv_free_type; const pv_koala_version = exports.pv_koala_version as pv_koala_version_type; - const pv_koala_delay_sample = exports.pv_koala_delay_sample as pv_koala_delay_sample_type; + const pv_koala_delay_sample = + exports.pv_koala_delay_sample as pv_koala_delay_sample_type; const pv_koala_process = exports.pv_koala_process as pv_koala_process_type; const pv_koala_reset = exports.pv_koala_reset as pv_koala_reset_type; const pv_koala_delete = exports.pv_koala_delete as pv_koala_delete_type; const pv_koala_init = exports.pv_koala_init as pv_koala_init_type; - const pv_status_to_string = exports.pv_status_to_string as pv_status_to_string_type; - const pv_koala_frame_length = exports.pv_koala_frame_length as pv_koala_frame_length_type; + const pv_status_to_string = + exports.pv_status_to_string as pv_status_to_string_type; + const pv_koala_frame_length = + exports.pv_koala_frame_length as pv_koala_frame_length_type; const pv_sample_rate = exports.pv_sample_rate as pv_sample_rate_type; + const pv_set_sdk = exports.pv_set_sdk as pv_set_sdk_type; + const pv_get_error_stack = + exports.pv_get_error_stack as pv_get_error_stack_type; + const pv_free_error_stack = + exports.pv_free_error_stack as pv_free_error_stack_type; const objectAddressAddress = await aligned_alloc( Int32Array.BYTES_PER_ELEMENT, - Int32Array.BYTES_PER_ELEMENT, + Int32Array.BYTES_PER_ELEMENT ); if (objectAddressAddress === 0) { - throw new Error('malloc failed: Cannot allocate memory'); + throw new KoalaErrors.KoalaOutOfMemoryError( + 'malloc failed: Cannot allocate memory' + ); } const accessKeyAddress = await aligned_alloc( Uint8Array.BYTES_PER_ELEMENT, - (accessKey.length + 1) * Uint8Array.BYTES_PER_ELEMENT, + (accessKey.length + 1) * Uint8Array.BYTES_PER_ELEMENT ); if (accessKeyAddress === 0) { - throw new Error('malloc failed: Cannot allocate memory'); + throw new KoalaErrors.KoalaOutOfMemoryError( + 'malloc failed: Cannot allocate memory' + ); } for (let i = 0; i < accessKey.length; i++) { @@ -418,82 +514,140 @@ export class Koala { const modelPathEncoded = new TextEncoder().encode(modelPath); const modelPathAddress = await aligned_alloc( Uint8Array.BYTES_PER_ELEMENT, - (modelPathEncoded.length + 1) * Uint8Array.BYTES_PER_ELEMENT, + (modelPathEncoded.length + 1) * Uint8Array.BYTES_PER_ELEMENT ); if (modelPathAddress === 0) { - throw new Error('malloc failed: Cannot allocate memory'); + throw new KoalaErrors.KoalaOutOfMemoryError( + 'malloc failed: Cannot allocate memory' + ); } memoryBufferUint8.set(modelPathEncoded, modelPathAddress); memoryBufferUint8[modelPathAddress + modelPathEncoded.length] = 0; + const sdkEncoded = new TextEncoder().encode(this._sdk); + const sdkAddress = await aligned_alloc( + Uint8Array.BYTES_PER_ELEMENT, + (sdkEncoded.length + 1) * Uint8Array.BYTES_PER_ELEMENT + ); + if (!sdkAddress) { + throw new KoalaErrors.KoalaOutOfMemoryError( + 'malloc failed: Cannot allocate memory' + ); + } + memoryBufferUint8.set(sdkEncoded, sdkAddress); + memoryBufferUint8[sdkAddress + sdkEncoded.length] = 0; + await pv_set_sdk(sdkAddress); + + const messageStackDepthAddress = await aligned_alloc( + Int32Array.BYTES_PER_ELEMENT, + Int32Array.BYTES_PER_ELEMENT + ); + if (!messageStackDepthAddress) { + throw new KoalaErrors.KoalaOutOfMemoryError( + 'malloc failed: Cannot allocate memory' + ); + } + + const messageStackAddressAddressAddress = await aligned_alloc( + Int32Array.BYTES_PER_ELEMENT, + Int32Array.BYTES_PER_ELEMENT + ); + if (!messageStackAddressAddressAddress) { + throw new KoalaErrors.KoalaOutOfMemoryError( + 'malloc failed: Cannot allocate memory' + ); + } + let status = await pv_koala_init( accessKeyAddress, modelPathAddress, - objectAddressAddress); - if (status !== PV_STATUS_SUCCESS) { - const msg = `'pv_koala_init' failed with status ${arrayBufferToStringAtIndex( - memoryBufferUint8, - await pv_status_to_string(status), - )}`; + objectAddressAddress + ); + + await pv_free(accessKeyAddress); + await pv_free(modelPathAddress); + + if (status !== PvStatus.SUCCESS) { + const messageStack = await Koala.getMessageStack( + pv_get_error_stack, + pv_free_error_stack, + messageStackAddressAddressAddress, + messageStackDepthAddress, + memoryBufferView, + memoryBufferUint8 + ); - throw new Error( - `${msg}\nDetails: ${pvError.getErrorString()}` + throw pvStatusToException( + status, + 'Initialization failed', + messageStack, + pvError ); } - const memoryBufferView = new DataView(memory.buffer); + const objectAddress = memoryBufferView.getInt32(objectAddressAddress, true); const delaySampleAddress = await aligned_alloc( Int32Array.BYTES_PER_ELEMENT, - Int32Array.BYTES_PER_ELEMENT, + Int32Array.BYTES_PER_ELEMENT ); if (delaySampleAddress === 0) { - throw new Error('malloc failed: Cannot allocate memory'); + throw new KoalaErrors.KoalaOutOfMemoryError( + 'malloc failed: Cannot allocate memory' + ); } status = await pv_koala_delay_sample(objectAddress, delaySampleAddress); if (status !== PV_STATUS_SUCCESS) { - const msg = `'pv_koala_delay_sample' failed with status ${arrayBufferToStringAtIndex( - memoryBufferUint8, - await pv_status_to_string(status), - )}`; + const messageStack = await Koala.getMessageStack( + pv_get_error_stack, + pv_free_error_stack, + messageStackAddressAddressAddress, + messageStackDepthAddress, + memoryBufferView, + memoryBufferUint8 + ); - throw new Error( - `${msg}\nDetails: ${pvError.getErrorString()}` + throw pvStatusToException( + status, + 'Get Koala delay samples failed', + messageStack, + pvError ); } const delaySample = memoryBufferView.getInt32(delaySampleAddress, true); + await pv_free(delaySample); const frameLength = await pv_koala_frame_length(); const sampleRate = await pv_sample_rate(); const versionAddress = await pv_koala_version(); const version = arrayBufferToStringAtIndex( memoryBufferUint8, - versionAddress, + versionAddress ); - await pv_free(accessKeyAddress); - await pv_free(modelPathAddress); - await pv_free(delaySample); - const inputBufferAddress = await aligned_alloc( Int16Array.BYTES_PER_ELEMENT, - frameLength * Int16Array.BYTES_PER_ELEMENT, + frameLength * Int16Array.BYTES_PER_ELEMENT ); if (inputBufferAddress === 0) { - throw new Error('malloc failed: Cannot allocate memory'); + throw new KoalaErrors.KoalaOutOfMemoryError( + 'malloc failed: Cannot allocate memory' + ); } const outputBufferAddress = await aligned_alloc( Int16Array.BYTES_PER_ELEMENT, - frameLength * Int16Array.BYTES_PER_ELEMENT, + frameLength * Int16Array.BYTES_PER_ELEMENT ); if (outputBufferAddress === 0) { - throw new Error('malloc failed: Cannot allocate memory'); + throw new KoalaErrors.KoalaOutOfMemoryError( + 'malloc failed: Cannot allocate memory' + ); } return { @@ -501,17 +655,64 @@ export class Koala { memory: memory, pvFree: pv_free, objectAddress: objectAddress, + messageStackAddressAddressAddress: messageStackAddressAddressAddress, + messageStackDepthAddress: messageStackDepthAddress, pvKoalaDelete: pv_koala_delete, pvKoalaProcess: pv_koala_process, pvKoalaReset: pv_koala_reset, pvStatusToString: pv_status_to_string, + pvGetErrorStack: pv_get_error_stack, + pvFreeErrorStack: pv_free_error_stack, delaySample: delaySample, frameLength: frameLength, sampleRate: sampleRate, version: version, inputBufferAddress: inputBufferAddress, outputBufferAddress: outputBufferAddress, - pvError: pvError + pvError: pvError, }; } + + private static async getMessageStack( + pv_get_error_stack: pv_get_error_stack_type, + pv_free_error_stack: pv_free_error_stack_type, + messageStackAddressAddressAddress: number, + messageStackDepthAddress: number, + memoryBufferView: DataView, + memoryBufferUint8: Uint8Array + ): Promise { + const status = await pv_get_error_stack( + messageStackAddressAddressAddress, + messageStackDepthAddress + ); + if (status !== PvStatus.SUCCESS) { + throw pvStatusToException(status, 'Unable to get Koala error state'); + } + + const messageStackAddressAddress = memoryBufferView.getInt32( + messageStackAddressAddressAddress, + true + ); + + const messageStackDepth = memoryBufferView.getInt32( + messageStackDepthAddress, + true + ); + const messageStack: string[] = []; + for (let i = 0; i < messageStackDepth; i++) { + const messageStackAddress = memoryBufferView.getInt32( + messageStackAddressAddress + i * Int32Array.BYTES_PER_ELEMENT, + true + ); + const message = arrayBufferToStringAtIndex( + memoryBufferUint8, + messageStackAddress + ); + messageStack.push(message); + } + + await pv_free_error_stack(messageStackAddressAddress); + + return messageStack; + } } diff --git a/binding/web/src/koala_errors.ts b/binding/web/src/koala_errors.ts new file mode 100644 index 0000000..515f578 --- /dev/null +++ b/binding/web/src/koala_errors.ts @@ -0,0 +1,252 @@ +// +// Copyright 2023 Picovoice Inc. +// +// You may not use this file except in compliance with the license. A copy of the license is located in the "LICENSE" +// file accompanying this source. +// +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// + +import { PvError } from '@picovoice/web-utils'; +import { PvStatus } from './types'; + +class KoalaError extends Error { + private readonly _status: PvStatus; + private readonly _shortMessage: string; + private readonly _messageStack: string[]; + + constructor( + status: PvStatus, + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(KoalaError.errorToString(message, messageStack, pvError)); + this._status = status; + this.name = 'KoalaError'; + this._shortMessage = message; + this._messageStack = messageStack; + } + + get status(): PvStatus { + return this._status; + } + + get shortMessage(): string { + return this._shortMessage; + } + + get messageStack(): string[] { + return this._messageStack; + } + + private static errorToString( + initial: string, + messageStack: string[], + pvError: PvError | null = null + ): string { + let msg = initial; + + if (pvError) { + const pvErrorMessage = pvError.getErrorString(); + if (pvErrorMessage.length > 0) { + msg += `\nDetails: ${pvErrorMessage}`; + } + } + + if (messageStack.length > 0) { + msg += `: ${messageStack.reduce( + (acc, value, index) => acc + '\n [' + index + '] ' + value, + '' + )}`; + } + + return msg; + } +} + +class KoalaOutOfMemoryError extends KoalaError { + constructor( + message: string, + messageStack?: string[], + pvError: PvError | null = null + ) { + super(PvStatus.OUT_OF_MEMORY, message, messageStack, pvError); + this.name = 'KoalaOutOfMemoryError'; + } +} + +class KoalaIOError extends KoalaError { + constructor( + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(PvStatus.IO_ERROR, message, messageStack, pvError); + this.name = 'KoalaIOError'; + } +} + +class KoalaInvalidArgumentError extends KoalaError { + constructor( + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(PvStatus.INVALID_ARGUMENT, message, messageStack, pvError); + this.name = 'KoalaInvalidArgumentError'; + } +} + +class KoalaStopIterationError extends KoalaError { + constructor( + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(PvStatus.STOP_ITERATION, message, messageStack, pvError); + this.name = 'KoalaStopIterationError'; + } +} + +class KoalaKeyError extends KoalaError { + constructor( + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(PvStatus.KEY_ERROR, message, messageStack, pvError); + this.name = 'KoalaKeyError'; + } +} + +class KoalaInvalidStateError extends KoalaError { + constructor( + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(PvStatus.INVALID_STATE, message, messageStack, pvError); + this.name = 'KoalaInvalidStateError'; + } +} + +class KoalaRuntimeError extends KoalaError { + constructor( + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(PvStatus.RUNTIME_ERROR, message, messageStack, pvError); + this.name = 'KoalaRuntimeError'; + } +} + +class KoalaActivationError extends KoalaError { + constructor( + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(PvStatus.ACTIVATION_ERROR, message, messageStack, pvError); + this.name = 'KoalaActivationError'; + } +} + +class KoalaActivationLimitReachedError extends KoalaError { + constructor( + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(PvStatus.ACTIVATION_LIMIT_REACHED, message, messageStack, pvError); + this.name = 'KoalaActivationLimitReachedError'; + } +} + +class KoalaActivationThrottledError extends KoalaError { + constructor( + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(PvStatus.ACTIVATION_THROTTLED, message, messageStack, pvError); + this.name = 'KoalaActivationThrottledError'; + } +} + +class KoalaActivationRefusedError extends KoalaError { + constructor( + message: string, + messageStack: string[] = [], + pvError: PvError | null = null + ) { + super(PvStatus.ACTIVATION_REFUSED, message, messageStack, pvError); + this.name = 'KoalaActivationRefusedError'; + } +} + +export { + KoalaError, + KoalaOutOfMemoryError, + KoalaIOError, + KoalaInvalidArgumentError, + KoalaStopIterationError, + KoalaKeyError, + KoalaInvalidStateError, + KoalaRuntimeError, + KoalaActivationError, + KoalaActivationLimitReachedError, + KoalaActivationThrottledError, + KoalaActivationRefusedError, +}; + +export function pvStatusToException( + pvStatus: PvStatus, + errorMessage: string, + messageStack: string[] = [], + pvError: PvError | null = null +): KoalaError { + switch (pvStatus) { + case PvStatus.OUT_OF_MEMORY: + return new KoalaOutOfMemoryError(errorMessage, messageStack, pvError); + case PvStatus.IO_ERROR: + return new KoalaIOError(errorMessage, messageStack, pvError); + case PvStatus.INVALID_ARGUMENT: + return new KoalaInvalidArgumentError(errorMessage, messageStack, pvError); + case PvStatus.STOP_ITERATION: + return new KoalaStopIterationError(errorMessage, messageStack, pvError); + case PvStatus.KEY_ERROR: + return new KoalaKeyError(errorMessage, messageStack, pvError); + case PvStatus.INVALID_STATE: + return new KoalaInvalidStateError(errorMessage, messageStack, pvError); + case PvStatus.RUNTIME_ERROR: + return new KoalaRuntimeError(errorMessage, messageStack, pvError); + case PvStatus.ACTIVATION_ERROR: + return new KoalaActivationError(errorMessage, messageStack, pvError); + case PvStatus.ACTIVATION_LIMIT_REACHED: + return new KoalaActivationLimitReachedError( + errorMessage, + messageStack, + pvError + ); + case PvStatus.ACTIVATION_THROTTLED: + return new KoalaActivationThrottledError( + errorMessage, + messageStack, + pvError + ); + case PvStatus.ACTIVATION_REFUSED: + return new KoalaActivationRefusedError( + errorMessage, + messageStack, + pvError + ); + default: + // eslint-disable-next-line no-console + console.warn(`Unmapped error code: ${pvStatus}`); + return new KoalaError(pvStatus, errorMessage); + } +} diff --git a/binding/web/src/koala_worker.ts b/binding/web/src/koala_worker.ts index 1d62ff5..f17b1c6 100644 --- a/binding/web/src/koala_worker.ts +++ b/binding/web/src/koala_worker.ts @@ -17,8 +17,10 @@ import { KoalaWorkerInitResponse, KoalaWorkerProcessResponse, KoalaWorkerReleaseResponse, + PvStatus, } from './types'; import { loadModel } from '@picovoice/web-utils'; +import { pvStatusToException } from './koala_errors'; export class KoalaWorker { private readonly _worker: Worker; @@ -26,11 +28,18 @@ export class KoalaWorker { private readonly _frameLength: number; private readonly _sampleRate: number; private readonly _delaySample: number; + private static _sdk: string = 'web'; private static _wasm: string; private static _wasmSimd: string; - private constructor(worker: Worker, version: string, frameLength: number, sampleRate: number, delaySample: number) { + private constructor( + worker: Worker, + version: string, + frameLength: number, + sampleRate: number, + delaySample: number + ) { this._worker = worker; this._version = version; this._frameLength = frameLength; @@ -94,6 +103,10 @@ export class KoalaWorker { } } + public static setSdk(sdk: string): void { + KoalaWorker._sdk = sdk; + } + /** * Creates an instance of the Picovoice Koala Noise Suppression Engine. * Behind the scenes, it requires the WebAssembly code to load and initialize before @@ -121,63 +134,96 @@ export class KoalaWorker { accessKey: string, processCallback: (enhancedPcm: Int16Array) => void, model: KoalaModel, - options: KoalaOptions = {}, + options: KoalaOptions = {} ): Promise { - const { processErrorCallback, ...rest } = options; + const { processErrorCallback, ...workerOptions } = options; - const customWritePath = (model.customWritePath) ? model.customWritePath : 'koala_model'; + const customWritePath = model.customWritePath + ? model.customWritePath + : 'koala_model'; const modelPath = await loadModel({ ...model, customWritePath }); const worker = new PvWorker(); - const returnPromise: Promise = new Promise((resolve, reject) => { - // @ts-ignore - block from GC - this.worker = worker; - worker.onmessage = (event: MessageEvent): void => { - switch (event.data.command) { - case 'ok': - worker.onmessage = (ev: MessageEvent): void => { - switch (ev.data.command) { - case 'ok': - processCallback(ev.data.enhancedPcm); - break; - case 'failed': - case 'error': - if (processErrorCallback) { - processErrorCallback(ev.data.message); - } else { - // eslint-disable-next-line no-console - console.error(ev.data.message); - } - break; - default: + const returnPromise: Promise = new Promise( + (resolve, reject) => { + // @ts-ignore - block from GC + this.worker = worker; + worker.onmessage = ( + event: MessageEvent + ): void => { + switch (event.data.command) { + case 'ok': + worker.onmessage = ( + ev: MessageEvent + ): void => { + switch (ev.data.command) { + case 'ok': + processCallback(ev.data.enhancedPcm); + break; + case 'failed': + case 'error': + { + const error = pvStatusToException( + ev.data.status, + ev.data.shortMessage, + ev.data.messageStack + ); + if (processErrorCallback) { + processErrorCallback(error); + } else { + // eslint-disable-next-line no-console + console.error(error); + } + } + break; + default: + // @ts-ignore + processErrorCallback( + pvStatusToException( + PvStatus.RUNTIME_ERROR, + `Unrecognized command: ${event.data.command}` + ) + ); + } + }; + resolve( + new KoalaWorker( + worker, + event.data.version, + event.data.frameLength, + event.data.sampleRate, + event.data.delaySample + ) + ); + break; + case 'failed': + case 'error': + reject( + pvStatusToException( + event.data.status, + event.data.shortMessage, + event.data.messageStack + ) + ); + break; + default: + reject( + pvStatusToException( + PvStatus.RUNTIME_ERROR, // @ts-ignore - processErrorCallback(`Unrecognized command: ${event.data.command}`); - } - }; - resolve( - new KoalaWorker( - worker, - event.data.version, - event.data.frameLength, - event.data.sampleRate, - event.data.delaySample)); - break; - case 'failed': - case 'error': - reject(event.data.message); - break; - default: - // @ts-ignore - reject(`Unrecognized command: ${event.data.command}`); - } - }; - }); + `Unrecognized command: ${event.data.command}` + ) + ); + } + }; + } + ); worker.postMessage({ command: 'init', accessKey: accessKey, modelPath: modelPath, - options: rest, + options: workerOptions, wasm: this._wasm, wasmSimd: this._wasmSimd, }); @@ -206,7 +252,7 @@ export class KoalaWorker { */ public async reset(): Promise { this._worker.postMessage({ - command: 'reset' + command: 'reset', }); } @@ -215,18 +261,31 @@ export class KoalaWorker { */ public release(): Promise { const returnPromise: Promise = new Promise((resolve, reject) => { - this._worker.onmessage = (event: MessageEvent): void => { + this._worker.onmessage = ( + event: MessageEvent + ): void => { switch (event.data.command) { case 'ok': resolve(); break; case 'failed': case 'error': - reject(event.data.message); + reject( + pvStatusToException( + event.data.status, + event.data.shortMessage, + event.data.messageStack + ) + ); break; default: - // @ts-ignore - reject(`Unrecognized command: ${event.data.command}`); + reject( + pvStatusToException( + PvStatus.RUNTIME_ERROR, + // @ts-ignore + `Unrecognized command: ${event.data.command}` + ) + ); } }; }); diff --git a/binding/web/src/koala_worker_handler.ts b/binding/web/src/koala_worker_handler.ts index 2e2b19f..3fd5dbe 100644 --- a/binding/web/src/koala_worker_handler.ts +++ b/binding/web/src/koala_worker_handler.ts @@ -11,7 +11,8 @@ /// import { Koala } from './koala'; -import { KoalaWorkerRequest } from './types'; +import { KoalaWorkerRequest, PvStatus } from './types'; +import { KoalaError } from './koala_errors'; let koala: Koala | null = null; @@ -22,10 +23,12 @@ const processCallback = (enhancedPcm: Int16Array): void => { }); }; -const processErrorCallback = (error: string): void => { +const processErrorCallback = (error: KoalaError): void => { self.postMessage({ command: 'error', - message: error, + status: error.status, + shortMessage: error.shortMessage, + messageStack: error.messageStack, }); }; @@ -40,7 +43,8 @@ self.onmessage = async function ( if (koala !== null) { self.postMessage({ command: 'error', - message: 'Koala already initialized', + status: PvStatus.INVALID_STATE, + shortMessage: 'Koala already initialized', }); return; } @@ -58,20 +62,31 @@ self.onmessage = async function ( version: koala.version, frameLength: koala.frameLength, sampleRate: koala.sampleRate, - delaySample: koala.delaySample + delaySample: koala.delaySample, }); } catch (e: any) { - self.postMessage({ - command: 'error', - message: e.message, - }); + if (e instanceof KoalaError) { + self.postMessage({ + command: 'error', + status: e.status, + shortMessage: e.shortMessage, + messageStack: e.messageStack, + }); + } else { + self.postMessage({ + command: 'error', + status: PvStatus.RUNTIME_ERROR, + shortMessage: e.message, + }); + } } break; case 'process': if (koala === null) { self.postMessage({ command: 'error', - message: 'Koala not initialized', + status: PvStatus.INVALID_STATE, + shortMessage: 'Koala not initialized', }); return; } @@ -81,7 +96,8 @@ self.onmessage = async function ( if (koala === null) { self.postMessage({ command: 'error', - message: 'Koala not initialized', + status: PvStatus.INVALID_STATE, + shortMessage: 'Koala not initialized', }); return; } @@ -100,8 +116,9 @@ self.onmessage = async function ( default: self.postMessage({ command: 'failed', + status: PvStatus.RUNTIME_ERROR, // @ts-ignore - message: `Unrecognized command: ${event.data.command}`, + shortMessage: `Unrecognized command: ${event.data.command}`, }); } }; diff --git a/binding/web/src/types.ts b/binding/web/src/types.ts index cbc3591..3ee4f2b 100644 --- a/binding/web/src/types.ts +++ b/binding/web/src/types.ts @@ -11,6 +11,23 @@ import { PvModel } from "@picovoice/web-utils"; +import {KoalaError} from "./koala_errors"; + +export enum PvStatus { + SUCCESS = 10000, + OUT_OF_MEMORY, + IO_ERROR, + INVALID_ARGUMENT, + STOP_ITERATION, + KEY_ERROR, + INVALID_STATE, + RUNTIME_ERROR, + ACTIVATION_ERROR, + ACTIVATION_LIMIT_REACHED, + ACTIVATION_THROTTLED, + ACTIVATION_REFUSED, +} + /** * KoalaModel types */ @@ -18,7 +35,7 @@ export type KoalaModel = PvModel; export type KoalaOptions = { /** @defaultValue undefined */ - processErrorCallback?: (error: string) => void + processErrorCallback?: (error: KoalaError) => void; }; export type KoalaWorkerInitRequest = { @@ -51,7 +68,9 @@ export type KoalaWorkerRequest = export type KoalaWorkerFailureResponse = { command: 'failed' | 'error'; - message: string; + status: PvStatus; + shortMessage: string; + messageStack: string[]; }; export type KoalaWorkerInitResponse = KoalaWorkerFailureResponse | { diff --git a/binding/web/test/koala.test.ts b/binding/web/test/koala.test.ts index c6b4d3e..bc3f0da 100644 --- a/binding/web/test/koala.test.ts +++ b/binding/web/test/koala.test.ts @@ -1,4 +1,4 @@ -import { Koala, KoalaWorker } from "../"; +import { Koala, KoalaWorker } from '../'; // @ts-ignore import koalaParams from './koala_params'; @@ -21,59 +21,71 @@ async function runTest( ) { const errorFrames: number[] = []; - const runProcess = () => new Promise(async (resolve, reject) => { - let numFrames = 0; - let numProcessed = 0; + const runProcess = () => + new Promise(async (resolve, reject) => { + let numFrames = 0; + let numProcessed = 0; - const koala = await instance.create( - ACCESS_KEY, - enhancedPcm => { - const frameStart = numProcessed * koala.frameLength; - const frameEnergy = rootMeanSquare(enhancedPcm); - - let energyDeviation: number; - if (referencePcm === undefined || frameStart < koala.delaySample) { - energyDeviation = frameEnergy; - } else { - const referenceFrame = referencePcm.slice(frameStart - koala.delaySample, frameStart - koala.delaySample + koala.frameLength); - energyDeviation = Math.abs(frameEnergy - rootMeanSquare(referenceFrame)); - } - - try { - expect(energyDeviation).to.be.lessThan(tolerance); - } catch (e) { - errorFrames.push(numProcessed); - } + const koala = await instance.create( + ACCESS_KEY, + enhancedPcm => { + const frameStart = numProcessed * koala.frameLength; + const frameEnergy = rootMeanSquare(enhancedPcm); - numProcessed += 1; - if (numFrames === numProcessed) { - if (koala instanceof KoalaWorker) { - koala.terminate(); + let energyDeviation: number; + if (referencePcm === undefined || frameStart < koala.delaySample) { + energyDeviation = frameEnergy; } else { - koala.release(); + const referenceFrame = referencePcm.slice( + frameStart - koala.delaySample, + frameStart - koala.delaySample + koala.frameLength + ); + energyDeviation = Math.abs( + frameEnergy - rootMeanSquare(referenceFrame) + ); } - if (errorFrames.length !== 0) { - reject(`Failed comparison for frames: '${errorFrames.join(",")}'`); - } else { - resolve(); + try { + expect(energyDeviation).to.be.lessThan(tolerance); + } catch (e) { + errorFrames.push(numProcessed); } + + numProcessed += 1; + if (numFrames === numProcessed) { + if (koala instanceof KoalaWorker) { + koala.terminate(); + } else { + koala.release(); + } + + if (errorFrames.length !== 0) { + reject( + `Failed comparison for frames: '${errorFrames.join(',')}'` + ); + } else { + resolve(); + } + } + }, + { publicPath: '/test/koala_params.pv', forceWrite: true }, + { + processErrorCallback: (error: string) => { + reject(error); + }, } - }, - { publicPath: '/test/koala_params.pv', forceWrite: true }, - { - processErrorCallback: (error: string) => { - reject(error); - } - } - ); + ); - numFrames = Math.round(inputPcm.length / koala.frameLength) - 1; - await koala.reset(); - for (let i = 0; i < (inputPcm.length - koala.frameLength + 1); i += koala.frameLength) { - await koala.process(inputPcm.slice(i, i + koala.frameLength)); - } - }); + numFrames = Math.round(inputPcm.length / koala.frameLength) - 1; + await koala.reset(); + for ( + let i = 0; + i < inputPcm.length - koala.frameLength + 1; + i += koala.frameLength + ) { + await koala.process(inputPcm.slice(i, i + koala.frameLength)); + } + }); try { await runProcess(); @@ -99,24 +111,33 @@ async function testReset( numFrames = Math.round(inputPcm.length / koala.frameLength) - 1; await koala.reset(); - for (let i = 0; i < (inputPcm.length - koala.frameLength + 1); i += koala.frameLength) { + for ( + let i = 0; + i < inputPcm.length - koala.frameLength + 1; + i += koala.frameLength + ) { await koala.process(inputPcm.slice(i, i + koala.frameLength)); } - const waitUntil = (): Promise => new Promise(resolve => { - setInterval(() => { - if (numFrames === frames.length) { - resolve(); - } - }, 100); - }); + const waitUntil = (): Promise => + new Promise(resolve => { + setInterval(() => { + if (numFrames === frames.length) { + resolve(); + } + }, 100); + }); await waitUntil(); const originalFrames = [...frames]; frames = []; await koala.reset(); - for (let i = 0; i < (inputPcm.length - koala.frameLength + 1); i += koala.frameLength) { + for ( + let i = 0; + i < inputPcm.length - koala.frameLength + 1; + i += koala.frameLength + ) { await koala.process(inputPcm.slice(i, i + koala.frameLength)); } @@ -134,137 +155,107 @@ async function testReset( } describe('Koala Binding', function () { - it('should be able to init with public path', async () => { - try { - const koala = await Koala.create( - ACCESS_KEY, - _ => {}, - { publicPath: '/test/koala_params.pv', forceWrite: true } - ); - expect(koala).to.not.be.undefined; - expect(koala.frameLength).to.be.greaterThan(0); - expect(koala.delaySample).to.be.gte(0); - expect(typeof koala.version).to.eq('string'); - expect(koala.version).length.to.be.greaterThan(0); - await koala.release(); - } catch (e) { - expect(e).to.be.undefined; - } - }); - - it('should be able to init with public path (worker)', async () => { - try { - const koala = await KoalaWorker.create( - ACCESS_KEY, - _ => {}, - { publicPath: '/test/koala_params.pv', forceWrite: true } - ); - expect(koala).to.not.be.undefined; - expect(koala.frameLength).to.be.greaterThan(0); - expect(koala.delaySample).to.be.gte(0); - expect(typeof koala.version).to.eq('string'); - expect(koala.version).length.to.be.greaterThan(0); - await koala.terminate(); - } catch (e) { - expect(e).to.be.undefined; - } - }); - - it('should be able to init with base64', async () => { - try { - const koala = await Koala.create( - ACCESS_KEY, - _ => {}, - { base64: koalaParams, forceWrite: true } - ); - expect(koala).to.not.be.undefined; - expect(koala.frameLength).to.be.greaterThan(0); - expect(koala.delaySample).to.be.gte(0); - expect(typeof koala.version).to.eq('string'); - expect(koala.version).length.to.be.greaterThan(0); - await koala.release(); - } catch (e) { - expect(e).to.be.undefined; - } - }); - - it('should be able to init with base64 (worker)', async () => { - try { - const koala = await KoalaWorker.create( - ACCESS_KEY, - _ => {}, - { base64: koalaParams, forceWrite: true } - ); - expect(koala).to.not.be.undefined; - expect(koala.frameLength).to.be.greaterThan(0); - expect(koala.delaySample).to.be.gte(0); - expect(typeof koala.version).to.eq('string'); - expect(koala.version).length.to.be.greaterThan(0); - await koala.release(); - } catch (e) { - expect(e).to.be.undefined; - } - }); - - it('should be able to process pure speech', () => { - cy.getFramesFromFile('audio_samples/test.wav').then( async inputPcm => { - await runTest(Koala, inputPcm, inputPcm); + for (const instance of [Koala, KoalaWorker]) { + const instanceString = instance === KoalaWorker ? 'worker' : 'main'; + it(`should be able to init with public path (${instanceString})`, async () => { + try { + const koala = await instance.create(ACCESS_KEY, _ => {}, { + publicPath: '/test/koala_params.pv', + forceWrite: true, + }); + expect(koala).to.not.be.undefined; + expect(koala.frameLength).to.be.greaterThan(0); + expect(koala.delaySample).to.be.gte(0); + expect(typeof koala.version).to.eq('string'); + expect(koala.version).length.to.be.greaterThan(0); + if (koala instanceof KoalaWorker) { + koala.terminate(); + } else { + await koala.release(); + } + } catch (e) { + expect(e).to.be.undefined; + } }); - }); - it('should be able to process pure speech (worker)', () => { - cy.getFramesFromFile('audio_samples/test.wav').then( async inputPcm => { - await runTest(KoalaWorker, inputPcm, inputPcm); + it(`should be able to init with base64 (${instanceString})`, async () => { + try { + const koala = await Koala.create(ACCESS_KEY, _ => {}, { + base64: koalaParams, + forceWrite: true, + }); + expect(koala).to.not.be.undefined; + expect(koala.frameLength).to.be.greaterThan(0); + expect(koala.delaySample).to.be.gte(0); + expect(typeof koala.version).to.eq('string'); + expect(koala.version).length.to.be.greaterThan(0); + if (koala instanceof KoalaWorker) { + koala.terminate(); + } else { + await koala.release(); + } + } catch (e) { + expect(e).to.be.undefined; + } }); - }); - it('should be able to process noise speech', () => { - cy.getFramesFromFile('audio_samples/noise.wav').then( async inputPcm => { - await runTest(Koala, inputPcm); + it(`should be able to process pure speech (${instanceString})`, () => { + cy.getFramesFromFile('audio_samples/test.wav').then(async inputPcm => { + await runTest(instance, inputPcm, inputPcm); + }); }); - }); - it('should be able to process noise speech (worker)', () => { - cy.getFramesFromFile('audio_samples/noise.wav').then( async inputPcm => { - await runTest(KoalaWorker, inputPcm); + it(`should be able to process noise speech (${instanceString})`, () => { + cy.getFramesFromFile('audio_samples/noise.wav').then(async inputPcm => { + await runTest(instance, inputPcm); + }); }); - }); - it('should be able to process mixed speech', () => { - cy.getFramesFromFile('audio_samples/noise.wav').then( inputPcm => { - cy.getFramesFromFile('audio_samples/test.wav').then(async referencePcm => { - const noisyPcm = new Int16Array(inputPcm.length); - for (let i = 0; i < inputPcm.length; i++) { - noisyPcm[i] = inputPcm[i] + referencePcm[i]; - } + it(`should be able to process mixed speech (${instanceString})`, () => { + cy.getFramesFromFile('audio_samples/noise.wav').then(inputPcm => { + cy.getFramesFromFile('audio_samples/test.wav').then( + async referencePcm => { + const noisyPcm = new Int16Array(inputPcm.length); + for (let i = 0; i < inputPcm.length; i++) { + noisyPcm[i] = inputPcm[i] + referencePcm[i]; + } - await runTest(Koala, noisyPcm, referencePcm); + await runTest(instance, noisyPcm, referencePcm); + } + ); }); }); - }); - it('should be able to process mixed speech (worker)', () => { - cy.getFramesFromFile('audio_samples/noise.wav').then( inputPcm => { - cy.getFramesFromFile('audio_samples/test.wav').then(async referencePcm => { - const noisyPcm = new Int16Array(inputPcm.length); - for (let i = 0; i < inputPcm.length; i++) { - noisyPcm[i] = inputPcm[i] + referencePcm[i]; - } - - await runTest(KoalaWorker, noisyPcm, referencePcm); + it(`should be able to reset (${instanceString})`, () => { + cy.getFramesFromFile('audio_samples/test.wav').then(async inputPcm => { + await testReset(instance, inputPcm); }); }); - }); - it('should be able to reset', () => { - cy.getFramesFromFile('audio_samples/test.wav').then( async inputPcm => { - await testReset(Koala, inputPcm); - }); - }); + it(`should return correct error message stack (${instanceString})`, async () => { + let messageStack = []; + try { + const koala = await instance.create('invalidAccessKey', _ => {}, { + publicPath: '/test/koala_params.pv', + forceWrite: true, + }); + expect(koala).to.be.undefined; + } catch (e: any) { + messageStack = e.messageStack; + } - it('should be able to reset (worker)', () => { - cy.getFramesFromFile('audio_samples/test.wav').then( async inputPcm => { - await testReset(KoalaWorker, inputPcm); + expect(messageStack.length).to.be.gt(0); + expect(messageStack.length).to.be.lte(8); + + try { + const koala = await instance.create('invalidAccessKey', _ => {}, { + publicPath: '/test/koala_params.pv', + forceWrite: true, + }); + expect(koala).to.be.undefined; + } catch (e: any) { + expect(messageStack.length).to.be.eq(e.messageStack.length); + } }); - }); + } }); diff --git a/demo/web/package.json b/demo/web/package.json index 7b1f5d0..c7c788f 100644 --- a/demo/web/package.json +++ b/demo/web/package.json @@ -1,6 +1,6 @@ { "name": "koala-web-demo", - "version": "1.0.0", + "version": "2.0.0", "description": "A basic demo to show how to use Koala for web browsers, using the IIFE version of the library", "main": "index.js", "private": true, @@ -18,7 +18,7 @@ "author": "Picovoice Inc", "license": "Apache-2.0", "dependencies": { - "@picovoice/koala-web": "~1.0.3", + "@picovoice/koala-web": "file:../../binding/web", "@picovoice/web-voice-processor": "~4.0.8" }, "devDependencies": { diff --git a/demo/web/yarn.lock b/demo/web/yarn.lock index 01ac5c3..c3acfb0 100644 --- a/demo/web/yarn.lock +++ b/demo/web/yarn.lock @@ -2,10 +2,8 @@ # yarn lockfile v1 -"@picovoice/koala-web@~1.0.3": - version "1.0.3" - resolved "https://registry.yarnpkg.com/@picovoice/koala-web/-/koala-web-1.0.3.tgz#95dd441f34f1697d7d1a03d5b61326ad3a1488ac" - integrity sha512-S1LFNn1CaWsBZ0jNo3IsfFfhAyjl7aZjCwO4eyFdJdHsybFNORUeKAWEfaPoyELmiHUooR4jtRVn51f+woGvyA== +"@picovoice/koala-web@file:../../binding/web": + version "2.0.0" dependencies: "@picovoice/web-utils" "=1.3.1" From e7e446bbd6b60efb9967f860744172e4203292a7 Mon Sep 17 00:00:00 2001 From: Ian Date: Mon, 30 Oct 2023 16:22:31 -0700 Subject: [PATCH 2/8] fix --- .github/workflows/web-demos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/web-demos.yml b/.github/workflows/web-demos.yml index 9764178..bf31629 100644 --- a/.github/workflows/web-demos.yml +++ b/.github/workflows/web-demos.yml @@ -36,7 +36,7 @@ jobs: node-version: ${{ matrix.node-version }} - name: Build Web SDK - run: yarn && yarn copywasm && yarn copyppn && yarn build + run: yarn && yarn copywasm && yarn build working-directory: binding/web - name: Pre-build dependencies From 85731847bc27c3986c34bd79e46a59c5566a4260 Mon Sep 17 00:00:00 2001 From: Ian Date: Mon, 30 Oct 2023 16:30:15 -0700 Subject: [PATCH 3/8] fix --- binding/web/cypress.config.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/binding/web/cypress.config.ts b/binding/web/cypress.config.ts index bc17521..01bb065 100644 --- a/binding/web/cypress.config.ts +++ b/binding/web/cypress.config.ts @@ -1,14 +1,15 @@ -import { defineConfig } from "cypress"; +import { defineConfig } from 'cypress'; export default defineConfig({ env: { - "NUM_TEST_ITERATIONS": 20, - "PROC_PERFORMANCE_THRESHOLD_SEC": 0.8 + NUM_TEST_ITERATIONS: 20, + PROC_PERFORMANCE_THRESHOLD_SEC: 0.8, }, e2e: { - supportFile: "cypress/support/index.ts", - specPattern: "test/*.test.{js,jsx,ts,tsx}", + defaultCommandTimeout: 30000, + supportFile: 'cypress/support/index.ts', + specPattern: 'test/*.test.{js,jsx,ts,tsx}', video: false, - screenshotOnRunFailure: false + screenshotOnRunFailure: false, }, }); From 2fd9bb1c8aa813fc9323d4ee483c497ee78b6cc0 Mon Sep 17 00:00:00 2001 From: Ian Date: Mon, 30 Oct 2023 16:45:21 -0700 Subject: [PATCH 4/8] lint --- binding/web/.eslintignore | 5 +++ binding/web/.eslintrc.js | 5 ++- binding/web/src/types.ts | 70 ++++++++++++++++------------- binding/web/test/koala_perf.test.ts | 32 ++++++++----- 4 files changed, 67 insertions(+), 45 deletions(-) create mode 100644 binding/web/.eslintignore diff --git a/binding/web/.eslintignore b/binding/web/.eslintignore new file mode 100644 index 0000000..2a3f354 --- /dev/null +++ b/binding/web/.eslintignore @@ -0,0 +1,5 @@ +node_modules +dist +lib +rollup.config.js +.eslintrc.js \ No newline at end of file diff --git a/binding/web/.eslintrc.js b/binding/web/.eslintrc.js index 112ad3c..00f929e 100644 --- a/binding/web/.eslintrc.js +++ b/binding/web/.eslintrc.js @@ -34,7 +34,8 @@ module.exports = { ignoreParameters: true, ignoreProperties: true } - ] + ], + '@typescript-eslint/no-shadow': 2 } }, { @@ -257,7 +258,7 @@ module.exports = { // disallow shadowing of names such as arguments 'no-shadow-restricted-names': 2, // disallow declaration of variables already declared in the outer scope - 'no-shadow': 2, + 'no-shadow': 0, // disallow use of undefined when initializing variables 'no-undef-init': 0, // disallow use of undeclared variables unless mentioned in a /*global */ block diff --git a/binding/web/src/types.ts b/binding/web/src/types.ts index 3ee4f2b..fb31a35 100644 --- a/binding/web/src/types.ts +++ b/binding/web/src/types.ts @@ -9,9 +9,9 @@ specific language governing permissions and limitations under the License. */ -import { PvModel } from "@picovoice/web-utils"; +import { PvModel } from '@picovoice/web-utils'; -import {KoalaError} from "./koala_errors"; +import { KoalaError } from './koala_errors'; export enum PvStatus { SUCCESS = 10000, @@ -54,17 +54,17 @@ export type KoalaWorkerProcessRequest = { export type KoalaWorkerResetRequest = { command: 'reset'; -} +}; export type KoalaWorkerReleaseRequest = { command: 'release'; }; export type KoalaWorkerRequest = - KoalaWorkerInitRequest | - KoalaWorkerProcessRequest | - KoalaWorkerResetRequest | - KoalaWorkerReleaseRequest; + | KoalaWorkerInitRequest + | KoalaWorkerProcessRequest + | KoalaWorkerResetRequest + | KoalaWorkerReleaseRequest; export type KoalaWorkerFailureResponse = { command: 'failed' | 'error'; @@ -73,29 +73,37 @@ export type KoalaWorkerFailureResponse = { messageStack: string[]; }; -export type KoalaWorkerInitResponse = KoalaWorkerFailureResponse | { - command: 'ok'; - frameLength: number; - sampleRate: number; - version: string; - delaySample: number; -}; - -export type KoalaWorkerProcessResponse = KoalaWorkerFailureResponse | { - command: 'ok'; - enhancedPcm: Int16Array; -}; - -export type KoalaWorkerResetResponse = KoalaWorkerFailureResponse | { - command: 'ok'; -}; - -export type KoalaWorkerReleaseResponse = KoalaWorkerFailureResponse | { - command: 'ok'; -}; +export type KoalaWorkerInitResponse = + | KoalaWorkerFailureResponse + | { + command: 'ok'; + frameLength: number; + sampleRate: number; + version: string; + delaySample: number; + }; + +export type KoalaWorkerProcessResponse = + | KoalaWorkerFailureResponse + | { + command: 'ok'; + enhancedPcm: Int16Array; + }; + +export type KoalaWorkerResetResponse = + | KoalaWorkerFailureResponse + | { + command: 'ok'; + }; + +export type KoalaWorkerReleaseResponse = + | KoalaWorkerFailureResponse + | { + command: 'ok'; + }; export type KoalaWorkerResponse = - KoalaWorkerInitResponse | - KoalaWorkerProcessResponse | - KoalaWorkerResetResponse | - KoalaWorkerReleaseResponse; + | KoalaWorkerInitResponse + | KoalaWorkerProcessResponse + | KoalaWorkerResetResponse + | KoalaWorkerReleaseResponse; diff --git a/binding/web/test/koala_perf.test.ts b/binding/web/test/koala_perf.test.ts index 7990ed5..238fd44 100644 --- a/binding/web/test/koala_perf.test.ts +++ b/binding/web/test/koala_perf.test.ts @@ -1,8 +1,10 @@ -import { Koala, KoalaWorker } from "../"; +import { Koala, KoalaWorker } from '../'; const ACCESS_KEY = Cypress.env('ACCESS_KEY'); const NUM_TEST_ITERATIONS = Number(Cypress.env('NUM_TEST_ITERATIONS')); -const PROC_PERFORMANCE_THRESHOLD_SEC = Number(Cypress.env('PROC_PERFORMANCE_THRESHOLD_SEC')); +const PROC_PERFORMANCE_THRESHOLD_SEC = Number( + Cypress.env('PROC_PERFORMANCE_THRESHOLD_SEC') +); async function testPerformance( instance: typeof Koala | typeof KoalaWorker, @@ -23,17 +25,22 @@ async function testPerformance( numFrames = Math.round(inputPcm.length / koala.frameLength) - 1; - const waitUntil = (): Promise => new Promise(resolve => { - setInterval(() => { - if (numFrames === processedFrames) { - resolve(); - } - }, 100); - }); + const waitUntil = (): Promise => + new Promise(resolve => { + setInterval(() => { + if (numFrames === processedFrames) { + resolve(); + } + }, 100); + }); await koala.reset(); const start = Date.now(); - for (let i = 0; i < (inputPcm.length - koala.frameLength + 1); i += koala.frameLength) { + for ( + let i = 0; + i < inputPcm.length - koala.frameLength + 1; + i += koala.frameLength + ) { await koala.process(inputPcm.slice(i, i + koala.frameLength)); } @@ -49,6 +56,7 @@ async function testPerformance( } const avgPerf = perfResults.reduce((a, b) => a + b) / NUM_TEST_ITERATIONS; + // eslint-disable-next-line no-console console.log(`Average proc performance: ${avgPerf} seconds`); expect(avgPerf).to.be.lessThan(PROC_PERFORMANCE_THRESHOLD_SEC); } @@ -57,13 +65,13 @@ describe('Koala binding performance test', () => { Cypress.config('defaultCommandTimeout', 60000); it(`should be lower than performance threshold (${PROC_PERFORMANCE_THRESHOLD_SEC}s)`, () => { - cy.getFramesFromFile('audio_samples/test.wav').then( async inputPcm => { + cy.getFramesFromFile('audio_samples/test.wav').then(async inputPcm => { await testPerformance(Koala, inputPcm); }); }); it(`should be lower than performance threshold (${PROC_PERFORMANCE_THRESHOLD_SEC}s) (worker)`, () => { - cy.getFramesFromFile('audio_samples/test.wav').then( async inputPcm => { + cy.getFramesFromFile('audio_samples/test.wav').then(async inputPcm => { await testPerformance(KoalaWorker, inputPcm); }); }); From 4d4a9068f1bb9e854833d1d8fb39d5f79b4c8a8a Mon Sep 17 00:00:00 2001 From: Ian Date: Mon, 30 Oct 2023 17:09:01 -0700 Subject: [PATCH 5/8] back to promise --- binding/web/src/koala.ts | 81 +++++++++++++++-------------- binding/web/test/koala_perf.test.ts | 18 +++---- 2 files changed, 50 insertions(+), 49 deletions(-) diff --git a/binding/web/src/koala.ts b/binding/web/src/koala.ts index 930b996..cad829d 100644 --- a/binding/web/src/koala.ts +++ b/binding/web/src/koala.ts @@ -381,51 +381,56 @@ export class Koala { * Call this function in between calls to `process` that do not provide consecutive frames of audio. */ public async reset(): Promise { - this._processMutex - .runExclusive(async () => { - if (this._wasmMemory === undefined) { - throw new KoalaErrors.KoalaInvalidStateError( - 'Attempted to call Koala reset after release.' - ); - } - - const status = await this._pvKoalaReset(this._objectAddress); + return new Promise((resolve, reject) => { + this._processMutex + .runExclusive(async () => { + if (this._wasmMemory === undefined) { + throw new KoalaErrors.KoalaInvalidStateError( + 'Attempted to call Koala reset after release.' + ); + } - if (status !== PvStatus.SUCCESS) { - const memoryBufferUint8 = new Uint8Array(this._wasmMemory.buffer); - const memoryBufferView = new DataView(this._wasmMemory.buffer); - const messageStack = await Koala.getMessageStack( - this._pvGetErrorStack, - this._pvFreeErrorStack, - this._messageStackAddressAddressAddress, - this._messageStackDepthAddress, - memoryBufferView, - memoryBufferUint8 - ); + const status = await this._pvKoalaReset(this._objectAddress); + + if (status !== PvStatus.SUCCESS) { + const memoryBufferUint8 = new Uint8Array(this._wasmMemory.buffer); + const memoryBufferView = new DataView(this._wasmMemory.buffer); + const messageStack = await Koala.getMessageStack( + this._pvGetErrorStack, + this._pvFreeErrorStack, + this._messageStackAddressAddressAddress, + this._messageStackDepthAddress, + memoryBufferView, + memoryBufferUint8 + ); + + const error = pvStatusToException( + status, + 'Reset failed', + messageStack + ); + if (this._processErrorCallback) { + this._processErrorCallback(error); + } else { + // eslint-disable-next-line no-console + console.error(error); + } + } - const error = pvStatusToException( - status, - 'Reset failed', - messageStack - ); + resolve(); + }) + .catch((error: any) => { if (this._processErrorCallback) { - this._processErrorCallback(error); + this._processErrorCallback( + pvStatusToException(PvStatus.RUNTIME_ERROR, error.toString()) + ); } else { // eslint-disable-next-line no-console console.error(error); } - } - }) - .catch((error: any) => { - if (this._processErrorCallback) { - this._processErrorCallback( - pvStatusToException(PvStatus.RUNTIME_ERROR, error.toString()) - ); - } else { - // eslint-disable-next-line no-console - console.error(error); - } - }); + reject(error); + }); + }); } /** diff --git a/binding/web/test/koala_perf.test.ts b/binding/web/test/koala_perf.test.ts index 238fd44..7d73821 100644 --- a/binding/web/test/koala_perf.test.ts +++ b/binding/web/test/koala_perf.test.ts @@ -62,17 +62,13 @@ async function testPerformance( } describe('Koala binding performance test', () => { - Cypress.config('defaultCommandTimeout', 60000); + for (const instance of [Koala, KoalaWorker]) { + Cypress.config('defaultCommandTimeout', 60000); - it(`should be lower than performance threshold (${PROC_PERFORMANCE_THRESHOLD_SEC}s)`, () => { - cy.getFramesFromFile('audio_samples/test.wav').then(async inputPcm => { - await testPerformance(Koala, inputPcm); - }); - }); - - it(`should be lower than performance threshold (${PROC_PERFORMANCE_THRESHOLD_SEC}s) (worker)`, () => { - cy.getFramesFromFile('audio_samples/test.wav').then(async inputPcm => { - await testPerformance(KoalaWorker, inputPcm); + it(`should be lower than performance threshold (${PROC_PERFORMANCE_THRESHOLD_SEC}s)`, () => { + cy.getFramesFromFile('audio_samples/test.wav').then(async inputPcm => { + await testPerformance(instance, inputPcm); + }); }); - }); + } }); From d5e376501d804fa3623c06656880cb8da61d1c31 Mon Sep 17 00:00:00 2001 From: Kwangsoo Yeo Date: Fri, 3 Nov 2023 11:12:47 -0700 Subject: [PATCH 6/8] fix koala + koala test --- binding/web/src/koala.ts | 2 +- binding/web/test/koala.test.ts | 48 +++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/binding/web/src/koala.ts b/binding/web/src/koala.ts index cad829d..8c69970 100644 --- a/binding/web/src/koala.ts +++ b/binding/web/src/koala.ts @@ -145,7 +145,7 @@ export class Koala { this._inputBufferAddress = handleWasm.inputBufferAddress; this._outputBufferAddress = handleWasm.outputBufferAddress; this._messageStackAddressAddressAddress = - handleWasm.messageStackDepthAddress; + handleWasm.messageStackAddressAddressAddress; this._messageStackDepthAddress = handleWasm.messageStackDepthAddress; this._pvError = handleWasm.pvError; diff --git a/binding/web/test/koala.test.ts b/binding/web/test/koala.test.ts index bc3f0da..3621383 100644 --- a/binding/web/test/koala.test.ts +++ b/binding/web/test/koala.test.ts @@ -2,6 +2,7 @@ import { Koala, KoalaWorker } from '../'; // @ts-ignore import koalaParams from './koala_params'; +import { KoalaError } from "../dist/types/koala_errors"; const ACCESS_KEY = Cypress.env('ACCESS_KEY'); @@ -13,6 +14,10 @@ function rootMeanSquare(pcm: Int16Array): number { return Math.sqrt(sumSquares / pcm.length); } +function delay(time: number) { + return new Promise(resolve => setTimeout(resolve, time)); +} + async function runTest( instance: typeof Koala | typeof KoalaWorker, inputPcm: Int16Array, @@ -70,7 +75,7 @@ async function runTest( }, { publicPath: '/test/koala_params.pv', forceWrite: true }, { - processErrorCallback: (error: string) => { + processErrorCallback: (error: KoalaError) => { reject(error); }, } @@ -155,6 +160,47 @@ async function testReset( } describe('Koala Binding', function () { + it(`should return process error message stack`, async () => { + let error: KoalaError | null = null; + + const runProcess = () => new Promise(async resolve => { + const koala = await Koala.create( + ACCESS_KEY, + () => { }, + { + publicPath: '/test/koala_params.pv', + forceWrite: true, + }, + { + processErrorCallback: (e: KoalaError) => { + error = e; + resolve(); + } + } + ); + const testPcm = new Int16Array(koala.frameLength); + // @ts-ignore + const objectAddress = koala._objectAddress; + + // @ts-ignore + koala._objectAddress = 0; + await koala.process(testPcm); + + await delay(1000); + + // @ts-ignore + koala._objectAddress = objectAddress; + await koala.release(); + }); + + await runProcess(); + expect(error).to.not.be.null; + if (error) { + expect((error as KoalaError).messageStack.length).to.be.gt(0); + expect((error as KoalaError).messageStack.length).to.be.lte(8); + } + }); + for (const instance of [Koala, KoalaWorker]) { const instanceString = instance === KoalaWorker ? 'worker' : 'main'; it(`should be able to init with public path (${instanceString})`, async () => { From 60c443a849077ed5145a1250a020a80fc7163c47 Mon Sep 17 00:00:00 2001 From: Ian Date: Fri, 3 Nov 2023 15:41:22 -0700 Subject: [PATCH 7/8] improvements --- binding/web/src/koala.ts | 26 +---- binding/web/src/koala_worker.ts | 9 +- binding/web/src/koala_worker_handler.ts | 146 ++++++++++++++---------- binding/web/src/types.ts | 4 +- 4 files changed, 101 insertions(+), 84 deletions(-) diff --git a/binding/web/src/koala.ts b/binding/web/src/koala.ts index 8c69970..568a1be 100644 --- a/binding/web/src/koala.ts +++ b/binding/web/src/koala.ts @@ -391,7 +391,6 @@ export class Koala { } const status = await this._pvKoalaReset(this._objectAddress); - if (status !== PvStatus.SUCCESS) { const memoryBufferUint8 = new Uint8Array(this._wasmMemory.buffer); const memoryBufferView = new DataView(this._wasmMemory.buffer); @@ -404,30 +403,13 @@ export class Koala { memoryBufferUint8 ); - const error = pvStatusToException( - status, - 'Reset failed', - messageStack - ); - if (this._processErrorCallback) { - this._processErrorCallback(error); - } else { - // eslint-disable-next-line no-console - console.error(error); - } + throw pvStatusToException(status, 'Reset failed', messageStack); } - + }) + .then(() => { resolve(); }) .catch((error: any) => { - if (this._processErrorCallback) { - this._processErrorCallback( - pvStatusToException(PvStatus.RUNTIME_ERROR, error.toString()) - ); - } else { - // eslint-disable-next-line no-console - console.error(error); - } reject(error); }); }); @@ -440,6 +422,8 @@ export class Koala { await this._pvKoalaDelete(this._objectAddress); await this._pvFree(this._inputBufferAddress); await this._pvFree(this._outputBufferAddress); + await this._pvFree(this._messageStackAddressAddressAddress); + await this._pvFree(this._messageStackDepthAddress); delete this._wasmMemory; this._wasmMemory = undefined; } diff --git a/binding/web/src/koala_worker.ts b/binding/web/src/koala_worker.ts index f17b1c6..8c5c970 100644 --- a/binding/web/src/koala_worker.ts +++ b/binding/web/src/koala_worker.ts @@ -17,6 +17,7 @@ import { KoalaWorkerInitResponse, KoalaWorkerProcessResponse, KoalaWorkerReleaseResponse, + KoalaWorkerResetResponse, PvStatus, } from './types'; import { loadModel } from '@picovoice/web-utils'; @@ -154,12 +155,16 @@ export class KoalaWorker { switch (event.data.command) { case 'ok': worker.onmessage = ( - ev: MessageEvent + ev: MessageEvent< + KoalaWorkerProcessResponse | KoalaWorkerResetResponse + > ): void => { switch (ev.data.command) { - case 'ok': + case 'ok-process': processCallback(ev.data.enhancedPcm); break; + case 'ok-reset': + break; case 'failed': case 'error': { diff --git a/binding/web/src/koala_worker_handler.ts b/binding/web/src/koala_worker_handler.ts index 3fd5dbe..e7dda10 100644 --- a/binding/web/src/koala_worker_handler.ts +++ b/binding/web/src/koala_worker_handler.ts @@ -11,14 +11,14 @@ /// import { Koala } from './koala'; -import { KoalaWorkerRequest, PvStatus } from './types'; +import { KoalaWorkerRequest, KoalaWorkerInitRequest, PvStatus } from './types'; import { KoalaError } from './koala_errors'; let koala: Koala | null = null; const processCallback = (enhancedPcm: Int16Array): void => { self.postMessage({ - command: 'ok', + command: 'ok-process', enhancedPcm: enhancedPcm, }); }; @@ -32,6 +32,88 @@ const processErrorCallback = (error: KoalaError): void => { }); }; +const initRequest = async (request: KoalaWorkerInitRequest): Promise => { + if (koala !== null) { + return { + command: 'error', + status: PvStatus.INVALID_STATE, + shortMessage: 'Koala already initialized', + }; + } + try { + Koala.setWasm(request.wasm); + Koala.setWasmSimd(request.wasmSimd); + koala = await Koala._init( + request.accessKey, + processCallback, + request.modelPath, + { ...request.options, processErrorCallback } + ); + return { + command: 'ok', + version: koala.version, + frameLength: koala.frameLength, + sampleRate: koala.sampleRate, + delaySample: koala.delaySample, + }; + } catch (e: any) { + if (e instanceof KoalaError) { + return { + command: 'error', + status: e.status, + shortMessage: e.shortMessage, + messageStack: e.messageStack, + }; + } + return { + command: 'error', + status: PvStatus.RUNTIME_ERROR, + shortMessage: e.message, + }; + } +}; + +const resetRequest = async (): Promise => { + if (koala === null) { + return { + command: 'error', + status: PvStatus.INVALID_STATE, + shortMessage: 'Koala not initialized', + }; + } + try { + await koala.reset(); + return { + command: 'ok-reset', + }; + } catch (e: any) { + if (e instanceof KoalaError) { + return { + command: 'error', + status: e.status, + shortMessage: e.shortMessage, + messageStack: e.messageStack, + }; + } + return { + command: 'error', + status: PvStatus.RUNTIME_ERROR, + shortMessage: e.message, + }; + } +}; + +const releaseRequest = async (): Promise => { + if (koala !== null) { + await koala.release(); + koala = null; + close(); + } + return { + command: 'ok', + }; +}; + /** * Koala worker handler. */ @@ -40,46 +122,7 @@ self.onmessage = async function ( ): Promise { switch (event.data.command) { case 'init': - if (koala !== null) { - self.postMessage({ - command: 'error', - status: PvStatus.INVALID_STATE, - shortMessage: 'Koala already initialized', - }); - return; - } - try { - Koala.setWasm(event.data.wasm); - Koala.setWasmSimd(event.data.wasmSimd); - koala = await Koala._init( - event.data.accessKey, - processCallback, - event.data.modelPath, - { ...event.data.options, processErrorCallback } - ); - self.postMessage({ - command: 'ok', - version: koala.version, - frameLength: koala.frameLength, - sampleRate: koala.sampleRate, - delaySample: koala.delaySample, - }); - } catch (e: any) { - if (e instanceof KoalaError) { - self.postMessage({ - command: 'error', - status: e.status, - shortMessage: e.shortMessage, - messageStack: e.messageStack, - }); - } else { - self.postMessage({ - command: 'error', - status: PvStatus.RUNTIME_ERROR, - shortMessage: e.message, - }); - } - } + self.postMessage(await initRequest(event.data)); break; case 'process': if (koala === null) { @@ -93,25 +136,10 @@ self.onmessage = async function ( await koala.process(event.data.inputFrame); break; case 'reset': - if (koala === null) { - self.postMessage({ - command: 'error', - status: PvStatus.INVALID_STATE, - shortMessage: 'Koala not initialized', - }); - return; - } - await koala.reset(); + self.postMessage(await resetRequest()); break; case 'release': - if (koala !== null) { - await koala.release(); - koala = null; - close(); - } - self.postMessage({ - command: 'ok', - }); + self.postMessage(await releaseRequest()); break; default: self.postMessage({ diff --git a/binding/web/src/types.ts b/binding/web/src/types.ts index fb31a35..2c93b49 100644 --- a/binding/web/src/types.ts +++ b/binding/web/src/types.ts @@ -86,14 +86,14 @@ export type KoalaWorkerInitResponse = export type KoalaWorkerProcessResponse = | KoalaWorkerFailureResponse | { - command: 'ok'; + command: 'ok-process'; enhancedPcm: Int16Array; }; export type KoalaWorkerResetResponse = | KoalaWorkerFailureResponse | { - command: 'ok'; + command: 'ok-reset'; }; export type KoalaWorkerReleaseResponse = From c135a82a6908ed31a4e77eb5088a740a954d70ac Mon Sep 17 00:00:00 2001 From: Ian Date: Fri, 3 Nov 2023 15:59:06 -0700 Subject: [PATCH 8/8] add sdk to worker --- binding/web/src/koala_worker.ts | 1 + binding/web/src/koala_worker_handler.ts | 1 + binding/web/src/types.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/binding/web/src/koala_worker.ts b/binding/web/src/koala_worker.ts index 8c5c970..5f62765 100644 --- a/binding/web/src/koala_worker.ts +++ b/binding/web/src/koala_worker.ts @@ -231,6 +231,7 @@ export class KoalaWorker { options: workerOptions, wasm: this._wasm, wasmSimd: this._wasmSimd, + sdk: this._sdk, }); return returnPromise; diff --git a/binding/web/src/koala_worker_handler.ts b/binding/web/src/koala_worker_handler.ts index e7dda10..c0d566e 100644 --- a/binding/web/src/koala_worker_handler.ts +++ b/binding/web/src/koala_worker_handler.ts @@ -43,6 +43,7 @@ const initRequest = async (request: KoalaWorkerInitRequest): Promise => { try { Koala.setWasm(request.wasm); Koala.setWasmSimd(request.wasmSimd); + Koala.setSdk(request.sdk); koala = await Koala._init( request.accessKey, processCallback, diff --git a/binding/web/src/types.ts b/binding/web/src/types.ts index 2c93b49..f2faf3f 100644 --- a/binding/web/src/types.ts +++ b/binding/web/src/types.ts @@ -45,6 +45,7 @@ export type KoalaWorkerInitRequest = { options: KoalaOptions; wasm: string; wasmSimd: string; + sdk: string; }; export type KoalaWorkerProcessRequest = {