Skip to content

Commit

Permalink
port logic of speech2text
Browse files Browse the repository at this point in the history
  • Loading branch information
@foxriver76
foxriver76 committed Dec 14, 2023
1 parent 7812f1d commit e264943
Show file tree
Hide file tree
Showing 8 changed files with 332 additions and 277 deletions.
3 changes: 2 additions & 1 deletion src/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"@reduxjs/toolkit": "^2.0.1",
"@sentry/browser": "^7.85.0",
"@sentry/integrations": "^7.85.0",
"@types/dom-speech-recognition": "^0.0.4",
"ace-builds": "^1.32.0",
"craco-module-federation": "^1.1.0",
"dayjs": "^1.11.10",
Expand Down Expand Up @@ -71,4 +72,4 @@
"not ie <= 11",
"not op_mini all"
]
}
}
260 changes: 0 additions & 260 deletions src/public/widgets/basic.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1835,241 +1835,6 @@
refresh(evData.val, evData.x, evData.y);
});
},
speech2text: function (wid, view, data, style) {
var $div = $('#' + wid);
// if nothing found => wait
if (!$div.length) {
return setTimeout(function () {
vis.binds.basic.speech2text(wid, view, data, style);
}, 100);
}

if (typeof webkitSpeechRecognition === 'undefined') {
console.error('Browser does not support speech 2 text');
$div.find('.mic-info-text').html(_('info_upgrade'));
return;
}

var _data ={
continuous: data.speechMode === 'continuous' || data.speechMode === 'startstop',
speechMode: data.speechMode,
oid: data.oid,
language: data.language,
finalTranscript: '',
lastText: '',
recognition: null,
ignoreOnEnd: false,
timer: null,
words: data.keywords ? data.keywords.toLowerCase().split(/[\s,;]+/g) : null,
textSentColor: data.textSentColor,
keyWordColor: data.keyWordColor,
imageInactive: data.imageInactive,
imageActive: data.imageActive || data.imageInactive,
imageStarted: data.imageStarted || data.imageInactive,
imageDetected: data.imageDetected || data.imageInactive,
imageSent: data.imageSent || data.imageInactive,
$text: $div.find('.mic-info-text'),
$result: $div.find('.mic-results'),
$image: $div.find('.mic-image')
};
data = null;

function findKeyWord(text) {
var found = false;
if (_data.words) {
found = false;
var wwords = text.toLowerCase().split(' ');
for (var w = 0; w < _data.words.length; w++) {
var pos = wwords.indexOf(_data.words[w]);
if (pos !== -1) {
var offset = 0;
for (var p = 0; p < pos; p++) {
offset += wwords[p].length + 1;
}
text = text.substring(0, offset) + '<b>' + text.substring(offset, offset + wwords[pos].length) + '</b>' + text.substring(offset + wwords[pos].length);
found = true;
break;
}
}
} else {
return true;
}
return found ? text : false;
}

function startRecognition() {
if (_data.recognition) {
if (_data.speechMode === 'startstop') {
_data.recognition.onend = null;
_data.recognition.stop();
_data.recognition = null;
_data.$text.html(_('info_start')).show();
_data.$image.attr('src', _data.imageInactive);
_data.$result.hide();
}
return;
}

_data.recognition = new webkitSpeechRecognition();
_data.recognition.continuous = _data.continuous;
_data.recognition.interimResults = true;
_data.recognition.maxAlternatives = 1;
var startTimestamp = new Date();


_data.recognition.onstart = function() {
_data.$text.html(_('info_speak_now')).show();
_data.$image.attr('src', _data.imageActive);
_data.$result.hide();
};

_data.recognition.onerror = function(event) {
_data.$result.hide();
_data.$text.show();
_data.$image.attr('src', _data.imageInactive);
if (event.error === 'no-speech') {
_data.$text.html(_('info_no_speech'));
//_data.ignoreOnEnd = true;
}
if (event.error === 'audio-capture') {
_data.$text.html(_('info_no_microphone'));
_data.ignoreOnEnd = true;
}
if (event.error === 'not-allowed') {
if (event.timeStamp - startTimestamp < 100) {
_data.$text.html(_('info_blocked'));
} else {
_data.$text.html(_('info_denied'));
}
_data.ignoreOnEnd = true;
}
};

_data.recognition.onend = function() {
// console.log('onend');
_data.recognition.stop();
_data.recognition = null;
if (!_data.ignoreOnEnd) {
setTimeout(function () {
_data.$image.attr('src', _data.imageInactive);
if (_data.continuous) {
_data.$result.hide();
_data.$text.html(_('info_speak_now')).show();
startRecognition();
} else {
_data.$result.hide();
_data.$text.html(_('info_start')).show();
}
}, 1000);
}
};

_data.recognition.onresult = function(event) {
var interimTranscript = '';

if (typeof event.results === 'undefined') {
_data.recognition.onend = null;
_data.recognition.stop();
return;
}

for (var i = event.resultIndex; i < event.results.length; ++i) {
if (event.results[i].isFinal) {
_data.finalTranscript += event.results[i][0].transcript;
} else {
interimTranscript += event.results[i][0].transcript;
}
}

if (_data.$image.attr('src') === _data.imageActive) _data.$image.attr('src', _data.imageStarted);

//final_transcript = capitalize(final_transcript);
_data.$text.hide();
var text = _data.finalTranscript || interimTranscript;
var found = findKeyWord(text);

if (!_data.originalColor) _data.originalColor = _data.$result.css('color');

if (found) {
text = found;
if (_data.$image.attr('src') === _data.imageStarted) _data.$image.attr('src', _data.imageDetected);
_data.$result.addClass('mic-keyword-found');
if (_data.keyWordColor) _data.$result.css({color: _data.keyWordColor});
}

_data.$result.html(text).show();

if (_data.finalTranscript) {
if (_data.timer) clearTimeout(_data.timer);

if (found) {
_data.$image.attr('src', _data.imageSent);
if (_data.textSentColor) _data.$result.css({color: _data.textSentColor});
_data.$result.addClass('mic-text-sent');
vis.setValue(_data.oid, _data.finalTranscript);
}

_data.finalTranscript = '';
_data.lastText = '';

setTimeout(function () {
_data.$image.attr('src', _data.imageActive);
_data.$result.css({color: _data.originalColor});
_data.$result.hide().removeClass('mic-keyword-found mic-text-sent');
_data.$text.html(_('info_speak_now')).show();
}, 1000);
} else {
_data.lastText = interimTranscript;

if (_data.timer) clearTimeout(_data.timer);
_data.timer = setTimeout(function () {
_data.timer = null;
_data.recognition.onend = null;
_data.recognition.stop();
_data.recognition = null;

var found = findKeyWord(_data.lastText);
if (found) {
_data.$image.attr('src', _data.imageSent);
if (_data.textSentColor) _data.$result.css({color: _data.textSentColor});
_data.$result.addClass('mic-text-sent');
vis.setValue(_data.oid, _data.finalTranscript);
}

_data.finalTranscript = '';
_data.lastText = '';

setTimeout(function () {
_data.$image.attr('src', _data.imageActive);
_data.$result.css({color: _data.originalColor});
_data.$result.hide().removeClass('mic-keyword-found mic-text-sent');
_data.$text.html(_('info_speak_now')).show();
startRecognition();
}, 1000);
}, 3000);
}
};

_data.recognition.lang = _data.language;

_data.recognition.start();
}

if (_data.speechMode === 'continuous') {
if (!vis.editMode) {
startRecognition();
} else {
$div.find('.mic-info-text').html(_('info_start'));
}
} else {
$div.find('.mic-info-text').html(_('info_start'));
if (!vis.editMode) {
$div.click(function () {
startRecognition();
});
}
}
},
getTimeInterval: function (oldTime, hoursToShow) {
var result = '';

Expand Down Expand Up @@ -3027,31 +2792,6 @@
</div>
</script-->

<script id="tplSpeech2Text"
type="text/ejs"
class="vis-tpl"
data-vis-set="basic"
data-vis-type="val"
data-vis-prev='<img src="widgets/basic/img/Prev_Speech2Text.png"></img>'
data-vis-name="Speech to text"
data-vis-no-gestures="true"
data-vis-attrs="oid;speechMode[single]/select,single,startstop,continuous;language/select,,en-US,de,ru-RU;keywords;"
data-vis-attrs0="group.image;noImage/checkbox;imageInactive[widgets~basic~img~micInactive.svg]/image;imageActive[widgets~basic~img~micActive.svg]/image;imageStarted[widgets~basic~img~micStarted.svg]/image;imageDetected[widgets~basic~img~micDetected.svg]/image;imageSent[widgets~basic~img~micSent.svg]/image;imageHeightPx[70]/slider,0,200,1;imageWidthPx[70]/slider,0,200,1;"
data-vis-attrs1="group.text;noText/checkbox;noResults/checkbox;keyWordColor[#FFB051]/color;textSentColor[#7E88D3]/color;">
<div class="vis-widget <%== this.data.attr('class') %>" style="height: 77px; width: 500px" id="<%= this.data.attr('wid') %>">
<div class="vis-widget-body">
<table style="height: 100%; width: 100%"><tr>
<td style="<%= this.data.attr('noImage') ? 'display: none' : '' %>"><img class="mic-image" style="height: <%= this.data.attr('imageHeightPx') %>px; width: <%= this.data.attr('imageWidthPx') %>px" src="<%= this.data.attr('imageInactive') %>" /></td>
<td class="mic-text" style="width: 100%">
<div class="mic-info-text" style="<%= this.data.attr('noText') ? 'display: none' : '' %>"></div>
<div class="mic-results" style="<%= this.data.attr('noResults') ? 'display: none' : '' %>"></div>
</td>
</tr></table>
</div>
</div>
<% vis.binds.basic.speech2text(this.data.wid, this.view, this.data, this.style); %>
</script>

<script id="tplFullScreen"
type="text/ejs"
class="vis-tpl"
Expand Down
2 changes: 1 addition & 1 deletion src/src/Vis/Widgets/Basic/BasicIFrame.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ export default class BasicIFrame extends VisRxWidget<RxData> {

onPropertyUpdate() {
const src = this.state.rxData.src || '';
const refreshInterval = parseFloat(this.state.rxData.refreshInterval) || 0;
const refreshInterval = Number(this.state.rxData.refreshInterval) || 0;
const refreshOnViewChange = this.state.rxData.refreshOnViewChange === true;
const refreshOnWakeUp = this.state.rxData.refreshOnWakeUp === true;

Expand Down
2 changes: 1 addition & 1 deletion src/src/Vis/Widgets/Basic/BasicImage.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ export default class BasicImage extends VisRxWidget<RxData> {

onPropertyUpdate() {
const src = this.state.rxData.src || '';
const refreshInterval = parseFloat(this.state.rxData.refreshInterval) || 0;
const refreshInterval = Number(this.state.rxData.refreshInterval) || 0;
const refreshOnViewChange = this.state.rxData.refreshOnViewChange === true;
const refreshOnWakeUp = this.state.rxData.refreshOnWakeUp === true;

Expand Down
Loading

0 comments on commit e264943

Please sign in to comment.