ben.carrasco/jambonz-feature-server
1
0
Fork 0
mirror of https://github.com/jambonz/jambonz-feature-server.git synced 2025-12-20 08:40:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

initial changes for soniox (#270)

Browse Source 
* initial changes for soniox

* changes to gather for soniox

* parse soniox stt results

* handle <end> token for soniox

* soniox: handle empty array of words

* support for soniox hints

* add soniox storage options

* update to verb specs

* add support for transcribe

* compile soniox transcripts

* gather: kill no input timer for soniox when we get interim results

* fix buffering of soniox transcripts

* fix for compiling soniox transcript

* another fix for compiling soniox transcript

* another fix

* handling of <end> token

* fix soniox bug

* gather: fixes for soniox continous asr

* fix undefined variable reference

* fix prev commit

* bugfix: allow verb_status requests

* gather: for soniox no need to restart transcription after final transcription received

* update verb specs

* update verb specs, fixes for continuous asr:
This commit is contained in:
Dave Horton
2023-03-03 13:37:55 -05:00
committed by GitHub
parent ab1947e23e
commit 1c683f1142
10 changed files with 272 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
lib/utils/transcription-utils.js
View File

@@ -5,6 +5,7 @@ const {
AwsTranscriptionEvents,
NuanceTranscriptionEvents,
DeepgramTranscriptionEvents,
SonioxTranscriptionEvents,
NvidiaTranscriptionEvents
} = require('./constants');
@@ -88,9 +89,70 @@ const stickyVars = {
],
nvidia: [
'NVIDIA_HINTS'
],
soniox: [
'SONIOX_PROFANITY_FILTER',
'SONIOX_MODEL'
]
};
const compileSonioxTranscripts = (finalWordChunks, channel, language) => {
const words = finalWordChunks.flat();
const transcript = words.reduce((acc, word) => {
if (word.text === '<end>') return acc;
if ([',', '.', '?', '!'].includes(word.text)) return `${acc}${word.text}`;
return `${acc} ${word.text}`;
}, '').trim();
const realWords = words.filter((word) => ![',.!?;'].includes(word.text) && word.text !== '<end>');
const confidence = realWords.reduce((acc, word) => acc + word.confidence, 0) / realWords.length;
const alternatives = [{transcript, confidence}];
return {
language_code: language,
channel_tag: channel,
is_final: true,
alternatives,
vendor: {
name: 'soniox',
evt: words
}
};
};
const normalizeSoniox = (evt, channel, language) => {
const copy = JSON.parse(JSON.stringify(evt));
/* an <end> token indicates the end of an utterance */
const endTokenPos = evt.words.map((w) => w.text).indexOf('<end>');
const endpointReached = endTokenPos !== -1;
const words = endpointReached ? evt.words.slice(0, endTokenPos) : evt.words;
/* note: we can safely ignore words after the <end> token as they will be returned again */
const finalWords = words.filter((word) => word.is_final);
const nonFinalWords = words.filter((word) => !word.is_final);
const is_final = endpointReached && finalWords.length > 0;
const transcript = words.reduce((acc, word) => {
if ([',', '.', '?', '!'].includes(word.text)) return `${acc}${word.text}`;
else return `${acc} ${word.text}`;
}, '').trim();
const realWords = words.filter((word) => ![',.!?;'].includes(word.text) && word.text !== '<end>');
const confidence = realWords.reduce((acc, word) => acc + word.confidence, 0) / realWords.length;
const alternatives = [{transcript, confidence}];
return {
language_code: language,
channel_tag: channel,
is_final,
alternatives,
vendor: {
name: 'soniox',
endpointReached,
evt: copy,
finalWords,
nonFinalWords
}
};
};
const normalizeDeepgram = (evt, channel, language) => {
const copy = JSON.parse(JSON.stringify(evt));
const alternatives = (evt.channel?.alternatives || [])
@@ -237,6 +299,8 @@ module.exports = (logger) => {
return normalizeIbm(evt, channel, language);
case 'nvidia':
return normalizeNvidia(evt, channel, language);
case 'soniox':
return normalizeSoniox(evt, channel, language);
default:
logger.error(`Unknown vendor ${vendor}`);
return evt;
@@ -441,6 +505,29 @@ module.exports = (logger) => {
{DEEPGRAM_SPEECH_TAG: deepgramOptions.tag}
};
}
else if ('soniox' === rOpts.vendor) {
const {sonioxOptions = {}} = rOpts;
const {storage = {}} = sonioxOptions;
opts = {
...opts,
...(sttCredentials.api_key) &&
{SONIOX_API_KEY: sttCredentials.api_key},
...(rOpts.hints.length > 0 && typeof rOpts.hints[0] === 'string' &&
{SONIOX_HINTS: rOpts.hints.join(',')}),
...(rOpts.hints.length > 0 && typeof rOpts.hints[0] === 'object' &&
{SONIOX_HINTS: JSON.stringify(rOpts.hints)}),
...(typeof rOpts.hintsBoost === 'number' &&
{SONIOX_HINTS_BOOST: rOpts.hintsBoost}),
...(sonioxOptions.model) &&
{SONIOX_MODEL: sonioxOptions.model},
...((sonioxOptions.profanityFilter || rOpts.profanityFilter) && {SONIOX_PROFANITY_FILTER: 1}),
...(storage?.id && {SONIOX_STORAGE_ID: storage.id}),
...(storage?.id && storage?.title && {SONIOX_STORAGE_TITLE: storage.title}),
...(storage?.id && storage?.disableStoreAudio && {SONIOX_STORAGE_DISABLE_AUDIO: 1}),
...(storage?.id && storage?.disableStoreTranscript && {SONIOX_STORAGE_DISABLE_TRANSCRIPT: 1}),
...(storage?.id && storage?.disableSearch && {SONIOX_STORAGE_DISABLE_SEARCH: 1})
};
}
else if ('ibm' === rOpts.vendor) {
const {ibmOptions = {}} = rOpts;
opts = {
@@ -524,6 +611,9 @@ module.exports = (logger) => {
ep.removeCustomEventListener(DeepgramTranscriptionEvents.Connect);
ep.removeCustomEventListener(DeepgramTranscriptionEvents.ConnectFailure);
ep.removeCustomEventListener(SonioxTranscriptionEvents.Transcription);
ep.removeCustomEventListener(SonioxTranscriptionEvents.Error);
ep.removeCustomEventListener(NvidiaTranscriptionEvents.Transcription);
ep.removeCustomEventListener(NvidiaTranscriptionEvents.TranscriptionComplete);
ep.removeCustomEventListener(NvidiaTranscriptionEvents.StartOfSpeech);
@@ -534,8 +624,9 @@ module.exports = (logger) => {
const setSpeechCredentialsAtRuntime = (recognizer) => {
if (!recognizer) return;
if (recognizer.vendor === 'nuance') {
const {clientId, secret} = recognizer.nuanceOptions || {};
const {clientId, secret, kryptonEndpoint} = recognizer.nuanceOptions || {};
if (clientId && secret) return {client_id: clientId, secret};
if (kryptonEndpoint) return {krypton_endpoint: kryptonEndpoint};
}
else if (recognizer.vendor === 'nvidia') {
const {rivaUri} = recognizer.nvidiaOptions || {};
@@ -545,6 +636,10 @@ module.exports = (logger) => {
const {apiKey} = recognizer.deepgramOptions || {};
if (apiKey) return {api_key: apiKey};
}
else if (recognizer.vendor === 'soniox') {
const {apiKey} = recognizer.sonioxOptions || {};
if (apiKey) return {api_key: apiKey};
}
else if (recognizer.vendor === 'ibm') {
const {ttsApiKey, ttsRegion, sttApiKey, sttRegion, instanceId} = recognizer.ibmOptions || {};
if (ttsApiKey || sttApiKey) return {
@@ -561,6 +656,7 @@ module.exports = (logger) => {
normalizeTranscription,
setChannelVarsForStt,
removeSpeechListeners,
setSpeechCredentialsAtRuntime
setSpeechCredentialsAtRuntime,
compileSonioxTranscripts
};
};