cx-analysis update

This commit is contained in:
surajshivakumar
2024-08-08 16:40:11 -04:00
parent dcb57709eb
commit 4e4372203f
2 changed files with 415 additions and 195 deletions
+340
View File
@@ -0,0 +1,340 @@
const { request } = require('undici');
const { pipeline } = require('stream');
const fs = require('fs');
const path = require('path');
const os = require('os');
const fsP = require('fs').promises;
const createAudioService = require('..');
const audioService = createAudioService();
// Constants
const JAMBONZ_API_BASE_URL = 'https://jambonz.one/api/v1';
// Utility functions
async function readJSONFile(filePath) {
try {
const data = await fsP.readFile(filePath, 'utf8');
return JSON.parse(data);
} catch (err) {
throw new Error(`Error reading JSON file: ${err.message}`);
}
}
async function writeJSONFile(filePath, data) {
try {
await fs.writeFileSync(filePath, JSON.stringify(data, null, 2));
} catch (err) {
throw new Error(`Error writing JSON file: ${err.message}`);
}
}
function extractJambonzTrace(data, audioService) {
const participants = [];
let duration = 0;
const uniqueNumbers = new Set();
const result = {
conversation: {
'as heard': {
'transcripts': [],
'transcription vendor': '',
}
}
};
data.resourceSpans.forEach((resourceSpan) => {
resourceSpan.instrumentationLibrarySpans.forEach((librarySpan) => {
librarySpan.spans.forEach((span) => {
const startTime = span.startTimeUnixNano;
const endTime = span.endTimeUnixNano;
span.attributes.forEach((attribute) => {
if (attribute.value && attribute.value.stringValue) {
switch (attribute.key) {
case 'from':
if (!uniqueNumbers.has(attribute.value.stringValue)) {
uniqueNumbers.add(attribute.value.stringValue);
participants.push({
'type': 'human',
'initiatedConversation': true,
'id': {
'name': null,
'phone': attribute.value.stringValue
}
});
}
break;
case 'duration':
duration += parseInt(attribute.value.stringValue || '0', 10);
break;
case 'to':
if (!uniqueNumbers.has(attribute.value.stringValue)) {
uniqueNumbers.add(attribute.value.stringValue);
participants.push({
'type': 'machine',
'initiatedConversation': false,
'id': {
'name': 'jambonz.one',
'phone': attribute.value.stringValue
}
});
}
break;
case 'http.body':
if (attribute.value) {
const httpBody = JSON.parse(attribute.value.stringValue);
if (httpBody.speech && httpBody.speech.alternatives && httpBody.speech.alternatives.length > 0) {
const alternative = httpBody.speech.alternatives[0];
result.conversation['as heard']['transcripts'].push({
transcript: alternative.transcript,
confidence: alternative.confidence,
vendor:httpBody.speech.vendor.name,
startTime: parseInt(startTime, 10),
endTime: parseInt(endTime, 10)
});
result.conversation['as heard']['transcription vendor'] = httpBody.speech.vendor.name;
}
}
break;
case 'stt.result':
if (attribute.value) {
const sttResult = JSON.parse(attribute.value.stringValue);
sttResult.alternatives.forEach((alternative) => {
result.conversation['as heard']['transcripts'].push({
transcript: alternative.transcript,
confidence: alternative.confidence,
startTime: parseInt(startTime, 10),
endTime: parseInt(endTime, 10)
});
if (!result.conversation['as heard']['transcription vendor']) {
result.conversation['as heard']['transcription vendor'] = sttResult.vendor.name;
}
});
}
break;
}
}
});
});
});
});
// Sort the transcripts by start time
result.conversation['as heard']['transcripts'].sort((a, b) => a.startTime - b.startTime);
console.log(result.conversation['as heard'].transcripts);
return { 'participants': participants, 'duration': duration, 'conversation': result.conversation };
}
async function fetchCallDetails(callId) {
const headers = {
'Authorization': `Bearer ${process.env.JAMBONZ_API_TOKEN}`,
'Accept': '*/*',
};
// eslint-disable-next-line max-len
const url = `${JAMBONZ_API_BASE_URL}/Accounts/${process.env.ACCOUNT_SID}/RecentCalls?page=1&count=25&filter=${callId}`;
try {
const { statusCode, body } = await request(url, { headers });
if (statusCode !== 200) {
throw new Error(`Request failed with status code: ${statusCode}`);
}
var data = await body.json();
data = data.data[0];
const response = {
'call_start' : data.answered_at,
'duration_ms':data.duration * 1000,
'trace_id': data.trace_id,
'recording_url' : data.recording_url
};
// await writeJSONFile('response_call_details.json', targetCall);
console.log('Call details has been received');
return response;
} catch (error) {
console.error('Error fetching data:', error);
throw error;
}
}
async function fetchJambonzTrace(trace_id) {
const url = `${JAMBONZ_API_BASE_URL}/Accounts/${process.env.ACCOUNT_SID}/RecentCalls/trace/${trace_id}`;
const headers = {
'Authorization': `Bearer ${process.env.JAMBONZ_API_TOKEN}`,
'Accept': '*/*',
};
try {
const { statusCode, body } = await request(url, { headers });
if (statusCode !== 200) {
throw new Error(`Request failed with status code: ${statusCode}`);
}
const data = await body.json();
await writeJSONFile('trace.json', data);
console.log('Trace received');
const jsonData = await readJSONFile('./trace.json');
return extractJambonzTrace(jsonData, audioService);
} catch (error) {
console.error('Error fetching trace data:', error);
throw error;
}
}
async function downloadFile(url, tempFilePath, finalFilePath, token, callDetails) {
try {
const { statusCode, body } = await request(url, {
method: 'GET',
headers: {
'Authorization': `Bearer ${token}`,
'Accept': '*/*'
}
});
if (statusCode !== 200) {
throw new Error(`Failed to download file: ${statusCode}`);
}
const tempWriteStream = fs.createWriteStream(tempFilePath);
pipeline(
body,
tempWriteStream,
async(err) => {
if (err) {
console.error('Pipeline failed:', err);
} else {
try {
await processAndRedactFile(tempFilePath, finalFilePath, callDetails);
} catch (processingError) {
console.error('Error processing file:', processingError);
} finally {
// Clean up the temporary file
fs.unlink(tempFilePath, (err) => {
if (err) {
console.error('Error deleting temp file:', err);
} else {
console.log('Temporary file deleted.');
}
});
}
}
}
);
} catch (error) {
console.error('Error fetching file:', error);
}
}
async function processAndRedactFile(audioFilePath, outputFilePath, callDetails) {
const credentials = { 'vendor': 'deepgram', 'apiKey': process.env.DEEPGRAM_API_KEY };
try {
const transcriptionResults = await audioService.transcribe(credentials, audioFilePath);
// const timestampsToRedact = transcriptionResults.redactionTimestamps;
// await audioService.redact({
// credentials: credentials,
// transcriptionData: timestampsToRedact,
// audioPath: audioFilePath,
// audioOutputPath: outputFilePath
// });
const jambonzTrace = await fetchJambonzTrace(callDetails.trace_id);
const {'redactionTimestamps':_, ...transcript} = transcriptionResults;
jambonzTrace.duration = callDetails.duration_ms;
jambonzTrace.call_start = callDetails.call_start;
jambonzTrace.transcript = transcript;
const callStartMillis = new Date(callDetails.call_start).getTime();
jambonzTrace.transcript.speechEvents = jambonzTrace.transcript.speechEvents.map((event) => {
const startTimestamp = new Date(callStartMillis + event.start * 1000).toISOString();
// const endTimestamp = new Date(callStartMillis + event.end * 1000).toISOString();
return {
spokenAt: startTimestamp,
...event
// start: startTimestamp,
// end: endTimestamp,
};
});
const inputData = jambonzTrace;
const deepgramTranscripts = inputData.transcript.speechEvents;
const asHeardTranscripts = inputData.conversation['as heard'].transcripts;
// Determine the minimum length to avoid out-of-bounds errors
const minLength = Math.min(deepgramTranscripts.length, asHeardTranscripts.length);
// Transform the data
const turns = [];
for (let i = 0; i < minLength ; i++) {
const offer = deepgramTranscripts[i];
const response = deepgramTranscripts[i + 1];
const follow_up = deepgramTranscripts[i + 2];
const offerAsHeard = asHeardTranscripts[i];
const responseAsHeard = asHeardTranscripts[i + 1];
turns.push({
offer: offer.sentence,
response: {
asTranscribed: {
transcript: response.sentence,
vendor: "deepgram",
confidence: null
},
asHeard: {
transcript: responseAsHeard ? responseAsHeard.transcript : '',
vendor: responseAsHeard ? responseAsHeard.vendor : '',
confidence: responseAsHeard ? responseAsHeard.confidence : 0
}
},
follow_up: follow_up.sentence
});
i++;
}
// Write the output to a file
const outputData = {
startTime: inputData.call_start,
endTime: new Date(new Date(inputData.call_start).getTime() + inputData.duration).toISOString(),
recognizer: "deepgram",
turns: turns
};
await writeJSONFile(`${process.env.OUTPUT_PATH}/transcription.json`, jambonzTrace);
await writeJSONFile(`${process.env.OUTPUT_PATH}/cx_transcription.json`, outputData);
console.log('File redacted and saved successfully.');
} catch (error) {
console.error('Failed to redact audio:', error);
throw error;
}
}
// Main execution
async function main() {
try {
const [,, callId] = process.argv;
if (!callId) {
throw new Error('Missing required command line arguments. Usage: node app.js <callId>');
}
const callDetails = await fetchCallDetails(callId);
const dateFormat = callDetails.recording_url.split('record/')[1];
const url = `${JAMBONZ_API_BASE_URL}/Accounts/${process.env.ACCOUNT_SID}/RecentCalls/${callId}/record/${dateFormat}`;
//
//
// // console.log(callDetails);
const finalFilePath = path.resolve(__dirname, path.join(process.env.OUTPUT_PATH, 'redacted_audio.wav'));
const tempFolder = process.env.TEMP_FOLDER || os.tmpdir();
const tempFilePath = path.join(tempFolder, 'tempDownloadedFile.mp3');
// //
// //
await downloadFile(url, tempFilePath, finalFilePath, process.env.JAMBONZ_API_TOKEN, callDetails);
} catch (error) {
console.error('Error in main process:', error);
process.exit(1);
}
}
main();
+75 -195
View File
@@ -1769,11 +1769,11 @@ export TEMP_FOLDER=optional_temp_folder_path
}
```
### CX-Analysis (WIP):
### CX-Analysis :
```json
{
"startTime": "2024-07-19T03:01:23.644Z",
"endTime": "2024-07-19T03:04:04.644Z",
"startTime": "2024-08-08T17:57:03.791Z",
"endTime": "2024-08-08T17:59:31.791Z",
"recognizer": "deepgram",
"turns": [
{
@@ -1781,280 +1781,160 @@ export TEMP_FOLDER=optional_temp_folder_path
"response": {
"asTranscribed": {
"transcript": "Hi. I have a question about my recent order.",
"vendor": "deepgram"
"vendor": "deepgram",
"confidence": null
},
"asHeard": {
"transcript": "hi I have a question about my recent order",
"confidence": 0.9734987616539001
"confidence": 0.9675881266593933,
"vendor": "google"
}
},
"follow_up": "I'd be happy to help you. Can I please have your full name and order number?"
"follow_up": "I'd be happy to help you with that. Can I please have your full name and order number to look up your information?"
},
{
"offer": "Hi. I have a question about my recent order.",
"response": {
"asTranscribed": {
"transcript": "I'd be happy to help you. Can I please have your full name and order number?",
"vendor": "deepgram"
},
"asHeard": {
"transcript": " I'd be happy to help you can I please have your full name and order number",
"confidence": 0.9531590938568115
}
},
"follow_up": "Sure. My name is [NAME_1], and my order number is [NUMERICAL_PII_1]."
},
{
"offer": "I'd be happy to help you. Can I please have your full name and order number?",
"offer": "I'd be happy to help you with that. Can I please have your full name and order number to look up your information?",
"response": {
"asTranscribed": {
"transcript": "Sure. My name is [NAME_1], and my order number is [NUMERICAL_PII_1].",
"vendor": "deepgram"
"vendor": "deepgram",
"confidence": null
},
"asHeard": {
"transcript": " sure my name is John Smith and my order number is 123456789",
"confidence": 0.8831360936164856
"transcript": "sure my name is John Smith and my order number is 1 2 3 4 5 6 7 8 9",
"confidence": 0.8548645973205566,
"vendor": "google"
}
},
"follow_up": "Thank you. For verification purposes, could you also provide me with the last four digits of your credit card used for the purchase?"
"follow_up": "Thank you, [NAME_GIVEN_2]. For verification purposes, could you also provide me with the last four digits of the credit card used for the purchase?"
},
{
"offer": "Sure. My name is [NAME_1], and my order number is [NUMERICAL_PII_1].",
"response": {
"asTranscribed": {
"transcript": "Thank you. For verification purposes, could you also provide me with the last four digits of your credit card used for the purchase?",
"vendor": "deepgram"
},
"asHeard": {
"transcript": " thank you for verification purposes could you also provide me with the last four digits of your credit card used for the purchase",
"confidence": 0.9808544516563416
}
},
"follow_up": "Of course. It's 9876."
},
{
"offer": "Thank you. For verification purposes, could you also provide me with the last four digits of your credit card used for the purchase?",
"offer": "Thank you, [NAME_GIVEN_2]. For verification purposes, could you also provide me with the last four digits of the credit card used for the purchase?",
"response": {
"asTranscribed": {
"transcript": "Of course. It's 9876.",
"vendor": "deepgram"
"vendor": "deepgram",
"confidence": null
},
"asHeard": {
"transcript": " of course it's 9876",
"confidence": 0.8269519209861755
"transcript": "of course it's 9876",
"confidence": 0.8935980200767517,
"vendor": "google"
}
},
"follow_up": "Thank you. Please give me a moment to pull up your details. Okay. I see your order here. It looks like you purchased the smartwatch on [DATE_1]. How can I assist you with that?"
"follow_up": "Thank you. Please give me a moment to pull up your order details. Pause for a few seconds. Okay. I see your order here. It looks like you purchased a smartwatch on [DATE_1]. How can I assist you with this order?"
},
{
"offer": "Of course. It's 9876.",
"offer": "Thank you. Please give me a moment to pull up your order details. Pause for a few seconds. Okay. I see your order here. It looks like you purchased a smartwatch on [DATE_1]. How can I assist you with this order?",
"response": {
"asTranscribed": {
"transcript": "Thank you. Please give me a moment to pull up your details. Okay. I see your order here. It looks like you purchased the smartwatch on [DATE_1]. How can I assist you with that?",
"vendor": "deepgram"
"transcript": "I received the product, but it's not functioning properly. It seems to be defective.",
"vendor": "deepgram",
"confidence": null
},
"asHeard": {
"transcript": " thank you please give me a moment to pull up your details",
"confidence": 0.9891281127929688
"transcript": "I received the product but it's not functioning properly it seems to be defective",
"confidence": 0.9702370762825012,
"vendor": "google"
}
},
"follow_up": "Yeah. I received the product, but it's not functioning properly. It seems to be defective."
"follow_up": "I'm sorry to hear that. We can definitely get this sorted out for you. Could you describe the issue you're experiencing with the product?"
},
{
"offer": "Thank you. Please give me a moment to pull up your details. Okay. I see your order here. It looks like you purchased the smartwatch on [DATE_1]. How can I assist you with that?",
"offer": "I'm sorry to hear that. We can definitely get this sorted out for you. Could you describe the issue you're experiencing with the product?",
"response": {
"asTranscribed": {
"transcript": "Yeah. I received the product, but it's not functioning properly. It seems to be defective.",
"vendor": "deepgram"
"transcript": "The screen keeps flickering sometimes, and it won't turn on at all.",
"vendor": "deepgram",
"confidence": null
},
"asHeard": {
"transcript": " yeah I received the product but it's not function properly it seems to be defective",
"confidence": 0.9271853566169739
"transcript": "the screen keeps flickering sometimes and it won't turn on at all",
"confidence": 0.9558009505271912,
"vendor": "google"
}
},
"follow_up": "I'm sorry to hear that. We can definitely get this sorted for you. Can you describe the issue we are facing?"
"follow_up": "That sounds frustrating. Let's proceed with a return and replacement. Could you confirm your shipping address for me to arrange the return?"
},
{
"offer": "Yeah. I received the product, but it's not functioning properly. It seems to be defective.",
"response": {
"asTranscribed": {
"transcript": "I'm sorry to hear that. We can definitely get this sorted for you. Can you describe the issue we are facing?",
"vendor": "deepgram"
},
"asHeard": {
"transcript": " okay I see your order here it looks like you purchased this Smartwatch on July 1st how can I assist you with that",
"confidence": 0.8799465894699097
}
},
"follow_up": "Yes. The screen keeps flickering, and sometimes I won't turn on at all."
},
{
"offer": "I'm sorry to hear that. We can definitely get this sorted for you. Can you describe the issue we are facing?",
"response": {
"asTranscribed": {
"transcript": "Yes. The screen keeps flickering, and sometimes I won't turn on at all.",
"vendor": "deepgram"
},
"asHeard": {
"transcript": " I'm sorry to hear that we can definitely get this sorted for you can you describe the issue your facing",
"confidence": 0.9006506204605103
}
},
"follow_up": "That sounds frustrating. Let's, proceed with the return and replacement. Could you confirm shipping address?"
},
{
"offer": "Yes. The screen keeps flickering, and sometimes I won't turn on at all.",
"response": {
"asTranscribed": {
"transcript": "That sounds frustrating. Let's, proceed with the return and replacement. Could you confirm shipping address?",
"vendor": "deepgram"
},
"asHeard": {
"transcript": " yes the screen keeps licking and sometimes I won't turn on at all",
"confidence": 0.937334418296814
}
},
"follow_up": "Sure. It's [LOCATION_ADDRESS_1]."
},
{
"offer": "That sounds frustrating. Let's, proceed with the return and replacement. Could you confirm shipping address?",
"offer": "That sounds frustrating. Let's proceed with a return and replacement. Could you confirm your shipping address for me to arrange the return?",
"response": {
"asTranscribed": {
"transcript": "Sure. It's [LOCATION_ADDRESS_1].",
"vendor": "deepgram"
"vendor": "deepgram",
"confidence": null
},
"asHeard": {
"transcript": " that sounds frustrating let's proceed with the return and replacement could you confirm shipping address",
"confidence": 0.9030960202217102
"transcript": "sure it's 1 to 3 Maple Street Apartment 4B Springfield Illinois 62704",
"confidence": 0.8138220906257629,
"vendor": "google"
}
},
"follow_up": "Thank you, [NAME_GIVEN_2]. We will send a prepaid return label to your email address on file, which is [EMAIL_ADDRESS_1]. Is"
"follow_up": "Thank you, [NAME_GIVEN_2]. We will send a prepaid return label to your email address on file, which is [EMAIL_ADDRESS_1]. Is that correct?"
},
{
"offer": "Sure. It's [LOCATION_ADDRESS_1].",
"offer": "Thank you, [NAME_GIVEN_2]. We will send a prepaid return label to your email address on file, which is [EMAIL_ADDRESS_1]. Is that correct?",
"response": {
"asTranscribed": {
"transcript": "Thank you, [NAME_GIVEN_2]. We will send a prepaid return label to your email address on file, which is [EMAIL_ADDRESS_1]. Is",
"vendor": "deepgram"
},
"asHeard": {
"transcript": " sure it's one two three Maple Street Apartment 4B Spring Field Illinois 62704",
"confidence": 0.7194310426712036
}
},
"follow_up": "Yes."
},
{
"offer": "Thank you, [NAME_GIVEN_2]. We will send a prepaid return label to your email address on file, which is [EMAIL_ADDRESS_1]. Is",
"response": {
"asTranscribed": {
"transcript": "Yes.",
"vendor": "deepgram"
},
"asHeard": {
"transcript": "thank you John we will send a prepaid return label to your email address on file which is John dot Smith at example.com is that correct",
"confidence": 0.8666307926177979
}
},
"follow_up": "that correct?"
},
{
"offer": "Yes.",
"response": {
"asTranscribed": {
"transcript": "that correct?",
"vendor": "deepgram"
"transcript": "Yes. That's correct.",
"vendor": "deepgram",
"confidence": null
},
"asHeard": {
"transcript": "yes that's correct",
"confidence": 0.9237890243530273
"confidence": 0.9711934924125671,
"vendor": "google"
}
},
"follow_up": "That's correct."
"follow_up": "Great. Once you receive the return label, please send the defective product back to us. As soon as we receive it, we will ship out a replacement to your address at [LOCATION_ADDRESS_1]. You will receive an email with the tracking information."
},
{
"offer": "that correct?",
"offer": "Great. Once you receive the return label, please send the defective product back to us. As soon as we receive it, we will ship out a replacement to your address at [LOCATION_ADDRESS_1]. You will receive an email with the tracking information.",
"response": {
"asTranscribed": {
"transcript": "That's correct.",
"vendor": "deepgram"
"transcript": "Thank you very much. How long will the entire process take?",
"vendor": "deepgram",
"confidence": null
},
"asHeard": {
"transcript": " great once you receive the return label please send the defective product back to us as soon as we receive it we will ship out a replacement to your address which I'm really confirming as one two three Maple Street Apartment 4B Springfield Illinois 62704",
"confidence": 0.8999208211898804
"transcript": "thank you very much how long will the entire process take",
"confidence": 0.9617440700531006,
"vendor": "google"
}
},
"follow_up": "Great. Once you receive the return label, please send the defective product back to us. As soon as we receive it, we will ship out a replacement to your address, which I'm reconfirming as [LOCATION_ADDRESS_1]."
"follow_up": "Once we receive the defective product, the replacement should arrive within [DURATION_1]. You'll be notified via email throughout the process."
},
{
"offer": "That's correct.",
"response": {
"asTranscribed": {
"transcript": "Great. Once you receive the return label, please send the defective product back to us. As soon as we receive it, we will ship out a replacement to your address, which I'm reconfirming as [LOCATION_ADDRESS_1].",
"vendor": "deepgram"
},
"asHeard": {
"transcript": " no yes thank you very much her how long will it how long will the entire process take",
"confidence": 0.9343628287315369
}
},
"follow_up": "Yes. Thank you very much. How long will it, how long will the entire process take?"
},
{
"offer": "Great. Once you receive the return label, please send the defective product back to us. As soon as we receive it, we will ship out a replacement to your address, which I'm reconfirming as [LOCATION_ADDRESS_1].",
"response": {
"asTranscribed": {
"transcript": "Yes. Thank you very much. How long will it, how long will the entire process take?",
"vendor": "deepgram"
},
"asHeard": {
"transcript": " once we receive the defective product the replacement should arrive within 5 to 7 business days",
"confidence": 0.8692674040794373
}
},
"follow_up": "Once we receive the defective product, the replacement should arrive within [DURATION_1]."
},
{
"offer": "Yes. Thank you very much. How long will it, how long will the entire process take?",
"response": {
"asTranscribed": {
"transcript": "Once we receive the defective product, the replacement should arrive within [DURATION_1].",
"vendor": "deepgram"
},
"asHeard": {
"transcript": " perfect thank you for your help",
"confidence": 0.9718412756919861
}
},
"follow_up": "Perfect. Thank you for your help."
},
{
"offer": "Once we receive the defective product, the replacement should arrive within [DURATION_1].",
"offer": "Once we receive the defective product, the replacement should arrive within [DURATION_1]. You'll be notified via email throughout the process.",
"response": {
"asTranscribed": {
"transcript": "Perfect. Thank you for your help.",
"vendor": "deepgram"
"vendor": "deepgram",
"confidence": null
},
"asHeard": {
"transcript": " you're welcome John if you have any other questions please reach out",
"confidence": 0.9155542254447937
"transcript": "perfect thank you for your help",
"confidence": 0.958747386932373,
"vendor": "google"
}
},
"follow_up": "You're welcome, [NAME_GIVEN_2]. If you have any other questions, please"
"follow_up": "You're welcome, [NAME_GIVEN_2]. If you have any other questions or need further assistance, please don't hesitate to call us back. Have a great day."
},
{
"offer": "Perfect. Thank you for your help.",
"offer": "You're welcome, [NAME_GIVEN_2]. If you have any other questions or need further assistance, please don't hesitate to call us back. Have a great day.",
"response": {
"asTranscribed": {
"transcript": "You're welcome, [NAME_GIVEN_2]. If you have any other questions, please",
"vendor": "deepgram"
"transcript": "You too.",
"vendor": "deepgram",
"confidence": null
},
"asHeard": {
"transcript": " sure thank you goodbye",
"confidence": 0.9046414494514465
"transcript": "YouTube",
"confidence": 0.5463442206382751,
"vendor": "google"
}
},
"follow_up": "Sure."
"follow_up": "Goodbye."
}
]
}