Compare commits

...

13 Commits

Author SHA1 Message Date
Quan HL
66388617e6 fix 2023-07-28 21:14:07 +07:00
Quan HL
9211c9461c fix 2023-07-28 20:51:27 +07:00
Quan HL
35435340c1 fix 2023-07-28 20:33:10 +07:00
Quan HL
8c9f377671 fix 2023-07-28 20:29:27 +07:00
Quan HL
6e7154a31b add tags to google storage 2023-07-28 19:39:06 +07:00
Quan HL
1f54b10d72 add metadata to google storage 2023-07-28 17:56:59 +07:00
Quan HL
7e1f85ec14 add metadata to google storage 2023-07-28 17:55:33 +07:00
Quan HL
e173c12d43 add metadata to google storage 2023-07-28 17:54:47 +07:00
Quan HL
a507f67fbc add google storage writablestream 2023-07-28 17:51:22 +07:00
Quan HL
f2a0c93ba8 add google storage writablestream 2023-07-28 17:48:44 +07:00
Quan HL
cb483a74c2 add google storage writablestream 2023-07-28 17:48:35 +07:00
Quan HL
1372f8fc4c feat google storage 2023-07-28 15:19:33 +07:00
Quan HL
e02269c004 feat google storage 2023-07-28 15:17:00 +07:00
10 changed files with 326 additions and 66 deletions

View File

@@ -0,0 +1,41 @@
const { Storage } = require('@google-cloud/storage');
const { Writable } = require('stream');
class GoogleStorageUploadStream extends Writable {
constructor(logger, opts) {
super(opts);
this.logger = logger;
this.metadata = opts.metadata;
const storage = new Storage(opts.bucketCredential);
this.gcsFile = storage.bucket(opts.bucketName).file(opts.Key);
this.writeStream = this.gcsFile.createWriteStream();
this.writeStream.on('error', (err) => this.logger.error(err));
this.writeStream.on('finish', () => {
this.logger.info('google storage Upload completed.');
this._addMetadata();
});
}
_write(chunk, encoding, callback) {
this.writeStream.write(chunk, encoding, callback);
}
_final(callback) {
this.writeStream.end();
this.writeStream.once('finish', callback);
}
async _addMetadata() {
try {
await this.gcsFile.setMetadata({metadata: this.metadata});
this.logger.info('Google storage Upload and metadata setting completed.');
} catch (err) {
this.logger.error(err, 'Google storage An error occurred while setting metadata');
}
}
}
module.exports = GoogleStorageUploadStream;

View File

@@ -1,17 +1,6 @@
const path = require('node:path');
async function record(logger, socket, url) {
const p = path.basename(url);
const idx = p.lastIndexOf('/');
const vendor = p.substring(idx + 1);
switch (vendor) {
case 'aws_s3':
return require('./s3')(logger, socket);
default:
logger.info(`unknown bucket vendor: ${vendor}`);
socket.send(`unknown bucket vendor: ${vendor}`);
socket.close();
}
async function record(logger, socket) {
return require('./upload')(logger, socket);
}
module.exports = record;

View File

@@ -1,8 +1,8 @@
const Account = require('../models/account');
const Websocket = require('ws');
const PCMToMP3Encoder = require('./encoder');
const S3MultipartUploadStream = require('./s3-multipart-upload-stream');
const wav = require('wav');
const { getUploader } = require('./utils');
async function upload(logger, socket) {
@@ -43,19 +43,11 @@ async function upload(logger, socket) {
Key += `/${day.getDate().toString().padStart(2, '0')}/${callSid}.${account[0].record_format}`;
// Uploader
const uploaderOpts = {
bucketName: obj.name,
Key,
metadata,
bucketCredential: {
credentials: {
accessKeyId: obj.access_key_id,
secretAccessKey: obj.secret_access_key,
},
region: obj.region || 'us-east-1'
}
};
const uploadStream = new S3MultipartUploadStream(logger, uploaderOpts);
const uploadStream = getUploader(Key, metadata, obj, logger);
if (!uploadStream) {
logger.info('There is no available record uploader, close the socket.');
socket.close();
}
/**encoder */
let encoder;

40
lib/record/utils.js Normal file
View File

@@ -0,0 +1,40 @@
const GoogleStorageUploadStream = require('./google-storage');
const S3MultipartUploadStream = require('./s3-multipart-upload-stream');
const getUploader = (Key, metadata, bucket_credential, logger) => {
const uploaderOpts = {
bucketName: bucket_credential.name,
Key,
metadata
};
switch (bucket_credential.vendor) {
case 'aws_s3':
uploaderOpts.bucketCredential = {
credentials: {
accessKeyId: bucket_credential.access_key_id,
secretAccessKey: bucket_credential.secret_access_key,
},
region: bucket_credential.region || 'us-east-1'
};
return new S3MultipartUploadStream(logger, uploaderOpts);
case 'google':
const serviceKey = JSON.parse(bucket_credential.service_key);
uploaderOpts.bucketCredential = {
projectId: serviceKey.project_id,
credentials: {
client_email: serviceKey.client_email,
private_key: serviceKey.private_key
}
};
return new GoogleStorageUploadStream(logger, uploaderOpts);
default:
logger.error(`unknown bucket vendor: ${bucket_credential.vendor}`);
break;
}
return null;
};
module.exports = {
getUploader
};

View File

@@ -1,7 +1,7 @@
const router = require('express').Router();
const assert = require('assert');
const request = require('request');
const {DbErrorBadRequest, DbErrorForbidden, DbErrorUnprocessableRequest, DbError} = require('../../utils/errors');
const {DbErrorBadRequest, DbErrorForbidden, DbErrorUnprocessableRequest} = require('../../utils/errors');
const Account = require('../../models/account');
const Application = require('../../models/application');
const Webhook = require('../../models/webhook');
@@ -23,8 +23,8 @@ const {
} = require('./utils');
const short = require('short-uuid');
const VoipCarrier = require('../../models/voip-carrier');
const { encrypt, decrypt } = require('../../utils/encrypt-decrypt');
const { testAwsS3 } = require('../../utils/storage-utils');
const { encrypt } = require('../../utils/encrypt-decrypt');
const { testAwsS3, testGoogleStorage } = require('../../utils/storage-utils');
const translator = short();
let idx = 0;
@@ -541,7 +541,8 @@ function encryptBucketCredential(obj) {
name,
access_key_id,
secret_access_key,
tags
tags,
service_key
} = obj.bucket_credential;
switch (vendor) {
@@ -554,6 +555,11 @@ function encryptBucketCredential(obj) {
secret_access_key, tags});
obj.bucket_credential = encrypt(awsData);
break;
case 'google':
assert(service_key, 'invalid aws S3 bucket credential: service_key is required');
const googleData = JSON.stringify({vendor, name, service_key, tags});
obj.bucket_credential = encrypt(googleData);
break;
case 'none':
obj.bucket_credential = null;
break;
@@ -708,35 +714,20 @@ router.post('/:sid/BucketCredentialTest', async(req, res) => {
try {
const account_sid = parseAccountSid(req);
await validateRequest(req, account_sid);
let {vendor, name, region, access_key_id, secret_access_key} = req.body;
const {vendor, name, region, access_key_id, secret_access_key, service_key} = req.body;
const ret = {
status: 'not tested'
};
if (secret_access_key.endsWith('XXXXXX')) {
// this is when the password already saved in account
const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
const results = await Account.retrieve(account_sid, service_provider_sid);
if (results.length === 0) throw new DbError('Invalid Account Sid');
const {bucket_credential} = results[0];
if (bucket_credential) {
const o = JSON.parse(decrypt(bucket_credential));
vendor = o.vendor;
switch (vendor) {
case 'aws_s3':
name = o.name;
region = o.region;
access_key_id = o.access_key_id;
secret_access_key = o.secret_access_key;
break;
}
}
}
switch (vendor) {
case 'aws_s3':
await testAwsS3(logger, {vendor, name, region, access_key_id, secret_access_key});
ret.status = 'ok';
break;
case 'google':
await testGoogleStorage(logger, {vendor, name, service_key});
ret.status = 'ok';
break;
default:
throw new DbErrorBadRequest(`Does not support test for ${vendor}`);
}

View File

@@ -4,7 +4,7 @@ const {DbErrorBadRequest} = require('../../utils/errors');
const {getHomerApiKey, getHomerSipTrace, getHomerPcap} = require('../../utils/homer-utils');
const {getJaegerTrace} = require('../../utils/jaeger-utils');
const Account = require('../../models/account');
const { getS3Object } = require('../../utils/storage-utils');
const { getS3Object, getGoogleStorageObject } = require('../../utils/storage-utils');
const parseAccountSid = (url) => {
const arr = /Accounts\/([^\/]*)/.exec(url);
@@ -124,22 +124,26 @@ router.get('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
const r = await Account.retrieve(account_sid);
if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
const {bucket_credential} = r[0];
const getOptions = {
...bucket_credential,
key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
};
let stream;
switch (bucket_credential.vendor) {
case 'aws_s3':
const getS3Options = {
...bucket_credential,
key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
};
const stream = await getS3Object(logger, getS3Options);
res.set({
'Content-Type': `audio/${format || 'mp3'}`
});
stream.pipe(res);
stream = await getS3Object(logger, getOptions);
break;
case 'google':
stream = await getGoogleStorageObject(logger, getOptions);
break;
default:
logger.error(`There is no handler for fetching record from ${bucket_credential.vendor}`);
return res.sendStatus(500);
}
res.set({
'Content-Type': `audio/${format || 'mp3'}`
});
stream.pipe(res);
} catch (err) {
logger.error({err}, ` error retrieving recording ${call_sid}`);
res.sendStatus(404);

View File

@@ -0,0 +1 @@
Hello From Jambonz. This file was created because Record all call bucket credential test.

View File

@@ -1,4 +1,42 @@
const { S3Client, PutObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3');
const {Storage} = require('@google-cloud/storage');
const fs = require('fs');
function testGoogleStorage(logger, opts) {
return new Promise((resolve, reject) => {
const serviceKey = JSON.parse(opts.service_key);
const storage = new Storage({
projectId: serviceKey.project_id,
credentials: {
client_email: serviceKey.client_email,
private_key: serviceKey.private_key
},
});
const blob = storage.bucket(opts.name).file('jambonz-sample.text');
fs.createReadStream(`${__dirname}/jambonz-sample.text`)
.pipe(blob.createWriteStream())
.on('error', (err) => reject(err))
.on('finish', () => resolve());
});
}
async function getGoogleStorageObject(logger, opts) {
const serviceKey = JSON.parse(opts.service_key);
const storage = new Storage({
projectId: serviceKey.project_id,
credentials: {
client_email: serviceKey.client_email,
private_key: serviceKey.private_key
},
});
const bucket = storage.bucket(opts.name);
const file = bucket.file(opts.key);
return file.createReadStream();
}
async function testAwsS3(logger, opts) {
const s3 = new S3Client({
@@ -38,5 +76,7 @@ async function getS3Object(logger, opts) {
module.exports = {
testAwsS3,
getS3Object
getS3Object,
testGoogleStorage,
getGoogleStorageObject
};

167
package-lock.json generated
View File

@@ -13,6 +13,7 @@
"@aws-sdk/client-transcribe": "^3.363.0",
"@deepgram/sdk": "^1.21.0",
"@google-cloud/speech": "^5.2.0",
"@google-cloud/storage": "^6.12.0",
"@jambonz/db-helpers": "^0.9.0",
"@jambonz/lamejs": "^1.2.2",
"@jambonz/realtimedb-helpers": "^0.8.6",
@@ -1520,6 +1521,18 @@
"node": ">=12.0.0"
}
},
"node_modules/@google-cloud/paginator": {
"version": "3.0.7",
"resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-3.0.7.tgz",
"integrity": "sha512-jJNutk0arIQhmpUUQJPJErsojqo834KcyB6X7a1mxuic8i1tKXxde8E69IZxNZawRIlZdIK2QY4WALvlK5MzYQ==",
"dependencies": {
"arrify": "^2.0.0",
"extend": "^3.0.2"
},
"engines": {
"node": ">=10"
}
},
"node_modules/@google-cloud/projectify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz",
@@ -1560,6 +1573,59 @@
"uuid": "dist/bin/uuid"
}
},
"node_modules/@google-cloud/storage": {
"version": "6.12.0",
"resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-6.12.0.tgz",
"integrity": "sha512-78nNAY7iiZ4O/BouWMWTD/oSF2YtYgYB3GZirn0To6eBOugjXVoK+GXgUXOl+HlqbAOyHxAVXOlsj3snfbQ1dw==",
"dependencies": {
"@google-cloud/paginator": "^3.0.7",
"@google-cloud/projectify": "^3.0.0",
"@google-cloud/promisify": "^3.0.0",
"abort-controller": "^3.0.0",
"async-retry": "^1.3.3",
"compressible": "^2.0.12",
"duplexify": "^4.0.0",
"ent": "^2.2.0",
"extend": "^3.0.2",
"fast-xml-parser": "^4.2.2",
"gaxios": "^5.0.0",
"google-auth-library": "^8.0.1",
"mime": "^3.0.0",
"mime-types": "^2.0.8",
"p-limit": "^3.0.1",
"retry-request": "^5.0.0",
"teeny-request": "^8.0.0",
"uuid": "^8.0.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@google-cloud/storage/node_modules/mime": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz",
"integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==",
"bin": {
"mime": "cli.js"
},
"engines": {
"node": ">=10.0.0"
}
},
"node_modules/@google-cloud/storage/node_modules/p-limit": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
"integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
"dependencies": {
"yocto-queue": "^0.1.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/@google-cloud/text-to-speech": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-4.2.2.tgz",
@@ -3034,6 +3100,14 @@
"resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
"integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ=="
},
"node_modules/async-retry": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz",
"integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==",
"dependencies": {
"retry": "0.13.1"
}
},
"node_modules/asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -3509,6 +3583,17 @@
"integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==",
"dev": true
},
"node_modules/compressible": {
"version": "2.0.18",
"resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
"integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==",
"dependencies": {
"mime-db": ">= 1.43.0 < 2"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -8101,6 +8186,14 @@
"through": "~2.3.4"
}
},
"node_modules/retry": {
"version": "0.13.1",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
"integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",
"engines": {
"node": ">= 4"
}
},
"node_modules/retry-axios": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-2.6.0.tgz",
@@ -9504,7 +9597,6 @@
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
"integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
"dev": true,
"engines": {
"node": ">=10"
},
@@ -10741,6 +10833,15 @@
"teeny-request": "^8.0.0"
}
},
"@google-cloud/paginator": {
"version": "3.0.7",
"resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-3.0.7.tgz",
"integrity": "sha512-jJNutk0arIQhmpUUQJPJErsojqo834KcyB6X7a1mxuic8i1tKXxde8E69IZxNZawRIlZdIK2QY4WALvlK5MzYQ==",
"requires": {
"arrify": "^2.0.0",
"extend": "^3.0.2"
}
},
"@google-cloud/projectify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz",
@@ -10771,6 +10872,46 @@
}
}
},
"@google-cloud/storage": {
"version": "6.12.0",
"resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-6.12.0.tgz",
"integrity": "sha512-78nNAY7iiZ4O/BouWMWTD/oSF2YtYgYB3GZirn0To6eBOugjXVoK+GXgUXOl+HlqbAOyHxAVXOlsj3snfbQ1dw==",
"requires": {
"@google-cloud/paginator": "^3.0.7",
"@google-cloud/projectify": "^3.0.0",
"@google-cloud/promisify": "^3.0.0",
"abort-controller": "^3.0.0",
"async-retry": "^1.3.3",
"compressible": "^2.0.12",
"duplexify": "^4.0.0",
"ent": "^2.2.0",
"extend": "^3.0.2",
"fast-xml-parser": "^4.2.2",
"gaxios": "^5.0.0",
"google-auth-library": "^8.0.1",
"mime": "^3.0.0",
"mime-types": "^2.0.8",
"p-limit": "^3.0.1",
"retry-request": "^5.0.0",
"teeny-request": "^8.0.0",
"uuid": "^8.0.0"
},
"dependencies": {
"mime": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz",
"integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="
},
"p-limit": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
"integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
"requires": {
"yocto-queue": "^0.1.0"
}
}
}
},
"@google-cloud/text-to-speech": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-4.2.2.tgz",
@@ -11997,6 +12138,14 @@
"resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz",
"integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ=="
},
"async-retry": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz",
"integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==",
"requires": {
"retry": "0.13.1"
}
},
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -12345,6 +12494,14 @@
"integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==",
"dev": true
},
"compressible": {
"version": "2.0.18",
"resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
"integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==",
"requires": {
"mime-db": ">= 1.43.0 < 2"
}
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -15774,6 +15931,11 @@
"through": "~2.3.4"
}
},
"retry": {
"version": "0.13.1",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
"integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="
},
"retry-axios": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-2.6.0.tgz",
@@ -16828,8 +16990,7 @@
"yocto-queue": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
"integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
"dev": true
"integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="
}
}
}

View File

@@ -53,7 +53,8 @@
"uuid": "^8.3.2",
"yamljs": "^0.3.0",
"ws": "^8.12.1",
"wav": "^1.0.2"
"wav": "^1.0.2",
"@google-cloud/storage" : "^6.12.0"
},
"devDependencies": {
"eslint": "^8.39.0",