diff --git a/package-lock.json b/package-lock.json index 0bb1984..445bbdc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -135,9 +135,9 @@ } }, "@types/express-serve-static-core": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.20.tgz", - "integrity": "sha512-8qqFN4W53IEWa9bdmuVrUcVkFemQWnt5DKPQ/oa8xKDYgtjCr2OO6NX5TIK49NLFr3mPYU2cLh92DQquC3oWWQ==", + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.21.tgz", + "integrity": "sha512-gwCiEZqW6f7EoR8TTEfalyEhb1zA5jQJnRngr97+3pzMaO1RKoI1w2bw07TK72renMUVWcWS5mLI6rk1NqN0nA==", "requires": { "@types/node": "*", "@types/qs": "*", @@ -1048,9 +1048,9 @@ } }, "es-abstract": { - "version": "1.18.2", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.2.tgz", - "integrity": "sha512-byRiNIQXE6HWNySaU6JohoNXzYgbBjztwFnBLUTiJmWXjaU9bSq3urQLUlNLQ292tc+gc07zYZXNZjaOoAX3sw==", + "version": "1.18.3", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.3.tgz", + "integrity": "sha512-nQIr12dxV7SSxE6r6f1l3DtAeEYdsGpps13dR0TwJg1S8gyp4ZPgy3FZcHBgbiQqnoqSTb+oC+kO4UQ0C/J8vw==", "requires": { "call-bind": "^1.0.2", "es-to-primitive": "^1.2.1", @@ -1539,9 +1539,9 @@ } }, "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==" }, "fresh": { "version": "0.5.2", @@ -1619,9 +1619,8 @@ } }, "genie-toolkit": { - "version": "0.8.0", - "resolved": "https://registry.npmjs.org/genie-toolkit/-/genie-toolkit-0.8.0.tgz", - "integrity": "sha512-bmh30nfQMu4Vf3+JzgEUcV40rregEnDlFJIH8Kw8Zu2D5Hk4OltO4k3n8a7AlM8jI4y8LFB+C7cvgzzmla2Xfw==", + "version": "github:stanford-oval/genie-toolkit#7cb03df54ecbc575c5ec89f8f67cc7779bef59b7", + "from": "github:stanford-oval/genie-toolkit", "requires": { "JSONStream": "^1.3.5", "adt": "^0.7.2", @@ -1656,10 +1655,10 @@ "stemmer": "^1.0.5", "string-interp": "^0.3.1", "thingpedia": "^2.9.0", - "thingtalk": "^2.0.0", + "thingtalk": "^2.0.1", "thingtalk-units": "^0.2.0", "twilio": "^3.63.0", - "typescript": "^4.2.4", + "typescript": "~4.2.4", "uuid": "^8.2.0", "ws": "^7.4.6" }, @@ -2877,11 +2876,11 @@ "dev": true }, "proxy-addr": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz", - "integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", "requires": { - "forwarded": "~0.1.2", + "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, @@ -3747,9 +3746,9 @@ } }, "thingtalk": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/thingtalk/-/thingtalk-2.0.0.tgz", - "integrity": "sha512-j7BC6hCWWUhltr/fdnyffI8tp1kH251rmcovY5AacrQbilvwKydlq24jcBIJ87hV7FB/wTZiM3RedCDCkAO9pQ==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/thingtalk/-/thingtalk-2.0.1.tgz", + "integrity": "sha512-gUof/zYjE3Dv/S4cL5cP66PMHUENIiGOXKnp8UmzSTrAotPBdH3/paZhs4K/Vr9/BTEqgbIzc11eQlorbO17eA==", "requires": { "@types/semver": "^7.3.6", "byline": "^5.0.0", @@ -3826,9 +3825,9 @@ "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, "twilio": { - "version": "3.63.0", - "resolved": "https://registry.npmjs.org/twilio/-/twilio-3.63.0.tgz", - "integrity": "sha512-ftZckbTBjJ5dgzdII9j0sqYw9SYq3wqTC9r6NmV7CRU0EXXDil5/AbKb78xNPLtMPx3+mn2N+2oTkQlTtWs9TQ==", + "version": "3.63.1", + "resolved": "https://registry.npmjs.org/twilio/-/twilio-3.63.1.tgz", + "integrity": "sha512-xwtOM78sO2jGxKg1AW+7XlJdrhTMW9dzr6665O+IB/VtNVQB7JQS48pLCZFnBaTvZOILVO0Q6t63wv24hIbr/A==", "requires": { "axios": "^0.21.1", "dayjs": "^1.8.29", @@ -3885,9 +3884,9 @@ } }, "typescript": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.2.tgz", - "integrity": "sha512-zZ4hShnmnoVnAHpVHWpTcxdv7dWP60S2FsydQLV8V5PbS3FifjWFFRiHSWpDJahly88PRyV5teTSLoq4eG7mKw==" + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.4.tgz", + "integrity": "sha512-V+evlYHZnQkaz8TRBuxTA92yZBPotr5H+WhQ7bD3hZUndx5tGOa1fuCgeSjxAzM1RiN5IzvadIXTVefuuwZCRg==" }, "uid-number": { "version": "0.0.6", @@ -3973,6 +3972,22 @@ "resolved": "https://registry.npmjs.org/weak-map/-/weak-map-1.0.5.tgz", "integrity": "sha1-eWkVhNmGB/UHC9O3CkDmuyLkAes=" }, + "webrtcvad": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/webrtcvad/-/webrtcvad-1.0.1.tgz", + "integrity": "sha512-oLfReCmGMpRducFWKP+o0GpKZLPj0u6qkln3P7wGaEzyjxtBiFuzvK28pmKF5SSihEkC1RGKO7Pi9C4VfY1q4Q==", + "requires": { + "bindings": "^1.3.0", + "node-addon-api": "^1.7.1" + }, + "dependencies": { + "node-addon-api": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-1.7.2.tgz", + "integrity": "sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==" + } + } + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/package.json b/package.json index cf5b1e0..aae8d05 100644 --- a/package.json +++ b/package.json @@ -23,12 +23,13 @@ "byline": "^5.0.0", "canberra": "^0.1.2", "dbus-native": "^0.4.0", - "genie-toolkit": "~0.8.0", + "genie-toolkit": "github:stanford-oval/genie-toolkit", "keytar": "^7.7.0", "node-gettext": "^3.0.0", "pulseaudio2": "^0.5.0", "snowboy": "^1.3.1", - "thingpedia": "~2.9.0" + "thingpedia": "~2.9.0", + "webrtcvad": "^1.0.1" }, "devDependencies": { "eslint": "^7.27.0" diff --git a/service/config.js b/service/config.js index d423ddb..fc392a6 100644 --- a/service/config.js +++ b/service/config.js @@ -21,5 +21,4 @@ module.exports.SEMPRE_URL = 'https://nlp.almond.stanford.edu'; module.exports.THINGPEDIA_URL = 'https://thingpedia.stanford.edu/thingpedia'; -module.exports.MS_SPEECH_RECOGNITION_PRIMARY_KEY = 'de1f02817356494483ba502b2ce95f6f'; -module.exports.MS_SPEECH_RECOGNITION_SECONDARY_KEY = '3dc6ce0b832940f0b0c984a1517c457e'; +module.exports.NL_URL = process.env.THINGENGINE_NL_URL || 'https://nlp-staging.almond.stanford.edu'; diff --git a/service/main.js b/service/main.js index 602674c..e4d77a6 100644 --- a/service/main.js +++ b/service/main.js @@ -338,7 +338,7 @@ class AppControlChannel extends events.EventEmitter { this._history = []; this._speechHandler = new Genie.SpeechHandler(this._conversation, _engine.platform, { - subscriptionKey: Config.MS_SPEECH_RECOGNITION_PRIMARY_KEY + nlUrl: Config.NL_URL }); let play; diff --git a/service/platform/index.js b/service/platform/index.js index 0522016..69da93f 100644 --- a/service/platform/index.js +++ b/service/platform/index.js @@ -258,6 +258,45 @@ class SystemSettings { } } +let webrtcvad; +try { + webrtcvad = require('webrtcvad').default; +} catch(e) { + console.log("VAD not available"); + webrtcvad = null; +} + +class VAD { + constructor() { + this._instance = null; + this.frameSize = 0; + } + + setup(bitrate, level) { + if (this._instance) + this._instance = null; + + if (webrtcvad) { + this._instance = new webrtcvad(bitrate, level); + // 16khz audio single-channel 16 bit: 10ms: 160b, 20ms: 320b, 30ms: 480b + this.frameSize = 320; + // console.log("setup VAD bitrate", bitrate, "level", level); + return true; + } + + return false; + } + + process(chunk) { + if (!this._instance) + return false; + let n = chunk.length % this.frameSize, r = 0; + for (let i = 0; i < n; i++) + r += this._instance.process(chunk.slice(i * this.frameSize, this.frameSize)); + return r; + } +} + class Platform extends Tp.BasePlatform { // Initialize the platform code // Will be called before instantiating the engine @@ -300,6 +339,9 @@ class Platform extends Tp.BasePlatform { this._ensurePulseConfig(); }); this._wakeWordDetector = new WakeWordDetector(); + this._voiceDetector = null; + if (webrtcvad && VAD) + this._voiceDetector = new VAD(); this._sqliteKey = null; } @@ -428,6 +470,9 @@ class Platform extends Tp.BasePlatform { case 'pulseaudio': return true; + case 'voice-detector': + return this._voiceDetector !== null; + case 'bluetooth': // temporarily disabled return false; @@ -468,7 +513,8 @@ class Platform extends Tp.BasePlatform { case 'wakeword-detector': return this._wakeWordDetector; - + case 'voice-detector': + return this._voiceDetector; case 'app-launcher': return this._appLauncher; case 'system-lock':