From 25d363cbf657c56f894c47110e56d9f1cca6cd3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81lvarez=20D=C3=ADez?= Date: Fri, 10 Jan 2025 01:07:56 +0100 Subject: [PATCH] [UI] Add voice dialog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miguel Álvarez Díez --- .../web/build/webpack.config.js | 20 +- bundles/org.openhab.ui/web/package-lock.json | 12 + bundles/org.openhab.ui/web/package.json | 1 + .../web/src/assets/i18n/common/en.json | 1 + .../src/assets/i18n/theme-switcher/en.json | 7 +- .../org.openhab.ui/web/src/components/app.vue | 8 +- .../web/src/components/dialog-mixin.js | 73 +++ .../web/src/components/theme-switcher.vue | 62 ++- .../org.openhab.ui/web/src/js/store/index.js | 9 +- .../web/src/js/voice/audio-main.js | 438 +++++++++++++++ .../web/src/js/voice/audio-types.js | 44 ++ .../web/src/js/voice/audio-worker.js | 514 ++++++++++++++++++ .../src/js/voice/audio/audio-sink-worklet.js | 154 ++++++ .../web/src/js/voice/audio/audio-sink.js | 94 ++++ .../js/voice/audio/audio-source-worklet.js | 14 + .../web/src/js/voice/audio/audio-source.js | 105 ++++ bundles/org.openhab.ui/web/src/pages/home.vue | 1 + 17 files changed, 1541 insertions(+), 16 deletions(-) create mode 100644 bundles/org.openhab.ui/web/src/components/dialog-mixin.js create mode 100644 bundles/org.openhab.ui/web/src/js/voice/audio-main.js create mode 100644 bundles/org.openhab.ui/web/src/js/voice/audio-types.js create mode 100644 bundles/org.openhab.ui/web/src/js/voice/audio-worker.js create mode 100644 bundles/org.openhab.ui/web/src/js/voice/audio/audio-sink-worklet.js create mode 100644 bundles/org.openhab.ui/web/src/js/voice/audio/audio-sink.js create mode 100644 bundles/org.openhab.ui/web/src/js/voice/audio/audio-source-worklet.js create mode 100644 bundles/org.openhab.ui/web/src/js/voice/audio/audio-source.js diff --git a/bundles/org.openhab.ui/web/build/webpack.config.js b/bundles/org.openhab.ui/web/build/webpack.config.js index 99646abb9e..b3a6418877 100644 --- a/bundles/org.openhab.ui/web/build/webpack.config.js +++ b/bundles/org.openhab.ui/web/build/webpack.config.js @@ -70,16 +70,16 @@ module.exports = { allowedHosts: "all", historyApiFallback: true, proxy: [ - { - context: ['/auth', '/rest', '/chart', '/proxy', '/icon', '/static', '/changePassword', '/createApiToken', '/audio'], - target: apiBaseUrl - }, - { - context: ['/ws/logs', '/ws/events'], - target: apiBaseUrl, - ws: true - } - ] + { + context: ['/auth', '/rest', '/chart', '/proxy', '/icon', '/static', '/changePassword', '/createApiToken', '/audio'], + target: apiBaseUrl + }, + { + context: ['/ws/logs', '/ws/events', '/ws/audio-pcm'], + target: apiBaseUrl, + ws: true + } + ] }, performance: { maxAssetSize: 2048000, diff --git a/bundles/org.openhab.ui/web/package-lock.json b/bundles/org.openhab.ui/web/package-lock.json index 06f57e9a88..ff9c05b661 100644 --- a/bundles/org.openhab.ui/web/package-lock.json +++ b/bundles/org.openhab.ui/web/package-lock.json @@ -43,6 +43,7 @@ "path-browserify": "^1.0.1", "pkce-challenge": "^3.1.0", "qrcode": "^1.5.4", + "reentrant-lock": "^3.0.0", "scope-css": "^1.2.1", "stream-browserify": "^3.0.0", "template7": "^1.4.2", @@ -18576,6 +18577,12 @@ "node": ">= 10.13.0" } }, + "node_modules/reentrant-lock": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/reentrant-lock/-/reentrant-lock-3.0.0.tgz", + "integrity": "sha512-YEGnY2CuYT7/a0sVWr/wobSTwQVow60QoiJvDjLCrTQjawhdk1KpuBJM1JqWOYEnxIRnKCj9XjKCQC0XejVtgw==", + "license": "MIT" + }, "node_modules/reflect.getprototypeof": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.6.tgz", @@ -35790,6 +35797,11 @@ "resolve": "^1.20.0" } }, + "reentrant-lock": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/reentrant-lock/-/reentrant-lock-3.0.0.tgz", + "integrity": "sha512-YEGnY2CuYT7/a0sVWr/wobSTwQVow60QoiJvDjLCrTQjawhdk1KpuBJM1JqWOYEnxIRnKCj9XjKCQC0XejVtgw==" + }, "reflect.getprototypeof": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.6.tgz", diff --git a/bundles/org.openhab.ui/web/package.json b/bundles/org.openhab.ui/web/package.json index 2fd9bf9534..de273ec3c4 100644 --- a/bundles/org.openhab.ui/web/package.json +++ b/bundles/org.openhab.ui/web/package.json @@ -84,6 +84,7 @@ "path-browserify": "^1.0.1", "pkce-challenge": "^3.1.0", "qrcode": "^1.5.4", + "reentrant-lock": "^3.0.0", "scope-css": "^1.2.1", "stream-browserify": "^3.0.0", "template7": "^1.4.2", diff --git a/bundles/org.openhab.ui/web/src/assets/i18n/common/en.json b/bundles/org.openhab.ui/web/src/assets/i18n/common/en.json index f76f4d9c9e..246f5292a4 100644 --- a/bundles/org.openhab.ui/web/src/assets/i18n/common/en.json +++ b/bundles/org.openhab.ui/web/src/assets/i18n/common/en.json @@ -33,6 +33,7 @@ "home.editHome": "Edit Home Page", "home.pinToHome": "Pin to Home", "home.exitToApp": "Return to App", + "home.triggerVoice": "Trigger voice dialog", "home.tip.otherApps": "Open the apps panel to launch other interfaces", "sidebar.noPages": "No pages", "sidebar.administration": "Administration", diff --git a/bundles/org.openhab.ui/web/src/assets/i18n/theme-switcher/en.json b/bundles/org.openhab.ui/web/src/assets/i18n/theme-switcher/en.json index 00e9816ad6..e497ed38a7 100644 --- a/bundles/org.openhab.ui/web/src/assets/i18n/theme-switcher/en.json +++ b/bundles/org.openhab.ui/web/src/assets/i18n/theme-switcher/en.json @@ -20,5 +20,10 @@ "about.miscellaneous.theme.disablePageTransition": "Disable page transition animations", "about.miscellaneous.webaudio.enable": "Enable Web Audio sink support", "about.miscellaneous.commandItem.title": "Listen for UI commands to ", - "about.miscellaneous.commandItem.selectItem": "Item" + "about.miscellaneous.commandItem.selectItem": "Item", + "about.dialog": "Voice Support", + "about.dialog.enable": "Enable Voice Dialog", + "about.dialog.id": "Connection Id", + "about.dialog.listeningItem": "Listening Item", + "about.dialog.locationItem": "Location Item" } diff --git a/bundles/org.openhab.ui/web/src/components/app.vue b/bundles/org.openhab.ui/web/src/components/app.vue index 6acd7b240d..bcc97dd142 100644 --- a/bundles/org.openhab.ui/web/src/components/app.vue +++ b/bundles/org.openhab.ui/web/src/components/app.vue @@ -276,6 +276,7 @@ import auth from './auth-mixin' import i18n from './i18n-mixin' import connectionHealth from './connection-health-mixin' import sseEvents from './sse-events-mixin' +import dialog from './dialog-mixin' import dayjs from 'dayjs' import dayjsLocales from 'dayjs/locale.json' @@ -283,7 +284,7 @@ import dayjsLocales from 'dayjs/locale.json' import { AddonIcons, AddonTitles } from '@/assets/addon-store' export default { - mixins: [auth, i18n, connectionHealth, sseEvents], + mixins: [auth, i18n, connectionHealth, sseEvents, dialog], components: { EmptyStatePlaceholder, PanelRight, @@ -700,11 +701,16 @@ export default { } }) + this.$f7.on('triggerDialog', () => { + this.triggerDialog() + }) + if (window) { window.addEventListener('keydown', this.keyDown) } this.startEventSource() + this.startAudioWebSocket() }) } } diff --git a/bundles/org.openhab.ui/web/src/components/dialog-mixin.js b/bundles/org.openhab.ui/web/src/components/dialog-mixin.js new file mode 100644 index 0000000000..66c854d003 --- /dev/null +++ b/bundles/org.openhab.ui/web/src/components/dialog-mixin.js @@ -0,0 +1,73 @@ +import { getAccessToken } from '@/js/openhab/auth.js' + +export default { + data () { + return { + audioMain: null + } + }, + methods: { + startAudioWebSocket () { + if (this.audioMain) { + return + } + const dialogEnabled = localStorage.getItem('openhab.ui:dialog.enabled') === 'true' + if (!dialogEnabled) { + return + } + const identifier = localStorage.getItem('openhab.ui:dialog.id') ?? 'anonymous' + const dialogListeningItem = localStorage.getItem('openhab.ui:dialog.listeningItem') ?? '' + const dialogLocationItem = localStorage.getItem('openhab.ui:dialog.locationItem') ?? '' + import('../js/voice/audio-main.js').then(({ AudioMain }) => { + if (this.audioMain) { + return + } + let port = '' + if (!((location.protocol === 'https:' && location.port === '443') || (location.protocol === 'http:' && location.port === '80'))) { + port = `:${location.port}` + } + const ohURL = `${location.protocol}//${location.hostname}${port}` + const updatePageIcon = (online, recording, playing) => { + let voiceIcon + if (!online) { + voiceIcon = 'f7:mic_slash_fill' + } else if (recording) { + voiceIcon = 'f7:mic_circle_fill' + } else if (playing) { + voiceIcon = 'f7:speaker_2_fill' + } else { + voiceIcon = 'f7:mic_circle' + } + this.$store.commit('setVoiceIcon', voiceIcon) + } + updatePageIcon(false) + this.audioMain = new AudioMain(ohURL, getAccessToken, { + onMessage: (...args) => { + console.debug('Voice: ' + args[0]) + }, + onRunningChange (io) { + updatePageIcon(io.isRunning(), io.isListening(), io.isSpeaking()) + }, + onListeningChange (io) { + updatePageIcon(io.isRunning(), io.isListening(), io.isSpeaking()) + }, + onSpeakingChange (io) { + updatePageIcon(io.isRunning(), io.isListening(), io.isSpeaking()) + } + }) + const events = ['touchstart', 'touchend', 'mousedown', 'keydown'] + const startAudio = () => { + clean() + this.audioMain.initialize(identifier, dialogListeningItem, dialogLocationItem) + } + const clean = () => events.forEach(e => document.body.removeEventListener(e, startAudio)) + events.forEach(e => document.body.addEventListener(e, startAudio, false)) + }) + }, + triggerDialog () { + if (this.audioMain != null) { + this.audioMain.sendSpot() + } + } + } +} diff --git a/bundles/org.openhab.ui/web/src/components/theme-switcher.vue b/bundles/org.openhab.ui/web/src/components/theme-switcher.vue index 4ef9c979b7..f6610de3fb 100644 --- a/bundles/org.openhab.ui/web/src/components/theme-switcher.vue +++ b/bundles/org.openhab.ui/web/src/components/theme-switcher.vue @@ -98,6 +98,26 @@ + + + + + + + + + + + + + + + + + + @@ -111,7 +131,6 @@