[UI] Add voice dialog

Signed-off-by: Miguel Álvarez Díez <[email protected]>

Author: GiviMAD

bundles/org.openhab.ui/web/build/webpack.config.js

@@ -70,16 +70,16 @@ module.exports = {
     allowedHosts: "all",
     historyApiFallback: true,
     proxy: [
-      {
-        context: ['/auth', '/rest', '/chart', '/proxy', '/icon', '/static', '/changePassword', '/createApiToken', '/audio'],
-        target: apiBaseUrl
-      },
-      {
-        context: ['/ws/logs', '/ws/events'],
-        target: apiBaseUrl,
-        ws: true
-      }
-    ]
+    {
+      context: ['/auth', '/rest', '/chart', '/proxy', '/icon', '/static', '/changePassword', '/createApiToken', '/audio'],
+      target: apiBaseUrl
+    },
+    {
+      context: ['/ws/logs', '/ws/events', '/ws/audio-pcm'],
+      target: apiBaseUrl,
+      ws: true
+    }
+  ]
   },
   performance: {
     maxAssetSize: 2048000,

bundles/org.openhab.ui/web/package-lock.json

@@ -84,6 +84,7 @@
     "path-browserify": "^1.0.1",
     "pkce-challenge": "^3.1.0",
     "qrcode": "^1.5.4",
+    "reentrant-lock": "^3.0.0",
     "scope-css": "^1.2.1",
     "stream-browserify": "^3.0.0",
     "template7": "^1.4.2",

bundles/org.openhab.ui/web/package.json

@@ -33,6 +33,7 @@
   "home.editHome": "Edit Home Page",
   "home.pinToHome": "Pin to Home",
   "home.exitToApp": "Return to App",
+  "home.triggerVoice": "Trigger voice dialog",
   "home.tip.otherApps": "Open the apps panel to launch other interfaces",
   "sidebar.noPages": "No pages",
   "sidebar.administration": "Administration",

bundles/org.openhab.ui/web/src/assets/i18n/common/en.json

@@ -20,5 +20,10 @@
   "about.miscellaneous.theme.disablePageTransition": "Disable page transition animations",
   "about.miscellaneous.webaudio.enable": "Enable Web Audio sink support",
   "about.miscellaneous.commandItem.title": "Listen for UI commands to ",
-  "about.miscellaneous.commandItem.selectItem": "Item"
+  "about.miscellaneous.commandItem.selectItem": "Item",
+  "about.dialog": "Voice Support",
+  "about.dialog.enable": "Enable Voice Dialog",
+  "about.dialog.id": "Connection Id",
+  "about.dialog.listeningItem": "Listening Item",
+  "about.dialog.locationItem": "Location Item"
 }

bundles/org.openhab.ui/web/src/assets/i18n/theme-switcher/en.json

@@ -276,14 +276,15 @@ import auth from './auth-mixin'
 import i18n from './i18n-mixin'
 import connectionHealth from './connection-health-mixin'
 import sseEvents from './sse-events-mixin'
+import dialog from './dialog-mixin'
 
 import dayjs from 'dayjs'
 import dayjsLocales from 'dayjs/locale.json'
 
 import { AddonIcons, AddonTitles } from '@/assets/addon-store'
 
 export default {
-  mixins: [auth, i18n, connectionHealth, sseEvents],
+  mixins: [auth, i18n, connectionHealth, sseEvents, dialog],
   components: {
     EmptyStatePlaceholder,
     PanelRight,
@@ -700,11 +701,16 @@ export default {
         }
       })
 
+      this.$f7.on('triggerDialog', () => {
+        this.triggerDialog()
+      })
+
       if (window) {
         window.addEventListener('keydown', this.keyDown)
       }
 
       this.startEventSource()
+      this.startAudioWebSocket()
     })
   }
 }

bundles/org.openhab.ui/web/src/components/app.vue

@@ -0,0 +1,73 @@
+import { getAccessToken } from '@/js/openhab/auth.js'
+
+export default {
+  data () {
+    return {
+      audioMain: null
+    }
+  },
+  methods: {
+    startAudioWebSocket () {
+      if (this.audioMain) {
+        return
+      }
+      const dialogEnabled = localStorage.getItem('openhab.ui:dialog.enabled') === 'true'
+      if (!dialogEnabled) {
+        return
+      }
+      const identifier = localStorage.getItem('openhab.ui:dialog.id') ?? 'anonymous'
+      const dialogListeningItem = localStorage.getItem('openhab.ui:dialog.listeningItem') ?? ''
+      const dialogLocationItem = localStorage.getItem('openhab.ui:dialog.locationItem') ?? ''
+      import('../js/voice/audio-main.js').then(({ AudioMain }) => {
+        if (this.audioMain) {
+          return
+        }
+        let port = ''
+        if (!((location.protocol === 'https:' && location.port === '443') || (location.protocol === 'http:' && location.port === '80'))) {
+          port = `:${location.port}`
+        }
+        const ohURL = `${location.protocol}//${location.hostname}${port}`
+        const updatePageIcon = (online, recording, playing) => {
+          let voiceIcon
+          if (!online) {
+            voiceIcon = 'f7:mic_slash_fill'
+          } else if (recording) {
+            voiceIcon = 'f7:mic_circle_fill'
+          } else if (playing) {
+            voiceIcon = 'f7:speaker_2_fill'
+          } else {
+            voiceIcon = 'f7:mic_circle'
+          }
+          this.$store.commit('setVoiceIcon', voiceIcon)
+        }
+        updatePageIcon(false)
+        this.audioMain = new AudioMain(ohURL, getAccessToken, {
+          onMessage: (...args) => {
+            console.debug('Voice: ' + args[0])
+          },
+          onRunningChange (io) {
+            updatePageIcon(io.isRunning(), io.isListening(), io.isSpeaking())
+          },
+          onListeningChange (io) {
+            updatePageIcon(io.isRunning(), io.isListening(), io.isSpeaking())
+          },
+          onSpeakingChange (io) {
+            updatePageIcon(io.isRunning(), io.isListening(), io.isSpeaking())
+          }
+        })
+        const events = ['touchstart', 'touchend', 'mousedown', 'keydown']
+        const startAudio = () => {
+          clean()
+          this.audioMain.initialize(identifier, dialogListeningItem, dialogLocationItem)
+        }
+        const clean = () => events.forEach(e => document.body.removeEventListener(e, startAudio))
+        events.forEach(e => document.body.addEventListener(e, startAudio, false))
+      })
+    },
+    triggerDialog () {
+      if (this.audioMain != null) {
+        this.audioMain.sendSpot()
+      }
+    }
+  }
+}

bundles/org.openhab.ui/web/src/components/dialog-mixin.js

@@ -98,6 +98,26 @@
         </f7-list>
       </f7-col>
     </f7-row>
+
+    <f7-row v-if="showDialogOptions">
+      <f7-col>
+        <f7-block-title v-t="'about.dialog'" />
+        <f7-list>
+          <f7-list-item>
+            <span v-t="'about.dialog.enable'" />
+            <f7-toggle :checked="dialog === 'true'" @toggle:change="setDialog" />
+          </f7-list-item>
+          <f7-list-item>
+            <span v-t="'about.dialog.id'" />
+            <f7-input type="button" :value="identifier" />
+          </f7-list-item>
+          <item-picker :title="$t('about.dialog.listeningItem')" :multiple="false" :value="listeningItem"
+                       @input="setDialogListeningItem" />
+          <item-picker :title="$t('about.dialog.locationItem')" :multiple="false" :value="locationItem"
+                       @input="setDialogLocationItem" />
+        </f7-list>
+      </f7-col>
+    </f7-row>
   </f7-block>
 </template>
 
@@ -111,7 +131,6 @@
 <script>
 import { loadLocaleMessages } from '@/js/i18n'
 import ItemPicker from '@/components/config/controls/item-picker.vue'
-
 export default {
   components: {
     ItemPicker
@@ -165,6 +184,18 @@ export default {
     setCommandItem (value) {
       localStorage.setItem('openhab.ui:commandItem', value)
       setTimeout(() => { location.reload() }, 50) // Delay reload, otherwise it doesn't work
+    },
+    setDialog (value) {
+      localStorage.setItem('openhab.ui:dialog.enabled', value)
+      setTimeout(() => { location.reload() }, 50) // Delay reload, otherwise it doesn't work
+    },
+    setDialogListeningItem (value) {
+      localStorage.setItem('openhab.ui:dialog.listeningItem', value)
+      setTimeout(() => { location.reload() }, 50) // Delay reload, otherwise it doesn't work
+    },
+    setDialogLocationItem (value) {
+      localStorage.setItem('openhab.ui:dialog.locationItem', value)
+      setTimeout(() => { location.reload() }, 50) // Delay reload, otherwise it doesn't work
     }
   },
   computed: {
@@ -197,6 +228,30 @@ export default {
     },
     commandItem () {
       return localStorage.getItem('openhab.ui:commandItem') || ''
+    },
+    showDialogOptions () {
+      const getUserMediaSupported = !!(window.navigator && window.navigator.mediaDevices && window.navigator.mediaDevices.getUserMedia)
+      return getUserMediaSupported &&
+        !!window.AudioContext &&
+        !!window.crypto
+    },
+    dialog () {
+      return localStorage.getItem('openhab.ui:dialog.enabled') || 'default'
+    },
+    identifier () {
+      const key = 'openhab.ui:dialog.id'
+      let id = localStorage.getItem(key)
+      if (!id) {
+        id = `ui-${Math.round(Math.random() * 100)}-${Math.round(Math.random() * 100)}`
+        localStorage.setItem(key, id)
+      }
+      return id
+    },
+    listeningItem () {
+      return localStorage.getItem('openhab.ui:dialog.listeningItem') || ''
+    },
+    locationItem () {
+      return localStorage.getItem('openhab.ui:dialog.locationItem') || ''
     }
   }
 }
@@ -284,5 +339,8 @@ export default {
 .nav-bars-picker-fill .demo-navbar:before,
 .nav-bars-picker-fill .demo-navbar:after
   background #fff
-
+.title-fixed .item-title
+  width: 200%
+.input-right input
+  text-align: right
 </style>

bundles/org.openhab.ui/web/src/components/theme-switcher.vue

@@ -33,11 +33,13 @@ const store = new Vuex.Store({
     },
     websiteUrl: null,
     developerDock: false,
-    pagePath: null
+    pagePath: null,
+    voiceIcon: null
   },
   getters: {
     apiEndpoint: (state) => (type) => (!state.apiEndpoints) ? null : state.apiEndpoints.find((e) => e.type === type),
-    locale: (state, getters) => state.locale ?? 'default'
+    locale: (state, getters) => state.locale ?? 'default',
+    voiceIcon: (state) => state.voiceIcon
   },
   mutations: {
     setRootResource (state, { rootResponse }) {
@@ -56,6 +58,9 @@ const store = new Vuex.Store({
     },
     setPagePath (state, value) {
       state.pagePath = value
+    },
+    setVoiceIcon (state, value) {
+      state.voiceIcon = value
     }
   },
   actions: {