From bcd4b8ba0491ffc109333fe0580c95e35d1253ca Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Tue, 17 Sep 2019 11:35:16 -0400 Subject: [PATCH 1/3] Moved windows dictation and speech data providers to core --- .../WindowsDictationDataProvider.cs | 310 ------------------ .../WindowsDictationDataProvider.cs.meta | 11 - .../Controllers/WindowsSpeechDataProvider.cs | 195 ----------- .../WindowsSpeechDataProvider.cs.meta | 11 - 4 files changed, 527 deletions(-) delete mode 100644 XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsDictationDataProvider.cs delete mode 100644 XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsDictationDataProvider.cs.meta delete mode 100644 XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsSpeechDataProvider.cs delete mode 100644 XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsSpeechDataProvider.cs.meta diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsDictationDataProvider.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsDictationDataProvider.cs deleted file mode 100644 index ab22758..0000000 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsDictationDataProvider.cs +++ /dev/null @@ -1,310 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. See LICENSE in the project root for license information. - -using System.Text; -using System.Threading.Tasks; -using UnityEngine; -using XRTK.Definitions.Controllers; -using XRTK.Interfaces.InputSystem; -using XRTK.Providers.Controllers; -using XRTK.Services; -using XRTK.Utilities.Async; - -#if UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN -using UnityEngine.Windows.Speech; -#endif // UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN - -namespace XRTK.WindowsMixedReality.Controllers -{ - /// - /// Dictation data provider for Windows 10 based platforms. - /// - public class WindowsDictationDataProvider : BaseDictationDataProvider - { - /// - /// Constructor. - /// - /// - /// - /// - public WindowsDictationDataProvider(string name, uint priority, BaseMixedRealityControllerDataProviderProfile profile) - : base(name, priority, profile) - { -#if UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN - if (dictationRecognizer == null) - { - dictationRecognizer = new DictationRecognizer(); - } -#endif // UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN - } - -#if UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN - - #region IMixedRealityService Implementation - - /// - public override void Enable() - { - if (!Application.isPlaying) { return; } - - inputSource = MixedRealityToolkit.InputSystem.RequestNewGenericInputSource(Name); - dictationResult = string.Empty; - - dictationRecognizer.DictationHypothesis += DictationRecognizer_DictationHypothesis; - dictationRecognizer.DictationResult += DictationRecognizer_DictationResult; - dictationRecognizer.DictationComplete += DictationRecognizer_DictationComplete; - dictationRecognizer.DictationError += DictationRecognizer_DictationError; - } - - /// - public override void Update() - { - if (!Application.isPlaying) { return; } - - if (!isTransitioning && IsListening && !Microphone.IsRecording(deviceName) && dictationRecognizer.Status == SpeechSystemStatus.Running) - { - // If the microphone stops as a result of timing out, make sure to manually stop the dictation recognizer. - StopRecording(); - } - - if (!hasFailed && dictationRecognizer.Status == SpeechSystemStatus.Failed) - { - hasFailed = true; - MixedRealityToolkit.InputSystem.RaiseDictationError(inputSource, "Dictation recognizer has failed!"); - } - } - - /// - public override async void Disable() - { - base.Disable(); - - if (!Application.isPlaying) { return; } - - if (!isTransitioning && IsListening) { await StopRecordingAsync(); } - - dictationRecognizer.DictationHypothesis -= DictationRecognizer_DictationHypothesis; - dictationRecognizer.DictationResult -= DictationRecognizer_DictationResult; - dictationRecognizer.DictationComplete -= DictationRecognizer_DictationComplete; - dictationRecognizer.DictationError -= DictationRecognizer_DictationError; - } - - protected override void OnDispose(bool finalizing) - { - if (finalizing) - { - dictationRecognizer.Dispose(); - } - - base.OnDispose(finalizing); - } - - #endregion IMixedRealityService Implementation - - #region IMixedRealityDictationDataProvider Implementation - - private bool hasFailed; - private bool hasListener; - private bool isTransitioning; - - private IMixedRealityInputSource inputSource = null; - - /// - /// Caches the text currently being displayed in dictation display text. - /// - private StringBuilder textSoFar; - - private string deviceName = string.Empty; - - /// - /// The device audio sampling rate. - /// - /// Set by UnityEngine.Microphone. - private int samplingRate; - - /// - /// String result of the current dictation. - /// - private string dictationResult; - - /// - /// Audio clip of the last dictation session. - /// - private AudioClip dictationAudioClip; - - private static DictationRecognizer dictationRecognizer; - - private readonly WaitUntil waitUntilPhraseRecognitionSystemHasStarted = new WaitUntil(() => PhraseRecognitionSystem.Status != SpeechSystemStatus.Stopped); - private readonly WaitUntil waitUntilPhraseRecognitionSystemHasStopped = new WaitUntil(() => PhraseRecognitionSystem.Status != SpeechSystemStatus.Running); - - private readonly WaitUntil waitUntilDictationRecognizerHasStarted = new WaitUntil(() => dictationRecognizer.Status != SpeechSystemStatus.Stopped); - private readonly WaitUntil waitUntilDictationRecognizerHasStopped = new WaitUntil(() => dictationRecognizer.Status != SpeechSystemStatus.Running); - - /// - public override bool IsListening { get; protected set; } = false; - - /// - public override async void StartRecording(GameObject listener = null, float initialSilenceTimeout = 5, float autoSilenceTimeout = 20, int recordingTime = 10, string micDeviceName = "") - { - await StartRecordingAsync(listener, initialSilenceTimeout, autoSilenceTimeout, recordingTime, micDeviceName); - } - - /// - public override async Task StartRecordingAsync(GameObject listener = null, float initialSilenceTimeout = 5f, float autoSilenceTimeout = 20f, int recordingTime = 10, string micDeviceName = "") - { - if (IsListening || isTransitioning || MixedRealityToolkit.InputSystem == null || !Application.isPlaying) - { - Debug.LogWarning("Unable to start recording"); - return; - } - - hasFailed = false; - IsListening = true; - isTransitioning = true; - - if (listener != null) - { - hasListener = true; - MixedRealityToolkit.InputSystem.PushModalInputHandler(listener); - } - - if (PhraseRecognitionSystem.Status == SpeechSystemStatus.Running) - { - PhraseRecognitionSystem.Shutdown(); - } - - await waitUntilPhraseRecognitionSystemHasStopped; - Debug.Assert(PhraseRecognitionSystem.Status == SpeechSystemStatus.Stopped); - - // Query the maximum frequency of the default microphone. - deviceName = micDeviceName; - Microphone.GetDeviceCaps(deviceName, out _, out samplingRate); - - dictationRecognizer.InitialSilenceTimeoutSeconds = initialSilenceTimeout; - dictationRecognizer.AutoSilenceTimeoutSeconds = autoSilenceTimeout; - dictationRecognizer.Start(); - - await waitUntilDictationRecognizerHasStarted; - Debug.Assert(dictationRecognizer.Status == SpeechSystemStatus.Running); - - if (dictationRecognizer.Status == SpeechSystemStatus.Failed) - { - MixedRealityToolkit.InputSystem.RaiseDictationError(inputSource, "Dictation recognizer failed to start!"); - return; - } - - // Start recording from the microphone. - dictationAudioClip = Microphone.Start(deviceName, false, recordingTime, samplingRate); - textSoFar = new StringBuilder(); - isTransitioning = false; - } - - /// - public override async void StopRecording() - { - await StopRecordingAsync(); - } - - /// - public override async Task StopRecordingAsync() - { - if (!IsListening || isTransitioning || !Application.isPlaying) - { - Debug.LogWarning("Unable to stop recording"); - return null; - } - - IsListening = false; - isTransitioning = true; - - if (hasListener) - { - MixedRealityToolkit.InputSystem.PopModalInputHandler(); - hasListener = false; - } - - Microphone.End(deviceName); - - if (dictationRecognizer.Status == SpeechSystemStatus.Running) - { - dictationRecognizer.Stop(); - } - - await waitUntilDictationRecognizerHasStopped; - Debug.Assert(dictationRecognizer.Status == SpeechSystemStatus.Stopped); - - PhraseRecognitionSystem.Restart(); - - await waitUntilPhraseRecognitionSystemHasStarted; - Debug.Assert(PhraseRecognitionSystem.Status == SpeechSystemStatus.Running); - - isTransitioning = false; - return dictationAudioClip; - } - - /// - /// This event is fired while the user is talking. As the recognizer listens, it provides text of what it's heard so far. - /// - /// The currently hypothesized recognition. - private void DictationRecognizer_DictationHypothesis(string text) - { - // We don't want to append to textSoFar yet, because the hypothesis may have changed on the next event. - dictationResult = $"{textSoFar} {text}..."; - - MixedRealityToolkit.InputSystem.RaiseDictationHypothesis(inputSource, dictationResult); - } - - /// - /// This event is fired after the user pauses, typically at the end of a sentence. The full recognized string is returned here. - /// - /// The text that was heard by the recognizer. - /// A representation of how confident (rejected, low, medium, high) the recognizer is of this recognition. - private void DictationRecognizer_DictationResult(string text, ConfidenceLevel confidence) - { - textSoFar.Append($"{text}. "); - - dictationResult = textSoFar.ToString(); - - MixedRealityToolkit.InputSystem.RaiseDictationResult(inputSource, dictationResult); - } - - /// - /// This event is fired when the recognizer stops, whether from StartRecording() being called, a timeout occurring, or some other error. - /// Typically, this will simply return "Complete". In this case, we check to see if the recognizer timed out. - /// - /// An enumerated reason for the session completing. - private void DictationRecognizer_DictationComplete(DictationCompletionCause cause) - { - // If Timeout occurs, the user has been silent for too long. - if (cause == DictationCompletionCause.TimeoutExceeded) - { - Microphone.End(deviceName); - - dictationResult = "Dictation has timed out. Please try again."; - } - - MixedRealityToolkit.InputSystem.RaiseDictationComplete(inputSource, dictationResult, dictationAudioClip); - textSoFar = null; - dictationResult = string.Empty; - } - - /// - /// This event is fired when an error occurs. - /// - /// The string representation of the error reason. - /// The int representation of the hresult. - private void DictationRecognizer_DictationError(string error, int hresult) - { - dictationResult = $"{error}\nHRESULT: {hresult}"; - - MixedRealityToolkit.InputSystem.RaiseDictationError(inputSource, dictationResult); - textSoFar = null; - dictationResult = string.Empty; - } - - #endregion IMixedRealityDictationDataProvider Implementation - -#endif // UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN - - } -} \ No newline at end of file diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsDictationDataProvider.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsDictationDataProvider.cs.meta deleted file mode 100644 index 7a8df2f..0000000 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsDictationDataProvider.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: c274f7b30cf34652a9e58f8f1b5e4e12 -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {fileID: 2800000, guid: 8ac5213854cf4dbabd140decf8df1946, type: 3} - userData: - assetBundleName: - assetBundleVariant: diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsSpeechDataProvider.cs b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsSpeechDataProvider.cs deleted file mode 100644 index fe93bd5..0000000 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsSpeechDataProvider.cs +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. See LICENSE in the project root for license information. - -using System; -using UnityEngine; -using XRTK.Definitions.Controllers; -using XRTK.Definitions.InputSystem; -using XRTK.Definitions.Utilities; -using XRTK.Interfaces.InputSystem; -using XRTK.Providers.Controllers; -using XRTK.Services; - -#if UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN -using UnityEngine.Windows.Speech; -#endif // UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN - -namespace XRTK.WindowsMixedReality.Controllers -{ - /// - /// Speech data provider for windows 10 based platforms. - /// - public class WindowsSpeechDataProvider : BaseSpeechDataProvider - { - /// - /// Constructor. - /// - /// - /// - /// - public WindowsSpeechDataProvider(string name, uint priority, BaseMixedRealityControllerDataProviderProfile profile) - : base(name, priority, profile) - { - if (MixedRealityToolkit.Instance.ActiveProfile.InputSystemProfile == null) - { - throw new Exception("Missing required input system profile!"); - } - - if (MixedRealityToolkit.Instance.ActiveProfile.InputSystemProfile.SpeechCommandsProfile == null) - { - throw new Exception("Missing required speech commands profile!"); - } - - if (MixedRealityToolkit.Instance.ActiveProfile.InputSystemProfile.SpeechCommandsProfile.SpeechCommands == null) - { - throw new Exception("Null speech commands in the speech commands profile!"); - } - -#if UNITY_WSA && UNITY_EDITOR - if (!UnityEditor.PlayerSettings.WSA.GetCapability(UnityEditor.PlayerSettings.WSACapability.Microphone)) - { - UnityEditor.PlayerSettings.WSA.SetCapability(UnityEditor.PlayerSettings.WSACapability.Microphone, true); - } -#endif - -#if UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN - var newKeywords = new string[Commands.Length]; - - for (int i = 0; i < Commands.Length; i++) - { - newKeywords[i] = Commands[i].Keyword; - } - - RecognitionConfidenceLevel = MixedRealityToolkit.Instance.ActiveProfile.InputSystemProfile.SpeechCommandsProfile.SpeechRecognitionConfidenceLevel; - - if (keywordRecognizer == null) - { - keywordRecognizer = new KeywordRecognizer(newKeywords, (ConfidenceLevel)RecognitionConfidenceLevel); - } -#endif // UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN - } - -#if UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN - - #region IMixedRealityService Implementation - - /// - public override void Enable() - { - base.Enable(); - - if (!Application.isPlaying || Commands.Length == 0) { return; } - - InputSource = MixedRealityToolkit.InputSystem?.RequestNewGenericInputSource("Windows Speech Input Source"); - - keywordRecognizer.OnPhraseRecognized += KeywordRecognizer_OnPhraseRecognized; - - if (MixedRealityToolkit.Instance.ActiveProfile.InputSystemProfile.SpeechCommandsProfile.SpeechRecognizerStartBehavior == AutoStartBehavior.AutoStart) - { - StartRecognition(); - } - } - - /// - public override void Update() - { - base.Update(); - - if (!keywordRecognizer.IsRunning) { return; } - - for (int i = 0; i < Commands.Length; i++) - { - if (Input.GetKeyDown(Commands[i].KeyCode)) - { - OnPhraseRecognized((ConfidenceLevel)RecognitionConfidenceLevel, TimeSpan.Zero, DateTime.UtcNow, Commands[i].Keyword); - } - } - } - - /// - public override void Disable() - { - base.Disable(); - - if (!Application.isPlaying || Commands.Length == 0) { return; } - - StopRecognition(); - - keywordRecognizer.OnPhraseRecognized -= KeywordRecognizer_OnPhraseRecognized; - } - - protected override void OnDispose(bool finalizing) - { - if (finalizing) - { - keywordRecognizer.Dispose(); - } - - base.OnDispose(finalizing); - } - - #endregion IMixedRealityService Implementation - - #region IMixedRealitySpeechDataProvider Implementation - - /// - /// The keywords to be recognized and optional keyboard shortcuts. - /// - private static SpeechCommands[] Commands => MixedRealityToolkit.Instance.ActiveProfile.InputSystemProfile.SpeechCommandsProfile.SpeechCommands; - - /// - /// The Input Source for Windows Speech Input. - /// - public IMixedRealityInputSource InputSource = null; - - private static KeywordRecognizer keywordRecognizer; - - /// - public override bool IsRecognitionActive => keywordRecognizer.IsRunning; - - /// - /// The that the is using. - /// - public RecognitionConfidenceLevel RecognitionConfidenceLevel { get; } - - /// - public override void StartRecognition() - { - if (!keywordRecognizer.IsRunning) - { - keywordRecognizer.Start(); - } - } - - /// - public override void StopRecognition() - { - if (keywordRecognizer.IsRunning) - { - keywordRecognizer.Stop(); - } - } - - private void KeywordRecognizer_OnPhraseRecognized(PhraseRecognizedEventArgs args) - { - OnPhraseRecognized(args.confidence, args.phraseDuration, args.phraseStartTime, args.text); - } - - private void OnPhraseRecognized(ConfidenceLevel confidence, TimeSpan phraseDuration, DateTime phraseStartTime, string text) - { - for (int i = 0; i < Commands.Length; i++) - { - if (Commands[i].Keyword == text) - { - MixedRealityToolkit.InputSystem.RaiseSpeechCommandRecognized(InputSource, Commands[i].Action, (RecognitionConfidenceLevel)confidence, phraseDuration, phraseStartTime, text); - break; - } - } - } - - #endregion IMixedRealitySpeechDataProvider Implementation - -#endif // UNITY_STANDALONE_WIN || UNITY_WSA || UNITY_EDITOR_WIN - - } -} diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsSpeechDataProvider.cs.meta b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsSpeechDataProvider.cs.meta deleted file mode 100644 index f1bf79c..0000000 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/Controllers/WindowsSpeechDataProvider.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: bfd7e6177f57407c82cefdc1495dc3f5 -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {fileID: 2800000, guid: 8ac5213854cf4dbabd140decf8df1946, type: 3} - userData: - assetBundleName: - assetBundleVariant: From 1392490ef7e084ffbad2a7cce29de153d6796d31 Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Tue, 17 Sep 2019 19:01:44 -0400 Subject: [PATCH 2/3] bumped version to 0.1.6 --- XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/package.json b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/package.json index fd43b6a..6872407 100644 --- a/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/package.json +++ b/XRTK.WindowsMixedReality/Packages/com.xrtk.wmr/package.json @@ -10,12 +10,12 @@ "mixed", "reality" ], - "version": "0.1.5", + "version": "0.1.6", "unity": "2019.1", "license": "MIT", "author": "XRTK Team (https://github.com/XRTK)", "dependencies": { - "com.xrtk.core": "0.1.19", + "com.xrtk.core": "0.1.20", "com.unity.xr.windowsmr.metro": "1.0.14" } } \ No newline at end of file From 96e439dff2364ac6a530fcc4c8afc3ceaa6c5d9b Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Wed, 18 Sep 2019 21:25:27 -0400 Subject: [PATCH 3/3] updated packages --- XRTK.WindowsMixedReality/Packages/manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/XRTK.WindowsMixedReality/Packages/manifest.json b/XRTK.WindowsMixedReality/Packages/manifest.json index ef3d4ea..81b1bb0 100644 --- a/XRTK.WindowsMixedReality/Packages/manifest.json +++ b/XRTK.WindowsMixedReality/Packages/manifest.json @@ -12,10 +12,10 @@ "com.unity.package-manager-ui": "2.1.2", "com.unity.textmeshpro": "2.0.1", "com.unity.xr.legacyinputhelpers": "2.0.6", - "com.unity.xr.oculus.standalone": "1.38.2", + "com.unity.xr.oculus.standalone": "1.38.3", "com.unity.xr.openvr.standalone": "1.0.5", "com.unity.xr.windowsmr.metro": "1.0.14", - "com.xrtk.core": "0.1.19", + "com.xrtk.core": "0.1.20", "com.unity.modules.ai": "1.0.0", "com.unity.modules.animation": "1.0.0", "com.unity.modules.assetbundle": "1.0.0",