New .NET/Unity SpeechlyClient audio pipeline

.gitignore

@@ -0,0 +1,2 @@
+[Oo]bj/
+docfx.console/

README.md

@@ -18,9 +18,10 @@ Download [speechly-client.unitypackage](https://github.com/speechly/speechly-uni
   - [speechly-unity/Assets/SpeechlyExamples/AudioFileToSpeechly/](speechly-unity/Assets/SpeechlyExamples/AudioFileToSpeechly/)
   - [speechly-unity/Assets/SpeechlyExamples/VoiceCommands/](speechly-unity/Assets/SpeechlyExamples/VoiceCommands/)
 
-## Unity usage
 
-Import `Speechly/` folder from `speechly-client.unitypackage` that contains code to use Speechly cloud API.
+## Getting Started with Unity
+
+Import `Speechly/` folder from [speechly-client.unitypackage](https://github.com/speechly/speechly-unity-dotnet/raw/main/speechly-client.unitypackage) that contains code to use Speechly cloud API.
 
 > If you want to skip straight to trying out a working sample scene, see [more code examples](#more-code-examples) below.
 
@@ -43,15 +44,11 @@ public class AudioFileToSpeechly : MonoBehaviour
 
   async void Start()
   {
-    // Get your app id from https://api.speechly.com/dashboard
     client = new SpeechlyClient(
-      appId: "ef84e8ba-c5a7-46c2-856e-8b853e2c77b1", // Basic speech-to-text configuration
-      config: new SpeechlyConfig {
-        deviceId = SystemInfo.deviceUniqueIdentifier
-      },
-      manualUpdate: true
+      manualUpdate: true,
+      debug: true
     );
-    
+
     // Set the desired callbacks.
     // OnSegmentChange fires on any change and keeps a record of all words, intents and entities until the end of utterance is signaled with `segment.isFinal`.
     // It's the recommended way to read SLU results.
@@ -60,12 +57,19 @@ public class AudioFileToSpeechly : MonoBehaviour
       Debug.Log(segment.ToString());
     };
 
-    // Connect should be only done once
-    await client.Connect();
+    // Get your app id from https://api.speechly.com/dashboard
+    decoder = new CloudDecoder(
+      appId: "ef84e8ba-c5a7-46c2-856e-8b853e2c77b1", // Basic ASR
+      deviceId: Platform.GetDeviceId(SystemInfo.deviceUniqueIdentifier),
+      debug: true
+    );
+
+    // Connect to CloudDecoder
+    await SpeechlyClient.Initialize(decoder);
 
     // Send test audio. Callback(s) will fire and log the results.
     await client.StartContext();
-    await client.SendAudioFile("Assets/Speechly/00_chinese_restaurant.raw");
+    client.ProcessAudioFile("Assets/Speechly/00_chinese_restaurant.raw");
     await client.StopContext();
   }
 
@@ -81,25 +85,29 @@ public class AudioFileToSpeechly : MonoBehaviour
 
 ### MicToSpeechly
 
-Import [SpeechlyExamples/MicToSpeechly/](speechly-unity/Assets/SpeechlyExamples/MicToSpeechly/) and `Speechly/` folders from `speechly-client.unitypackage` to run a Unity sample scene that streams data from microphone to Speechly using [MicToSpeechly.cs](https://github.com/speechly/speechly-unity-dotnet/blob/main/speechly-unity/Assets/Speechly/MicToSpeechly.cs) script running on a GameObject. App-specific logic is in `UseSpeechly.cs` which registers a callback and shows speech-to-text results in the UI.
+Import [SpeechlyExamples/MicToSpeechly/](speechly-unity/Assets/SpeechlyExamples/MicToSpeechly/) and `Speechly/` folders from [speechly-client.unitypackage](https://github.com/speechly/speechly-unity-dotnet/raw/main/speechly-client.unitypackage) to run a Unity sample scene that streams data from microphone to Speechly using [MicToSpeechly.cs](https://github.com/speechly/speechly-unity-dotnet/blob/main/speechly-unity/Assets/Speechly/MicToSpeechly.cs) script running on a GameObject. App-specific logic is in `UseSpeechly.cs` which registers a callback and shows speech-to-text results in the UI.
 
 ### VoiceCommands
 
 Import [SpeechlyExamples/VoiceCommands/](speechly-unity/Assets/SpeechlyExamples/VoiceCommands/) and `Speechly/` folders from `speechly-client.unitypackage` to run a Unity sample scene that showcases a point-and-talk interface: target an object and hold the mouse button to issue speech commands like "make it big and red" or "delete". Again, app-specific logic is in `UseSpeechly.cs` which registers a callback to respond to detected intents and keywords (entities).
 
-## OS X
+## API Documentation
+
+- [API documentation (DocFX generated)](https://speechly.github.io/speechly-unity-dotnet/)
+
+## OS X notes
 
 To enable microphone input on OS X, set `Player Settings > Settings for PC, Mac & Linux Standalone > Other Settings > Microphone Usage Description`, to for example, "Voice input is automatically processed by Speechly.com".
 
-## Android
+## Android notes
 
 ### Device testing
 
 To diagnose problems with device builds, you can do the following:
 
 - First try running MicToSpeechlyScene.unity in the editor without errors.
 - Change to Android player, set MicToSpeechlyScene.unity as the main scene and do a `build and run` to deploy the build to on a device.
-- On terminal, do `adb logcat -s Unity:E` to follow Unity-related error logs from the device.
+- On terminal, do `adb logcat -s Unity:D` to follow Unity-related logs from the device.
 - Run the app on device. Keep `Hold to talk` button pressed and say "ONE, TWO, THREE". Then release the button.
 - You should see "ONE, TWO, THREE" displayed in the top-left corner of the screen. If not, see the terminal for errors.
 
@@ -117,4 +125,15 @@ To diagnose problems with device builds, you can do the following:
 
 ## Developing and contributing
 
-- `link-speechly-sources.sh` shell script will create hard links from `speechly-client-net-standard-2.0/Speechly/` to `speechly-unity/Assets/Speechly/` so shared .NET code in `SLUClient` is in sync. Please run the script after checking out the repo and before making any changes. If you can't use the script please ensure that the files are identical manually before opening a PR.
+We are happy to receive community contributions! For small fixes, feel free to file a pull request. For bigger changes or new features start by filing an issue.
+
+- `./link-speechly-sources.sh` shell script will create hard links from `speechly-client-net-standard-2.0/Speechly/` to `speechly-unity/Assets/Speechly/` so shared .NET code in `SLUClient` is in sync. Please run the script after checking out the repo and before making any changes. If you can't use the script please ensure that the files are identical manually before opening a PR.
+- `./build-docs.sh` generates public API documentation using DocFX from triple-slash `///` comments with C# XML documentation tags.
+
+### Command line usage with `dotnet`
+
+SpeechlyClient features can be run with prerecorded audio on the command line in `speechly-client-net-standard-2.0/` folder:
+
+- `dotnet run test` processes an example file, sends to Speechly cloud SLU and prints the received results in console.
+- `dotnet run vad` processes an example file, sends the utterances audio to files in `temp/` folder as 16 bit raw and creates an utterance timestamp `.tsv` (tab-separated values) for each audio file processed.
+- `dotnet run vad myaudiofiles/*.raw` processes a set of files with VAD.

api/.manifest

@@ -0,0 +1,142 @@
+{
+  "Speechly.SLUClient": "Speechly.SLUClient.yml",
+  "Speechly.SLUClient.CloudDecoder": "Speechly.SLUClient.CloudDecoder.yml",
+  "Speechly.SLUClient.CloudDecoder.#ctor(System.String,System.String,System.String,System.String,System.Boolean)": "Speechly.SLUClient.CloudDecoder.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.BaselineEnergy": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.Enabled": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.Energy": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.IsSignalDetected": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.ProcessFrame(System.Single[],System.Int32,System.Int32)": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.VADActivation": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.VADControlListening": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.VADFrames": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.VADMinimumEnergy": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.VADNoiseHalftimeMillis": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.VADRelease": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.VADSignalToNoise": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.EnergyTresholdVAD.VADSustainMillis": "Speechly.SLUClient.EnergyTresholdVAD.yml",
+  "Speechly.SLUClient.IDecoder": "Speechly.SLUClient.IDecoder.yml",
+  "Speechly.SLUClient.Segment": "Speechly.SLUClient.Segment.yml",
+  "Speechly.SLUClient.Segment.#ctor(System.String,System.Int32)": "Speechly.SLUClient.Segment.yml",
+  "Speechly.SLUClient.Segment.contextId": "Speechly.SLUClient.Segment.yml",
+  "Speechly.SLUClient.Segment.entities": "Speechly.SLUClient.Segment.yml",
+  "Speechly.SLUClient.Segment.id": "Speechly.SLUClient.Segment.yml",
+  "Speechly.SLUClient.Segment.intent": "Speechly.SLUClient.Segment.yml",
+  "Speechly.SLUClient.Segment.isFinal": "Speechly.SLUClient.Segment.yml",
+  "Speechly.SLUClient.Segment.ToString": "Speechly.SLUClient.Segment.yml",
+  "Speechly.SLUClient.Segment.ToString(Speechly.Types.BeautifyIntent,Speechly.Types.BeautifyEntity,System.String)": "Speechly.SLUClient.Segment.yml",
+  "Speechly.SLUClient.Segment.words": "Speechly.SLUClient.Segment.yml",
+  "Speechly.SLUClient.SpeechlyClient": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.#ctor(System.Int32,System.Int32,System.Int32,System.Boolean,System.String,Speechly.SLUClient.EnergyTresholdVAD,System.Boolean)": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.AudioInputStreamIdentifier": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.Initialize(Speechly.SLUClient.IDecoder)": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.IsActive": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.IsAudioStreaming": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.IsReady": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnEntity": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnIntent": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnSegmentChange": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnStartContext": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnStartStream": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnStopContext": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnStopStream": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnTentativeEntity": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnTentativeIntent": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnTentativeTranscript": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.OnTranscript": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.ProcessAudio(Stream)": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.ProcessAudio(System.Single[],System.Int32,System.Int32,System.Boolean)": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.ProcessAudioFile(System.String)": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.SamplesSent": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.Shutdown": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.StartContext(System.String)": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.StartStream(System.String,System.Boolean)": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.StopContext": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.StopStream(System.Boolean)": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.StreamSamplePos": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.Update": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.UtteranceSerial": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.SLUClient.SpeechlyClient.Vad": "Speechly.SLUClient.SpeechlyClient.yml",
+  "Speechly.Tools": "Speechly.Tools.yml",
+  "Speechly.Tools.AudioTools": "Speechly.Tools.AudioTools.yml",
+  "Speechly.Tools.AudioTools.ConvertInt16ToFloat(System.Byte[]@,System.Single[]@,System.Int32,System.Int32,System.Int32)": "Speechly.Tools.AudioTools.yml",
+  "Speechly.Tools.AudioTools.Downsample(System.Single[]@,System.Single[]@,System.Int32,System.Int32,System.Int32,System.Int32)": "Speechly.Tools.AudioTools.yml",
+  "Speechly.Tools.AudioTools.GetAudioPeak(System.Single[]@,System.Int32,System.Int32)": "Speechly.Tools.AudioTools.yml",
+  "Speechly.Tools.AudioTools.GetEnergy(System.Single[]@,System.Int32,System.Int32)": "Speechly.Tools.AudioTools.yml",
+  "Speechly.Tools.JSON": "Speechly.Tools.JSON.yml",
+  "Speechly.Tools.JSON.Parse``1(System.String,``0)": "Speechly.Tools.JSON.yml",
+  "Speechly.Tools.JSON.ParseFromStream``1(Stream,``0)": "Speechly.Tools.JSON.yml",
+  "Speechly.Tools.JSON.Stringify``1(``0)": "Speechly.Tools.JSON.yml",
+  "Speechly.Tools.JSON.StringifyToStream``1(``0)": "Speechly.Tools.JSON.yml",
+  "Speechly.Tools.Logger": "Speechly.Tools.Logger.yml",
+  "Speechly.Tools.Logger.Log": "Speechly.Tools.Logger.yml",
+  "Speechly.Tools.Logger.LogError": "Speechly.Tools.Logger.yml",
+  "Speechly.Tools.Logger.LoggerDelegate": "Speechly.Tools.Logger.LoggerDelegate.yml",
+  "Speechly.Tools.Platform": "Speechly.Tools.Platform.yml",
+  "Speechly.Tools.Platform.ConfigPath": "Speechly.Tools.Platform.yml",
+  "Speechly.Tools.Platform.Fetch(System.String)": "Speechly.Tools.Platform.yml",
+  "Speechly.Tools.Platform.GetDeviceId(System.String)": "Speechly.Tools.Platform.yml",
+  "Speechly.Tools.Platform.GetPersistentStoragePath": "Speechly.Tools.Platform.yml",
+  "Speechly.Tools.Platform.GuidFromString(System.String)": "Speechly.Tools.Platform.yml",
+  "Speechly.Tools.Platform.RestoreOrCreateConfig``1(System.String)": "Speechly.Tools.Platform.yml",
+  "Speechly.Tools.Platform.SaveConfig``1(``0,System.String)": "Speechly.Tools.Platform.yml",
+  "Speechly.Tools.WsClient": "Speechly.Tools.WsClient.yml",
+  "Speechly.Tools.WsClient.ConnectAsync(System.String,System.String)": "Speechly.Tools.WsClient.yml",
+  "Speechly.Tools.WsClient.DisconnectAsync": "Speechly.Tools.WsClient.yml",
+  "Speechly.Tools.WsClient.Dispose": "Speechly.Tools.WsClient.yml",
+  "Speechly.Tools.WsClient.OnResponseReceived": "Speechly.Tools.WsClient.yml",
+  "Speechly.Tools.WsClient.ReceiveBufferSize": "Speechly.Tools.WsClient.yml",
+  "Speechly.Tools.WsClient.ResponseReceivedDelegate": "Speechly.Tools.WsClient.ResponseReceivedDelegate.yml",
+  "Speechly.Tools.WsClient.SendBytes(ArraySegment{System.Byte})": "Speechly.Tools.WsClient.yml",
+  "Speechly.Tools.WsClient.SendText(System.String)": "Speechly.Tools.WsClient.yml",
+  "Speechly.Types": "Speechly.Types.yml",
+  "Speechly.Types.BeautifyEntity": "Speechly.Types.BeautifyEntity.yml",
+  "Speechly.Types.BeautifyIntent": "Speechly.Types.BeautifyIntent.yml",
+  "Speechly.Types.Entity": "Speechly.Types.Entity.yml",
+  "Speechly.Types.Entity.endPosition": "Speechly.Types.Entity.yml",
+  "Speechly.Types.Entity.isFinal": "Speechly.Types.Entity.yml",
+  "Speechly.Types.Entity.startPosition": "Speechly.Types.Entity.yml",
+  "Speechly.Types.Entity.type": "Speechly.Types.Entity.yml",
+  "Speechly.Types.Entity.value": "Speechly.Types.Entity.yml",
+  "Speechly.Types.EntityDelegate": "Speechly.Types.EntityDelegate.yml",
+  "Speechly.Types.Intent": "Speechly.Types.Intent.yml",
+  "Speechly.Types.Intent.intent": "Speechly.Types.Intent.yml",
+  "Speechly.Types.Intent.isFinal": "Speechly.Types.Intent.yml",
+  "Speechly.Types.IntentDelegate": "Speechly.Types.IntentDelegate.yml",
+  "Speechly.Types.MsgCommon": "Speechly.Types.MsgCommon.yml",
+  "Speechly.Types.MsgCommon.audio_context": "Speechly.Types.MsgCommon.yml",
+  "Speechly.Types.MsgCommon.segment_id": "Speechly.Types.MsgCommon.yml",
+  "Speechly.Types.MsgCommon.type": "Speechly.Types.MsgCommon.yml",
+  "Speechly.Types.MsgEntity": "Speechly.Types.MsgEntity.yml",
+  "Speechly.Types.MsgEntity.data": "Speechly.Types.MsgEntity.yml",
+  "Speechly.Types.MsgIntent": "Speechly.Types.MsgIntent.yml",
+  "Speechly.Types.MsgIntent.data": "Speechly.Types.MsgIntent.yml",
+  "Speechly.Types.MsgIntent.Data": "Speechly.Types.MsgIntent.Data.yml",
+  "Speechly.Types.MsgIntent.Data.intent": "Speechly.Types.MsgIntent.Data.yml",
+  "Speechly.Types.MsgTentativeEntity": "Speechly.Types.MsgTentativeEntity.yml",
+  "Speechly.Types.MsgTentativeEntity.data": "Speechly.Types.MsgTentativeEntity.yml",
+  "Speechly.Types.MsgTentativeEntity.Data": "Speechly.Types.MsgTentativeEntity.Data.yml",
+  "Speechly.Types.MsgTentativeEntity.Data.entities": "Speechly.Types.MsgTentativeEntity.Data.yml",
+  "Speechly.Types.MsgTentativeTranscript": "Speechly.Types.MsgTentativeTranscript.yml",
+  "Speechly.Types.MsgTentativeTranscript.data": "Speechly.Types.MsgTentativeTranscript.yml",
+  "Speechly.Types.MsgTentativeTranscript.Data": "Speechly.Types.MsgTentativeTranscript.Data.yml",
+  "Speechly.Types.MsgTentativeTranscript.Data.transcript": "Speechly.Types.MsgTentativeTranscript.Data.yml",
+  "Speechly.Types.MsgTentativeTranscript.Data.words": "Speechly.Types.MsgTentativeTranscript.Data.yml",
+  "Speechly.Types.MsgTranscript": "Speechly.Types.MsgTranscript.yml",
+  "Speechly.Types.MsgTranscript.data": "Speechly.Types.MsgTranscript.yml",
+  "Speechly.Types.SegmentChangeDelegate": "Speechly.Types.SegmentChangeDelegate.yml",
+  "Speechly.Types.StartContextDelegate": "Speechly.Types.StartContextDelegate.yml",
+  "Speechly.Types.StartStreamDelegate": "Speechly.Types.StartStreamDelegate.yml",
+  "Speechly.Types.StopContextDelegate": "Speechly.Types.StopContextDelegate.yml",
+  "Speechly.Types.StopStreamDelegate": "Speechly.Types.StopStreamDelegate.yml",
+  "Speechly.Types.TentativeEntityDelegate": "Speechly.Types.TentativeEntityDelegate.yml",
+  "Speechly.Types.TentativeTranscriptDelegate": "Speechly.Types.TentativeTranscriptDelegate.yml",
+  "Speechly.Types.TranscriptDelegate": "Speechly.Types.TranscriptDelegate.yml",
+  "Speechly.Types.Word": "Speechly.Types.Word.yml",
+  "Speechly.Types.Word.endTimestamp": "Speechly.Types.Word.yml",
+  "Speechly.Types.Word.index": "Speechly.Types.Word.yml",
+  "Speechly.Types.Word.isFinal": "Speechly.Types.Word.yml",
+  "Speechly.Types.Word.startTimestamp": "Speechly.Types.Word.yml",
+  "Speechly.Types.Word.word": "Speechly.Types.Word.yml"
+}