Merge pull request #1 from speechly/fix/segment-modification-race

Optional energy-based noise gate to control listening on/off "hands free".

Author: arzga

speechly-client-net-standard-2.0/Speechly/SLUClient/Segment.cs

@@ -34,99 +34,114 @@ public string ToString(
         string confirmationMark
       ) {
       var sb = new StringBuilder();
-      var entityIds = new string[words.Length];
-      foreach (KeyValuePair<string, Entity> entity in entities) {
-        for (var i = entity.Value.startPosition; i < entity.Value.endPosition; i++) {
-          entityIds[i] = entity.Key;
+
+      lock(this) {
+        var entityIds = new string[words.Length];
+        foreach (KeyValuePair<string, Entity> entity in entities) {
+          for (var i = entity.Value.startPosition; i < entity.Value.endPosition; i++) {
+            entityIds[i] = entity.Key;
+          }
         }
-      }
 
-      sb.Append(intentTag(this.intent.intent));
-      
-      bool firstWord = sb.Length == 0;
-      string lastEntityId = null;
+        sb.Append(intentTag(this.intent.intent));
+        
+        bool firstWord = sb.Length == 0;
+        string lastEntityId = null;
 
-      for (int i = 0; i < words.Length; i++) {
-        var word = words[i];
-        if (word == null) continue;
+        for (int i = 0; i < words.Length; i++) {
+          var word = words[i];
+          if (word == null) continue;
 
 
-        if (entityIds[i] != lastEntityId) {
-          if (lastEntityId != null) {
-            var entity = entities[lastEntityId];
+          if (entityIds[i] != lastEntityId) {
+            if (lastEntityId != null) {
+              var entity = entities[lastEntityId];
+              if (!firstWord) sb.Append(" ");
+              sb.Append(entityTag(entity.value, entity.type));
+              firstWord = false;
+            }
+          }
+
+          if (entityIds[i] == null) {
             if (!firstWord) sb.Append(" ");
-            sb.Append(entityTag(entity.value, entity.type));
+            sb.Append(word.word);
             firstWord = false;
           }
+
+          lastEntityId = entityIds[i];
         }
 
-        if (entityIds[i] == null) {
+        if (lastEntityId != null) {
           if (!firstWord) sb.Append(" ");
-          sb.Append(word.word);
-          firstWord = false;
+          var entity = entities[lastEntityId];
+          sb.Append(entityTag(entity.value, entity.type));
         }
 
-        lastEntityId = entityIds[i];
-      }
-
-      if (lastEntityId != null) {
-        if (!firstWord) sb.Append(" ");
-        var entity = entities[lastEntityId];
-        sb.Append(entityTag(entity.value, entity.type));
+        if (this.isFinal) sb.Append(confirmationMark);
       }
 
-      if (this.isFinal) sb.Append(confirmationMark);
-
       return sb.ToString();
     }
 
     public void UpdateTranscript(Word word) {
-      if (word.index >= words.Length) {
-        Array.Resize(ref words, word.index + 1);
+      lock(this) {
+        if (word.index >= words.Length) {
+          Array.Resize(ref words, word.index + 1);
+        }
+        words[word.index] = word;
       }
-      words[word.index] = word;
     }
 
     public void UpdateTranscript(Word[] words) {
-      foreach(Word w in words) UpdateTranscript(w);
+      lock(this) {
+        foreach(Word w in words) UpdateTranscript(w);
+      }
     }
 
     public void UpdateEntity(Entity entity) {
-      entities[EntityMapKey(entity)] = entity;
+      lock(this) {
+        entities[EntityMapKey(entity)] = entity;
+      }
     }
 
     public void UpdateEntity(Entity[] entities) {
-      foreach(Entity entity in entities) UpdateEntity(entity);
+      lock(this) {
+        foreach(Entity entity in entities) UpdateEntity(entity);
+      }
     }
 
     public void UpdateIntent(string intent, bool isFinal) {
-      this.intent.intent = intent;
-      this.intent.isFinal = isFinal;
+      lock(this) {
+        this.intent.intent = intent;
+        this.intent.isFinal = isFinal;
+      }
     }
 
     public void EndSegment() {
-      // Filter away any entities which were not finalized.
-      foreach (KeyValuePair<string, Entity> entity in entities) {
-        if (!entity.Value.isFinal) {
-          this.entities.Remove(entity.Key);
+      lock(this) {
+        // Filter away any entities which were not finalized.
+        foreach (KeyValuePair<string, Entity> entity in entities) {
+          if (!entity.Value.isFinal) {
+            this.entities.Remove(entity.Key);
+          }
         }
-      }
 
-      // Filter away any transcripts which were not finalized. Keep indices intact.
-      for (int i = 0; i < words.Length; i++) {
-        if (words[i] != null) {
-          if (!words[i].isFinal) {
-            words[i] = null;
+        // Filter away any transcripts which were not finalized. Keep indices intact.
+        for (int i = 0; i < words.Length; i++) {
+          if (words[i] != null) {
+            if (!words[i].isFinal) {
+              words[i] = null;
+            }
           }
         }
-      }
 
-      if (!this.intent.isFinal) {
-        this.intent.intent = "";
-        this.intent.isFinal = true;
+        if (!this.intent.isFinal) {
+          this.intent.intent = "";
+          this.intent.isFinal = true;
+        }
+        // Mark as final.
+        this.isFinal = true;
       }
-      // Mark as final.
-      this.isFinal = true;
     }
 
     private string EntityMapKey(Entity e) {

speechly-client-net-standard-2.0/Speechly/SLUClient/SpeechlyClient.cs

@@ -8,7 +8,7 @@
 namespace Speechly.SLUClient {
 
   public class SpeechlyClient {
-    public static bool DEBUG_SAVE_AUDIO = false;
+    public static bool DEBUG_SAVE_AUDIO = true;
 
     public delegate void SegmentChangeDelegate(Segment segment);
     public delegate void StateChangeDelegate(ClientState state);
@@ -27,6 +27,7 @@ public class SpeechlyClient {
     public StateChangeDelegate OnStateChange = (ClientState state) => {};
 
     public bool IsListening { get; private set; } = false;
+    public int SamplesSent { get; private set; } = 0;
     public ClientState State { get; private set; } = ClientState.Disconnected;
     // Optional message queue should messages be run in the main thread
     private ConcurrentQueue<string> messageQueue = new ConcurrentQueue<string>();
@@ -122,6 +123,7 @@ public async Task<string> StartContext(string appId = null) {
         if (DEBUG_SAVE_AUDIO) {
           debugAudioStream = new FileStream($"utterance_{contextId}.raw", FileMode.CreateNew, FileAccess.Write, FileShare.None);
         }
+        SamplesSent = 0;
         SetState(ClientState.Recording);
         return contextId;
       } catch (Exception e) {
@@ -143,6 +145,7 @@ public async Task SendAudio(Stream fileStream) {
         if (DEBUG_SAVE_AUDIO) {
           debugAudioStream.Write(b, 0, bytesRead);
         }
+        SamplesSent += bytesRead / 2;
         await wsClient.SendBytes(new ArraySegment<byte>(b, 0, bytesRead));
       }
     }
@@ -166,6 +169,7 @@ public async Task SendAudio(float[] floats, int start = 0, int end = -1) {
         debugAudioStream.Write(buf, 0, buf.Length);
       }
 
+      SamplesSent += bufSize;
       await wsClient.SendBytes(new ArraySegment<byte>(buf));
     }
 
@@ -245,8 +249,9 @@ private void OnResponse(string msgString)
             break;
           }
         }
-      } catch (Exception e) {
-        throw new Exception($"Ouch. {e.GetType()} while handling message with content: {msgString}");
+      } catch {
+        Logger.LogError($"Error while handling message with content: {msgString}");
+        throw;
       }
     }