diff --git a/whisper/transcribe.py b/whisper/transcribe.py
index 84feb12a..cba59ecf 100644
--- a/whisper/transcribe.py
+++ b/whisper/transcribe.py
@@ -174,7 +174,11 @@ def decode_with_fallback(segment: torch.Tensor) -> DecodingResult:
                 and decode_result.avg_logprob < logprob_threshold
             ):
                 needs_fallback = True  # average log probability is too low
-
+            if (
+                no_speech_threshold is not None
+                and decode_result.no_speech_prob > no_speech_threshold
+            ):
+                needs_fallback = False  # silence
             if not needs_fallback:
                 break