diff --git a/tensorflow_asr/models/transducer.py b/tensorflow_asr/models/transducer.py
index bfc901157d..07a9c78855 100755
--- a/tensorflow_asr/models/transducer.py
+++ b/tensorflow_asr/models/transducer.py
@@ -56,7 +56,7 @@ def __init__(self,
                 name=f"{name}_{rnn_type}_{i}", return_state=True,
                 implementation=rnn_implementation,
                 kernel_regularizer=kernel_regularizer,
-                bias_regularizer=bias_regularizer
+                bias_regularizer=bias_regularizer, unroll=True
             )
             if layer_norm:
                 ln = tf.keras.layers.LayerNormalization(name=f"{name}_ln_{i}")