diff --git a/lm_eval/models/openai_completions.py b/lm_eval/models/openai_completions.py
index c50ae63b8f..a977725615 100644
--- a/lm_eval/models/openai_completions.py
+++ b/lm_eval/models/openai_completions.py
@@ -281,7 +281,7 @@ def sameuntil_chunks(xs, size):
                 **{
                     k: v
                     for k, v in request_args.items()
-                    if k not in ["do_sample", "max_gen_toks"]
+                    if k not in {"do_sample", "max_gen_toks", "until"}
                 },
             )
             for resp, (context, args_) in zip(response.choices, chunk):