Respect the maximum number of tokens in interactive. (#298)

tjohnman · Johnman · ggerganov · web-flow · commit 368d0c8a9eba · 2023-03-19T20:31:17.000+02:00
Co-authored-by: Johnman &lt;johnman@github&gt;
Co-authored-by: Georgi Gerganov &lt;ggerganov@gmail.com&gt;
diff --git a/main.cpp b/main.cpp
@@ -1062,7 +1062,6 @@ int main(int argc, char ** argv) {
         }
 
         // end of text token
-
         if (embd.back() == EOS_TOKEN_ID) {
             if (params.interactive) {
                 is_interacting = true;
@@ -1071,6 +1070,12 @@ int main(int argc, char ** argv) {
                 break;
             }
         }
+
+        // In interactive mode, respect the maximum number of tokens and drop back to user input when reached.
+        if (params.interactive && remaining_tokens <= 0) {
+            remaining_tokens = params.n_predict;
+            is_interacting = true;
+        }
     }
 
 #if defined (_WIN32)

Original file line number	Diff line number	Diff line change
`@@ -1062,7 +1062,6 @@ int main(int argc, char ** argv) {`
`1062`	`1062`	`}`
`1063`	`1063`
`1064`	`1064`	`// end of text token`
`1065`		`-`
`1066`	`1065`	`if (embd.back() == EOS_TOKEN_ID) {`
`1067`	`1066`	`if (params.interactive) {`
`1068`	`1067`	`is_interacting = true;`
`@@ -1071,6 +1070,12 @@ int main(int argc, char ** argv) {`
`1071`	`1070`	`break;`
`1072`	`1071`	`}`
`1073`	`1072`	`}`
	`1073`	`+`
	`1074`	`+ // In interactive mode, respect the maximum number of tokens and drop back to user input when reached.`
	`1075`	`+ if (params.interactive && remaining_tokens <= 0) {`
	`1076`	`+ remaining_tokens = params.n_predict;`
	`1077`	`+ is_interacting = true;`
	`1078`	`+ }`
`1074`	`1079`	`}`
`1075`	`1080`
`1076`	`1081`	`#if defined (_WIN32)`