Merge pull request ggml-org#28 from SlyEcho/master

karpathy · web-flow · commit 228c4ea3ea70 · 2023-07-24T07:23:07.000-07:00
Fix tokenizer reading on Windows
diff --git a/run.c b/run.c
@@ -425,7 +425,7 @@ int main(int argc, char *argv[]) {
     // read in the tokenizer.bin file
     char** vocab = (char**)malloc(config.vocab_size * sizeof(char*));
     {
-        FILE *file = fopen("tokenizer.bin", "r");
+        FILE *file = fopen("tokenizer.bin", "rb");
         if (!file) {
             printf("Unable to open the tokenizer file tokenizer.bin! Run "
             "python tokenizer.py to convert tokenizer.model -> tokenizer.bin\n");

Original file line number	Diff line number	Diff line change
`@@ -425,7 +425,7 @@ int main(int argc, char *argv[]) {`
`425`	`425`	`// read in the tokenizer.bin file`
`426`	`426`	`char vocab = (char)malloc(config.vocab_size * sizeof(char*));`
`427`	`427`	`{`
`428`		`- FILE *file = fopen("tokenizer.bin", "r");`
	`428`	`+ FILE *file = fopen("tokenizer.bin", "rb");`
`429`	`429`	`if (!file) {`
`430`	`430`	`printf("Unable to open the tokenizer file tokenizer.bin! Run "`
`431`	`431`	`"python tokenizer.py to convert tokenizer.model -> tokenizer.bin\n");`