Add more logging for missing argument for tokenizer artifact (pytorch#934)

larryliu0820 · web-flow · commit ef5f365aa5df · 2024-07-20T21:09:02.000-07:00
Summary:

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:
diff --git a/tokenizer/sentencepiece.cpp b/tokenizer/sentencepiece.cpp
@@ -38,7 +38,7 @@ void SPTokenizer::load(const std::string& tokenizer_path) {
   // read in the file
   const auto status = _processor->Load(tokenizer_path);
   if (!status.ok()) {
-    fprintf(stderr, "couldn't load %s\n", tokenizer_path.c_str());
+    fprintf(stderr, "couldn't load %s\n. If this tokenizer artifact is for llama3, please pass `-l 3`.", tokenizer_path.c_str());
     exit(EXIT_FAILURE);
   }
   // load vocab_size, bos_tok, eos_tok

Original file line number	Diff line number	Diff line change
`@@ -38,7 +38,7 @@ void SPTokenizer::load(const std::string& tokenizer_path) {`
`38`	`38`	`// read in the file`
`39`	`39`	`const auto status = _processor->Load(tokenizer_path);`
`40`	`40`	`if (!status.ok()) {`
`41`		`- fprintf(stderr, "couldn't load %s\n", tokenizer_path.c_str());`
	`41`	+ fprintf(stderr, "couldn't load %s\n. If this tokenizer artifact is for llama3, please pass `-l 3`.", tokenizer_path.c_str());
`42`	`42`	`exit(EXIT_FAILURE);`
`43`	`43`	`}`
`44`	`44`	`// load vocab_size, bos_tok, eos_tok`