Rename third_party folder to Avoid Copybara g3 Errors (#148)

* rename third_party * fix ut * fix ut
AI-Hypercomputer · Nov 8, 2024 · 15e3963 · 15e3963
1 parent 52d63a5
commit 15e3963
Show file tree

Hide file tree

Showing 10 changed files with 6 additions and 6 deletions.
diff --git a/Makefile b/Makefile
@@ -51,4 +51,4 @@ unit-tests:
 	coverage run -m unittest -v
 
 check-test-coverage:
-	coverage report -m --omit="jetstream/core/proto/*,jetstream/engine/tokenizer_pb2.py,jetstream/third_party/*" --fail-under=96
+	coverage report -m --omit="jetstream/core/proto/*,jetstream/engine/tokenizer_pb2.py,jetstream/external_tokenizers/*" --fail-under=96
diff --git a/benchmarks/benchmark_serving.py b/benchmarks/benchmark_serving.py
@@ -73,7 +73,7 @@
 from jetstream.core.proto import jetstream_pb2
 from jetstream.core.proto import jetstream_pb2_grpc
 from jetstream.engine.token_utils import load_vocab
-from jetstream.third_party.llama3 import llama3_tokenizer
+from jetstream.external_tokenizers.llama3 import llama3_tokenizer
 import numpy as np
 from tqdm.asyncio import tqdm  # pytype: disable=pyi-error
 import pandas

diff --git a/jetstream/engine/token_utils.py b/jetstream/engine/token_utils.py
@@ -28,7 +28,7 @@
 from jetstream.engine import mock_utils
 from jetstream.engine import tokenizer_api
 from jetstream.engine import tokenizer_pb2
-from jetstream.third_party.llama3 import llama3_tokenizer
+from jetstream.external_tokenizers.llama3 import llama3_tokenizer
 
 # ResultToken class to store tokens ids.
 ResultTokens = Any

diff --git a/jetstream/third_party/__init__.py → jetstream/external_tokenizers/__init__.py b/jetstream/third_party/__init__.py → jetstream/external_tokenizers/__init__.py
diff --git a/jetstream/third_party/llama3/__init__.py → ...am/external_tokenizers/llama3/__init__.py b/jetstream/third_party/llama3/__init__.py → ...am/external_tokenizers/llama3/__init__.py
diff --git a/...am/third_party/llama3/llama3_tokenizer.py → ...nal_tokenizers/llama3/llama3_tokenizer.py b/...am/third_party/llama3/llama3_tokenizer.py → ...nal_tokenizers/llama3/llama3_tokenizer.py
diff --git a/...engine/third_party/llama2/tokenizer.model → ...xternal_tokenizers/llama2/tokenizer.model b/...engine/third_party/llama2/tokenizer.model → ...xternal_tokenizers/llama2/tokenizer.model
diff --git a/...engine/third_party/llama3/tokenizer.model → ...xternal_tokenizers/llama3/tokenizer.model b/...engine/third_party/llama3/tokenizer.model → ...xternal_tokenizers/llama3/tokenizer.model
diff --git a/jetstream/tests/engine/test_token_utils.py b/jetstream/tests/engine/test_token_utils.py
@@ -55,7 +55,7 @@ def decode(self, t: int) -> str:
 class TokenUtilsTest(unittest.TestCase):
 
   def setup_sentencepiece(self):
-    self.tokenizer_path = "third_party/llama2/tokenizer.model"
+    self.tokenizer_path = "external_tokenizers/llama2/tokenizer.model"
     current_dir = os.path.dirname(__file__)
     self.tokenizer_path = os.path.join(current_dir, self.tokenizer_path)
     print(f"model_path: {self.tokenizer_path}")
@@ -66,7 +66,7 @@ def setup_sentencepiece(self):
     self.jt_tokenizer = JetStreamTokenizer(self.tokenizer_path)
 
   def setup_tiktoken(self):
-    self.tokenizer_path = "third_party/llama3/tokenizer.model"
+    self.tokenizer_path = "external_tokenizers/llama3/tokenizer.model"
     current_dir = os.path.dirname(__file__)
     self.tokenizer_path = os.path.join(current_dir, self.tokenizer_path)
     print(f"model_path: {self.tokenizer_path}")

diff --git a/pylintrc b/pylintrc
@@ -9,7 +9,7 @@
 [MAIN]
 
 # Files or directories to be skipped. They should be base names, not paths.
-ignore=third_party
+ignore=external_tokenizers
 
 # Files or directories matching the regex patterns are skipped. The regex
 # matches against base names, not paths.