add download tiktoken

langchain-ai · Sep 27, 2024 · 1fcf4b2 · 1fcf4b2
1 parent e359a42
commit 1fcf4b2
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 0 deletions.
diff --git a/.github/workflows/run_notebooks.yml b/.github/workflows/run_notebooks.yml
@@ -32,6 +32,10 @@ jobs:
       - name: Start services
         run: make start-services
 
+      - name: Pre-download tiktoken files
+        run: |
+          poetry run python docs/scripts/download_tiktoken.py
+
       - name: Prepare notebooks
         run: |
           if [ "${{ matrix.lib-version }}" = "development" ]; then

diff --git a/docs/_scripts/download_tiktoken.py b/docs/_scripts/download_tiktoken.py
@@ -0,0 +1,4 @@
+import tiktoken
+
+# This will trigger the download and caching of the necessary files
+encoding = tiktoken.encoding_for_model("gpt2")