debug

.github/workflows/tests.yml

@@ -3,7 +3,7 @@ name: Tests
 on:
   push:
     branches: [main]
-  pull_request_target:
+  pull_request:
   workflow_dispatch:
 
 env:
@@ -14,137 +14,15 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  authorize:
-    environment: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository && 'external' || 'internal' }}
-    runs-on: ubuntu-latest
-    steps:
-      - run: true
-
-  lint:
-    needs: authorize
-
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.event.pull_request.head.sha || github.ref }}
-
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.9'
-
-      - name: Setup uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: true
-          cache-suffix: lint
-          cache-dependency-glob: pyproject.toml
-
-      - name: Install nox
-        run: uv pip install nox --system
-
-      - name: Cache mypy
-        uses: actions/cache@v4
-        with:
-          path: .mypy_cache
-          key: mypy-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
-
-      - name: Cache pre-commit hooks
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pre-commit
-          key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
-
-      - name: Lint code
-        run: nox -s lint
-
-  datachain:
-    needs: authorize
-
-    timeout-minutes: 40
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest-8-cores]
-        pyv: ['3.9', '3.10', '3.11', '3.12']
-        include:
-          - os: macos-latest
-            pyv: '3.9'
-          - os: macos-latest
-            pyv: '3.12'
-          - os: windows-latest
-            pyv: '3.9'
-          - os: windows-latest
-            pyv: '3.12'
-
-    steps:
-      - name: Check out the repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: ${{ github.event.pull_request.head.sha || github.ref }}
-
-      - name: Set up Python ${{ matrix.pyv }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.pyv }}
-
-      - name: Setup uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: true
-          cache-suffix: tests-${{ matrix.pyv }}
-          cache-dependency-glob: pyproject.toml
-
-      - name: Install nox
-        run: uv pip install nox --system
-
-      - name: Skip flaky azure, gs remotes on macOS
-        if: runner.os == 'macOS'
-        run: echo 'DISABLE_REMOTES_ARG=--disable-remotes=azure,gs' >> "$GITHUB_ENV"
-
-      - name: Skip all remotes on Windows
-        if: runner.os == 'Windows'
-        run: echo 'DISABLE_REMOTES_ARG=--disable-remotes=azure,gs' >> $env:GITHUB_ENV
-
-      - name: Run tests
-        run: nox -s tests-${{ matrix.pyv }} -- $DISABLE_REMOTES_ARG
-        shell: bash
-
-      - name: Upload coverage report
-        uses: codecov/codecov-action@v5
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: coverage.xml
-          flags: datachain
-
-      - name: Build package
-        run: nox -s build
-
-      - name: Build docs
-        run: nox -s docs
-
   examples:
-    needs: authorize
-
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest]
-        pyv: ['3.9', '3.12']
-        group: ['get_started', 'llm_and_nlp or computer_vision', 'multimodal']
-        exclude:
-          - {os: ubuntu-latest, pyv: '3.9', group: 'multimodal'}
-          - {os: ubuntu-latest, pyv: '3.12', group: 'multimodal'}
-        include:
-          - {os: ubuntu-latest-4-cores, pyv: "3.9", group: multimodal}
-          - {os: ubuntu-latest-4-cores, pyv: "3.12", group: multimodal}
+        os: [ubuntu-latest]
+        pyv: ['3.9']
+        group: ['llm_and_nlp']
 
     steps:
       - uses: actions/checkout@v4
@@ -166,20 +44,5 @@ jobs:
       - name: Install nox
         run: uv pip install nox --system
 
-      # HF runs against actual API - thus run it only once
-      - name: Set hf token
-        if: matrix.os == 'ubuntu-latest' && matrix.pyv == '3.12'
-        run: echo 'HF_TOKEN=${{ secrets.HF_TOKEN }}' >> "$GITHUB_ENV"
-
       - name: Run examples
         run: nox -s examples -p ${{ matrix.pyv }} -- -m "${{ matrix.group }}"
-
-  check:
-    if: always()
-    needs: [lint, datachain, examples]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: re-actors/alls-green@release/v1
-        with:
-          allowed-failures: examples
-          jobs: ${{ toJSON(needs) }}

examples/llm_and_nlp/unstructured-summary-map.py

@@ -22,10 +22,13 @@
 
 
 def partition_object(file):
+    with file.open() as f:
+        print("no. of chars", len(f.read()))
     with file.open() as raw:
         elements = partition(
             file=raw, metadata_filename=file.name, strategy=PartitionStrategy.FAST
         )
+        print(f"{elements=}")
     title = str(elements[0])
     text = "\n\n".join([str(el) for el in elements])
     df = convert_to_dataframe(elements)

noxfile.py

@@ -78,7 +78,8 @@ def dev(session: nox.Session) -> None:
 
 @nox.session(python=["3.9", "3.10", "3.11", "3.12", "pypy3.9", "pypy3.10"])
 def examples(session: nox.Session) -> None:
-    session.install(".[examples]")
+    session.install(".[examples]", "--exclude-newer=2024-12-09")
+    session.run("uv", "pip", "freeze")
     session.run(
         "pytest",
         "-m",

pyproject.toml

@@ -111,7 +111,8 @@ examples = [
   "pdfplumber==0.11.4",
   "huggingface_hub[hf_transfer]",
   "onnx==1.16.1",
-  "ultralytics==8.3.37"
+  "ultralytics==8.3.37",
+  "scikit-learn<1.6"
 ]
 
 [project.urls]

tests/examples/test_examples.py

@@ -15,7 +15,7 @@
     ]
 )
 
-llm_and_nlp_examples = sorted(glob.glob("examples/llm_and_nlp/**/*.py", recursive=True))
+llm_and_nlp_examples = ["examples/llm_and_nlp/unstructured-summary-map.py"]
 
 multimodal_examples = sorted(glob.glob("examples/multimodal/**/*.py", recursive=True))