Merge branch 'main' into update-spice-remove-cost-tracker

.github/workflows/benchmarks.yml

@@ -20,9 +20,6 @@ jobs:
  python -m pip install .
  pip install -r dev-requirements.txt
 
- - name: Install universal-ctags
- run: sudo apt update && sudo apt install universal-ctags
-
  - name: Run and upload benchmarks
  run: ./scripts/run_and_upload_benchmarks.sh
  env:

.github/workflows/lint_and_test.yml

@@ -43,15 +43,6 @@ jobs:
  uses: actions/setup-node@v4
  with:
  node-version: "16"
- - name: Install universal-ctags (Ubuntu)
- if: runner.os == 'Linux'
- run: sudo apt update && sudo apt install universal-ctags
- - name: Install universal-ctags (OSX)
- if: runner.os == 'macOS'
- run: brew update && brew install universal-ctags
- - name: Install universal-ctags (Windows)
- if: runner.os == 'Windows'
- run: choco install universal-ctags
  - name: Set up Python ${{ matrix.python-version }}
  uses: actions/setup-python@v3
  with:

benchmarks/benchmark_runner.py

@@ -228,7 +228,7 @@ async def run(self, retries: int = 1) -> list[BenchmarkResult]:
  )
  try:
  if sample.context and self.config.auto_context_tokens:
- score = await run_auto_context_benchmark(sample, self.config, include_context=False)
+ score = await run_auto_context_benchmark(sample, self.config)
  result.context_results = {**score, "auto_context_tokens": self.config.auto_context_tokens}
  result.context_precision = score["precision"]
  result.context_recall = score["recall"]

benchmarks/context_benchmark.py

@@ -5,13 +5,13 @@
 from pathlib import Path
 from typing import Any
 
+from ragdaemon.daemon import Daemon
+
 from benchmarks.arg_parser import common_benchmark_parser
 from benchmarks.run_sample import setup_sample
 from benchmarks.swe_bench_runner import SWE_BENCH_SAMPLES_DIR, get_swe_samples
-from mentat import Mentat
 from mentat.config import Config
 from mentat.sampler.sample import Sample
-from mentat.session_context import SESSION_CONTEXT
 
 
 def _score(predicted: set[Path], expected: set[Path]) -> dict[str, Any]:
@@ -23,9 +23,7 @@ def _score(predicted: set[Path], expected: set[Path]) -> dict[str, Any]:
  return {"precision": precision, "recall": recall, "n_true": len(expected)}
 
 
-async def run_auto_context_benchmark(
- sample: Sample, config: Config, cwd: Path | str | None = None, include_context: bool = False
-) -> dict[str, Any]:
+async def run_auto_context_benchmark(sample: Sample, config: Config, cwd: Path | str | None = None) -> dict[str, Any]:
  """Run a sample using Mentat and return the resulting diff"""
  starting_dir = Path.cwd()
 
@@ -34,24 +32,24 @@ async def run_auto_context_benchmark(
  "In order to run the auto-context benchmark, sample.context must not "
  "be empty (ground truth) and config.auto_context_tokens must be > 0."
  )
- paths = [] if not include_context else [Path(a) for a in sample.context]
 
  try:
  _, cwd, _, _ = setup_sample(sample, None, skip_test_exec=True)
- exclude_paths = [cwd / ".venv"]
- mentat = Mentat(cwd=cwd, paths=paths, exclude_paths=exclude_paths, config=config or Config())
- await mentat.startup()
- await asyncio.sleep(0.01) # Required to initialize llm_api_handler for embeddings
+ ignore_patterns = [cwd / ".venv"]
+ annotators = {
+ "hierarchy": {"ignore_patterns": ignore_patterns},
+ "chunker_line": {"lines_per_chunk": 100},
+ }
+ daemon = Daemon(cwd=cwd, annotators=annotators)
+ await daemon.update()
 
  # TODO: If there's a conversation history, we might consider the cumulative context.
  # Setup a mock for the LLM response and run the conversation until this point.
- code_context = SESSION_CONTEXT.get().code_context
- _ = await code_context.get_code_message(0, sample.message_prompt)
- predicted = set(path.relative_to(cwd) for path in code_context.include_files.keys())
+ context = daemon.get_context(sample.message_prompt, auto_tokens=config.auto_context_tokens)
+ predicted = {Path(a) for a in context.context.keys()}
  actual = {Path(a) for a in sample.context}
  score = _score(predicted, actual)
 
- await mentat.shutdown()
  return score
  finally:
  os.chdir(starting_dir)

benchmarks/run_sample.py

@@ -31,6 +31,15 @@ def setup_sample(
  )
  cwd = Path(repo.working_dir)
 
+ # Make sure there's a .gitignore file, and that '.ragdaemon/*' is in it
+ gitignore_path = cwd / ".gitignore"
+ if not gitignore_path.exists():
+ gitignore_path.write_text(".ragdaemon/*\n")
+ else:
+ gitignore_contents = gitignore_path.read_text()
+ if ".ragdaemon/*" not in gitignore_contents:
+ gitignore_path.write_text(gitignore_contents + ".ragdaemon/*\n")
+
  test_executable = None
  if not skip_test_exec and (sample.FAIL_TO_PASS or sample.PASS_TO_PASS):
  # If there's an environment_setup_commit, this is what it's needed for.

docs/source/developer/mentat.rst

@@ -8,7 +8,6 @@ Subpackages
  :maxdepth: 4
 
  mentat.command
- mentat.feature_filters
  mentat.parsers
  mentat.prompts
  mentat.python_client
@@ -98,14 +97,6 @@ mentat.conversation module
  :undoc-members:
  :show-inheritance:
 
-mentat.ctags module
--------------------
-
-.. automodule:: mentat.ctags
- :members:
- :undoc-members:
- :show-inheritance:
-
 mentat.diff\_context module
 ---------------------------
 
@@ -122,14 +113,6 @@ mentat.edit\_history module
  :undoc-members:
  :show-inheritance:
 
-mentat.embeddings module
-------------------------
-
-.. automodule:: mentat.embeddings
- :members:
- :undoc-members:
- :show-inheritance:
-
 mentat.errors module
 --------------------
 

docs/source/user/getting_started.rst

@@ -39,26 +39,6 @@ If you want to use the whisper transcription on an OS besides macOS or Windows y
 
  sudo apt-get install libportaudio2
 
-In order to use auto context you will need to install ctags:
-
-Windows:
-
-.. code-block:: bash
-
- choco install universal-ctags
-
-macOS:
-
-.. code-block:: bash
-
- brew install universal-ctags
-
-Ubuntu:
-
-.. code-block:: bash
-
- sudo apt install universal-ctags
-
 Basic Usage
 -----------