bug: `.gitignore` is ignored, so my file is not ignored

[MNeMoNiCuZ]

I'm ingesting this repo:
https://github.com/MNeMoNiCuZ/ComfyUI-mnemic-nodes

And I have added to my .gitignore the following file:
nodes/negativeprompt/merges.txt

But this is still being ingested unfortunately, creating a 100k line document filled with tokens :)

Is this .gitignore working as expected?
Or maybe I'm using it incorrectly.

[NicolasIRAGNE]

Are you using the webui?

We do not exclude gitignored files by default and the option does not work on the webui

However, specifying nodes/negativeprompt/merges.txt in the Exclude text box works as intented:

Repository: mnemonicuz/comfyui-mnemic-nodes
Files analyzed: 49

Estimated tokens: 555.6k

not ideal but should work for your use case at this moment

[MNeMoNiCuZ]

Are you using the webui?

We do not exclude gitignored files by default and the option does not work on the webui

However, specifying nodes/negativeprompt/merges.txt in the Exclude text box works as intented:

Repository: mnemonicuz/comfyui-mnemic-nodes
Files analyzed: 49

Estimated tokens: 555.6k

not ideal but should work for your use case at this moment

Thanks for your reply.
No I was not using the webUI, I'm using the Python Package version (https://pypi.org/project/gitingest) in an MCP that fetches the documents from a GitHub repo and stores it for later retrieval.
I still get that large merges.txt-file in my pulled file, even though it's set to be ignored by the .gitignore in on that GitHub repo.

Maybe I am using it wrong.
Should I route the .gitignore into some specific exclude argument when using the package?
The PyPI-page mentions:

By default, files listed in .gitignore are skipped. Use --include-gitignored if you need those files in the digest.

So it sounds like it should work automatically and it should ignore whatever the gitignore is already ignoring.

[NicolasIRAGNE]

Sounds like a bug, it should work as you expect it to

I will try to investigate

[MNeMoNiCuZ]

Thanks!
As a reference if it helps, my MCP tool that does this is below.

from pydantic import Field
from tools.mcp_instance import mcp
import re
import os
from datetime import datetime, timedelta
from urllib.parse import urlparse
from fuzzywuzzy import process

from gitingest import ingest, ingest_async

# --- Configuration Constants ---
# Directory to store ingested GitHub repository data.
# This path should be absolute and accessible by the tool.
INGESTED_FILES_DIR = os.path.join(os.getcwd(), "tools/mcp/coding/github_repo_files")
# Number of days after which ingested files will be automatically deleted.
CLEANUP_DAYS = 7

# Ensure the ingestion directory exists
os.makedirs(INGESTED_FILES_DIR, exist_ok=True)


async def _cleanup_old_ingested_files():
    """
    Deletes ingested files older than CLEANUP_DAYS from INGESTED_FILES_DIR.
    """
    now = datetime.now()
    
    if not os.path.exists(INGESTED_FILES_DIR):
        return

    for filename in os.listdir(INGESTED_FILES_DIR):
        filepath = os.path.join(INGESTED_FILES_DIR, filename)
        if os.path.isfile(filepath):
            file_mod_time = datetime.fromtimestamp(os.path.getmtime(filepath))
            if (now - file_mod_time) > timedelta(days=CLEANUP_DAYS):
                try:
                    os.remove(filepath)
                except OSError as e:
                    pass


def _parse_ingested_content(full_content: str) -> dict:
    """
    Parses the full ingested content string into summary, tree, and file_contents.
    """
    summary_end_marker = "\n\nDirectory structure:\n"
    tree_end_marker = "\n\n================================================\nFILE: "

    summary = ""
    tree = ""
    file_contents = ""

    # Find summary
    summary_match = re.match(r"(Repository:.*?)\n\nDirectory structure:", full_content, re.DOTALL)
    if summary_match:
        summary = summary_match.group(1).strip()
        remaining_content = full_content[summary_match.end():]
    else:
        remaining_content = full_content  # Fallback if summary format is unexpected

    # Find tree
    tree_match = re.match(r"(Directory structure:.*?)\n\n={20,}\nFILE:", remaining_content, re.DOTALL)
    if tree_match:
        tree = tree_match.group(1).strip()
        file_contents = remaining_content[tree_match.end():].strip()
    else:
        file_contents = remaining_content.strip()  # Fallback if tree format is unexpected

    return {
        "summary": summary,
        "directory_tree": tree,
        "file_contents": file_contents
    }

@mcp.tool()
async def github_repo_info(
    github_url: str = Field(None, description="Optional: The URL of the GitHub repository to ingest (e.g., 'https://github.com/owner/repo'). If provided, content will be ingested and stored locally. If no URL is provided, instead use the fuzzy_search_ingested_file to search for a matching file"),
    ingested_file_path: str = Field(None, description="Optional: The absolute path to a previously ingested repository file. Use this to analyze already ingested data without re-ingesting."),
    fuzzy_search_ingested_file: str = Field(None, description="Optional: A fuzzy search pattern to find a previously ingested repository file by its *filename*. If a good match is found, its content will be used for analysis."),
    search_pattern: str = Field(None, description="Optional: A regex pattern to search within the 'file_contents' of the repository."),
    file_path_filter: str = Field(None, description="Optional: A regex pattern to filter files by their path when searching or getting content."),
    get_file_content: str = Field(None, description="Optional: The exact file path (e.g., 'src/main.js') within the ingested repository to retrieve content for. This will return the full content of that specific file."),
) -> dict:
    """
    A tool to ingest GitHub repository content, store it locally, and read the information.
    Data can be loaded either by providing a GitHub URL for fresh ingestion or by referencing
    a previously ingested and cached repository file (either by its exact path or by a fuzzy
    match of its filename). Use this tool when the user asks about information from a GitHub repo. If no URL is given, it's likely the user intends you to use fuzzy search to find cached data.
    """
    
    await _cleanup_old_ingested_files()

    full_repo_content = ""
    current_ingested_file = ""

    if github_url:
        try:
            # Generate a unique filename based on repo name and timestamp
            parsed_url = urlparse(github_url)
            repo_name = os.path.basename(parsed_url.path).replace(".git", "")  # Remove .git if present
            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            filename = f"{repo_name}_{timestamp}.txt"
            current_ingested_file = os.path.join(INGESTED_FILES_DIR, filename)

            summary, tree, content = await ingest_async(github_url)

            full_repo_content = f"{summary}\n\n{tree}\n\n{content}"

            with open(current_ingested_file, "w", encoding="utf-8") as f:
                f.write(full_repo_content)

        except Exception as e:
            return {
                "status": "error",
                "message": f"Failed to ingest repository from {github_url}: {str(e)}"
            }
    elif ingested_file_path:
        if not os.path.exists(ingested_file_path):
            return {
                "status": "error",
                "message": f"Ingested file not found at: {ingested_file_path}"
            }
        try:
            with open(ingested_file_path, "r", encoding="utf-8") as f:
                full_repo_content = f.read()
            current_ingested_file = ingested_file_path
        except Exception as e:
            return {
                "status": "error",
                "message": f"Failed to read ingested file {ingested_file_path}: {e}"
            }
    elif fuzzy_search_ingested_file:
        available_files = [f for f in os.listdir(INGESTED_FILES_DIR) if os.path.isfile(os.path.join(INGESTED_FILES_DIR, f))]
        if not available_files:
            return {
                "status": "error",
                "message": "No ingested files found to perform fuzzy search."
            }
        
        best_match = process.extractOne(fuzzy_search_ingested_file, available_files)
        if best_match and best_match[1] >= 70:  # 70 is a common threshold for a good match
            matched_filename = best_match[0]
            ingested_file_path = os.path.join(INGESTED_FILES_DIR, matched_filename)
            try:
                with open(ingested_file_path, "r", encoding="utf-8") as f:
                    full_repo_content = f.read()
                current_ingested_file = ingested_file_path
            except Exception as e:
                return {
                    "status": "error",
                    "message": f"Failed to read fuzzy matched file {ingested_file_path}: {e}"
                }
        else:
            return {
                "status": "error",
                "message": f"No good match found for '{fuzzy_search_ingested_file}' in ingested files. Consider providing a more precise name or ingesting the repository."
            }
    else:
        return {
            "status": "error",
            "message": "Either 'github_url', 'ingested_file_path', or 'fuzzy_search_ingested_file' must be provided."
        }

    # Parse the full content for analysis
    parsed_content = _parse_ingested_content(full_repo_content)
    summary = parsed_content["summary"]
    directory_tree = parsed_content["directory_tree"]
    file_contents_raw = parsed_content["file_contents"]

    results = {
        "ingested_file_path": current_ingested_file,
        "summary": summary,
        "directory_tree": directory_tree
    }

    if search_pattern:
        found_matches = []
        # Split the content by the file delimiters
        files = re.split(r"={20,}\nFILE: (.*?)\n={20,}\n", file_contents_raw)
        # The first element is usually empty or preamble, skip it
        for i in range(1, len(files), 2):
            file_path = files[i].strip()
            content = files[i+1].strip()

            if file_path_filter and not re.search(file_path_filter, file_path):
                continue  # Skip if file_path_filter is provided and doesn't match

            for line_num, line in enumerate(content.splitlines()):
                if re.search(search_pattern, line):
                    found_matches.append({
                        "file": file_path,
                        "line_number": line_num + 1,
                        "line_content": line
                    })
        results["search_results"] = found_matches

    if get_file_content:
        # Split the content by the file delimiters
        files = re.split(r"={20,}\nFILE: (.*?)\n={20,}\n", file_contents_raw)
        found_file_content = None
        for i in range(1, len(files), 2):
            file_path = files[i].strip()
            content = files[i+1].strip()
            if file_path == get_file_content:
                found_file_content = {"file": file_path, "content": content}
                break
        if found_file_content:
            results["requested_file_content"] = found_file_content
        else:
            results["requested_file_content"] = {"file": get_file_content, "content": "File not found in ingested data or path is incorrect."}

    if not search_pattern and not get_file_content:
        results["message"] = "No specific analysis requested. Returning summary and directory tree."

    return {"status": "success", "analysis_results": results}

[ix-56h]

Found the source of the bug.
I'll do some more testing and make a PR.