Code clean up.

.github/workflows/coverage.yml

@@ -7,6 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python
         uses: actions/setup-python@v2
         with:

.gitignore

@@ -54,7 +54,17 @@ Pipfile.lock
 # Temporarily Ignored
 .github/workflows/ci.yml
 readmeai/markdown/data/icons.json
-readmeai/settings/prompts.toml
 readmeai/llms/base.py
+readmeai/llms/chunk.py
 readmeai/llms/factory.py
+
+# Templates and Settings
 templates/
+readmeai/settings/markdown.toml
+readmeai/settings/models.toml
+readmeai/settings/prompts.toml
+readmeai/settings/quickstart.toml
+readmeai/settings/quickstart_wip.toml
+
+# dalle images
+examples/images/dalle

Makefile

@@ -1,10 +1,11 @@
 # Makefile
 
+COMMITS := 10
 SHELL := /bin/bash
 VENV := readmeai
 VV := \
 
-.PHONY: help clean format lint conda-recipe git-rm-cache nox pytest poetry-reqs word-search
+.PHONY: help clean format lint conda-recipe git-rm-cache git-log nox pytest poetry-reqs search
 
 help:
 	@echo "Commands:"
@@ -13,10 +14,11 @@ help:
 	@echo "lint         : executes code linting."
 	@echo "conda-recipe : builds conda package."
 	@echo "git-rm-cache : fix git untracked files."
+	@echo "git-log      : displays git log."
 	@echo "nox          : executes nox test suite."
 	@echo "pytest       : executes tests."
 	@echo "poetry-reqs  : generates requirements.txt file."
-	@echo "word-search  : searches for a word in the repository."
+	@echo "search       : searches word in directory."
 
 .PHONY: clean
 clean: format
@@ -43,13 +45,18 @@ conda-recipe:
 git-rm-cache:
 	git rm -r --cached .
 
+.PHONY: git-log
+git-log:
+	git log -n ${COMMITS} --pretty=tformat: --shortstat
+
 .PHONY: nox
 nox:
 	nox -f noxfile.py
 
 .PHONY: pytest
 pytest:
-	pytest ${VV} -n auto \
+	pytest ${VV} \
+	-n auto \
 	--asyncio-mode=auto \
 	--cov=. \
 	--cov-report=xml \
@@ -60,7 +67,7 @@ pytest:
 poetry-reqs:
 	poetry export -f requirements.txt --output setup/requirements.txt --without-hashes
 
-.PHONY: word-search
-word-search: clean
+.PHONY: search
+search: clean
 	@echo -e "\nSearching for: ${WORD} in directory: ${CURDIR}"
 	grep -Ril ${WORD} readmeai tests scripts setup

readmeai/core/preprocess.py

@@ -103,6 +103,7 @@ def generate_file_info(
     def get_dependencies(self, contents: List[FileContext]) -> List[str]:
         """Returns a list of dependencies."""
         try:
+            dependency_dict = {}
             dependencies = set()
             dependency_files = self.config_helper.dependency_files.get(
                 "dependency_files"
@@ -112,12 +113,17 @@ def get_dependencies(self, contents: List[FileContext]) -> List[str]:
                 dependencies.update(file_data.dependencies)
                 dependencies.add(file_data.language)
                 dependencies.add(file_data.file_ext)
+
                 if file_data.file_name in dependency_files:
                     dependencies.add(file_data.file_name)
+                    dependency_dict[
+                        file_data.file_name
+                    ] = file_data.dependencies
+
                 if GITHUB_WORKFLOWS_PATH in str(file_data.file_path):
                     dependencies.add("github actions")
 
-            return list(dependencies)
+            return list(dependencies), dependency_dict
 
         except Exception as exc:
             logger.error(f"Error getting dependencies: {exc}")
@@ -199,14 +205,23 @@ def process_repository(
     repo_context = repo_processor.generate_contents(temp_dir)
     repo_context = repo_processor.tokenize_content(repo_context)
     repo_context = repo_processor.language_mapper(repo_context)
-    dependencies = repo_processor.get_dependencies(repo_context)
+
+    dependencies, dependency_dict = repo_processor.get_dependencies(
+        repo_context
+    )
 
     raw_files = [
-        (context.file_path, context.content) for context in repo_context
+        (str(context.file_path), context.content) for context in repo_context
     ]
 
     config.md.tree = ReadmeBuilder(
         config, config_helper, dependencies, raw_files, temp_dir
     ).md_tree
 
-    return repo_context, dependencies, raw_files, config.md.tree
+    return (
+        repo_context,
+        dependencies,
+        raw_files,
+        config.md.tree,
+        dependency_dict,
+    )

readmeai/core/utils.py

@@ -111,28 +111,3 @@ def is_file_ignored(conf_helper: ConfigHelper, file_path: Path) -> bool:
         return True
 
     return False
-
-
-def is_valid_url(url: str) -> bool:
-    """Validate a URL string.
-
-    Parameters
-    ----------
-    url
-        The URL string to validate.
-
-    Returns
-    -------
-        True if the URL is valid, False otherwise.
-    """
-    regex = re.compile(
-        r"^(?:http|ftp)s?://"  # http:// or https://
-        r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|"  # domain
-        r"localhost|"  # localhost
-        r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|"  # ipv4
-        r"\[?[A-F0-9]*:[A-F0-9:]+\]?)"  # ipv6
-        r"(?::\d+)?"  # optional port
-        r"(?:/?|[/?]\S+)$",
-        re.IGNORECASE,
-    )
-    return re.match(regex, url) is not None

readmeai/parsers/python.py

@@ -95,7 +95,10 @@ def parse(self, content: str) -> List[str]:
                         dependencies.extend(
                             self.extract_package_names(group_deps)
                         )
-                return dependencies
+            else:
+                dependencies = []
+
+            return dependencies
 
         except toml.TomlDecodeError as exc:
             return self.handle_parsing_error(f"pyproject.toml: {str(exc)}")

scripts/run_batch.sh

@@ -1,14 +1,13 @@
 #!/usr/bin/env bash
 
-version="0.5.0"
-run_date=$(date +"%Y%m%d")
+#version="0.5.0"
+#run_date=$(date +"%Y%m%d")
 filenames=(
     #"readme-litellm"
-    "readme-fal-js"
+    #"readme-fal-js"
     #gitmate-2
     #gitlab
     "readme-local"
-    "readme-pyflink"
     "readme-python"
     "readme-streamlit"
     "readme-postgres"
@@ -22,10 +21,9 @@ filenames=(
 )
 repositories=(
     #"https://github.com/BerriAI/litellm"
-    "https://github.com/fal-ai/fal-js"
+    #"https://github.com/fal-ai/fal-js"
     #https://gitlab.com/gitmate/open-source/gitmate-2
     #https://gitlab.com/bavarder/bavarder/
-    "/Users/k01101011/Documents/GitHub/gpt-scripts"
     "/Users/k01101011/Documents/GitHub/pyflink-poc"
     "https://github.com/eli64s/readme-ai"
     "https://github.com/eli64s/readme-ai-streamlit"
@@ -40,12 +38,14 @@ repositories=(
 )
 align=("left" "center")
 badge_styles=("default" "flat" "flat-square" "plastic" "for-the-badge" "skills" "skills-light")
-image=("default" "black" "cloud" "gradient" "grey" "purple" "yellow")
+image=("blue" "black" "cloud" "gradient" "grey" "purple")
+badge_color=("blue" "green" "red" "yellow" "orange" "pink" "purple" "blueviolet" "white" "black" "brightgreen" "ff69b4" "999999")
 
 for index in "${!repositories[@]}"; do
     repo="${repositories[$index]}"
-    filename="${filenames[$index]}_v${version}_${run_date}.md"
+    filename="${filenames[$index]}.md" #_v${version}_${run_date}.md"
     random_badge=${badge_styles[$RANDOM % ${#badge_styles[@]}]}
+    random_badge_color=${badge_color[$RANDOM % ${#badge_color[@]}]}
     image_style=${image[$RANDOM % ${#image[@]}]}
     alignment=${align[$RANDOM % ${#align[@]}]}
     rand_choice=$((RANDOM % 2))
@@ -56,15 +56,19 @@ for index in "${!repositories[@]}"; do
     if [ "$random_badge" != "default" ]; then
         cmd+=" -b \"$random_badge\""
     fi
-    if [ "$image_style" != "default" ]; then
+    if [ "$image_style" != "blue" ]; then
         cmd+=" -i \"$image_style\""
     fi
-    if [ "$alignment" != "center" ]; then # Assuming 'center' is the default alignment
+    if [ "$alignment" != "center" ]; then
         cmd+=" -a \"$alignment\""
     fi
     if [ $rand_choice -eq 1 ]; then
         cmd+=" -e"
     fi
+    if [ "$random_badge_color" != "blue" ]; then
+        cmd+=" --badge-color \"$random_badge_color\""
+    fi
 
     eval $cmd
+
 done

tests/test_app.py

@@ -8,6 +8,12 @@
 from readmeai.exceptions import ReadmeGeneratorError
 
 
+@pytest.fixture
+def mock_output_file(tmp_path):
+    """Fixture for a mock output file."""
+    return tmp_path / "test_readme.md"
+
+
 @patch("readmeai.app.clone_repository")
 @patch("readmeai.app.process_repository")
 @patch("readmeai.app.ModelHandler")
@@ -22,6 +28,7 @@ async def test_readme_generator_online(
     mock_config_helper,
     mock_dependencies,
     mock_summaries,
+    mock_output_file,
     tmp_path,
 ):
     """Test the readme_generator function."""
@@ -32,6 +39,7 @@ async def test_readme_generator_online(
         mock_dependencies,
         mock_summaries,
         "test_tree",
+        {"dependency1": "command1", "dependency2": "command2"},
     )
     mock_config.git.repository = tmp_path
     mock_model_handler.return_value.use_api.return_value.__aenter__.return_value.batch_request.return_value = (
@@ -42,7 +50,7 @@ async def test_readme_generator_online(
     )
     mock_build_readme_md.return_value = "test_readme_md"
 
-    await readme_generator(mock_config, mock_config_helper)
+    await readme_generator(mock_config, mock_config_helper, mock_output_file)
 
     mock_clone_repository.assert_called_once()
     mock_process_repository.assert_called_once()
@@ -64,6 +72,7 @@ async def test_readme_generator_offline(
     mock_config_helper,
     mock_dependencies,
     mock_summaries,
+    mock_output_file,
     tmp_path,
 ):
     """Test the readme_generator function."""
@@ -74,11 +83,12 @@ async def test_readme_generator_offline(
         mock_dependencies,
         mock_summaries,
         "test_tree",
+        {"dependency1": "command1", "dependency2": "command2"},
     )
     mock_config.git.repository = tmp_path
     mock_build_readme_md.return_value = "test_readme_md"
 
-    await readme_generator(mock_config, mock_config_helper)
+    await readme_generator(mock_config, mock_config_helper, mock_output_file)
 
     mock_clone_repository.assert_called_once()
     mock_process_repository.assert_called_once()
@@ -92,11 +102,14 @@ async def test_readme_generator_exception_handling(
     mock_clone_repository,
     mock_config,
     mock_config_helper,
+    mock_output_file,
 ):
     """Test the readme_generator function exception handling."""
     mock_clone_repository.side_effect = Exception("Test Exception")
     with pytest.raises(ReadmeGeneratorError):
-        await readme_generator(mock_config, mock_config_helper)
+        await readme_generator(
+            mock_config, mock_config_helper, mock_output_file
+        )
     assert mock_clone_repository.call_count == 1
 
 

tests/test_config/test_enums.py

@@ -57,5 +57,5 @@ def test_badge_options():
 def test_image_options():
     """Test the CLI options for header images."""
     assert ImageOptions.CUSTOM == "CUSTOM"
-    assert isinstance(ImageOptions.DEFAULT, str)
+    assert isinstance(ImageOptions.BLUE, str)
     assert isinstance(ImageOptions.BLACK, str)

tests/test_config/test_validators.py

@@ -1,40 +1,14 @@
 """Tests for validator methods used on command-line arguments."""
 
-import os
 from pathlib import Path
 from unittest.mock import patch
 
 import pytest
 
-from readmeai.config.validators import GitValidator, ModelValidator
+from readmeai.config.validators import GitValidator
 from readmeai.exceptions import GitValidationError
 
 
-def test_set_environment_with_key(monkeypatch, caplog):
-    """Test setting the environment with a provided API key."""
-    monkeypatch.setattr(os, "environ", {})
-    api_key = "test_api_key"
-    ModelValidator.set_environment(api_key, {})
-    assert os.environ["OPENAI_API_KEY"] == api_key
-    assert "Provided API key exported to environment." in caplog.text
-
-
-def test_set_environment_with_existing_key(monkeypatch, caplog):
-    """Test using existing API key in the environment."""
-    monkeypatch.setenv("OPENAI_API_KEY", "existing_key")
-    result = ModelValidator.set_environment(None, {})
-    assert result == "existing_key"
-
-
-def test_set_environment_no_key(monkeypatch, caplog):
-    """Test setting environment in offline mode when no API key is provided."""
-    monkeypatch.setattr(os, "environ", {})
-    values = {}
-    ModelValidator.set_environment(None, values)
-    assert values["offline"] is True
-    assert "No API key found. Running in offline mode." in caplog.text
-
-
 @pytest.mark.parametrize(
     "url",
     [