Merge pull request #25 from chriswritescode-dev/rst-support

Rst support - UI Refactor

Author: chriswritescode-dev

CHANGELOG.md

@@ -5,7 +5,33 @@ All notable changes to CodeDox will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## 0.2.8 
+
+### Added
+- **reStructuredText (RST) Support**: Complete RST document processing with intelligent code extraction
+  - Support for `.. code-block::` directive with language specification
+  - Support for `.. code::` and `.. sourcecode::` directives
+  - Literal block extraction using `::` syntax
+  - Automatic language detection from RST directives
+  - Comprehensive test coverage with various RST formats
+  - Integration with existing upload and processing pipeline
+
+### Improved
+- **Code Extraction Pipeline**: Enhanced to handle RST documents alongside existing formats
+
+### Technical Details
+- **RST Code Extraction**: Implemented `RSTCodeExtractor` class in `src/api/routes/upload_utils.py`
+- **Supported RST Features**:
+  - `.. code-block:: python` with language specification and options
+  - `.. code:: javascript` for shorter syntax
+  - `.. sourcecode:: ruby` for alternative directive format
+  - Literal blocks using `::` syntax with preserved indentation
+  - Automatic filtering of directive options and metadata
+  - Context preservation around code blocks
+- **Test Coverage**: Added comprehensive test suite in `tests/test_rst_extraction.py`
+- **File Support**: RST files (.rst, .rest, .restx, .rtxt, .rstx) now fully supported in upload processing
+
+## [0.2.7]
 
 ### Added
 - **GitHub Repository Processing**: Direct cloning and processing of GitHub repositories
@@ -17,12 +43,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Source URL generation for each processed file
 - **New CLI Command**: `upload-repo` command for GitHub repository processing
 - **Comprehensive GitHub Documentation**: Added detailed documentation for GitHub features
+- **Enhanced Search with Markdown Fallback**: Automatically searches documentation content when code snippet searches return limited results
+- **Markdown Search with Highlighting**: Full-text search across markdown documentation with intelligent highlighting and auto-scroll to matches
+- **Web UI Search Improvements**: Toggle between code-only and enhanced search modes in the UI
+- **Comprehensive Documentation**: Added detailed feature documentation with screenshots
 
 ### Improved
 - **Documentation Structure**: Better organization with dedicated GitHub processing section
 - **Markdown Formatting**: Fixed list formatting issues for better mkdocs rendering
+- **Code Quality**: Fixed import ordering, removed unused imports, and improved type hints throughout the codebase
+- **Test Infrastructure**: Enhanced test fixtures and database cleanup procedures
+- **Search Performance**: Optimized PostgreSQL full-text search with better indexing and query strategies
+- **MCP Tools**: Improved error handling and response formatting for better AI assistant integration
+
+### Fixed
+- **Code Extraction**: Resolved issues with HTML code extraction and duplicate detection
+- **Upload UI**: Fixed spacing and layout issues in the upload interface
+- **Database Schema**: Cleaned up schema setup and migration handling
+
+### Changed
+- **Screenshot Organization**: Consolidated all screenshots into docs/screenshots directory for single source of truth
+- **Search Defaults**: Changed default search behavior to use enhanced mode for better results
 
-## [0.2.6] - 2025-01-01
+## [0.2.6]
 
 ### Added
 - **Enhanced Search with Markdown Fallback**: Automatically searches documentation content when code snippet searches return limited results
@@ -45,7 +88,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **Screenshot Organization**: Consolidated all screenshots into docs/screenshots directory for single source of truth
 - **Search Defaults**: Changed default search behavior to use enhanced mode for better results
 
-## [0.2.5] - 2024-12-31
+## [0.2.5]
 
 ### Added
 - Initial public release with core functionality
@@ -54,4 +97,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - PostgreSQL full-text search
 - MCP server integration
 - React/TypeScript Web UI
-- Upload support for markdown files
+- Upload support for markdown files

cli.py

@@ -199,14 +199,12 @@ async def run_upload():
         final_source_url = source_url if source_url else f"file://{os.path.abspath(file_path)}"
         final_name = name if name else os.path.basename(file_path)
 
-        # Determine content type from extension
-        content_type = "markdown"
-        if file_path.endswith(".rst"):
-            content_type = "restructuredtext"
-        elif file_path.endswith(".adoc"):
-            content_type = "asciidoc"
-        elif file_path.endswith(".txt"):
-            content_type = "text"
+        # Determine content type from extension using centralized config
+        from src.constants import get_content_type_for_extension
+        
+        content_type = get_content_type_for_extension(file_path)
+        if not content_type:
+            content_type = "text"  # Default fallback
 
         processor = UploadProcessor()
 
@@ -278,15 +276,16 @@ async def run_upload_repo():
             task = progress.add_task("Cloning repository...", total=None)
 
             # Auto-generate name from repo URL if not provided
-            if not name:
+            repo_name = name
+            if not repo_name:
                 import re
 
                 match = re.search(r"/([^/]+?)(?:\.git)?$", repo_url)
-                name = match.group(1) if match else "Repository Documentation"
+                repo_name = match.group(1) if match else "Repository Documentation"
 
             config = GitHubRepoConfig(
                 repo_url=repo_url,
-                name=name,
+                name=repo_name,
                 path=path,
                 branch=branch,
                 token=token,

docs/getting-started/upgrade.md

@@ -50,7 +50,7 @@ docker-compose down
 git fetch origin
 
 # For specific version
-git checkout v0.2.6
+git checkout v0.2.8
 
 # Or for latest main branch
 git pull origin main
@@ -130,7 +130,7 @@ python cli.py search "test query"
 
 ## Version-Specific Upgrade Notes
 
-### Upgrading to 0.2.6
+### Upgrading to 0.2.8
 
 **New Features:**
 - Enhanced markdown search with fallback

docs/index.md

@@ -32,7 +32,7 @@ CodeDox solves this by:
 ### Intelligent Code Extraction
 
 - **AI-Enhanced Processing**: LLM-powered language detection, title generation, and intelligent descriptions
-- **Multi-Format Support**: Extracts from Markdown, HTML, text files, and 20+ documentation frameworks
+- **Multi-Format Support**: Extracts from Markdown, HTML, text files
 - **Smart Context Preservation**: Maintains relationships and surrounding context for better code understanding
 - **Batch Processing**: Efficient parallel processing with configurable concurrency
 

frontend/src/components/ConfirmationDialog.tsx

@@ -1,4 +1,5 @@
 import React from 'react';
+import { useKeyboardShortcut } from '../hooks/useKeyboardShortcut';
 
 interface ConfirmationDialogProps {
   isOpen: boolean;
@@ -23,6 +24,9 @@ export const ConfirmationDialog: React.FC<ConfirmationDialogProps> = ({
   variant = 'default',
   isConfirming = false,
 }) => {
+  // Handle Escape key to cancel
+  useKeyboardShortcut('Escape', onCancel, { isEnabled: isOpen && !isConfirming });
+  
   if (!isOpen) return null;
 
   return (

frontend/src/components/DocumentSearchModal.tsx

@@ -2,6 +2,7 @@ import React, { useState, useEffect, useRef } from "react";
 import { Search, X, FileText, ExternalLink, Loader2 } from "lucide-react";
 import { api } from "../lib/api";
 import { FullPageModal } from "./FullPageModal";
+import { useKeyboardShortcut } from "../hooks/useKeyboardShortcut";
 
 interface DocumentSearchModalProps {
   isOpen: boolean;
@@ -34,6 +35,8 @@ export const DocumentSearchModal: React.FC<DocumentSearchModalProps> = ({
     }
   }, [isOpen]);
 
+  useKeyboardShortcut("Escape", onClose, isOpen);
+
   const performSearch = async (searchQuery: string) => {
     if (!searchQuery.trim()) {
       setResults([]);
@@ -73,11 +76,7 @@ export const DocumentSearchModal: React.FC<DocumentSearchModalProps> = ({
     setFullPageModalOpen(true);
   };
 
-  const handleKeyDown = (e: React.KeyboardEvent) => {
-    if (e.key === "Escape") {
-      onClose();
-    }
-  };
+
 
   if (!isOpen) return null;
 
@@ -92,7 +91,6 @@ export const DocumentSearchModal: React.FC<DocumentSearchModalProps> = ({
               type="text"
               value={query}
               onChange={(e) => setQuery(e.target.value)}
-              onKeyDown={handleKeyDown}
               placeholder="Search pages by title, URL, or content..."
               className="flex-1 bg-transparent outline-none text-gray-900 dark:text-gray-100 placeholder-gray-500"
               autoFocus

frontend/src/components/FullPageModal.tsx

@@ -12,6 +12,7 @@ import {
 } from "lucide-react";
 import { api } from "../lib/api";
 import { Link } from "react-router-dom";
+import { useKeyboardShortcut } from "../hooks/useKeyboardShortcut";
 
 interface FullPageModalProps {
   url: string;
@@ -44,25 +45,14 @@ export const FullPageModal: React.FC<FullPageModalProps> = ({
   }, [isOpen, url]);
 
   // Handle keyboard shortcuts
-  useEffect(() => {
-    const handleKeyDown = (e: KeyboardEvent) => {
-      if (!isOpen) return;
-
-      // ESC to close modal
-      if (e.key === "Escape") {
-        onClose();
-      }
-
-      // Ctrl/Cmd + F to focus search
-      if ((e.ctrlKey || e.metaKey) && e.key === "f") {
-        e.preventDefault();
-        searchInputRef.current?.focus();
-      }
-    };
-
-    window.addEventListener("keydown", handleKeyDown);
-    return () => window.removeEventListener("keydown", handleKeyDown);
-  }, [isOpen, onClose]);
+  useKeyboardShortcut('Escape', onClose, { isEnabled: isOpen });
+  
+  useKeyboardShortcut('f', () => {
+    searchInputRef.current?.focus();
+  }, { 
+    isEnabled: isOpen,
+    ctrlKey: true 
+  });
 
   const fetchPageMarkdown = async () => {
     setLoading(true);

frontend/src/components/Layout.tsx

@@ -100,14 +100,14 @@ export default function Layout() {
           <div className="px-3 pt-4 pb-2 space-y-2">
             <button
               onClick={() => setCrawlDialogOpen(true)}
-              className="w-full flex items-center gap-2 px-3 py-2.5 text-sm bg-primary hover:bg-primary/90 text-primary-foreground rounded-lg transition-all font-medium shadow-sm hover:shadow-md"
+              className="cursor-pointer w-full flex items-center gap-2 px-3 py-2.5 text-sm bg-primary hover:bg-primary/90 text-primary-foreground rounded-lg transition-all font-medium shadow-sm hover:shadow-md"
             >
               <Plus className="h-4 w-4" />
               <span className="flex-1 text-left">New Crawl</span>
             </button>
             <button
               onClick={() => setDocumentSearchOpen(true)}
-              className="w-full flex items-center gap-2 px-3 py-2 text-sm bg-gray-100 dark:bg-gray-800 hover:bg-gray-200 dark:hover:bg-gray-700 rounded-lg transition-colors text-gray-600 dark:text-gray-400"
+              className="cursor-pointer w-full flex items-center gap-2 px-3 py-2 text-sm bg-gray-100 dark:bg-gray-800 hover:bg-gray-200 dark:hover:bg-gray-700 rounded-lg transition-colors text-gray-600 dark:text-gray-400"
             >
               <FileSearch className="h-4 w-4" />
               <span className="flex-1 text-left">Markdown Pages</span>

frontend/src/components/NewCrawlDialog.tsx

@@ -1,6 +1,7 @@
 import React, { useState } from 'react';
 import { X } from 'lucide-react';
 import { useToast } from '../hooks/useToast';
+import { useKeyboardShortcut } from '../hooks/useKeyboardShortcut';
 
 interface NewCrawlDialogProps {
   isOpen: boolean;
@@ -38,6 +39,15 @@ export const NewCrawlDialog: React.FC<NewCrawlDialogProps> = ({
 
   const [maxConcurrentInput, setMaxConcurrentInput] = useState('5');
 
+  const handleClose = () => {
+    setFormData({ name: '', version: '', base_url: '', max_pages: undefined, max_depth: 1, domain_filter: '', url_patterns: '', max_concurrent_crawls: 5 });
+    setMaxConcurrentInput('5');
+    onClose();
+  };
+
+  // Handle Escape key to close dialog
+  useKeyboardShortcut('Escape', handleClose, { isEnabled: isOpen && !isSubmitting });
+
   if (!isOpen) return null;
 
   const handleSubmit = (e: React.FormEvent) => {
@@ -64,12 +74,6 @@ export const NewCrawlDialog: React.FC<NewCrawlDialogProps> = ({
     onSubmit(submitData);
   };
 
-  const handleClose = () => {
-    setFormData({ name: '', version: '', base_url: '', max_pages: undefined, max_depth: 1, domain_filter: '', url_patterns: '', max_concurrent_crawls: 5 });
-    setMaxConcurrentInput('5');
-    onClose();
-  };
-
   return (
     <div className="fixed inset-0 bg-slate-600/50 backdrop-blur-sm flex items-center justify-center z-50">
       <div className="bg-card border-2 border-border rounded-lg w-full max-w-md max-h-[90vh] shadow-2xl animate-slide-up flex flex-col">

frontend/src/components/RecrawlDialog.tsx

@@ -1,5 +1,6 @@
 import { useState } from "react";
 import { X, RefreshCw, AlertTriangle } from "lucide-react";
+import { useKeyboardShortcut } from "../hooks/useKeyboardShortcut";
 
 interface RecrawlDialogProps {
   isOpen: boolean;
@@ -20,6 +21,9 @@ export function RecrawlDialog({
 }: RecrawlDialogProps) {
   const [ignoreHash, setIgnoreHash] = useState(false);
 
+  // Handle Escape key to cancel
+  useKeyboardShortcut('Escape', onCancel, { isEnabled: isOpen && !isRecrawling });
+
   if (!isOpen) return null;
 
   return (