diff --git a/.gitignore b/.gitignore index ba0430d..e3aeaa2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,52 @@ -__pycache__/ \ No newline at end of file +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtual environments +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Results and outputs +results/ +*.png +*.log +*.csv +*.sqlite +*.nsys-rep + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0c007c8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2025 University of Washington Efficient Systems Lab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..42d6f87 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,33 @@ +# Include essential documentation +include README.md +include LICENSE +include requirements.txt + +# Include all config files +recursive-include configs *.yml *.yaml *.json + +# Include all script files +recursive-include scripts *.py *.sh + +# Include application files +recursive-include applications *.py *.yml *.yaml *.json *.txt *.md + +# Include inference backend files +recursive-include inference_backends *.py *.yml *.yaml *.json + +# Include monitoring utilities +recursive-include monitors *.py + +# Exclude build artifacts and caches +global-exclude __pycache__ +global-exclude *.py[cod] +global-exclude *$py.class +global-exclude *.so +global-exclude .DS_Store + +# Exclude environment and results +prune env +prune venv +prune results +prune .git + diff --git a/README.md b/README.md index e244e5e..b534c75 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,33 @@ ConsumerBench is a comprehensive benchmarking framework that evaluates the runtime performance of user-defined GenAI applications under realistic conditions on end-user devices. -## 🚀 Benchmark Setup +## 🚀 Quick Start + +### Option 1: Install from PyPI (Recommended) + +```bash +# Install ConsumerBench +pip install consumer-bench + +# Run a benchmark +consumerbench --config path/to/your/config.yml +``` + +### Option 2: Install from Source + +```bash +# Clone the repository +git clone https://github.com/your-org/ConsumerBench.git +cd ConsumerBench + +# Install in development mode +pip install -e . + +# Run a benchmark +consumerbench --config configs/sleep_test.yml +``` + +### Option 3: Development Setup ```bash # Clone the repository @@ -23,8 +49,23 @@ Follow instructions mentioned in `applications/` Add your own yml workflow in `configs/` ### Running benchmark -Run the benchmark using the command + +```bash +# Basic usage +consumerbench --config + +# With custom results directory +consumerbench --config configs/workflow_chatbot.yml --results ./my_results + +# With visualization enabled (generates DAG plots) +consumerbench --config configs/workflow_imagegen.yml --visualize + +# View help +consumerbench --help ``` + +### Legacy Command (still supported) +```bash python src/scripts/run_consumerbench.py --config ``` diff --git a/configs/sleep_test.yml b/configs/sleep_test.yml new file mode 100644 index 0000000..1931976 --- /dev/null +++ b/configs/sleep_test.yml @@ -0,0 +1,17 @@ +sleep1: + type: SleepApplication + sleep_time: 1.5 + num_requests: 3 + +sleep2: + type: SleepApplication + sleep_time: 2.5 + num_requests: 2 + +workflows: + first_sleep: + uses: sleep1 + + second_sleep: + uses: sleep2 + depend_on: ["first_sleep"] \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..64ffd32 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,97 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "consumer-bench" +version = "0.1.0" +description = "A comprehensive benchmarking framework for evaluating GenAI applications on end-user devices" +readme = "README.md" +requires-python = ">=3.10" +license = {text = "MIT"} +authors = [ + {name = "Yile Gu"}, + {name = "Rohan Kadekodi"}, + {name = "Hoang Nguyen"}, + {name = "Keisuke Kamahori"}, + {name = "Yiyu Liu"}, + {name = "Baris Kasikci"} +] +keywords = ["benchmark", "genai", "performance", "llm", "inference"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] + +dependencies = [ + "accelerate>=1.5.0", + "aiohttp>=3.9.0", + "anthropic>=0.52.0", + "datasets>=3.0.0", + "diffusers>=0.30.0", + "fastapi>=0.115.0", + "huggingface-hub>=0.30.0", + "httpx>=0.27.0", + "instructor>=1.8.0", + "ipykernel>=6.29.0", + "librosa>=0.11.0", + "matplotlib>=3.10.0", + "mcp>=1.9.0", + "mcp-server-fetch>=2025.4.0", + "numba>=0.61.0", + "numpy>=2.0.0", + "nvidia-ml-py>=12.0.0", + "openai>=1.80.0", + "pandas>=2.2.0", + "pillow>=11.0.0", + "psutil>=7.0.0", + "pydantic>=2.11.0", + "pynvml>=12.0.0", + "PyYAML>=6.0.0", + "requests>=2.32.0", + "scikit-learn>=1.6.0", + "scipy>=1.15.0", + "seaborn>=0.13.0", + "soundfile>=0.13.0", + "torch>=2.6.0", + "torchaudio>=2.6.0", + "torchvision>=0.21.0", + "tqdm>=4.67.0", + "transformers>=4.50.0", + "uvicorn>=0.34.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", + "flake8>=6.0.0", +] + +[project.urls] +Homepage = "https://consumerbench.github.io/" +Documentation = "https://consumerbench.github.io/" +Repository = "https://github.com/efeslab/ConsumerBench" +Issues = "https://github.com/efeslab/ConsumerBench/issues" + +[project.scripts] +consumerbench = "consumerbench.cli:main" + +[tool.setuptools] +packages = ["consumerbench", "consumerbench.applications", "consumerbench.monitors", "consumerbench.scripts", "consumerbench.inference_backends"] + +[tool.setuptools.package-dir] +consumerbench = "src" +"consumerbench.applications" = "applications" +"consumerbench.monitors" = "monitors" +"consumerbench.scripts" = "scripts" +"consumerbench.inference_backends" = "inference_backends" + +[tool.setuptools.package-data] +consumerbench = ["*.yml", "*.yaml", "*.json"] + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..91065d5 --- /dev/null +++ b/setup.py @@ -0,0 +1,11 @@ +""" +ConsumerBench Setup File +This file provides backward compatibility with older pip versions. +Modern installations should use pyproject.toml. +""" +from setuptools import setup + +# All configuration is in pyproject.toml +# This file exists for backward compatibility +setup() + diff --git a/src/cli.py b/src/cli.py new file mode 100644 index 0000000..940220d --- /dev/null +++ b/src/cli.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +ConsumerBench CLI Entry Point +""" +import sys +import os +import argparse + +# Add the package directories to the path +repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, repo_dir) + +from applications.SleepApplication.SleepApplication import SleepApplication +from applications.ImageGen.ImageGen import ImageGen +from applications.DeepResearch.DeepResearch import DeepResearch +from applications.Chatbot.Chatbot import Chatbot +from applications.LiveCaptions.LiveCaptions import LiveCaptions +from applications.MCPServer.MCPServer import MCPServer +from src.workflow import Workflow +import src.globals as globals + +from monitors.memory_util import GpuMemoryMonitor + + +def main(): + """Main entry point for ConsumerBench CLI""" + parser = argparse.ArgumentParser( + description='ConsumerBench - Benchmark GenAI applications on end-user devices', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + consumerbench --config configs/sleep_test.yml + consumerbench --config configs/workflow_chatbot.yml --results ./my_results + consumerbench --config configs/workflow_imagegen.yml --visualize + consumerbench --config configs/workflow_live_captions.yml --mcp_trace trace.jsonl + +For more information, visit: https://github.com/your-org/ConsumerBench + """ + ) + parser.add_argument( + '--config', + type=str, + help='Path to the YAML config file (required)', + required=True + ) + parser.add_argument( + '--mcp_trace', + type=str, + help='Path to MCP trace JSONL file (optional)', + default=None + ) + parser.add_argument( + '--results', + type=str, + help='Path to save results (default: ./results)', + default=f"{repo_dir}/results" + ) + parser.add_argument( + '--visualize', + action='store_true', + help='Generate visualization plots for the benchmark DAG (default: False)' + ) + parser.add_argument( + '--version', + action='version', + version='ConsumerBench 0.1.0' + ) + + args = parser.parse_args() + config_file = args.config + mcp_trace_file = args.mcp_trace + + print("=" * 60) + print("=== ConsumerBench - GenAI Application Benchmarking ===") + print("=" * 60) + print(f"\nUsing config file: {config_file}") + if mcp_trace_file: + print(f"Using MCP trace file: {mcp_trace_file}") + print(f"Results will be saved to: {args.results}\n") + + # Initialize globals + globals.set_start_time() + globals.set_results_dir(f"{args.results}") + + # Create application instances + sleepApplication1 = SleepApplication() + imageGen = ImageGen() + deepResearch = DeepResearch() + chatbot = Chatbot() + liveCaptions = LiveCaptions() + mcpServer = MCPServer(mcp_trace_file=mcp_trace_file, config_file=config_file) + + # Create workflow from YAML + workflow = Workflow(config_file) + + # Register applications + workflow.register_application("SleepApplication", sleepApplication1) + workflow.register_application("ImageGen", imageGen) + workflow.register_application("DeepResearch", deepResearch) + workflow.register_application("Chatbot", chatbot) + workflow.register_application("LiveCaptions", liveCaptions) + workflow.register_application("MCPServer", mcpServer) + + print("\nRegistered applications:") + for app_name, app in workflow.applications.items(): + print(f" - {app_name}: {type(app).__name__}") + print() + + # Load workflow configuration + workflow.load_workflow_unit_config() + print("Loaded workflow configuration:") + for unit_name, unit_config in workflow.workflow_unit_map.items(): + print(f" - {unit_name}: {unit_config['type']} (count: {unit_config['count']})") + print() + + # Generate task queue + workflow.generate_task_queue() + print("Generated task queue:") + for k, v in workflow.tasks_map_queue.items(): + print(f"Task group {k}:") + for unit in v: + print(f" - {unit.type} (ID: {unit.id})") + print() + + # Generate benchmark + bm = workflow.generate_benchmark() + print("Benchmark generated successfully.") + + # Visualize the benchmark (optional) + if args.visualize: + bm.visualize("complex_workflow_benchmark.png") + print("Benchmark visualization saved to 'complex_workflow_benchmark.png'") + else: + print("Skipping visualization (use --visualize to generate DAG plots)") + + # Set up GPU memory monitoring + gpu_monitor = GpuMemoryMonitor(gpu_id=0, interval=0.01, results_dir=args.results) + import threading + monitor_thread = threading.Thread(target=gpu_monitor.start_monitoring) + monitor_thread.daemon = True + monitor_thread.start() + + # Run the benchmark + print("\n" + "=" * 60) + print("=== Running Benchmark ===") + print("=" * 60) + total_time = bm.run_concurrent() + print(f"Total execution time: {total_time:.4f} seconds") + + # Display results + print("\n=== Results ===") + bm.display_results() + + # Stop GPU memory monitoring + gpu_monitor.running = False + monitor_thread.join() + + print("\n" + "=" * 60) + print("Benchmark completed successfully!") + print(f"Results are saved in {globals.get_results_dir()}") + print("=" * 60) + + +if __name__ == "__main__": + main() +