Add upload feature

Author: ahadnagy

.github/workflows/benchmark_v2.yml

@@ -28,6 +28,21 @@ on:
         required: false
         type: string
         default: ''
+      upload_to_hf:
+        description: 'HuggingFace Dataset to upload results to (e.g., "org/benchmark-results")'
+        required: false
+        type: string
+        default: ''
+      run_id:
+        description: 'Custom run ID for organizing results (auto-generated if not provided)'
+        required: false
+        type: string
+        default: ''
+      benchmark_repo_id:
+        description: 'HuggingFace Dataset to upload results to (e.g., "org/benchmark-results")'
+        required: false
+        type: string
+        default: ''
 
 env:
   HF_HOME: /mnt/cache
@@ -93,6 +108,15 @@ jobs:
             args="$args --commit-id '${{ github.sha }}'"
           fi
           
+          # Add HuggingFace upload parameters if specified
+          if [ -n "${{ inputs.upload_to_hf }}" ]; then
+            args="$args --upload-to-hf '${{ inputs.upload_to_hf }}'"
+          fi
+          
+          if [ -n "${{ inputs.run_id }}" ]; then
+            args="$args --run-id '${{ inputs.run_id }}'"
+          fi
+          
           echo "benchmark_args=$args" >> $GITHUB_OUTPUT
           echo "Benchmark arguments: $args"
 

.github/workflows/benchmark_v2_caller.yml

@@ -52,4 +52,7 @@ jobs:
       measurement_iterations: ${{ inputs.measurement_iterations || 5 }}
       num_tokens_to_generate: ${{ inputs.num_tokens_to_generate || 100 }}
       commit_sha: ${{ github.sha }}
+      upload_to_hf: true
+      run_id: ${{ github.run_id }}
+      benchmark_repo_id: ahadnagy/transformers-benchmarks
     secrets: inherit

benchmark_v2/README.md

@@ -21,6 +21,36 @@ python run_benchmarks.py \
     --num-tokens-to-generate 200
 ```
 
+### Uploading Results to HuggingFace Dataset
+
+You can automatically upload benchmark results to a HuggingFace Dataset for tracking and analysis:
+
+```bash
+# Upload to a public dataset with auto-generated run ID
+python run_benchmarks.py --upload-to-hf username/benchmark-results
+
+# Upload with a custom run ID for easy identification
+python run_benchmarks.py --upload-to-hf username/benchmark-results --run-id experiment_v1
+```
+
+**Dataset Directory Structure:**
+```
+dataset_name/
+├── 2025-01-15/
+│   ├── runs/                       # Non-scheduled runs (manual, PR, etc.)
+│   │   └── 123-1245151651/         # GitHub run number and ID
+│   │       └── benchmark_results/
+│   │           ├── benchmark_summary_20250115_143022.json
+│   │           └── model-name/
+│   │               └── model-name_benchmark_20250115_143022.json
+│   └── benchmark_results_abc123de/ # Scheduled runs (daily CI)
+│       ├── benchmark_summary_20250115_143022.json
+│       └── model-name/
+│           └── model-name_benchmark_20250115_143022.json
+└── 2025-01-16/
+    └── ...
+```
+
 ### Running Specific Benchmarks
 
 ```bash

benchmark_v2/requirements.txt

@@ -3,4 +3,5 @@ psutil>=5.8.0
 gpustat>=1.0.0
 torch>=2.0.0
 transformers>=4.30.0
-datasets>=2.10.0 
+datasets>=2.10.0
+huggingface_hub>=0.16.0 

benchmark_v2/run_benchmarks.py

@@ -24,6 +24,7 @@
 import os
 import sys
 import json
+import uuid
 from datetime import datetime
 from pathlib import Path
 from typing import Dict, List, Any, Optional
@@ -94,6 +95,9 @@ def discover_benchmarks(benches_dir: str) -> List[Dict[str, Any]]:
             else:
                 logging.warning(f"No runner function found in {py_file}")
 
+        except ImportError as e:
+            logging.error(f"Failed to import {py_file}: Missing dependency - {e}")
+            logging.error(f"Make sure to install: pip install -r requirements.txt")
         except Exception as e:
             logging.error(f"Failed to import {py_file}: {e}")
 
@@ -168,7 +172,8 @@ def run_single_benchmark(
 def generate_summary_report(
     output_dir: str, 
     benchmark_results: Dict[str, Any],
-    logger: logging.Logger
+    logger: logging.Logger,
+    benchmark_run_uuid: Optional[str] = None
 ) -> str:
     """Generate a summary report of all benchmark runs."""
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
@@ -177,6 +182,7 @@ def generate_summary_report(
     summary_data = {
         "run_metadata": {
             "timestamp": datetime.utcnow().isoformat(),
+            "benchmark_run_uuid": benchmark_run_uuid,
             "total_benchmarks": len(benchmark_results),
             "successful_benchmarks": len([r for r in benchmark_results.values() if r is not None]),
             "failed_benchmarks": len([r for r in benchmark_results.values() if r is None])
@@ -192,10 +198,111 @@ def generate_summary_report(
     return summary_file
 
 
+def upload_results_to_hf_dataset(
+    output_dir: str,
+    summary_file: str,
+    dataset_name: str,
+    run_id: Optional[str] = None,
+    logger: logging.Logger = None
+) -> Optional[str]:
+    """
+    Upload benchmark results to a HuggingFace Dataset.
+    Based on upload_collated_report() from utils/collated_reports.py
+    
+    Args:
+        output_dir: Local output directory containing results
+        summary_file: Path to the summary file
+        dataset_name: Name of the HuggingFace dataset to upload to
+        run_id: Unique run identifier (if None, will generate one)
+        logger: Logger instance
+        
+    Returns:
+        The run_id used for the upload, None if upload failed
+    """
+    if logger is None:
+        logger = logging.getLogger(__name__)
+    
+    import os
+    from huggingface_hub import HfApi
+    
+    api = HfApi()
+    
+    if run_id is None:
+        github_run_number = os.getenv("GITHUB_RUN_NUMBER")
+        github_run_id = os.getenv("GITHUB_RUN_ID")
+        if github_run_number and github_run_id:
+            run_id = f"{github_run_number}-{github_run_id}"
+    
+    date_folder = datetime.now().strftime("%Y-%m-%d")
+    
+    github_event_name = os.getenv("GITHUB_EVENT_NAME")
+    if github_event_name != "schedule":
+        # Non-scheduled runs go under a runs subfolder
+        repo_path = f"{date_folder}/runs/{run_id}/benchmark_results"
+    else:
+        # Scheduled runs go directly under the date
+        repo_path = f"{date_folder}/{run_id}/benchmark_results"
+    
+    logger.info(f"Uploading benchmark results to dataset '{dataset_name}' at path '{repo_path}'")
+    
+    try:
+        # Get the authentication token (prioritize specific token, fallback to HF_TOKEN)
+        token = os.getenv("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN") or os.getenv("HF_TOKEN")
+        
+        # Upload all files in the output directory
+        from pathlib import Path
+        output_path = Path(output_dir)
+        
+        for file_path in output_path.rglob("*"):
+            if file_path.is_file():
+                # Calculate relative path from output_dir
+                relative_path = file_path.relative_to(output_path)
+                path_in_repo = f"{repo_path}/{relative_path}"
+                
+                logger.debug(f"Uploading {file_path} to {path_in_repo}")
+                
+                api.upload_file(
+                    path_or_fileobj=str(file_path),
+                    path_in_repo=path_in_repo,
+                    repo_id=dataset_name,
+                    repo_type="dataset",
+                    token=token,
+                    commit_message=f"Upload benchmark results for run {run_id}"
+                )
+        
+        logger.info(f"Successfully uploaded results to: https://huggingface.co/datasets/{dataset_name}/tree/main/{repo_path}")
+        
+        return run_id
+        
+    except Exception as upload_error:
+        logger.error(f"Failed to upload results: {upload_error}")
+        import traceback
+        logger.debug(traceback.format_exc())
+        return None
+
+
 def main():
     """Main entry point for the benchmarking script."""
+    # Generate a unique UUID for this benchmark run
+    benchmark_run_uuid = str(uuid.uuid4())[:8]
+    
     parser = argparse.ArgumentParser(
-        description="Run all benchmarks in the ./benches directory"
+        description="Run all benchmarks in the ./benches directory",
+        epilog="""
+Examples:
+  # Run all available benchmarks
+  python3 run_benchmarks.py
+  
+  # Run with specific model and upload to HuggingFace Dataset
+  python3 run_benchmarks.py --model-id meta-llama/Llama-2-7b-hf --upload-to-hf username/benchmark-results
+  
+  # Run with custom run ID and upload to HuggingFace Dataset
+  python3 run_benchmarks.py --run-id experiment_v1 --upload-to-hf org/benchmarks
+  
+  # Run only specific benchmarks with file logging
+  python3 run_benchmarks.py --include llama --enable-file-logging
+        """,
+        formatter_class=argparse.RawDescriptionHelpFormatter
     )
 
     parser.add_argument(
@@ -261,12 +368,6 @@ def main():
         help="Exclude benchmarks matching these names"
     )
 
-    parser.add_argument(
-        "--enable-mock",
-        action="store_true",
-        help="Enable mock benchmark (skipped by default)"
-    )
-    
     parser.add_argument(
         "--enable-file-logging",
         action="store_true",
@@ -279,12 +380,25 @@ def main():
         help="Git commit ID for metadata (if not provided, will auto-detect from git)"
     )
 
+    parser.add_argument(
+        "--upload-to-hub",
+        type=str,
+        help="Upload results to HuggingFace Dataset (provide dataset name, e.g., 'username/benchmark-results')"
+    )
+    
+    parser.add_argument(
+        "--run-id",
+        type=str,
+        help="Custom run ID for organizing results (if not provided, will generate a unique ID)"
+    )
+    
     args = parser.parse_args()
 
     # Setup logging
     logger = setup_logging(args.log_level, args.enable_file_logging)
 
     logger.info("Starting benchmark discovery and execution")
+    logger.info(f"Benchmark run UUID: {benchmark_run_uuid}")
     logger.info(f"Output directory: {args.output_dir}")
     logger.info(f"Benches directory: {args.benches_dir}")
 
@@ -327,9 +441,6 @@ def main():
         if args.model_id:
             benchmark_kwargs['model_id'] = args.model_id
 
-        # Add enable_mock flag for mock benchmark
-        benchmark_kwargs['enable_mock'] = args.enable_mock
-        
         # Add commit_id if provided
         if args.commit_id:
             benchmark_kwargs['commit_id'] = args.commit_id
@@ -352,7 +463,27 @@ def main():
                 successful_count += 1
 
         # Generate summary report
-        summary_file = generate_summary_report(args.output_dir, benchmark_results, logger)
+        summary_file = generate_summary_report(args.output_dir, benchmark_results, logger, benchmark_run_uuid)
+        
+        # Upload results to HuggingFace Dataset if requested
+        upload_run_id = None
+        if args.upload_to_hf:
+            logger.info("=" * 60)
+            logger.info("UPLOADING TO HUGGINGFACE DATASET")
+            logger.info("=" * 60)
+            # Use provided run_id or fallback to benchmark run UUID
+            effective_run_id = args.run_id or benchmark_run_uuid
+            upload_run_id = upload_results_to_hf_dataset(
+                output_dir=args.output_dir,
+                summary_file=summary_file,
+                dataset_name=args.upload_to_hf,
+                run_id=effective_run_id,
+                logger=logger
+            )
+            if upload_run_id:
+                logger.info(f"Upload completed with run ID: {upload_run_id}")
+            else:
+                logger.warning("Upload failed - continuing with local results")
 
         # Final summary
         total_benchmarks = len(filtered_benchmarks)
@@ -367,6 +498,14 @@ def main():
         logger.info(f"Output directory: {args.output_dir}")
         logger.info(f"Summary report: {summary_file}")
 
+        if args.upload_to_hf:
+            if upload_run_id:
+                logger.info(f"HuggingFace Dataset: {args.upload_to_hf}")
+                logger.info(f"Run ID: {upload_run_id}")
+                logger.info(f"View results: https://huggingface.co/datasets/{args.upload_to_hf}/tree/main/{datetime.now().strftime('%Y-%m-%d')}/runs/{upload_run_id}")
+            else:
+                logger.warning("Upload to HuggingFace Dataset failed")
+        
         if failed_count > 0:
             logger.warning(f"{failed_count} benchmark(s) failed. Check logs for details.")
             return 1