Merge pull request #41 from roboflow/develop

Merge develop into main, F1 score & Yolo v11
roboflow · Sep 30, 2024 · cadc27a · cadc27a
2 parents 4689a0c + 15c53a9
commit cadc27a
Show file tree

Hide file tree

Showing 61 changed files with 5,172 additions and 963 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,9 @@ data/*
 
 # yolov9 repo
 **/yolov9-repo
+# mmyolo-weights
+**/mmyolo-weights
+
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -30,7 +30,7 @@ repos:
           additional_dependencies: ["bandit[toml]"]
 
   -   repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.6.4
+      rev: v0.6.7
       hooks:
       -   id: ruff
           args: [--fix, --exit-non-zero-on-fix]

diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@ This project is a **model leaderboard** website for evaluating a variety of mode
 
 ## Project Structure
 
-```
+```text
 model-leaderboard/
 │
 ├── data/                  # Directory for storing datasets (e.g., COCO)
@@ -25,7 +25,7 @@ model-leaderboard/
 
 Each model is expected to have:
 
-```
+```text
 model_name/
 │
 ├── requirements.txt        # Dependencies for running the model
@@ -40,14 +40,14 @@ Each model is expected to be run in a separate python virtual environment, with
 Before we automate the models to be run regularly, the naming standards are relaxed.
 The only requirements is to store results for `results.json` in the model directory. For consistency, we advice to keep the scripts in `run.py`.
 
-### Key Files:
+### Key Files
 
 1. **`download_data.py`**: Downloads the dataset (currently configured for COCO) and places it into the `data/` directory.
 2. **`build_static_site.py`**: Aggregates the results of the models to be shown on the GitHub Pages site.
 3. **`run_overnight.sh`**: An early version of a script to run the entire process, generating model results and comparing to downloaded data. We hacked it together for the first iteration of the leaderboard. Requires `uv`.
 4. **`gradio_app.py`**: The initial version of the leaderboard UI. Displays model results in a gradio page.
 
-5. **Model-Specific Folders**:
+5. **Model-Specific Folders**"
 
    - Each object detection model is housed in its own folder under `models/object_detection/`. These folders include `run.py`, which generates evaluation results in the form of a `results.json` file.
 

diff --git a/gradio_app.py b/gradio_app.py
@@ -1,24 +1,35 @@
 import json
+import sys
 from pathlib import Path
 from typing import Any, Dict, List
 
 import gradio as gr
 
+TITLE = """<h1 align="center">Model Leaderboard </h1>"""
+DESC = """
+<div style="text-align: center; display: flex; justify-content: center; align-items: center;">
+    <italic>powered by: &nbsp<a href='https://github.com/roboflow/supervision'>
+    <img src='https://supervision.roboflow.com/latest/assets/supervision-lenny.png'
+    height=24 width=24 style='display: inline-block'> supervision</a></italic>
+    &nbsp&nbsp&nbsp&nbsp
+        <a href="https://github.com/roboflow/supervision">
+    <img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/roboflow/supervision"
+        style="margin-right: 10px;">
+    </a>
+</div>
+"""  # noqa: E501 title/docs
+
 
 def load_results() -> List[Dict]:
-    results_list = []
-    for model_dir in Path("models/object_detection").iterdir():
-        if model_dir.is_file() or model_dir.name.startswith("_"):
-            continue
-        results_file = model_dir / "results.json"
-        if not results_file.exists():
-            print(f"Results file not found for {model_dir.name}")
-            continue
-        with open(results_file) as f:
-            results = json.load(f)
-            results_list.append(results)
-    results_list.sort(key=lambda x: x["metadata"]["model"])
-    return results_list
+    results: List[Dict] = []
+    results_file: Path = Path("static/aggregate_results.json")
+    if not results_file.exists():
+        print("aggregate_results.json file not found")
+        sys.exit(1)
+    with open(results_file) as f:
+        results = json.load(f)
+    results.sort(key=lambda x: x["metadata"]["model"])
+    return results
 
 
 def get_result_header() -> List[str]:
@@ -31,6 +42,14 @@ def get_result_header() -> List[str]:
         "mAP 50:95 (Small)",
         "mAP 50:95 (Medium)",
         "mAP 50:95 (Large)",
+        "F1 50",
+        "F1 75",
+        "F1 50 (Small)",
+        "F1 75 (Small)",
+        "F1 50 (Medium)",
+        "F1 75 (Medium)",
+        "F1 50 (Large)",
+        "F1 75 (Large)",
     ]
 
 
@@ -50,6 +69,14 @@ def parse_result(result: Dict) -> List[Any]:
         round(result["small_objects"]["map50_95"], round_digits),
         round(result["medium_objects"]["map50_95"], round_digits),
         round(result["large_objects"]["map50_95"], round_digits),
+        round(result["f1_50"], round_digits),
+        round(result["f1_75"], round_digits),
+        round(result["f1_small_objects"]["f1_50"], round_digits),
+        round(result["f1_small_objects"]["f1_75"], round_digits),
+        round(result["f1_medium_objects"]["f1_50"], round_digits),
+        round(result["f1_medium_objects"]["f1_75"], round_digits),
+        round(result["f1_large_objects"]["f1_50"], round_digits),
+        round(result["f1_large_objects"]["f1_75"], round_digits),
     ]
 
 
@@ -59,13 +86,8 @@ def parse_result(result: Dict) -> List[Any]:
 
 with gr.Blocks() as demo:
     gr.Markdown("# Model Leaderboard")
-    gr.HTML(
-        """
-        <italic>powered by: &nbsp<a href='https://github.com/roboflow/supervision'>
-        <img src='https://supervision.roboflow.com/latest/assets/supervision-lenny.png'
-        height=24 width=24 style='display: inline-block'> supervision</a></italic>
-        """
-    )
+    gr.HTML(TITLE)
+    gr.HTML(DESC)
     gr.DataFrame(headers=header, value=results)
 
 demo.launch()
diff --git a/index.html b/index.html
@@ -9,8 +9,9 @@
     <link rel="stylesheet" href="static/dark-theme.css" id="theme-link">
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.2/css/all.min.css">
     <link rel="stylesheet" href="https://cdn.datatables.net/2.1.5/css/dataTables.dataTables.min.css">
-    <link href="https://fonts.googleapis.com/css2?family=Source+Code+Pro:ital,wght@0,200..900;1,200..900&display=swap"
-        rel="stylesheet">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Source+Code+Pro:ital,wght@0,200..900;1,200..900&display=swap">
+
+
 </head>
 
 <body>
@@ -46,28 +47,36 @@ <h1>Model Leaderboard</h1>
         <table id="leaderboard">
             <thead>
                 <tr>
-                    <th>Model</th>
-                    <th>Parameters (M)</th>
-                    <th>mAP 50:95</th>
-                    <th>mAP 50</th>
-                    <th>mAP 75</th>
-                    <th>mAP 50:95 (Small)</th>
-                    <th>mAP 50:95 (Medium)</th>
-                    <th>mAP 50:95 (Large)</th>
-                    <th>License</th>
+                    <th data-header-tooltip="Model">Model</th>
+                    <th data-header-tooltip="Parameters (M)">Parameters (M)</th>
+                    <th data-header-tooltip="Mean Average Precision at IoU 50:95">mAP 50:95</th>
+                    <th data-header-tooltip="Mean Average Precision at IoU 50">mAP 50</th>
+                    <th data-header-tooltip="Mean Average Precision at IoU 75">mAP 75</th>
+                    <th data-header-tooltip="Mean Average Precision at IoU 50:95 for Small Objects">mAP 50:95 (Small)</th>
+                    <th data-header-tooltip="Mean Average Precision at IoU 50:95 for Medium Objects">mAP 50:95 (Medium)</th>
+                    <th data-header-tooltip="Mean Average Precision at IoU 50:95 for Large Objects">mAP 50:95 (Large)</th>
+                    <th data-header-tooltip="F1 Score at IoU 50">F1 50</th>
+                    <th data-header-tooltip="F1 Score at IoU 75">F1 75</th>
+                    <th data-header-tooltip="F1 Score at IoU 50 for Small Objects">F1 50 (Small)</th>
+                    <th data-header-tooltip="F1 Score at IoU 75 for Small Objects">F1 75 (Small)</th>
+                    <th data-header-tooltip="F1 Score at IoU 50 for Medium Objects">F1 50 (Medium)</th>
+                    <th data-header-tooltip="F1 Score at IoU 75 for Medium Objects">F1 75 (Medium)</th>
+                    <th data-header-tooltip="F1 Score at IoU 50 for Large Objects">F1 50 (Large)</th>
+                    <th data-header-tooltip="F1 Score at IoU 75 for Large Objects">F1 75 (Large)</th>
+                    <th data-header-tooltip="License">License</th>
                 </tr>
             </thead>
             <tbody id="table-body">
             </tbody>
         </table>
     </div>
 
+    <div class="tooltip"></div>
 
-
-    <script src="https://cdn.jsdelivr.net/npm/@floating-ui/core@1.6.7"></script>
-    <script src="https://cdn.jsdelivr.net/npm/@floating-ui/dom@1.6.10"></script>
     <script src="https://code.jquery.com/jquery-3.7.1.min.js"
         integrity="sha256-/JqT3SQfawRcv/BIHPThkBvs0OEvtFFmqPF/lYI/Cxo=" crossorigin="anonymous"></script>
+    <script src="https://cdn.jsdelivr.net/npm/@floating-ui/core@1.6.7"></script>
+    <script src="https://cdn.jsdelivr.net/npm/@floating-ui/dom@1.6.10"></script>
     <script src="https://cdn.datatables.net/2.1.5/js/dataTables.min.js"></script>
     <script src="static/aggregate_results.js"></script>
     <script src="static/script.js"></script>

diff --git a/models/object_detection/configs.py b/models/object_detection/configs.py
@@ -1 +1,2 @@
 CONFIDENCE_THRESHOLD = 0.001
+DATASET_DIR = "../../../data/coco-val-2017"
diff --git a/models/object_detection/rt-detr/requirements.txt b/models/object_detection/rt-detr/requirements.txt
@@ -2,5 +2,6 @@ torch>=2.0.1
 torchvision>=0.15.2
 PyYAML
 tensorboard
-supervision>=0.24.0rc1
+supervision @ git+https://github.com/roboflow/supervision.git@develop
 tqdm
+pycocotools
diff --git a/models/object_detection/rt-detr/results_rtdetr_r101vd.json b/models/object_detection/rt-detr/results_rtdetr_r101vd.json
@@ -1,30 +1,30 @@
 {
     "metadata": {
-        "model": "RT-DETRv1 (r101vd)",
+        "model": "RT-DETRv1 r101vd",
         "license": "Apache-2.0",
         "run_parameters": {
             "imgsz": 640,
             "conf": 0.001
         },
         "param_count": 92486124,
-        "run_date": "2024-09-10T16:21:37.375759+00:00"
+        "run_date": "2024-09-23T00:21:53.772495+00:00"
     },
     "map50_95": 0.5243300327374072,
-    "map50": 0.703951889531495,
-    "map75": 0.5678049556440812,
+    "map50": 0.7039518895314949,
+    "map75": 0.567804955644081,
     "small_objects": {
-        "map50_95": 0.2478549542363811,
-        "map50": 0.40066746534705366,
+        "map50_95": 0.24785495423638104,
+        "map50": 0.4006674653470536,
         "map75": 0.2710220439441845
     },
     "medium_objects": {
-        "map50_95": 0.4768582593911274,
-        "map50": 0.6625457345615452,
-        "map75": 0.5350029755816491
+        "map50_95": 0.47685825939112725,
+        "map50": 0.6625457345615451,
+        "map75": 0.535002975581649
     },
     "large_objects": {
-        "map50_95": 0.6758202036651619,
-        "map50": 0.8126340925145896,
+        "map50_95": 0.675820203665162,
+        "map50": 0.8126340925145898,
         "map75": 0.7285503198138372
     },
     "iou_thresholds": [
@@ -38,5 +38,31 @@
         0.85,
         0.8999999999999999,
         0.95
+    ],
+    "f1_50": 0.055002367478898175,
+    "f1_75": 0.044801037891291674,
+    "f1_small_objects": {
+        "f1_50": 0.05454378845982463,
+        "f1_75": 0.03349339245023226
+    },
+    "f1_medium_objects": {
+        "f1_50": 0.09350859896632002,
+        "f1_75": 0.07973878658212949
+    },
+    "f1_large_objects": {
+        "f1_50": 0.07539115839654872,
+        "f1_75": 0.07182353249020428
+    },
+    "f1_iou_thresholds": [
+        0.5,
+        0.55,
+        0.6,
+        0.65,
+        0.7,
+        0.75,
+        0.8,
+        0.85,
+        0.8999999999999999,
+        0.95
     ]
 }
diff --git a/models/object_detection/rt-detr/results_rtdetr_r18vd.json b/models/object_detection/rt-detr/results_rtdetr_r18vd.json
@@ -1,31 +1,31 @@
 {
     "metadata": {
-        "model": "RT-DETRv1 (r18vd)",
+        "model": "RT-DETRv1 r18vd",
         "license": "Apache-2.0",
         "run_parameters": {
             "imgsz": 640,
             "conf": 0.001
         },
         "param_count": 21955472,
-        "run_date": "2024-09-10T16:11:52.358781+00:00"
+        "run_date": "2024-09-23T00:13:40.227469+00:00"
     },
-    "map50_95": 0.4475030676297469,
-    "map50": 0.6156237779572045,
-    "map75": 0.48407456419504113,
+    "map50_95": 0.4475030676297468,
+    "map50": 0.6156237779572044,
+    "map75": 0.4840745641950413,
     "small_objects": {
-        "map50_95": 0.19035576042172211,
+        "map50_95": 0.1903557604217221,
         "map50": 0.3089236487784832,
-        "map75": 0.2030844478437464
+        "map75": 0.20308444784374644
     },
     "medium_objects": {
-        "map50_95": 0.3952415547608772,
-        "map50": 0.5612930297373171,
-        "map75": 0.442782433290807
+        "map50_95": 0.39524155476087713,
+        "map50": 0.561293029737317,
+        "map75": 0.4427824332908067
     },
     "large_objects": {
         "map50_95": 0.5894997261827915,
         "map50": 0.7265982743927857,
-        "map75": 0.6411468239795821
+        "map75": 0.6411468239795823
     },
     "iou_thresholds": [
         0.5,
@@ -38,5 +38,31 @@
         0.85,
         0.8999999999999999,
         0.95
+    ],
+    "f1_50": 0.053371116537810466,
+    "f1_75": 0.04239322395192036,
+    "f1_small_objects": {
+        "f1_50": 0.05425970668777465,
+        "f1_75": 0.031303239161977504
+    },
+    "f1_medium_objects": {
+        "f1_50": 0.08500153986627708,
+        "f1_75": 0.07000788375061538
+    },
+    "f1_large_objects": {
+        "f1_50": 0.07171836156682274,
+        "f1_75": 0.06735933629335475
+    },
+    "f1_iou_thresholds": [
+        0.5,
+        0.55,
+        0.6,
+        0.65,
+        0.7,
+        0.75,
+        0.8,
+        0.85,
+        0.8999999999999999,
+        0.95
     ]
 }
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		CONFIDENCE_THRESHOLD = 0.001
		DATASET_DIR = "../../../data/coco-val-2017"