Merge pull request #39 from roboflow/feature/f1-scores

feat: 🚀 f1 metrics are added into jsons and new yolo11 metric/F1 also added
roboflow · Sep 30, 2024 · 15c53a9 · 15c53a9
2 parents bd6a42f + aa3a178
commit 15c53a9
Show file tree

Hide file tree

Showing 54 changed files with 4,604 additions and 1,252 deletions.
diff --git a/gradio_app.py b/gradio_app.py
@@ -1,24 +1,35 @@
 import json
+import sys
 from pathlib import Path
 from typing import Any, Dict, List
 
 import gradio as gr
 
+TITLE = """<h1 align="center">Model Leaderboard </h1>"""
+DESC = """
+<div style="text-align: center; display: flex; justify-content: center; align-items: center;">
+    <italic>powered by: &nbsp<a href='https://github.com/roboflow/supervision'>
+    <img src='https://supervision.roboflow.com/latest/assets/supervision-lenny.png'
+    height=24 width=24 style='display: inline-block'> supervision</a></italic>
+    &nbsp&nbsp&nbsp&nbsp
+        <a href="https://github.com/roboflow/supervision">
+    <img alt="GitHub Repo stars" src="https://img.shields.io/github/stars/roboflow/supervision"
+        style="margin-right: 10px;">
+    </a>
+</div>
+"""  # noqa: E501 title/docs
+
 
 def load_results() -> List[Dict]:
-    results_list = []
-    for model_dir in Path("models/object_detection").iterdir():
-        if model_dir.is_file() or model_dir.name.startswith("_"):
-            continue
-        results_file = model_dir / "results.json"
-        if not results_file.exists():
-            print(f"Results file not found for {model_dir.name}")
-            continue
-        with open(results_file) as f:
-            results = json.load(f)
-            results_list.append(results)
-    results_list.sort(key=lambda x: x["metadata"]["model"])
-    return results_list
+    results: List[Dict] = []
+    results_file: Path = Path("static/aggregate_results.json")
+    if not results_file.exists():
+        print("aggregate_results.json file not found")
+        sys.exit(1)
+    with open(results_file) as f:
+        results = json.load(f)
+    results.sort(key=lambda x: x["metadata"]["model"])
+    return results
 
 
 def get_result_header() -> List[str]:
@@ -31,6 +42,14 @@ def get_result_header() -> List[str]:
         "mAP 50:95 (Small)",
         "mAP 50:95 (Medium)",
         "mAP 50:95 (Large)",
+        "F1 50",
+        "F1 75",
+        "F1 50 (Small)",
+        "F1 75 (Small)",
+        "F1 50 (Medium)",
+        "F1 75 (Medium)",
+        "F1 50 (Large)",
+        "F1 75 (Large)",
     ]
 
 
@@ -50,6 +69,14 @@ def parse_result(result: Dict) -> List[Any]:
         round(result["small_objects"]["map50_95"], round_digits),
         round(result["medium_objects"]["map50_95"], round_digits),
         round(result["large_objects"]["map50_95"], round_digits),
+        round(result["f1_50"], round_digits),
+        round(result["f1_75"], round_digits),
+        round(result["f1_small_objects"]["f1_50"], round_digits),
+        round(result["f1_small_objects"]["f1_75"], round_digits),
+        round(result["f1_medium_objects"]["f1_50"], round_digits),
+        round(result["f1_medium_objects"]["f1_75"], round_digits),
+        round(result["f1_large_objects"]["f1_50"], round_digits),
+        round(result["f1_large_objects"]["f1_75"], round_digits),
     ]
 
 
@@ -59,13 +86,8 @@ def parse_result(result: Dict) -> List[Any]:
 
 with gr.Blocks() as demo:
     gr.Markdown("# Model Leaderboard")
-    gr.HTML(
-        """
-        <italic>powered by: &nbsp<a href='https://github.com/roboflow/supervision'>
-        <img src='https://supervision.roboflow.com/latest/assets/supervision-lenny.png'
-        height=24 width=24 style='display: inline-block'> supervision</a></italic>
-        """
-    )
+    gr.HTML(TITLE)
+    gr.HTML(DESC)
     gr.DataFrame(headers=header, value=results)
 
 demo.launch()
diff --git a/index.html b/index.html
@@ -9,8 +9,9 @@
     <link rel="stylesheet" href="static/dark-theme.css" id="theme-link">
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.2/css/all.min.css">
     <link rel="stylesheet" href="https://cdn.datatables.net/2.1.5/css/dataTables.dataTables.min.css">
-    <link href="https://fonts.googleapis.com/css2?family=Source+Code+Pro:ital,wght@0,200..900;1,200..900&display=swap"
-        rel="stylesheet">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Source+Code+Pro:ital,wght@0,200..900;1,200..900&display=swap">
+
+
 </head>
 
 <body>
@@ -54,6 +55,14 @@ <h1>Model Leaderboard</h1>
                     <th data-header-tooltip="Mean Average Precision at IoU 50:95 for Small Objects">mAP 50:95 (Small)</th>
                     <th data-header-tooltip="Mean Average Precision at IoU 50:95 for Medium Objects">mAP 50:95 (Medium)</th>
                     <th data-header-tooltip="Mean Average Precision at IoU 50:95 for Large Objects">mAP 50:95 (Large)</th>
+                    <th data-header-tooltip="F1 Score at IoU 50">F1 50</th>
+                    <th data-header-tooltip="F1 Score at IoU 75">F1 75</th>
+                    <th data-header-tooltip="F1 Score at IoU 50 for Small Objects">F1 50 (Small)</th>
+                    <th data-header-tooltip="F1 Score at IoU 75 for Small Objects">F1 75 (Small)</th>
+                    <th data-header-tooltip="F1 Score at IoU 50 for Medium Objects">F1 50 (Medium)</th>
+                    <th data-header-tooltip="F1 Score at IoU 75 for Medium Objects">F1 75 (Medium)</th>
+                    <th data-header-tooltip="F1 Score at IoU 50 for Large Objects">F1 50 (Large)</th>
+                    <th data-header-tooltip="F1 Score at IoU 75 for Large Objects">F1 75 (Large)</th>
                     <th data-header-tooltip="License">License</th>
                 </tr>
             </thead>

diff --git a/models/object_detection/configs.py b/models/object_detection/configs.py
@@ -1 +1,2 @@
 CONFIDENCE_THRESHOLD = 0.001
+DATASET_DIR = "../../../data/coco-val-2017"
diff --git a/models/object_detection/rt-detr/requirements.txt b/models/object_detection/rt-detr/requirements.txt
@@ -2,5 +2,6 @@ torch>=2.0.1
 torchvision>=0.15.2
 PyYAML
 tensorboard
-supervision>=0.24.0rc1
+supervision @ git+https://github.com/roboflow/supervision.git@develop
 tqdm
+pycocotools
diff --git a/models/object_detection/rt-detr/results_rtdetr_r101vd.json b/models/object_detection/rt-detr/results_rtdetr_r101vd.json
@@ -1,30 +1,30 @@
 {
     "metadata": {
-        "model": "RT-DETRv1 (r101vd)",
+        "model": "RT-DETRv1 r101vd",
         "license": "Apache-2.0",
         "run_parameters": {
             "imgsz": 640,
             "conf": 0.001
         },
         "param_count": 92486124,
-        "run_date": "2024-09-10T16:21:37.375759+00:00"
+        "run_date": "2024-09-23T00:21:53.772495+00:00"
     },
     "map50_95": 0.5243300327374072,
-    "map50": 0.703951889531495,
-    "map75": 0.5678049556440812,
+    "map50": 0.7039518895314949,
+    "map75": 0.567804955644081,
     "small_objects": {
-        "map50_95": 0.2478549542363811,
-        "map50": 0.40066746534705366,
+        "map50_95": 0.24785495423638104,
+        "map50": 0.4006674653470536,
         "map75": 0.2710220439441845
     },
     "medium_objects": {
-        "map50_95": 0.4768582593911274,
-        "map50": 0.6625457345615452,
-        "map75": 0.5350029755816491
+        "map50_95": 0.47685825939112725,
+        "map50": 0.6625457345615451,
+        "map75": 0.535002975581649
     },
     "large_objects": {
-        "map50_95": 0.6758202036651619,
-        "map50": 0.8126340925145896,
+        "map50_95": 0.675820203665162,
+        "map50": 0.8126340925145898,
         "map75": 0.7285503198138372
     },
     "iou_thresholds": [
@@ -38,5 +38,31 @@
         0.85,
         0.8999999999999999,
         0.95
+    ],
+    "f1_50": 0.055002367478898175,
+    "f1_75": 0.044801037891291674,
+    "f1_small_objects": {
+        "f1_50": 0.05454378845982463,
+        "f1_75": 0.03349339245023226
+    },
+    "f1_medium_objects": {
+        "f1_50": 0.09350859896632002,
+        "f1_75": 0.07973878658212949
+    },
+    "f1_large_objects": {
+        "f1_50": 0.07539115839654872,
+        "f1_75": 0.07182353249020428
+    },
+    "f1_iou_thresholds": [
+        0.5,
+        0.55,
+        0.6,
+        0.65,
+        0.7,
+        0.75,
+        0.8,
+        0.85,
+        0.8999999999999999,
+        0.95
     ]
 }
diff --git a/models/object_detection/rt-detr/results_rtdetr_r18vd.json b/models/object_detection/rt-detr/results_rtdetr_r18vd.json
@@ -1,31 +1,31 @@
 {
     "metadata": {
-        "model": "RT-DETRv1 (r18vd)",
+        "model": "RT-DETRv1 r18vd",
         "license": "Apache-2.0",
         "run_parameters": {
             "imgsz": 640,
             "conf": 0.001
         },
         "param_count": 21955472,
-        "run_date": "2024-09-10T16:11:52.358781+00:00"
+        "run_date": "2024-09-23T00:13:40.227469+00:00"
     },
-    "map50_95": 0.4475030676297469,
-    "map50": 0.6156237779572045,
-    "map75": 0.48407456419504113,
+    "map50_95": 0.4475030676297468,
+    "map50": 0.6156237779572044,
+    "map75": 0.4840745641950413,
     "small_objects": {
-        "map50_95": 0.19035576042172211,
+        "map50_95": 0.1903557604217221,
         "map50": 0.3089236487784832,
-        "map75": 0.2030844478437464
+        "map75": 0.20308444784374644
     },
     "medium_objects": {
-        "map50_95": 0.3952415547608772,
-        "map50": 0.5612930297373171,
-        "map75": 0.442782433290807
+        "map50_95": 0.39524155476087713,
+        "map50": 0.561293029737317,
+        "map75": 0.4427824332908067
     },
     "large_objects": {
         "map50_95": 0.5894997261827915,
         "map50": 0.7265982743927857,
-        "map75": 0.6411468239795821
+        "map75": 0.6411468239795823
     },
     "iou_thresholds": [
         0.5,
@@ -38,5 +38,31 @@
         0.85,
         0.8999999999999999,
         0.95
+    ],
+    "f1_50": 0.053371116537810466,
+    "f1_75": 0.04239322395192036,
+    "f1_small_objects": {
+        "f1_50": 0.05425970668777465,
+        "f1_75": 0.031303239161977504
+    },
+    "f1_medium_objects": {
+        "f1_50": 0.08500153986627708,
+        "f1_75": 0.07000788375061538
+    },
+    "f1_large_objects": {
+        "f1_50": 0.07171836156682274,
+        "f1_75": 0.06735933629335475
+    },
+    "f1_iou_thresholds": [
+        0.5,
+        0.55,
+        0.6,
+        0.65,
+        0.7,
+        0.75,
+        0.8,
+        0.85,
+        0.8999999999999999,
+        0.95
     ]
 }
diff --git a/models/object_detection/rt-detr/results_rtdetr_r34vd.json b/models/object_detection/rt-detr/results_rtdetr_r34vd.json
@@ -1,30 +1,30 @@
 {
     "metadata": {
-        "model": "RT-DETRv1 (r34vd)",
+        "model": "RT-DETRv1 r34vd",
         "license": "Apache-2.0",
         "run_parameters": {
             "imgsz": 640,
             "conf": 0.001
         },
         "param_count": 33212676,
-        "run_date": "2024-09-10T16:14:42.535005+00:00"
+        "run_date": "2024-09-23T00:15:53.484978+00:00"
     },
-    "map50_95": 0.4711144281973547,
-    "map50": 0.6444973848870644,
-    "map75": 0.5089359115940468,
+    "map50_95": 0.47111442819735466,
+    "map50": 0.6444973848870641,
+    "map75": 0.508935911594047,
     "small_objects": {
-        "map50_95": 0.2105158447838686,
-        "map50": 0.3410547104144312,
+        "map50_95": 0.21051584478386856,
+        "map50": 0.34105471041443125,
         "map75": 0.22164556950501305
     },
     "medium_objects": {
-        "map50_95": 0.4214592794562458,
-        "map50": 0.5926057024799766,
-        "map75": 0.46982935350097305
+        "map50_95": 0.42145927945624584,
+        "map50": 0.5926057024799771,
+        "map75": 0.4698293535009732
     },
     "large_objects": {
-        "map50_95": 0.6165598066953087,
-        "map50": 0.7552632596160003,
+        "map50_95": 0.6165598066953086,
+        "map50": 0.755263259616,
         "map75": 0.667420318296039
     },
     "iou_thresholds": [
@@ -38,5 +38,31 @@
         0.85,
         0.8999999999999999,
         0.95
+    ],
+    "f1_50": 0.053745432714569996,
+    "f1_75": 0.04303912632299453,
+    "f1_small_objects": {
+        "f1_50": 0.05440093689866381,
+        "f1_75": 0.032358600850793104
+    },
+    "f1_medium_objects": {
+        "f1_50": 0.08658033405706038,
+        "f1_75": 0.07186071097655883
+    },
+    "f1_large_objects": {
+        "f1_50": 0.073251962690366,
+        "f1_75": 0.06901239395674584
+    },
+    "f1_iou_thresholds": [
+        0.5,
+        0.55,
+        0.6,
+        0.65,
+        0.7,
+        0.75,
+        0.8,
+        0.85,
+        0.8999999999999999,
+        0.95
     ]
 }
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		CONFIDENCE_THRESHOLD = 0.001
		DATASET_DIR = "../../../data/coco-val-2017"