add preparation doc

open-mmlab · Sep 19, 2023 · 748432d · 748432d
1 parent d89a141
commit 748432d
Show file tree

Hide file tree

Showing 2 changed files with 181 additions and 67 deletions.
diff --git a/configs/_base_/datasets/ubody3d.py b/configs/_base_/datasets/ubody3d.py
@@ -169,7 +169,7 @@
             swap='R_Thumb_3'),
         28:
         dict(
-            name='L_Thumb4',
+            name='L_Thumb_4',
             id=28,
             color=[255, 128, 0],
             type='',
@@ -313,7 +313,7 @@
             id=48,
             color=[255, 128, 0],
             type='',
-            swap='L_Thumb4'),
+            swap='L_Thumb_4'),
         49:
         dict(
             name='R_Index_1',
@@ -872,71 +872,87 @@
             swap='Face_56'),
     },
     skeleton_info={
-        0: dict(link=('L_Hip', 'R_Hip'), id=0, color=[0, 255, 0]),
-        1: dict(link=('L_Knee', 'R_Knee'), id=1, color=[0, 255, 0]),
-        2: dict(link=('L_Ankle', 'R_Ankle'), id=2, color=[0, 255, 0]),
-        3: dict(link=('L_Shoulder', 'R_Shoulder'), id=3, color=[0, 255, 0]),
-        4: dict(link=('L_Elbow', 'R_Elbow'), id=4, color=[0, 255, 0]),
-        5: dict(link=('L_Wrist', 'R_Wrist'), id=5, color=[0, 255, 0]),
-        6: dict(link=('L_Big_toe', 'R_Big_toe'), id=6, color=[0, 255, 0]),
-        7: dict(link=('L_Small_toe', 'R_Small_toe'), id=7, color=[0, 255, 0]),
-        8: dict(link=('L_Heel', 'R_Heel'), id=8, color=[0, 255, 0]),
-        9: dict(link=('L_Ear', 'R_Ear'), id=9, color=[0, 255, 0]),
-        10: dict(link=('L_Eye', 'R_Eye'), id=10, color=[0, 255, 0]),
-        11: dict(link=('L_Thumb_1', 'R_Thumb_1'), id=11, color=[255, 128, 0]),
-        12: dict(link=('L_Thumb_2', 'R_Thumb_2'), id=12, color=[255, 128, 0]),
-        13: dict(link=('L_Thumb_3', 'R_Thumb_3'), id=13, color=[255, 128, 0]),
-        14: dict(link=('L_Thumb4', 'R_Thumb_4'), id=14, color=[255, 128, 0]),
-        15: dict(link=('L_Index_1', 'R_Index_1'), id=15, color=[255, 128, 0]),
-        16: dict(link=('L_Index_2', 'R_Index_2'), id=16, color=[255, 128, 0]),
-        17: dict(link=('L_Index_3', 'R_Index_3'), id=17, color=[255, 128, 0]),
-        18: dict(link=('L_Index_4', 'R_Index_4'), id=18, color=[255, 128, 0]),
-        19:
-        dict(link=('L_Middle_1', 'R_Middle_1'), id=19, color=[255, 128, 0]),
-        20:
-        dict(link=('L_Middle_2', 'R_Middle_2'), id=20, color=[255, 128, 0]),
-        21:
-        dict(link=('L_Middle_3', 'R_Middle_3'), id=21, color=[255, 128, 0]),
-        22:
-        dict(link=('L_Middle_4', 'R_Middle_4'), id=22, color=[255, 128, 0]),
-        23: dict(link=('L_Ring_1', 'R_Ring_1'), id=23, color=[255, 128, 0]),
-        24: dict(link=('L_Ring_2', 'R_Ring_2'), id=24, color=[255, 128, 0]),
-        25: dict(link=('L_Ring_3', 'R_Ring_3'), id=25, color=[255, 128, 0]),
-        26: dict(link=('L_Ring_4', 'R_Ring_4'), id=26, color=[255, 128, 0]),
-        27: dict(link=('L_Pinky_1', 'R_Pinky_1'), id=27, color=[255, 128, 0]),
-        28: dict(link=('L_Pinky_2', 'R_Pinky_2'), id=28, color=[255, 128, 0]),
-        29: dict(link=('L_Pinky_3', 'R_Pinky_3'), id=29, color=[255, 128, 0]),
-        30: dict(link=('L_Pinky_4', 'R_Pinky_4'), id=30, color=[255, 128, 0]),
-        31: dict(link=('Face_3', 'Face_4'), id=31, color=[255, 255, 255]),
-        32: dict(link=('Face_5', 'Face_14'), id=32, color=[255, 255, 255]),
-        33: dict(link=('Face_6', 'Face_13'), id=33, color=[255, 255, 255]),
-        34: dict(link=('Face_7', 'Face_12'), id=34, color=[255, 255, 255]),
-        35: dict(link=('Face_8', 'Face_11'), id=35, color=[255, 255, 255]),
-        36: dict(link=('Face_9', 'Face_10'), id=36, color=[255, 255, 255]),
-        37: dict(link=('Face_19', 'Face_23'), id=37, color=[255, 255, 255]),
-        38: dict(link=('Face_20', 'Face_22'), id=38, color=[255, 255, 255]),
-        39: dict(link=('Face_24', 'Face_33'), id=39, color=[255, 255, 255]),
-        40: dict(link=('Face_25', 'Face_32'), id=40, color=[255, 255, 255]),
-        41: dict(link=('Face_26', 'Face_31'), id=41, color=[255, 255, 255]),
-        42: dict(link=('Face_27', 'Face_30'), id=42, color=[255, 255, 255]),
-        43: dict(link=('Face_28', 'Face_35'), id=43, color=[255, 255, 255]),
-        44: dict(link=('Face_29', 'Face_34'), id=44, color=[255, 255, 255]),
-        45: dict(link=('Face_36', 'Face_42'), id=45, color=[255, 255, 255]),
-        46: dict(link=('Face_37', 'Face_41'), id=46, color=[255, 255, 255]),
-        47: dict(link=('Face_38', 'Face_40'), id=47, color=[255, 255, 255]),
-        48: dict(link=('Face_43', 'Face_47'), id=48, color=[255, 255, 255]),
-        49: dict(link=('Face_44', 'Face_46'), id=49, color=[255, 255, 255]),
-        50: dict(link=('Face_48', 'Face_52'), id=50, color=[255, 255, 255]),
-        51: dict(link=('Face_49', 'Face_51'), id=51, color=[255, 255, 255]),
-        52: dict(link=('Face_53', 'Face_55'), id=52, color=[255, 255, 255]),
-        53: dict(link=('Face_56', 'Face_72'), id=53, color=[255, 255, 255]),
-        54: dict(link=('Face_57', 'Face_71'), id=54, color=[255, 255, 255]),
-        55: dict(link=('Face_58', 'Face_70'), id=55, color=[255, 255, 255]),
-        56: dict(link=('Face_59', 'Face_69'), id=56, color=[255, 255, 255]),
-        57: dict(link=('Face_60', 'Face_68'), id=57, color=[255, 255, 255]),
-        58: dict(link=('Face_61', 'Face_67'), id=58, color=[255, 255, 255]),
-        59: dict(link=('Face_62', 'Face_66'), id=59, color=[255, 255, 255]),
-        60: dict(link=('Face_63', 'Face_65'), id=60, color=[255, 255, 255]),
+        0: dict(link=('L_Ankle', 'L_Knee'), id=0, color=[0, 255, 0]),
+        1: dict(link=('L_Knee', 'L_Hip'), id=1, color=[0, 255, 0]),
+        2: dict(link=('R_Ankle', 'R_Knee'), id=2, color=[0, 255, 0]),
+        3: dict(link=('R_Knee', 'R_Hip'), id=3, color=[0, 255, 0]),
+        4: dict(link=('L_Hip', 'R_Hip'), id=4, color=[0, 255, 0]),
+        5: dict(link=('L_Shoulder', 'L_Hip'), id=5, color=[0, 255, 0]),
+        6: dict(link=('R_Shoulder', 'R_Hip'), id=6, color=[0, 255, 0]),
+        7: dict(link=('L_Shoulder', 'R_Shoulder'), id=7, color=[0, 255, 0]),
+        8: dict(link=('L_Shoulder', 'L_Elbow'), id=8, color=[0, 255, 0]),
+        9: dict(link=('R_Shoulder', 'R_Elbow'), id=9, color=[0, 255, 0]),
+        10: dict(link=('L_Elbow', 'L_Wrist'), id=10, color=[0, 255, 0]),
+        11: dict(link=('R_Elbow', 'R_Wrist'), id=11, color=[255, 128, 0]),
+        12: dict(link=('L_Eye', 'R_Eye'), id=12, color=[255, 128, 0]),
+        13: dict(link=('Nose', 'L_Eye'), id=13, color=[255, 128, 0]),
+        14: dict(link=('Nose', 'R_Eye'), id=14, color=[255, 128, 0]),
+        15: dict(link=('L_Eye', 'L_Ear'), id=15, color=[255, 128, 0]),
+        16: dict(link=('R_Eye', 'R_Ear'), id=16, color=[255, 128, 0]),
+        17: dict(link=('L_Ear', 'L_Shoulder'), id=17, color=[255, 128, 0]),
+        18: dict(link=('R_Ear', 'R_Shoulder'), id=18, color=[255, 128, 0]),
+        19: dict(link=('L_Ankle', 'L_Big_toe'), id=19, color=[255, 128, 0]),
+        20: dict(link=('L_Ankle', 'L_Small_toe'), id=20, color=[255, 128, 0]),
+        21: dict(link=('L_Ankle', 'L_Heel'), id=21, color=[255, 128, 0]),
+        22: dict(link=('R_Ankle', 'R_Big_toe'), id=22, color=[255, 128, 0]),
+        23: dict(link=('R_Ankle', 'R_Small_toe'), id=23, color=[255, 128, 0]),
+        24: dict(link=('R_Ankle', 'R_Heel'), id=24, color=[255, 128, 0]),
+        25: dict(link=('L_Wrist', 'L_Thumb_1'), id=25, color=[255, 128, 0]),
+        26: dict(link=('L_Thumb_1', 'L_Thumb_2'), id=26, color=[255, 128, 0]),
+        27: dict(link=('L_Thumb_2', 'L_Thumb_3'), id=27, color=[255, 128, 0]),
+        28: dict(link=('L_Thumb_3', 'L_Thumb_4'), id=28, color=[255, 128, 0]),
+        29: dict(link=('L_Wrist', 'L_Index_1'), id=29, color=[255, 128, 0]),
+        30: dict(link=('L_Index_1', 'L_Index_2'), id=30, color=[255, 128, 0]),
+        31:
+        dict(link=('L_Index_2', 'L_Index_3'), id=31, color=[255, 255, 255]),
+        32:
+        dict(link=('L_Index_3', 'L_Index_4'), id=32, color=[255, 255, 255]),
+        33: dict(link=('L_Wrist', 'L_Middle_1'), id=33, color=[255, 255, 255]),
+        34:
+        dict(link=('L_Middle_1', 'L_Middle_2'), id=34, color=[255, 255, 255]),
+        35:
+        dict(link=('L_Middle_2', 'L_Middle_3'), id=35, color=[255, 255, 255]),
+        36:
+        dict(link=('L_Middle_3', 'L_Middle_4'), id=36, color=[255, 255, 255]),
+        37: dict(link=('L_Wrist', 'L_Ring_1'), id=37, color=[255, 255, 255]),
+        38: dict(link=('L_Ring_1', 'L_Ring_2'), id=38, color=[255, 255, 255]),
+        39: dict(link=('L_Ring_2', 'L_Ring_3'), id=39, color=[255, 255, 255]),
+        40: dict(link=('L_Ring_3', 'L_Ring_4'), id=40, color=[255, 255, 255]),
+        41: dict(link=('L_Wrist', 'L_Pinky_1'), id=41, color=[255, 255, 255]),
+        42:
+        dict(link=('L_Pinky_1', 'L_Pinky_2'), id=42, color=[255, 255, 255]),
+        43:
+        dict(link=('L_Pinky_2', 'L_Pinky_3'), id=43, color=[255, 255, 255]),
+        44:
+        dict(link=('L_Pinky_3', 'L_Pinky_4'), id=44, color=[255, 255, 255]),
+        45: dict(link=('R_Wrist', 'R_Thumb_1'), id=45, color=[255, 255, 255]),
+        46:
+        dict(link=('R_Thumb_1', 'R_Thumb_2'), id=46, color=[255, 255, 255]),
+        47:
+        dict(link=('R_Thumb_2', 'R_Thumb_3'), id=47, color=[255, 255, 255]),
+        48:
+        dict(link=('R_Thumb_3', 'R_Thumb_4'), id=48, color=[255, 255, 255]),
+        49: dict(link=('R_Wrist', 'R_Index_1'), id=49, color=[255, 255, 255]),
+        50:
+        dict(link=('R_Index_1', 'R_Index_2'), id=50, color=[255, 255, 255]),
+        51:
+        dict(link=('R_Index_2', 'R_Index_3'), id=51, color=[255, 255, 255]),
+        52:
+        dict(link=('R_Index_3', 'R_Index_4'), id=52, color=[255, 255, 255]),
+        53: dict(link=('R_Wrist', 'R_Middle_1'), id=53, color=[255, 255, 255]),
+        54:
+        dict(link=('R_Middle_1', 'R_Middle_2'), id=54, color=[255, 255, 255]),
+        55:
+        dict(link=('R_Middle_2', 'R_Middle_3'), id=55, color=[255, 255, 255]),
+        56:
+        dict(link=('R_Middle_3', 'R_Middle_4'), id=56, color=[255, 255, 255]),
+        57: dict(link=('R_Wrist', 'R_Pinky_1'), id=57, color=[255, 255, 255]),
+        58:
+        dict(link=('R_Pinky_1', 'R_Pinky_2'), id=58, color=[255, 255, 255]),
+        59:
+        dict(link=('R_Pinky_2', 'R_Pinky_3'), id=59, color=[255, 255, 255]),
+        60:
+        dict(link=('R_Pinky_3', 'R_Pinky_4'), id=60, color=[255, 255, 255]),
     },
     joint_weights=[1.] * 137,
     sigmas=[])
diff --git a/docs/en/dataset_zoo/3d_body_keypoint.md b/docs/en/dataset_zoo/3d_body_keypoint.md
@@ -8,6 +8,7 @@ MMPose supported datasets:
 - [Human3.6M](#human36m) \[ [Homepage](http://vision.imar.ro/human3.6m/description.php) \]
 - [CMU Panoptic](#cmu-panoptic) \[ [Homepage](http://domedb.perception.cs.cmu.edu/) \]
 - [Campus/Shelf](#campus-and-shelf) \[ [Homepage](http://campar.in.tum.de/Chair/MultiHumanPose) \]
+- [UBody](#ubody3d) \[ [Homepage](https://osx-ubody.github.io/) \]
 
 ## Human3.6M
 
@@ -197,3 +198,100 @@ mmpose
     |   ├── pred_shelf_maskrcnn_hrnet_coco.pkl
     |   ├── actorsGT.mat
 ```
+
+## UBody3d
+
+<details>
+<summary align="right"><a href="https://arxiv.org/abs/2303.16160">UBody (CVPR'2023)</a></summary>
+
+```bibtex
+@article{lin2023one,
+  title={One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer},
+  author={Lin, Jing and Zeng, Ailing and Wang, Haoqian and Zhang, Lei and Li, Yu},
+  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
+  year={2023},
+}
+```
+
+</details>
+
+<div align="center">
+  <img src="https://github.com/open-mmlab/mmpose/assets/15952744/0c97e43a-46a9-46a3-a5dd-b84bf9d6d6f2" height="300px">
+</div>
+
+For [Ubody](https://github.com/IDEA-Research/OSX) dataset, videos and annotations can be downloaded from [OSX homepage](https://github.com/IDEA-Research/OSX).
+
+Download and extract them under $MMPOSE/data, and make them look like this:
+
+```text
+mmpose
+├── mmpose
+├── docs
+├── tests
+├── tools
+├── configs
+`── data
+    │── UBody
+        ├── annotations
+        │   ├── ConductMusic
+        │   ├── Entertainment
+        │   ├── Fitness
+        │   ├── Interview
+        │   ├── LiveVlog
+        │   ├── Magic_show
+        │   ├── Movie
+        │   ├── Olympic
+        │   ├── Online_class
+        │   ├── SignLanguage
+        │   ├── Singing
+        │   ├── Speech
+        │   ├── TVShow
+        │   ├── TalkShow
+        │   └── VideoConference
+        ├── splits
+        │   ├── inter_scene_test_list.npy
+        │   └── intra_scene_test_list.npy
+        ├── videos
+        │   ├── ConductMusic
+        │   ├── Entertainment
+        │   ├── Fitness
+        │   ├── Interview
+        │   ├── LiveVlog
+        │   ├── Magic_show
+        │   ├── Movie
+        │   ├── Olympic
+        │   ├── Online_class
+        │   ├── SignLanguage
+        │   ├── Singing
+        │   ├── Speech
+        │   ├── TVShow
+        │   ├── TalkShow
+        │   └── VideoConference
+```
+
+Convert videos to images then split them into train/val set:
+
+```shell
+python tools/dataset_converters/ubody_kpts_to_coco.py
+```
+
+Before generating 3D keypoints, you need to install SMPLX tools and download human models, please refer to [Github](https://github.com/vchoutas/smplx#installation) and [SMPLX](https://smpl-x.is.tue.mpg.de/download.php).
+
+```shell
+pip install smplx
+```
+
+The directory tree of human models should be like this:
+
+```text
+human_model_path
+|── smplx
+    ├── SMPLX_NEUTRAL.npz
+    ├── SMPLX_NEUTRAL.pkl
+```
+
+After the above preparations are finished, execute the following script:
+
+```shell
+python tools/dataset_converters/ubody_smplx_to_coco.py --data-root {$MMPOSE/data/UBody} --human-model-path {$MMPOSE/data/human_model_path/}
+```