diff --git a/configs/_base_/datasets/ubody3d.py b/configs/_base_/datasets/ubody3d.py index e2dfe0c570..9242559ea1 100644 --- a/configs/_base_/datasets/ubody3d.py +++ b/configs/_base_/datasets/ubody3d.py @@ -169,7 +169,7 @@ swap='R_Thumb_3'), 28: dict( - name='L_Thumb4', + name='L_Thumb_4', id=28, color=[255, 128, 0], type='', @@ -313,7 +313,7 @@ id=48, color=[255, 128, 0], type='', - swap='L_Thumb4'), + swap='L_Thumb_4'), 49: dict( name='R_Index_1', @@ -872,71 +872,87 @@ swap='Face_56'), }, skeleton_info={ - 0: dict(link=('L_Hip', 'R_Hip'), id=0, color=[0, 255, 0]), - 1: dict(link=('L_Knee', 'R_Knee'), id=1, color=[0, 255, 0]), - 2: dict(link=('L_Ankle', 'R_Ankle'), id=2, color=[0, 255, 0]), - 3: dict(link=('L_Shoulder', 'R_Shoulder'), id=3, color=[0, 255, 0]), - 4: dict(link=('L_Elbow', 'R_Elbow'), id=4, color=[0, 255, 0]), - 5: dict(link=('L_Wrist', 'R_Wrist'), id=5, color=[0, 255, 0]), - 6: dict(link=('L_Big_toe', 'R_Big_toe'), id=6, color=[0, 255, 0]), - 7: dict(link=('L_Small_toe', 'R_Small_toe'), id=7, color=[0, 255, 0]), - 8: dict(link=('L_Heel', 'R_Heel'), id=8, color=[0, 255, 0]), - 9: dict(link=('L_Ear', 'R_Ear'), id=9, color=[0, 255, 0]), - 10: dict(link=('L_Eye', 'R_Eye'), id=10, color=[0, 255, 0]), - 11: dict(link=('L_Thumb_1', 'R_Thumb_1'), id=11, color=[255, 128, 0]), - 12: dict(link=('L_Thumb_2', 'R_Thumb_2'), id=12, color=[255, 128, 0]), - 13: dict(link=('L_Thumb_3', 'R_Thumb_3'), id=13, color=[255, 128, 0]), - 14: dict(link=('L_Thumb4', 'R_Thumb_4'), id=14, color=[255, 128, 0]), - 15: dict(link=('L_Index_1', 'R_Index_1'), id=15, color=[255, 128, 0]), - 16: dict(link=('L_Index_2', 'R_Index_2'), id=16, color=[255, 128, 0]), - 17: dict(link=('L_Index_3', 'R_Index_3'), id=17, color=[255, 128, 0]), - 18: dict(link=('L_Index_4', 'R_Index_4'), id=18, color=[255, 128, 0]), - 19: - dict(link=('L_Middle_1', 'R_Middle_1'), id=19, color=[255, 128, 0]), - 20: - dict(link=('L_Middle_2', 'R_Middle_2'), id=20, color=[255, 128, 0]), - 21: - dict(link=('L_Middle_3', 'R_Middle_3'), id=21, color=[255, 128, 0]), - 22: - dict(link=('L_Middle_4', 'R_Middle_4'), id=22, color=[255, 128, 0]), - 23: dict(link=('L_Ring_1', 'R_Ring_1'), id=23, color=[255, 128, 0]), - 24: dict(link=('L_Ring_2', 'R_Ring_2'), id=24, color=[255, 128, 0]), - 25: dict(link=('L_Ring_3', 'R_Ring_3'), id=25, color=[255, 128, 0]), - 26: dict(link=('L_Ring_4', 'R_Ring_4'), id=26, color=[255, 128, 0]), - 27: dict(link=('L_Pinky_1', 'R_Pinky_1'), id=27, color=[255, 128, 0]), - 28: dict(link=('L_Pinky_2', 'R_Pinky_2'), id=28, color=[255, 128, 0]), - 29: dict(link=('L_Pinky_3', 'R_Pinky_3'), id=29, color=[255, 128, 0]), - 30: dict(link=('L_Pinky_4', 'R_Pinky_4'), id=30, color=[255, 128, 0]), - 31: dict(link=('Face_3', 'Face_4'), id=31, color=[255, 255, 255]), - 32: dict(link=('Face_5', 'Face_14'), id=32, color=[255, 255, 255]), - 33: dict(link=('Face_6', 'Face_13'), id=33, color=[255, 255, 255]), - 34: dict(link=('Face_7', 'Face_12'), id=34, color=[255, 255, 255]), - 35: dict(link=('Face_8', 'Face_11'), id=35, color=[255, 255, 255]), - 36: dict(link=('Face_9', 'Face_10'), id=36, color=[255, 255, 255]), - 37: dict(link=('Face_19', 'Face_23'), id=37, color=[255, 255, 255]), - 38: dict(link=('Face_20', 'Face_22'), id=38, color=[255, 255, 255]), - 39: dict(link=('Face_24', 'Face_33'), id=39, color=[255, 255, 255]), - 40: dict(link=('Face_25', 'Face_32'), id=40, color=[255, 255, 255]), - 41: dict(link=('Face_26', 'Face_31'), id=41, color=[255, 255, 255]), - 42: dict(link=('Face_27', 'Face_30'), id=42, color=[255, 255, 255]), - 43: dict(link=('Face_28', 'Face_35'), id=43, color=[255, 255, 255]), - 44: dict(link=('Face_29', 'Face_34'), id=44, color=[255, 255, 255]), - 45: dict(link=('Face_36', 'Face_42'), id=45, color=[255, 255, 255]), - 46: dict(link=('Face_37', 'Face_41'), id=46, color=[255, 255, 255]), - 47: dict(link=('Face_38', 'Face_40'), id=47, color=[255, 255, 255]), - 48: dict(link=('Face_43', 'Face_47'), id=48, color=[255, 255, 255]), - 49: dict(link=('Face_44', 'Face_46'), id=49, color=[255, 255, 255]), - 50: dict(link=('Face_48', 'Face_52'), id=50, color=[255, 255, 255]), - 51: dict(link=('Face_49', 'Face_51'), id=51, color=[255, 255, 255]), - 52: dict(link=('Face_53', 'Face_55'), id=52, color=[255, 255, 255]), - 53: dict(link=('Face_56', 'Face_72'), id=53, color=[255, 255, 255]), - 54: dict(link=('Face_57', 'Face_71'), id=54, color=[255, 255, 255]), - 55: dict(link=('Face_58', 'Face_70'), id=55, color=[255, 255, 255]), - 56: dict(link=('Face_59', 'Face_69'), id=56, color=[255, 255, 255]), - 57: dict(link=('Face_60', 'Face_68'), id=57, color=[255, 255, 255]), - 58: dict(link=('Face_61', 'Face_67'), id=58, color=[255, 255, 255]), - 59: dict(link=('Face_62', 'Face_66'), id=59, color=[255, 255, 255]), - 60: dict(link=('Face_63', 'Face_65'), id=60, color=[255, 255, 255]), + 0: dict(link=('L_Ankle', 'L_Knee'), id=0, color=[0, 255, 0]), + 1: dict(link=('L_Knee', 'L_Hip'), id=1, color=[0, 255, 0]), + 2: dict(link=('R_Ankle', 'R_Knee'), id=2, color=[0, 255, 0]), + 3: dict(link=('R_Knee', 'R_Hip'), id=3, color=[0, 255, 0]), + 4: dict(link=('L_Hip', 'R_Hip'), id=4, color=[0, 255, 0]), + 5: dict(link=('L_Shoulder', 'L_Hip'), id=5, color=[0, 255, 0]), + 6: dict(link=('R_Shoulder', 'R_Hip'), id=6, color=[0, 255, 0]), + 7: dict(link=('L_Shoulder', 'R_Shoulder'), id=7, color=[0, 255, 0]), + 8: dict(link=('L_Shoulder', 'L_Elbow'), id=8, color=[0, 255, 0]), + 9: dict(link=('R_Shoulder', 'R_Elbow'), id=9, color=[0, 255, 0]), + 10: dict(link=('L_Elbow', 'L_Wrist'), id=10, color=[0, 255, 0]), + 11: dict(link=('R_Elbow', 'R_Wrist'), id=11, color=[255, 128, 0]), + 12: dict(link=('L_Eye', 'R_Eye'), id=12, color=[255, 128, 0]), + 13: dict(link=('Nose', 'L_Eye'), id=13, color=[255, 128, 0]), + 14: dict(link=('Nose', 'R_Eye'), id=14, color=[255, 128, 0]), + 15: dict(link=('L_Eye', 'L_Ear'), id=15, color=[255, 128, 0]), + 16: dict(link=('R_Eye', 'R_Ear'), id=16, color=[255, 128, 0]), + 17: dict(link=('L_Ear', 'L_Shoulder'), id=17, color=[255, 128, 0]), + 18: dict(link=('R_Ear', 'R_Shoulder'), id=18, color=[255, 128, 0]), + 19: dict(link=('L_Ankle', 'L_Big_toe'), id=19, color=[255, 128, 0]), + 20: dict(link=('L_Ankle', 'L_Small_toe'), id=20, color=[255, 128, 0]), + 21: dict(link=('L_Ankle', 'L_Heel'), id=21, color=[255, 128, 0]), + 22: dict(link=('R_Ankle', 'R_Big_toe'), id=22, color=[255, 128, 0]), + 23: dict(link=('R_Ankle', 'R_Small_toe'), id=23, color=[255, 128, 0]), + 24: dict(link=('R_Ankle', 'R_Heel'), id=24, color=[255, 128, 0]), + 25: dict(link=('L_Wrist', 'L_Thumb_1'), id=25, color=[255, 128, 0]), + 26: dict(link=('L_Thumb_1', 'L_Thumb_2'), id=26, color=[255, 128, 0]), + 27: dict(link=('L_Thumb_2', 'L_Thumb_3'), id=27, color=[255, 128, 0]), + 28: dict(link=('L_Thumb_3', 'L_Thumb_4'), id=28, color=[255, 128, 0]), + 29: dict(link=('L_Wrist', 'L_Index_1'), id=29, color=[255, 128, 0]), + 30: dict(link=('L_Index_1', 'L_Index_2'), id=30, color=[255, 128, 0]), + 31: + dict(link=('L_Index_2', 'L_Index_3'), id=31, color=[255, 255, 255]), + 32: + dict(link=('L_Index_3', 'L_Index_4'), id=32, color=[255, 255, 255]), + 33: dict(link=('L_Wrist', 'L_Middle_1'), id=33, color=[255, 255, 255]), + 34: + dict(link=('L_Middle_1', 'L_Middle_2'), id=34, color=[255, 255, 255]), + 35: + dict(link=('L_Middle_2', 'L_Middle_3'), id=35, color=[255, 255, 255]), + 36: + dict(link=('L_Middle_3', 'L_Middle_4'), id=36, color=[255, 255, 255]), + 37: dict(link=('L_Wrist', 'L_Ring_1'), id=37, color=[255, 255, 255]), + 38: dict(link=('L_Ring_1', 'L_Ring_2'), id=38, color=[255, 255, 255]), + 39: dict(link=('L_Ring_2', 'L_Ring_3'), id=39, color=[255, 255, 255]), + 40: dict(link=('L_Ring_3', 'L_Ring_4'), id=40, color=[255, 255, 255]), + 41: dict(link=('L_Wrist', 'L_Pinky_1'), id=41, color=[255, 255, 255]), + 42: + dict(link=('L_Pinky_1', 'L_Pinky_2'), id=42, color=[255, 255, 255]), + 43: + dict(link=('L_Pinky_2', 'L_Pinky_3'), id=43, color=[255, 255, 255]), + 44: + dict(link=('L_Pinky_3', 'L_Pinky_4'), id=44, color=[255, 255, 255]), + 45: dict(link=('R_Wrist', 'R_Thumb_1'), id=45, color=[255, 255, 255]), + 46: + dict(link=('R_Thumb_1', 'R_Thumb_2'), id=46, color=[255, 255, 255]), + 47: + dict(link=('R_Thumb_2', 'R_Thumb_3'), id=47, color=[255, 255, 255]), + 48: + dict(link=('R_Thumb_3', 'R_Thumb_4'), id=48, color=[255, 255, 255]), + 49: dict(link=('R_Wrist', 'R_Index_1'), id=49, color=[255, 255, 255]), + 50: + dict(link=('R_Index_1', 'R_Index_2'), id=50, color=[255, 255, 255]), + 51: + dict(link=('R_Index_2', 'R_Index_3'), id=51, color=[255, 255, 255]), + 52: + dict(link=('R_Index_3', 'R_Index_4'), id=52, color=[255, 255, 255]), + 53: dict(link=('R_Wrist', 'R_Middle_1'), id=53, color=[255, 255, 255]), + 54: + dict(link=('R_Middle_1', 'R_Middle_2'), id=54, color=[255, 255, 255]), + 55: + dict(link=('R_Middle_2', 'R_Middle_3'), id=55, color=[255, 255, 255]), + 56: + dict(link=('R_Middle_3', 'R_Middle_4'), id=56, color=[255, 255, 255]), + 57: dict(link=('R_Wrist', 'R_Pinky_1'), id=57, color=[255, 255, 255]), + 58: + dict(link=('R_Pinky_1', 'R_Pinky_2'), id=58, color=[255, 255, 255]), + 59: + dict(link=('R_Pinky_2', 'R_Pinky_3'), id=59, color=[255, 255, 255]), + 60: + dict(link=('R_Pinky_3', 'R_Pinky_4'), id=60, color=[255, 255, 255]), }, joint_weights=[1.] * 137, sigmas=[]) diff --git a/docs/en/dataset_zoo/3d_body_keypoint.md b/docs/en/dataset_zoo/3d_body_keypoint.md index 82e21010fc..3a35e2443b 100644 --- a/docs/en/dataset_zoo/3d_body_keypoint.md +++ b/docs/en/dataset_zoo/3d_body_keypoint.md @@ -8,6 +8,7 @@ MMPose supported datasets: - [Human3.6M](#human36m) \[ [Homepage](http://vision.imar.ro/human3.6m/description.php) \] - [CMU Panoptic](#cmu-panoptic) \[ [Homepage](http://domedb.perception.cs.cmu.edu/) \] - [Campus/Shelf](#campus-and-shelf) \[ [Homepage](http://campar.in.tum.de/Chair/MultiHumanPose) \] +- [UBody](#ubody3d) \[ [Homepage](https://osx-ubody.github.io/) \] ## Human3.6M @@ -197,3 +198,100 @@ mmpose | ├── pred_shelf_maskrcnn_hrnet_coco.pkl | ├── actorsGT.mat ``` + +## UBody3d + +
+UBody (CVPR'2023) + +```bibtex +@article{lin2023one, + title={One-Stage 3D Whole-Body Mesh Recovery with Component Aware Transformer}, + author={Lin, Jing and Zeng, Ailing and Wang, Haoqian and Zhang, Lei and Li, Yu}, + booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + year={2023}, +} +``` + +
+ +
+ +
+ +For [Ubody](https://github.com/IDEA-Research/OSX) dataset, videos and annotations can be downloaded from [OSX homepage](https://github.com/IDEA-Research/OSX). + +Download and extract them under $MMPOSE/data, and make them look like this: + +```text +mmpose +├── mmpose +├── docs +├── tests +├── tools +├── configs +`── data + │── UBody + ├── annotations + │   ├── ConductMusic + │   ├── Entertainment + │   ├── Fitness + │   ├── Interview + │   ├── LiveVlog + │   ├── Magic_show + │   ├── Movie + │   ├── Olympic + │   ├── Online_class + │   ├── SignLanguage + │   ├── Singing + │   ├── Speech + │   ├── TVShow + │   ├── TalkShow + │   └── VideoConference + ├── splits + │   ├── inter_scene_test_list.npy + │   └── intra_scene_test_list.npy + ├── videos + │   ├── ConductMusic + │   ├── Entertainment + │   ├── Fitness + │   ├── Interview + │   ├── LiveVlog + │   ├── Magic_show + │   ├── Movie + │   ├── Olympic + │   ├── Online_class + │   ├── SignLanguage + │   ├── Singing + │   ├── Speech + │   ├── TVShow + │   ├── TalkShow + │   └── VideoConference +``` + +Convert videos to images then split them into train/val set: + +```shell +python tools/dataset_converters/ubody_kpts_to_coco.py +``` + +Before generating 3D keypoints, you need to install SMPLX tools and download human models, please refer to [Github](https://github.com/vchoutas/smplx#installation) and [SMPLX](https://smpl-x.is.tue.mpg.de/download.php). + +```shell +pip install smplx +``` + +The directory tree of human models should be like this: + +```text +human_model_path +|── smplx + ├── SMPLX_NEUTRAL.npz + ├── SMPLX_NEUTRAL.pkl +``` + +After the above preparations are finished, execute the following script: + +```shell +python tools/dataset_converters/ubody_smplx_to_coco.py --data-root {$MMPOSE/data/UBody} --human-model-path {$MMPOSE/data/human_model_path/} +```