diff --git a/mmhuman3d/core/conventions/keypoints_mapping/__init__.py b/mmhuman3d/core/conventions/keypoints_mapping/__init__.py index 6503eee7..d099091e 100644 --- a/mmhuman3d/core/conventions/keypoints_mapping/__init__.py +++ b/mmhuman3d/core/conventions/keypoints_mapping/__init__.py @@ -12,6 +12,7 @@ crowdpose, face3d, flame, + freihand, gta, h36m, human_data, @@ -82,6 +83,7 @@ 'mediapipe_whole_body': mediapipe.MP_WHOLE_BODY_KEYPOINTS, 'mediapipe_body': mediapipe.MP_BODY_KEYPOINTS, 'interhand': interhand.INTERHAND_KEYPOINTS, + 'freihand': freihand.FREIHAND_KEYPOINTS, } __KEYPOINTS_MAPPING_CACHE__ = defaultdict(dict) diff --git a/mmhuman3d/core/conventions/keypoints_mapping/freihand.py b/mmhuman3d/core/conventions/keypoints_mapping/freihand.py new file mode 100644 index 00000000..7e999ffb --- /dev/null +++ b/mmhuman3d/core/conventions/keypoints_mapping/freihand.py @@ -0,0 +1,23 @@ +FREIHAND_KEYPOINTS = [ + 'right_wrist', + 'right_thumb_1', + 'right_thumb_2', + 'right_thumb_3', + 'right_thumb', + 'right_index_1', + 'right_index_2', + 'right_index_3', + 'right_index', + 'right_middle_1', + 'right_middle_2', + 'right_middle_3', + 'right_middle', + 'right_ring_1', + 'right_ring_2', + 'right_ring_3', + 'right_ring', + 'right_pinky_1', + 'right_pinky_2', + 'right_pinky_3', + 'right_pinky', +] \ No newline at end of file diff --git a/mmhuman3d/core/conventions/keypoints_mapping/interhand.py b/mmhuman3d/core/conventions/keypoints_mapping/interhand.py index fb0f33a3..90410ec1 100644 --- a/mmhuman3d/core/conventions/keypoints_mapping/interhand.py +++ b/mmhuman3d/core/conventions/keypoints_mapping/interhand.py @@ -1,45 +1,45 @@ INTERHAND_KEYPOINTS = [ + 'right_thumb', # 指尖 + 'right_thumb_3', + 'right_thumb_2', + 'right_thumb_1', + 'right_index', 'right_index_3', 'right_index_2', 'right_index_1', - 'right_index', - 'right_thumb_3', # 指尖 - 'right_thumb_2', - 'right_thumb_1', - 'right_thumb', + 'right_middle', 'right_middle_3', 'right_middle_2', 'right_middle_1', - 'right_middle', + 'right_ring', 'right_ring_3', 'right_ring_2', 'right_ring_1', - 'right_ring', + 'right_pinky', 'right_pinky_3', 'right_pinky_2', 'right_pinky_1', - 'right_pinky', 'right_wrist', - 'left_index_3', - 'left_index_2', - 'left_index_1', - 'left_index', + 'left_thumb', 'left_thumb_3', 'left_thumb_2', 'left_thumb_1', - 'left_thumb', + 'left_index', + 'left_index_3', + 'left_index_2', + 'left_index_1', + 'left_middle', 'left_middle_3', 'left_middle_2', 'left_middle_1', - 'left_middle', + 'left_ring', 'left_ring_3', 'left_ring_2', 'left_ring_1', - 'left_ring', + 'left_pinky', 'left_pinky_3', 'left_pinky_2', 'left_pinky_1', - 'left_pinky', 'left_wrist' ] diff --git a/mmhuman3d/data/data_converters/freihand.py b/mmhuman3d/data/data_converters/freihand.py index c6a604b6..8acbe1b6 100644 --- a/mmhuman3d/data/data_converters/freihand.py +++ b/mmhuman3d/data/data_converters/freihand.py @@ -81,7 +81,7 @@ def convert_by_mode(self, stored in HumanData() format """ - seed = '230828' + seed = '241024' # use HumanData to store all data @@ -160,13 +160,37 @@ def convert_by_mode(self, kps3d = xyz[index % num_green_bg] kps3d = np.hstack([kps3d, np.ones([kps3d.shape[0], 1])]) keypoints3d.append(kps3d) + + + # # test overlay j2d + # import cv2 + # img = cv2.imread(f'{dataset_path}/{img_path}') + # # resize to 3x resolution + # scale = 5 + # img = cv2.resize(img, (224*scale, 224*scale)) + + # j2d_orig = kps2d.reshape(-1, 3) + # for i in range(len(j2d_orig)): + # if j2d_orig[i, 2] == 0: + # continue + # # draw kps + # cv2.circle(img, (int(j2d_orig[i,0])*scale, int(j2d_orig[i,1])*scale), 3, (0,0,255), -1) + # # draw index i + # cv2.putText(img, str(i), (int(j2d_orig[i,0])*scale, int(j2d_orig[i,1])*scale), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1, cv2.LINE_AA) + + # pass + # # write image + # os.makedirs(f'/mnt/AFS_weichen/mmhuman3d/test_area', exist_ok=True) + # # cv2.imwrite(f'/mnt/AFS_weichen/mmhuman3d/test_area/{seq_name}_{frame_idx}_{hand_type}.jpg', img) + # cv2.imwrite(f'/mnt/AFS_weichen/mmhuman3d/test_area/smplx_hands.jpg', img) + # return keypoints2d = np.array(keypoints2d) keypoints2d, keypoints2d_mask = convert_kps( - keypoints2d, src='mano', dst='human_data') + keypoints2d, src='freihand', dst='human_data') keypoints3d = np.array(keypoints3d) keypoints3d, keypoints3d_mask = convert_kps( - keypoints3d, src='mano', dst='human_data') + keypoints3d, src='freihand', dst='human_data') # smplx['global_orient'] = np.array(smplx['global_orient']).reshape(-1, 3) # smplx['betas'] = np.array(smplx['betas']) diff --git a/mmhuman3d/data/data_converters/interhand26m.py b/mmhuman3d/data/data_converters/interhand26m.py index 9d2925e4..6a58e9af 100644 --- a/mmhuman3d/data/data_converters/interhand26m.py +++ b/mmhuman3d/data/data_converters/interhand26m.py @@ -183,7 +183,7 @@ def convert_by_mode(self, dataset_path: str, out_path: str, # use HumanData to store the data human_data = HumanData() - seed = '230828' + seed = '141017' size = 999999 # initialize @@ -197,12 +197,14 @@ def convert_by_mode(self, dataset_path: str, out_path: str, image_path_, keypoints2d_smplx_ = [], [] keypoints3d_smplx_ = [] meta_ = {} - for meta_key in ['principal_point', 'focal_length']: + for meta_key in ['principal_point', 'focal_length', 'right_hand_valid', 'left_hand_valid']: meta_[meta_key] = [] # save mano params for vis purpose mano_ = [] # pdb.set_trace() + + instance_num = 0 # sort by image path for seq in tqdm( @@ -294,6 +296,9 @@ def convert_by_mode(self, dataset_path: str, out_path: str, image_size=(height, width), )).to(self.device) + # mm2m + j3d_c = j3d_c / 1000 + j2d = camera.transform_points_screen( torch.tensor(j3d_c.reshape(1, -1, 3), device=self.device)) j2d_orig = j2d[0, :, :2].detach().cpu().numpy() @@ -321,7 +326,6 @@ def convert_by_mode(self, dataset_path: str, out_path: str, hand_param[hand_type]['pose'][3:]) bboxs_[f'{hand_type[0]}hand_bbox_xywh'].append( bbox_xywh + [1]) - else: smplx_[f'{hand_type}_hand_pose'].append( np.zeros((45)).tolist()) @@ -329,21 +333,44 @@ def convert_by_mode(self, dataset_path: str, out_path: str, bbox_xywh + [0]) bboxs_['bbox_xywh'].append([0, 0, 334, 512, 1]) - + right_hand_valid = True if 'right' in available_hand_types else False + left_hand_valid = True if 'left' in available_hand_types else False + meta_['focal_length'].append(focal) meta_['principal_point'].append(princpt) - + + meta_['right_hand_valid'].append(right_hand_valid) + meta_['left_hand_valid'].append(left_hand_valid) + # append mano params mano_.append(hand_param) + + instance_num += 1 + if instance_num > 20000: + break + + # j2d_mano, _ = convert_kps(j2d_orig.reshape(1, -1, 3), src='interhand', dst='smplx') + # j2d_mano = j2d_orig.reshape(-1, 3) - # test overlay j2d + # # test overlay j2d # img = cv2.imread(f'{dataset_path}/{image_path}') + # j2d_orig = j2d_mano.reshape(-1, 3) # for i in range(len(j2d_orig)): + # if j2d_orig[i, 2] == 0: + # continue + # # draw kps # cv2.circle(img, (int(j2d_orig[i,0]), int(j2d_orig[i,1])), 3, (0,0,255), -1) + # # draw index i + # cv2.putText(img, str(i), (int(j2d_orig[i,0]), int(j2d_orig[i,1])), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1, cv2.LINE_AA) + # pass # # write image - # os.makedirs(f'{out_path}/{mode}', exist_ok=True) - # cv2.imwrite(f'{out_path}/{mode}/{seq_name}_{frame_idx}_{hand_type}.jpg', img) + # os.makedirs(f'/mnt/AFS_weichen/mmhuman3d/test_area', exist_ok=True) + # # cv2.imwrite(f'/mnt/AFS_weichen/mmhuman3d/test_area/{seq_name}_{frame_idx}_{hand_type}.jpg', img) + # cv2.imwrite(f'/mnt/AFS_weichen/mmhuman3d/test_area/smplx_hands.jpg', img) + + # pdb.set_trace() + size_i = min(size, len(image_path_)) @@ -372,7 +399,7 @@ def convert_by_mode(self, dataset_path: str, out_path: str, keypoints2d_smplx = np.concatenate( keypoints2d_smplx_, axis=0).reshape(-1, 42, 3) keypoints2d_smplx, keypoints2d_smplx_mask = \ - convert_kps(keypoints2d_smplx, src='mano_hands', dst='human_data') + convert_kps(keypoints2d_smplx, src='interhand', dst='smplx') human_data['keypoints2d_smplx'] = keypoints2d_smplx human_data['keypoints2d_smplx_mask'] = keypoints2d_smplx_mask @@ -380,7 +407,7 @@ def convert_by_mode(self, dataset_path: str, out_path: str, keypoints3d_smplx = np.concatenate( keypoints3d_smplx_, axis=0).reshape(-1, 42, 4) keypoints3d_smplx, keypoints3d_smplx_mask = \ - convert_kps(keypoints3d_smplx, src='mano_hands', dst='human_data') + convert_kps(keypoints3d_smplx, src='interhand', dst='smplx') human_data['keypoints3d_smplx'] = keypoints3d_smplx human_data['keypoints3d_smplx_mask'] = keypoints3d_smplx_mask @@ -394,6 +421,6 @@ def convert_by_mode(self, dataset_path: str, out_path: str, size_i = min(len(seqs), int(size)) out_file = os.path.join( out_path, - f'interhand26m_{mode}_{fps_mode}_{seed}_{"{:06d}".format(size_i)}.npz' + f'interhand26m_{mode}_{fps_mode}_{seed}_{"{:06d}".format(size_i)}_sample.npz' ) human_data.dump(out_file)