Skip to content

Commit

Permalink
update hands 241028
Browse files Browse the repository at this point in the history
  • Loading branch information
WC committed Oct 28, 2024
1 parent 2f7a5d3 commit df3b655
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 30 deletions.
2 changes: 2 additions & 0 deletions mmhuman3d/core/conventions/keypoints_mapping/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
crowdpose,
face3d,
flame,
freihand,
gta,
h36m,
human_data,
Expand Down Expand Up @@ -82,6 +83,7 @@
'mediapipe_whole_body': mediapipe.MP_WHOLE_BODY_KEYPOINTS,
'mediapipe_body': mediapipe.MP_BODY_KEYPOINTS,
'interhand': interhand.INTERHAND_KEYPOINTS,
'freihand': freihand.FREIHAND_KEYPOINTS,
}

__KEYPOINTS_MAPPING_CACHE__ = defaultdict(dict)
Expand Down
23 changes: 23 additions & 0 deletions mmhuman3d/core/conventions/keypoints_mapping/freihand.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
FREIHAND_KEYPOINTS = [
'right_wrist',
'right_thumb_1',
'right_thumb_2',
'right_thumb_3',
'right_thumb',
'right_index_1',
'right_index_2',
'right_index_3',
'right_index',
'right_middle_1',
'right_middle_2',
'right_middle_3',
'right_middle',
'right_ring_1',
'right_ring_2',
'right_ring_3',
'right_ring',
'right_pinky_1',
'right_pinky_2',
'right_pinky_3',
'right_pinky',
]
32 changes: 16 additions & 16 deletions mmhuman3d/core/conventions/keypoints_mapping/interhand.py
View file
Original file line number Diff line number Diff line change
@@ -1,45 +1,45 @@
INTERHAND_KEYPOINTS = [
'right_thumb', # 指尖
'right_thumb_3',
'right_thumb_2',
'right_thumb_1',
'right_index',
'right_index_3',
'right_index_2',
'right_index_1',
'right_index',
'right_thumb_3', # 指尖
'right_thumb_2',
'right_thumb_1',
'right_thumb',
'right_middle',
'right_middle_3',
'right_middle_2',
'right_middle_1',
'right_middle',
'right_ring',
'right_ring_3',
'right_ring_2',
'right_ring_1',
'right_ring',
'right_pinky',
'right_pinky_3',
'right_pinky_2',
'right_pinky_1',
'right_pinky',
'right_wrist',
'left_index_3',
'left_index_2',
'left_index_1',
'left_index',
'left_thumb',
'left_thumb_3',
'left_thumb_2',
'left_thumb_1',
'left_thumb',
'left_index',
'left_index_3',
'left_index_2',
'left_index_1',
'left_middle',
'left_middle_3',
'left_middle_2',
'left_middle_1',
'left_middle',
'left_ring',
'left_ring_3',
'left_ring_2',
'left_ring_1',
'left_ring',
'left_pinky',
'left_pinky_3',
'left_pinky_2',
'left_pinky_1',
'left_pinky',
'left_wrist'
]

Expand Down
30 changes: 27 additions & 3 deletions mmhuman3d/data/data_converters/freihand.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def convert_by_mode(self,
stored in HumanData() format
"""

seed = '230828'
seed = '241024'


# use HumanData to store all data
Expand Down Expand Up @@ -160,13 +160,37 @@ def convert_by_mode(self,
kps3d = xyz[index % num_green_bg]
kps3d = np.hstack([kps3d, np.ones([kps3d.shape[0], 1])])
keypoints3d.append(kps3d)


# # test overlay j2d
# import cv2
# img = cv2.imread(f'{dataset_path}/{img_path}')
# # resize to 3x resolution
# scale = 5
# img = cv2.resize(img, (224*scale, 224*scale))

# j2d_orig = kps2d.reshape(-1, 3)
# for i in range(len(j2d_orig)):
# if j2d_orig[i, 2] == 0:
# continue
# # draw kps
# cv2.circle(img, (int(j2d_orig[i,0])*scale, int(j2d_orig[i,1])*scale), 3, (0,0,255), -1)
# # draw index i
# cv2.putText(img, str(i), (int(j2d_orig[i,0])*scale, int(j2d_orig[i,1])*scale), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1, cv2.LINE_AA)

# pass
# # write image
# os.makedirs(f'/mnt/AFS_weichen/mmhuman3d/test_area', exist_ok=True)
# # cv2.imwrite(f'/mnt/AFS_weichen/mmhuman3d/test_area/{seq_name}_{frame_idx}_{hand_type}.jpg', img)
# cv2.imwrite(f'/mnt/AFS_weichen/mmhuman3d/test_area/smplx_hands.jpg', img)
# return

keypoints2d = np.array(keypoints2d)
keypoints2d, keypoints2d_mask = convert_kps(
keypoints2d, src='mano', dst='human_data')
keypoints2d, src='freihand', dst='human_data')
keypoints3d = np.array(keypoints3d)
keypoints3d, keypoints3d_mask = convert_kps(
keypoints3d, src='mano', dst='human_data')
keypoints3d, src='freihand', dst='human_data')

# smplx['global_orient'] = np.array(smplx['global_orient']).reshape(-1, 3)
# smplx['betas'] = np.array(smplx['betas'])
Expand Down
49 changes: 38 additions & 11 deletions mmhuman3d/data/data_converters/interhand26m.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ def convert_by_mode(self, dataset_path: str, out_path: str,
# use HumanData to store the data
human_data = HumanData()

seed = '230828'
seed = '141017'
size = 999999

# initialize
Expand All @@ -197,12 +197,14 @@ def convert_by_mode(self, dataset_path: str, out_path: str,
image_path_, keypoints2d_smplx_ = [], []
keypoints3d_smplx_ = []
meta_ = {}
for meta_key in ['principal_point', 'focal_length']:
for meta_key in ['principal_point', 'focal_length', 'right_hand_valid', 'left_hand_valid']:
meta_[meta_key] = []
# save mano params for vis purpose
mano_ = []

# pdb.set_trace()

instance_num = 0

# sort by image path
for seq in tqdm(
Expand Down Expand Up @@ -294,6 +296,9 @@ def convert_by_mode(self, dataset_path: str, out_path: str,
image_size=(height, width),
)).to(self.device)

# mm2m
j3d_c = j3d_c / 1000

j2d = camera.transform_points_screen(
torch.tensor(j3d_c.reshape(1, -1, 3), device=self.device))
j2d_orig = j2d[0, :, :2].detach().cpu().numpy()
Expand Down Expand Up @@ -321,29 +326,51 @@ def convert_by_mode(self, dataset_path: str, out_path: str,
hand_param[hand_type]['pose'][3:])
bboxs_[f'{hand_type[0]}hand_bbox_xywh'].append(
bbox_xywh + [1])

else:
smplx_[f'{hand_type}_hand_pose'].append(
np.zeros((45)).tolist())
bboxs_[f'{hand_type[0]}hand_bbox_xywh'].append(
bbox_xywh + [0])
bboxs_['bbox_xywh'].append([0, 0, 334, 512, 1])


right_hand_valid = True if 'right' in available_hand_types else False
left_hand_valid = True if 'left' in available_hand_types else False

meta_['focal_length'].append(focal)
meta_['principal_point'].append(princpt)


meta_['right_hand_valid'].append(right_hand_valid)
meta_['left_hand_valid'].append(left_hand_valid)

# append mano params
mano_.append(hand_param)

instance_num += 1
if instance_num > 20000:
break

# j2d_mano, _ = convert_kps(j2d_orig.reshape(1, -1, 3), src='interhand', dst='smplx')
# j2d_mano = j2d_orig.reshape(-1, 3)

# test overlay j2d
# # test overlay j2d
# img = cv2.imread(f'{dataset_path}/{image_path}')
# j2d_orig = j2d_mano.reshape(-1, 3)
# for i in range(len(j2d_orig)):
# if j2d_orig[i, 2] == 0:
# continue
# # draw kps
# cv2.circle(img, (int(j2d_orig[i,0]), int(j2d_orig[i,1])), 3, (0,0,255), -1)
# # draw index i
# cv2.putText(img, str(i), (int(j2d_orig[i,0]), int(j2d_orig[i,1])), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1, cv2.LINE_AA)

# pass
# # write image
# os.makedirs(f'{out_path}/{mode}', exist_ok=True)
# cv2.imwrite(f'{out_path}/{mode}/{seq_name}_{frame_idx}_{hand_type}.jpg', img)
# os.makedirs(f'/mnt/AFS_weichen/mmhuman3d/test_area', exist_ok=True)
# # cv2.imwrite(f'/mnt/AFS_weichen/mmhuman3d/test_area/{seq_name}_{frame_idx}_{hand_type}.jpg', img)
# cv2.imwrite(f'/mnt/AFS_weichen/mmhuman3d/test_area/smplx_hands.jpg', img)

# pdb.set_trace()


size_i = min(size, len(image_path_))

Expand Down Expand Up @@ -372,15 +399,15 @@ def convert_by_mode(self, dataset_path: str, out_path: str,
keypoints2d_smplx = np.concatenate(
keypoints2d_smplx_, axis=0).reshape(-1, 42, 3)
keypoints2d_smplx, keypoints2d_smplx_mask = \
convert_kps(keypoints2d_smplx, src='mano_hands', dst='human_data')
convert_kps(keypoints2d_smplx, src='interhand', dst='smplx')
human_data['keypoints2d_smplx'] = keypoints2d_smplx
human_data['keypoints2d_smplx_mask'] = keypoints2d_smplx_mask

# keypoints3d_smplx
keypoints3d_smplx = np.concatenate(
keypoints3d_smplx_, axis=0).reshape(-1, 42, 4)
keypoints3d_smplx, keypoints3d_smplx_mask = \
convert_kps(keypoints3d_smplx, src='mano_hands', dst='human_data')
convert_kps(keypoints3d_smplx, src='interhand', dst='smplx')
human_data['keypoints3d_smplx'] = keypoints3d_smplx
human_data['keypoints3d_smplx_mask'] = keypoints3d_smplx_mask

Expand All @@ -394,6 +421,6 @@ def convert_by_mode(self, dataset_path: str, out_path: str,
size_i = min(len(seqs), int(size))
out_file = os.path.join(
out_path,
f'interhand26m_{mode}_{fps_mode}_{seed}_{"{:06d}".format(size_i)}.npz'
f'interhand26m_{mode}_{fps_mode}_{seed}_{"{:06d}".format(size_i)}_sample.npz'
)
human_data.dump(out_file)

0 comments on commit df3b655

Please sign in to comment.