-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathfaceprocessing.py
318 lines (279 loc) · 12.9 KB
/
faceprocessing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
import cv2
import gradio as gr
import mediapipe as mp
import dlib
import imutils
import numpy as np
mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_face_mesh = mp.solutions.face_mesh
mp_face_detection = mp.solutions.face_detection
def apply_media_pipe_face_detection(image):
with mp_face_detection.FaceDetection(
model_selection=1, min_detection_confidence=0.5) as face_detection:
results = face_detection.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
if not results.detections:
return image
annotated_image = image.copy()
for detection in results.detections:
mp_drawing.draw_detection(annotated_image, detection)
return annotated_image
def apply_media_pipe_facemesh(image):
with mp_face_mesh.FaceMesh(
static_image_mode=True,
max_num_faces=1,
refine_landmarks=True,
min_detection_confidence=0.5) as face_mesh:
results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
if not results.multi_face_landmarks:
return image
annotated_image = image.copy()
for face_landmarks in results.multi_face_landmarks:
mp_drawing.draw_landmarks(
image=annotated_image,
landmark_list=face_landmarks,
connections=mp_face_mesh.FACEMESH_TESSELATION,
landmark_drawing_spec=None,
connection_drawing_spec=mp_drawing_styles
.get_default_face_mesh_tesselation_style())
mp_drawing.draw_landmarks(
image=annotated_image,
landmark_list=face_landmarks,
connections=mp_face_mesh.FACEMESH_CONTOURS,
landmark_drawing_spec=None,
connection_drawing_spec=mp_drawing_styles
.get_default_face_mesh_contours_style())
mp_drawing.draw_landmarks(
image=annotated_image,
landmark_list=face_landmarks,
connections=mp_face_mesh.FACEMESH_IRISES,
landmark_drawing_spec=None,
connection_drawing_spec=mp_drawing_styles
.get_default_face_mesh_iris_connections_style())
return annotated_image
class FaceOrientation(object):
def __init__(self):
self.detect = dlib.get_frontal_face_detector()
self.predict = dlib.shape_predictor("model/shape_predictor_68_face_landmarks.dat")
def create_orientation(self, frame):
draw_rect1 = True
draw_rect2 = True
draw_lines = True
frame = imutils.resize(frame, width=800)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
subjects = self.detect(gray, 0)
for subject in subjects:
landmarks = self.predict(gray, subject)
size = frame.shape
# 2D image points. If you change the image, you need to change vector
image_points = np.array([
(landmarks.part(33).x, landmarks.part(33).y), # Nose tip
(landmarks.part(8).x, landmarks.part(8).y), # Chin
(landmarks.part(36).x, landmarks.part(36).y), # Left eye left corner
(landmarks.part(45).x, landmarks.part(45).y), # Right eye right corne
(landmarks.part(48).x, landmarks.part(48).y), # Left Mouth corner
(landmarks.part(54).x, landmarks.part(54).y) # Right mouth corner
], dtype="double")
# 3D model points.
model_points = np.array([
(0.0, 0.0, 0.0), # Nose tip
(0.0, -330.0, -65.0), # Chin
(-225.0, 170.0, -135.0), # Left eye left corner
(225.0, 170.0, -135.0), # Right eye right corne
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0) # Right mouth corner
])
# Camera internals
focal_length = size[1]
center = (size[1] / 2, size[0] / 2)
camera_matrix = np.array(
[[focal_length, 0, center[0]],
[0, focal_length, center[1]],
[0, 0, 1]], dtype="double"
)
dist_coeffs = np.zeros((4, 1)) # Assuming no lens distortion
(success, rotation_vector, translation_vector) = cv2.solvePnP(model_points, image_points, camera_matrix,
dist_coeffs)
(b1, jacobian) = cv2.projectPoints(np.array([(350.0, 270.0, 0.0)]), rotation_vector, translation_vector,
camera_matrix, dist_coeffs)
(b2, jacobian) = cv2.projectPoints(np.array([(-350.0, -270.0, 0.0)]), rotation_vector,
translation_vector, camera_matrix, dist_coeffs)
(b3, jacobian) = cv2.projectPoints(np.array([(-350.0, 270, 0.0)]), rotation_vector, translation_vector,
camera_matrix, dist_coeffs)
(b4, jacobian) = cv2.projectPoints(np.array([(350.0, -270.0, 0.0)]), rotation_vector,
translation_vector, camera_matrix, dist_coeffs)
(b11, jacobian) = cv2.projectPoints(np.array([(450.0, 350.0, 400.0)]), rotation_vector,
translation_vector, camera_matrix, dist_coeffs)
(b12, jacobian) = cv2.projectPoints(np.array([(-450.0, -350.0, 400.0)]), rotation_vector,
translation_vector, camera_matrix, dist_coeffs)
(b13, jacobian) = cv2.projectPoints(np.array([(-450.0, 350, 400.0)]), rotation_vector,
translation_vector, camera_matrix, dist_coeffs)
(b14, jacobian) = cv2.projectPoints(np.array([(450.0, -350.0, 400.0)]), rotation_vector,
translation_vector, camera_matrix, dist_coeffs)
b1 = (int(b1[0][0][0]), int(b1[0][0][1]))
b2 = (int(b2[0][0][0]), int(b2[0][0][1]))
b3 = (int(b3[0][0][0]), int(b3[0][0][1]))
b4 = (int(b4[0][0][0]), int(b4[0][0][1]))
b11 = (int(b11[0][0][0]), int(b11[0][0][1]))
b12 = (int(b12[0][0][0]), int(b12[0][0][1]))
b13 = (int(b13[0][0][0]), int(b13[0][0][1]))
b14 = (int(b14[0][0][0]), int(b14[0][0][1]))
if draw_rect1 == True:
cv2.line(frame, b1, b3, (255, 255, 0), 10)
cv2.line(frame, b3, b2, (255, 255, 0), 10)
cv2.line(frame, b2, b4, (255, 255, 0), 10)
cv2.line(frame, b4, b1, (255, 255, 0), 10)
if draw_rect2 == True:
cv2.line(frame, b11, b13, (255, 255, 0), 10)
cv2.line(frame, b13, b12, (255, 255, 0), 10)
cv2.line(frame, b12, b14, (255, 255, 0), 10)
cv2.line(frame, b14, b11, (255, 255, 0), 10)
if draw_lines == True:
cv2.line(frame, b11, b1, (0, 255, 0), 10)
cv2.line(frame, b13, b3, (0, 255, 0), 10)
cv2.line(frame, b12, b2, (0, 255, 0), 10)
cv2.line(frame, b14, b4, (0, 255, 0), 10)
return frame
face_orientation_obj = FaceOrientation()
class FaceProcessing(object):
def __init__(self, ui_obj):
self.name = "Face Image Processing"
self.description = "Call for Face Image and video Processing"
self.ui_obj = ui_obj
def take_webcam_photo(self, image):
return image
def take_webcam_video(self, images):
return images
def mp_webcam_photo(self, image):
return image
def mp_webcam_face_mesh(self, image):
face_mesh_image = apply_media_pipe_facemesh(image)
return face_mesh_image
def mp_webcam_face_detection(self, image):
face_detection_img = apply_media_pipe_face_detection(image)
return face_detection_img
def dlib_apply_face_orientation(self, image):
image = face_orientation_obj.create_orientation(image)
return image
def webcam_stream_update(self, video_frame):
video_out = face_orientation_obj.create_orientation(video_frame)
return video_out
def create_ui(self):
with self.ui_obj:
gr.Markdown("""
### 👨💻Made By Bishal Kumar Rauniyar👨💻
## Project: Face Processing based on DLIB Shape Predictor Model
""")
with gr.Tabs():
with gr.TabItem("Processing Webcam"):
with gr.Row():
webcam_image_in = gr.Image(label="Webcam Image Input")
webcam_video_in = gr.Video(label="Webcam Video Input")
with gr.Row():
webcam_photo_action = gr.Button("Take the Photo")
webcam_video_action = gr.Button("Take the Video")
with gr.Row():
webcam_photo_out = gr.Image(label="Webcam Photo Output")
webcam_video_out = gr.Video(label="Webcam Video")
with gr.TabItem("Mediapipe Facemesh with Webcam"):
with gr.Row():
with gr.Column():
mp_image_in = gr.Image(label="Webcam Image Input")
with gr.Column():
mp_photo_action = gr.Button("Take the Photo")
mp_apply_fm_action = gr.Button("Apply Face Mesh the Photo")
mp_apply_landmarks_action = gr.Button("Apply Face Landmarks the Photo")
with gr.Row():
mp_photo_out = gr.Image(label="Webcam Photo Output")
mp_fm_photo_out = gr.Image(label="Face Mesh Photo Output")
mp_lm_photo_out = gr.Image(label="Face Landmarks Photo Output")
with gr.TabItem("DLib Model Based Face Orientation"):
with gr.Row():
with gr.Column():
dlib_image_in = gr.Image(label="Webcam Image Input")
with gr.Column():
dlib_photo_action = gr.Button("Take the Photo")
dlib_apply_orientation_action = gr.Button("Apply Face Mesh the Photo")
with gr.Row():
dlib_photo_out = gr.Image(label="Webcam Photo Output")
dlib_orientation_photo_out = gr.Image(label="Face Mesh Photo Output")
with gr.TabItem("Face Orientation on Live Webcam Stream"):
with gr.Row():
webcam_stream_in = gr.Image(label="Webcam Stream Input",
streaming=True)
webcam_stream_out = gr.Image(label="Webcam Stream Output")
webcam_stream_in.change(
self.webcam_stream_update,
inputs=webcam_stream_in,
outputs=webcam_stream_out
)
dlib_photo_action.click(
self.mp_webcam_photo,
[
dlib_image_in
],
[
dlib_photo_out
]
)
dlib_apply_orientation_action.click(
self.dlib_apply_face_orientation,
[
dlib_image_in
],
[
dlib_orientation_photo_out
]
)
mp_photo_action.click(
self.mp_webcam_photo,
[
mp_image_in
],
[
mp_photo_out
]
)
mp_apply_fm_action.click(
self.mp_webcam_face_mesh,
[
mp_image_in
],
[
mp_fm_photo_out
]
)
mp_apply_landmarks_action.click(
self.mp_webcam_face_detection,
[
mp_image_in
],
[
mp_lm_photo_out
]
)
webcam_photo_action.click(
self.take_webcam_photo,
[
webcam_image_in
],
[
webcam_photo_out
]
)
webcam_video_action.click(
self.take_webcam_video,
[
webcam_video_in
],
[
webcam_video_out
]
)
def launch_ui(self):
self.ui_obj.launch()
if __name__ == '__main__':
my_app = gr.Blocks()
face_ui = FaceProcessing(my_app)
face_ui.create_ui()
face_ui.launch_ui()