From 0fd77e1c99a903878ef0029ea851ab8e5aabb2a2 Mon Sep 17 00:00:00 2001 From: Sefik Ilkin Serengil Date: Sat, 20 Jan 2024 19:23:28 +0000 Subject: [PATCH 1/3] facial recognition, detector and demography models are now using interface --- deepface/DeepFace.py | 68 +------ deepface/basemodels/ArcFace.py | 24 ++- deepface/basemodels/DeepID.py | 16 +- deepface/basemodels/DlibResNet.py | 21 ++ deepface/basemodels/DlibWrapper.py | 6 - deepface/basemodels/Facenet.py | 69 ++++++- deepface/basemodels/Facenet512.py | 40 ---- deepface/basemodels/FbDeepFace.py | 16 +- deepface/basemodels/OpenFace.py | 17 +- deepface/basemodels/SFace.py | 70 +++++-- deepface/basemodels/VGGFace.py | 27 ++- deepface/commons/functions.py | 5 +- deepface/detectors/DetectorWrapper.py | 67 ++++++ deepface/detectors/DlibWrapper.py | 156 +++++++------- deepface/detectors/FaceDetector.py | 149 -------------- deepface/detectors/FastMtcnnWrapper.py | 124 ++++++------ deepface/detectors/MediapipeWrapper.py | 146 ++++++------- deepface/detectors/MtcnnWrapper.py | 73 ++++--- deepface/detectors/OpenCvWrapper.py | 259 ++++++++++++------------ deepface/detectors/RetinaFaceWrapper.py | 111 +++++----- deepface/detectors/SsdWrapper.py | 185 +++++++++-------- deepface/detectors/YoloWrapper.py | 149 +++++++------- deepface/detectors/YunetWrapper.py | 192 +++++++++--------- deepface/extendedmodels/Age.py | 33 ++- deepface/extendedmodels/Emotion.py | 26 ++- deepface/extendedmodels/Gender.py | 24 ++- deepface/extendedmodels/Race.py | 22 +- deepface/models/Demography.py | 23 +++ deepface/models/Detector.py | 39 ++++ deepface/models/FacialRecognition.py | 28 +++ deepface/modules/demography.py | 26 +-- deepface/modules/modeling.py | 45 ++-- deepface/modules/representation.py | 24 +-- requirements.txt | 3 +- tests/visual-test.py | 5 + 35 files changed, 1223 insertions(+), 1065 deletions(-) delete mode 100644 deepface/basemodels/DlibWrapper.py delete mode 100644 deepface/basemodels/Facenet512.py create mode 100644 deepface/detectors/DetectorWrapper.py delete mode 100644 deepface/detectors/FaceDetector.py create mode 100644 deepface/models/Demography.py create mode 100644 deepface/models/Detector.py create mode 100644 deepface/models/FacialRecognition.py diff --git a/deepface/DeepFace.py b/deepface/DeepFace.py index 84d5f42f..a220d48e 100644 --- a/deepface/DeepFace.py +++ b/deepface/DeepFace.py @@ -8,7 +8,6 @@ import numpy as np import pandas as pd import tensorflow as tf -from deprecated import deprecated # package dependencies from deepface.commons import functions @@ -23,8 +22,6 @@ realtime, ) -# pylint: disable=no-else-raise, simplifiable-if-expression - logger = Logger(module="DeepFace") # ----------------------------------- @@ -40,6 +37,8 @@ from keras.models import Model # ----------------------------------- +functions.initialize_folder() + def build_model(model_name: str) -> Union[Model, Any]: """ @@ -413,69 +412,6 @@ def extract_faces( ) -# --------------------------- -# deprecated functions - - -@deprecated(version="0.0.78", reason="Use DeepFace.extract_faces instead of DeepFace.detectFace") -def detectFace( - img_path: Union[str, np.ndarray], - target_size: tuple = (224, 224), - detector_backend: str = "opencv", - enforce_detection: bool = True, - align: bool = True, -) -> Union[np.ndarray, None]: - """ - Deprecated function. Use extract_faces for same functionality. - - This function applies pre-processing stages of a face recognition pipeline - including detection and alignment - - Parameters: - img_path: exact image path, numpy array (BGR) or base64 encoded image. - Source image can have many face. Then, result will be the size of number - of faces appearing in that source image. - - target_size (tuple): final shape of facial image. black pixels will be - added to resize the image. - - detector_backend (string): face detection backends are retinaface, mtcnn, - opencv, ssd or dlib - - enforce_detection (boolean): function throws exception if face cannot be - detected in the fed image. Set this to False if you do not want to get - an exception and run the function anyway. - - align (boolean): alignment according to the eye positions. - - grayscale (boolean): extracting faces in rgb or gray scale - - Returns: - detected and aligned face as numpy array - - """ - logger.warn("Function detectFace is deprecated. Use extract_faces instead.") - face_objs = extract_faces( - img_path=img_path, - target_size=target_size, - detector_backend=detector_backend, - enforce_detection=enforce_detection, - align=align, - grayscale=False, - ) - - extracted_face = None - if len(face_objs) > 0: - extracted_face = face_objs[0]["face"] - return extracted_face - - -# --------------------------- -# main - -functions.initialize_folder() - - def cli() -> None: """ command line interface function will be offered in this block diff --git a/deepface/basemodels/ArcFace.py b/deepface/basemodels/ArcFace.py index b3059bda..f6472766 100644 --- a/deepface/basemodels/ArcFace.py +++ b/deepface/basemodels/ArcFace.py @@ -3,6 +3,7 @@ import tensorflow as tf from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.FacialRecognition import FacialRecognition logger = Logger(module="basemodels.ArcFace") @@ -42,10 +43,25 @@ Dense, ) +# pylint: disable=too-few-public-methods +class ArcFace(FacialRecognition): + """ + ArcFace model class + """ -def loadModel( + def __init__(self): + self.model = load_model() + self.model_name = "ArcFace" + + +def load_model( url="https://github.com/serengil/deepface_models/releases/download/v1.0/arcface_weights.h5", ) -> Model: + """ + Construct ArcFace model, download its weights and load + Returns: + model (Model) + """ base_model = ResNet34() inputs = base_model.inputs[0] arcface_model = base_model.outputs[0] @@ -81,7 +97,11 @@ def loadModel( def ResNet34() -> Model: - + """ + ResNet34 model + Returns: + model (Model) + """ img_input = Input(shape=(112, 112, 3)) x = ZeroPadding2D(padding=1, name="conv1_pad")(img_input) diff --git a/deepface/basemodels/DeepID.py b/deepface/basemodels/DeepID.py index fa128b08..0c74f0bc 100644 --- a/deepface/basemodels/DeepID.py +++ b/deepface/basemodels/DeepID.py @@ -3,6 +3,7 @@ import tensorflow as tf from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.FacialRecognition import FacialRecognition logger = Logger(module="basemodels.DeepID") @@ -38,10 +39,23 @@ # ------------------------------------- +# pylint: disable=too-few-public-methods +class DeepId(FacialRecognition): + """ + DeepId model class + """ -def loadModel( + def __init__(self): + self.model = load_model() + self.model_name = "DeepId" + + +def load_model( url="https://github.com/serengil/deepface_models/releases/download/v1.0/deepid_keras_weights.h5", ) -> Model: + """ + Construct DeepId model, download its weights and load + """ myInput = Input(shape=(55, 47, 3)) diff --git a/deepface/basemodels/DlibResNet.py b/deepface/basemodels/DlibResNet.py index 3cc57f10..c440f19a 100644 --- a/deepface/basemodels/DlibResNet.py +++ b/deepface/basemodels/DlibResNet.py @@ -4,12 +4,33 @@ import numpy as np from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.FacialRecognition import FacialRecognition logger = Logger(module="basemodels.DlibResNet") # pylint: disable=too-few-public-methods +class Dlib(FacialRecognition): + """ + Dlib model class + """ + + def __init__(self): + self.model = DlibResNet() + self.model_name = "Dlib" + + def find_embeddings(self, img: np.ndarray) -> list: + """ + Custom find embeddings function of Dlib different than FacialRecognition's one + Args: + img (np.ndarray) + Retunrs: + embeddings (list) + """ + return self.model.predict(img)[0].tolist() + + class DlibResNet: def __init__(self): diff --git a/deepface/basemodels/DlibWrapper.py b/deepface/basemodels/DlibWrapper.py deleted file mode 100644 index 51f0a0b9..00000000 --- a/deepface/basemodels/DlibWrapper.py +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Any -from deepface.basemodels.DlibResNet import DlibResNet - - -def loadModel() -> Any: - return DlibResNet() diff --git a/deepface/basemodels/Facenet.py b/deepface/basemodels/Facenet.py index 1ad4f879..84769090 100644 --- a/deepface/basemodels/Facenet.py +++ b/deepface/basemodels/Facenet.py @@ -3,6 +3,7 @@ import tensorflow as tf from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.FacialRecognition import FacialRecognition logger = Logger(module="basemodels.Facenet") @@ -42,12 +43,39 @@ # -------------------------------- +# pylint: disable=too-few-public-methods +class FaceNet128d(FacialRecognition): + """ + FaceNet-128d model class + """ + + def __init__(self): + self.model = load_facenet128d_model() + self.model_name = "FaceNet-128d" + + +class FaceNet512d(FacialRecognition): + """ + FaceNet-1512d model class + """ + + def __init__(self): + self.model = load_facenet512d_model() + self.model_name = "FaceNet-512d" + def scaling(x, scale): return x * scale -def InceptionResNetV2(dimension=128) -> Model: +def InceptionResNetV2(dimension: int = 128) -> Model: + """ + InceptionResNetV2 model + Args: + dimension (int): number of dimensions in the embedding layer + Returns: + model (Model) + """ inputs = Input(shape=(160, 160, 3)) x = Conv2D(32, 3, strides=2, padding="valid", use_bias=False, name="Conv2d_1a_3x3")(inputs) @@ -1618,9 +1646,16 @@ def InceptionResNetV2(dimension=128) -> Model: return model -def loadModel( +def load_facenet128d_model( url="https://github.com/serengil/deepface_models/releases/download/v1.0/facenet_weights.h5", ) -> Model: + """ + Construct FaceNet-128d model, download weights and then load weights + Args: + dimension (int): construct FaceNet-128d or FaceNet-512d models + Returns: + model (Model) + """ model = InceptionResNetV2() # ----------------------------------- @@ -1640,3 +1675,33 @@ def loadModel( # ----------------------------------- return model + + +def load_facenet512d_model( + url="https://github.com/serengil/deepface_models/releases/download/v1.0/facenet512_weights.h5", +) -> Model: + """ + Construct FaceNet-512d model, download its weights and load + Returns: + model (Model) + """ + + model = InceptionResNetV2(dimension=512) + + # ------------------------- + + home = functions.get_deepface_home() + + if os.path.isfile(home + "/.deepface/weights/facenet512_weights.h5") != True: + logger.info("facenet512_weights.h5 will be downloaded...") + + output = home + "/.deepface/weights/facenet512_weights.h5" + gdown.download(url, output, quiet=False) + + # ------------------------- + + model.load_weights(home + "/.deepface/weights/facenet512_weights.h5") + + # ------------------------- + + return model diff --git a/deepface/basemodels/Facenet512.py b/deepface/basemodels/Facenet512.py deleted file mode 100644 index 95aca65e..00000000 --- a/deepface/basemodels/Facenet512.py +++ /dev/null @@ -1,40 +0,0 @@ -import os -import gdown -import tensorflow as tf -from deepface.basemodels import Facenet -from deepface.commons import functions -from deepface.commons.logger import Logger - -logger = Logger(module="basemodels.Facenet512") - -tf_version = int(tf.__version__.split(".", maxsplit=1)[0]) - -if tf_version == 1: - from keras.models import Model -else: - from tensorflow.keras.models import Model - - -def loadModel( - url="https://github.com/serengil/deepface_models/releases/download/v1.0/facenet512_weights.h5", -) -> Model: - - model = Facenet.InceptionResNetV2(dimension=512) - - # ------------------------- - - home = functions.get_deepface_home() - - if os.path.isfile(home + "/.deepface/weights/facenet512_weights.h5") != True: - logger.info("facenet512_weights.h5 will be downloaded...") - - output = home + "/.deepface/weights/facenet512_weights.h5" - gdown.download(url, output, quiet=False) - - # ------------------------- - - model.load_weights(home + "/.deepface/weights/facenet512_weights.h5") - - # ------------------------- - - return model diff --git a/deepface/basemodels/FbDeepFace.py b/deepface/basemodels/FbDeepFace.py index 9f66a80e..eb49f7ef 100644 --- a/deepface/basemodels/FbDeepFace.py +++ b/deepface/basemodels/FbDeepFace.py @@ -4,6 +4,7 @@ import tensorflow as tf from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.FacialRecognition import FacialRecognition logger = Logger(module="basemodels.FbDeepFace") @@ -35,12 +36,23 @@ # ------------------------------------- -# pylint: disable=line-too-long +# pylint: disable=line-too-long, too-few-public-methods +class DeepFace(FacialRecognition): + """ + Fb's DeepFace model class + """ + def __init__(self): + self.model = load_model() + self.model_name = "DeepFace" -def loadModel( + +def load_model( url="https://github.com/swghosh/DeepFace/releases/download/weights-vggface2-2d-aligned/VGGFace2_DeepFace_weights_val-0.9034.h5.zip", ) -> Model: + """ + Construct DeepFace model, download its weights and load + """ base_model = Sequential() base_model.add( Convolution2D(32, (11, 11), activation="relu", name="C1", input_shape=(152, 152, 3)) diff --git a/deepface/basemodels/OpenFace.py b/deepface/basemodels/OpenFace.py index 9ba161e7..d829e301 100644 --- a/deepface/basemodels/OpenFace.py +++ b/deepface/basemodels/OpenFace.py @@ -3,6 +3,7 @@ import tensorflow as tf from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.FacialRecognition import FacialRecognition logger = Logger(module="basemodels.OpenFace") @@ -24,10 +25,24 @@ # --------------------------------------- +# pylint: disable=too-few-public-methods +class OpenFace(FacialRecognition): + """ + OpenFace model class + """ + def __init__(self): + self.model = load_model() + self.model_name = "OpenFace" -def loadModel( + +def load_model( url="https://github.com/serengil/deepface_models/releases/download/v1.0/openface_weights.h5", ) -> Model: + """ + Consturct OpenFace model, download its weights and load + Returns: + model (Model) + """ myInput = Input(shape=(96, 96, 3)) x = ZeroPadding2D(padding=(3, 3), input_shape=(96, 96, 3))(myInput) diff --git a/deepface/basemodels/SFace.py b/deepface/basemodels/SFace.py index 90b305cd..a9f36d14 100644 --- a/deepface/basemodels/SFace.py +++ b/deepface/basemodels/SFace.py @@ -7,20 +7,60 @@ from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.FacialRecognition import FacialRecognition logger = Logger(module="basemodels.SFace") # pylint: disable=line-too-long, too-few-public-methods -class _Layer: - input_shape = (None, 112, 112, 3) - output_shape = (None, 1, 128) +class SFace(FacialRecognition): + """ + SFace model class + """ + def __init__(self): + self.model = load_model() + self.model_name = "SFace" -class SFaceModel: - def __init__(self, model_path): + def find_embeddings(self, img: np.ndarray) -> list: + """ + Custom find embeddings function of SFace different than FacialRecognition's one + Args: + img (np.ndarray) + Retunrs: + embeddings (list) + """ + return self.model.predict(img)[0].tolist() + + +def load_model( + url="https://github.com/opencv/opencv_zoo/raw/main/models/face_recognition_sface/face_recognition_sface_2021dec.onnx", +) -> Any: + """ + Construct SFace model, download its weights and load + """ + + home = functions.get_deepface_home() + + file_name = home + "/.deepface/weights/face_recognition_sface_2021dec.onnx" + + if not os.path.isfile(file_name): + + logger.info("sface weights will be downloaded...") + + gdown.download(url, file_name, quiet=False) + + model = SFaceWrapper(model_path=file_name) + + return model + +class SFaceWrapper: + def __init__(self, model_path): + """ + SFace wrapper covering model construction, layer infos and predict + """ try: self.model = cv.FaceRecognizerSF.create( model=model_path, config="", backend_id=0, target_id=0 @@ -46,20 +86,6 @@ def predict(self, image: np.ndarray) -> np.ndarray: return embeddings -def load_model( - url="https://github.com/opencv/opencv_zoo/raw/main/models/face_recognition_sface/face_recognition_sface_2021dec.onnx", -) -> Any: - - home = functions.get_deepface_home() - - file_name = home + "/.deepface/weights/face_recognition_sface_2021dec.onnx" - - if not os.path.isfile(file_name): - - logger.info("sface weights will be downloaded...") - - gdown.download(url, file_name, quiet=False) - - model = SFaceModel(model_path=file_name) - - return model +class _Layer: + input_shape = (None, 112, 112, 3) + output_shape = (None, 1, 128) diff --git a/deepface/basemodels/VGGFace.py b/deepface/basemodels/VGGFace.py index a1494258..0a0df335 100644 --- a/deepface/basemodels/VGGFace.py +++ b/deepface/basemodels/VGGFace.py @@ -3,6 +3,7 @@ import tensorflow as tf from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.FacialRecognition import FacialRecognition logger = Logger(module="basemodels.VGGFace") @@ -37,8 +38,23 @@ # --------------------------------------- +# pylint: disable=too-few-public-methods +class VggFace(FacialRecognition): + """ + VGG-Face model class + """ -def baseModel() -> Sequential: + def __init__(self): + self.model = load_model() + self.model_name = "VGG-Face" + + +def base_model() -> Sequential: + """ + Base model of VGG-Face being used for classification - not to find embeddings + Returns: + model (Sequential): model was trained to classify 2622 identities + """ model = Sequential() model.add(ZeroPadding2D((1, 1), input_shape=(224, 224, 3))) model.add(Convolution2D(64, (3, 3), activation="relu")) @@ -87,11 +103,16 @@ def baseModel() -> Sequential: return model -def loadModel( +def load_model( url="https://github.com/serengil/deepface_models/releases/download/v1.0/vgg_face_weights.h5", ) -> Model: + """ + Final VGG-Face model being used for finding embeddings + Returns: + model (Model): returning 4096 dimensional vectors + """ - model = baseModel() + model = base_model() home = functions.get_deepface_home() output = home + "/.deepface/weights/vgg_face_weights.h5" diff --git a/deepface/commons/functions.py b/deepface/commons/functions.py index 46fb6d27..6113c759 100644 --- a/deepface/commons/functions.py +++ b/deepface/commons/functions.py @@ -12,7 +12,7 @@ from deprecated import deprecated # package dependencies -from deepface.detectors import FaceDetector +from deepface.detectors import DetectorWrapper from deepface.commons.logger import Logger logger = Logger(module="commons.functions") @@ -168,8 +168,7 @@ def extract_faces( if detector_backend == "skip": face_objs = [(img, img_region, 0)] else: - face_detector = FaceDetector.build_model(detector_backend) - face_objs = FaceDetector.detect_faces(face_detector, detector_backend, img, align) + face_objs = DetectorWrapper.detect_faces(detector_backend, img, align) # in case of no face found if len(face_objs) == 0 and enforce_detection is True: diff --git a/deepface/detectors/DetectorWrapper.py b/deepface/detectors/DetectorWrapper.py new file mode 100644 index 00000000..06cfca6a --- /dev/null +++ b/deepface/detectors/DetectorWrapper.py @@ -0,0 +1,67 @@ +from typing import Any +import numpy as np +from deepface.models.Detector import Detector +from deepface.detectors import ( + OpenCvWrapper, + SsdWrapper, + DlibWrapper, + MtcnnWrapper, + RetinaFaceWrapper, + MediapipeWrapper, + YoloWrapper, + YunetWrapper, + FastMtcnnWrapper, +) + + +def build_model(detector_backend: str) -> Any: + """ + Build a face detector model + Args: + detector_backend (str): backend detector name + Returns: + built detector (Any) + """ + global face_detector_obj # singleton design pattern + + backends = { + "opencv": OpenCvWrapper.OpenCv, + "mtcnn": MtcnnWrapper.MtCnn, + "ssd": SsdWrapper.Ssd, + "dlib": DlibWrapper.Dlib, + "retinaface": RetinaFaceWrapper.RetinaFace, + "mediapipe": MediapipeWrapper.MediaPipe, + "yolov8": YoloWrapper.Yolo, + "yunet": YunetWrapper.YuNet, + "fastmtcnn": FastMtcnnWrapper.FastMtCnn, + } + + if not "face_detector_obj" in globals(): + face_detector_obj = {} + + built_models = list(face_detector_obj.keys()) + if detector_backend not in built_models: + face_detector = backends.get(detector_backend) + + if face_detector: + face_detector = face_detector() + face_detector_obj[detector_backend] = face_detector + else: + raise ValueError("invalid detector_backend passed - " + detector_backend) + + return face_detector_obj[detector_backend] + + +def detect_faces(detector_backend: str, img: np.ndarray, align: bool = True) -> list: + """ + Detect face(s) from a given image + Args: + detector_backend (str): detector name + img (np.ndarray): pre-loaded image + alig (bool): enable or disable alignment after detection + Returns + result (list): tuple of face (np.ndarray), face region (list) + , confidence score (float) + """ + face_detector: Detector = build_model(detector_backend) + return face_detector.detect_faces(img=img, align=align) diff --git a/deepface/detectors/DlibWrapper.py b/deepface/detectors/DlibWrapper.py index 86e8f881..40f8eebd 100644 --- a/deepface/detectors/DlibWrapper.py +++ b/deepface/detectors/DlibWrapper.py @@ -3,106 +3,110 @@ import gdown import numpy as np from deepface.commons import functions +from deepface.models.Detector import Detector from deepface.commons.logger import Logger logger = Logger(module="detectors.DlibWrapper") -def build_model() -> dict: - """ - Build a dlib hog face detector model - Returns: - model (Any) - """ - home = functions.get_deepface_home() +class Dlib(Detector): + def __init__(self): + self.model = self.build_model() - # this is not a must dependency. do not import it in the global level. - try: - import dlib - except ModuleNotFoundError as e: - raise ImportError( - "Dlib is an optional detector, ensure the library is installed." - "Please install using 'pip install dlib' " - ) from e + def build_model(self) -> dict: + """ + Build a dlib hog face detector model + Returns: + model (Any) + """ + home = functions.get_deepface_home() - # check required file exists in the home/.deepface/weights folder - if os.path.isfile(home + "/.deepface/weights/shape_predictor_5_face_landmarks.dat") != True: + # this is not a must dependency. do not import it in the global level. + try: + import dlib + except ModuleNotFoundError as e: + raise ImportError( + "Dlib is an optional detector, ensure the library is installed." + "Please install using 'pip install dlib' " + ) from e - file_name = "shape_predictor_5_face_landmarks.dat.bz2" - logger.info(f"{file_name} is going to be downloaded") + # check required file exists in the home/.deepface/weights folder + if os.path.isfile(home + "/.deepface/weights/shape_predictor_5_face_landmarks.dat") != True: - url = f"http://dlib.net/files/{file_name}" - output = f"{home}/.deepface/weights/{file_name}" + file_name = "shape_predictor_5_face_landmarks.dat.bz2" + logger.info(f"{file_name} is going to be downloaded") - gdown.download(url, output, quiet=False) + url = f"http://dlib.net/files/{file_name}" + output = f"{home}/.deepface/weights/{file_name}" - zipfile = bz2.BZ2File(output) - data = zipfile.read() - newfilepath = output[:-4] # discard .bz2 extension - with open(newfilepath, "wb") as f: - f.write(data) + gdown.download(url, output, quiet=False) - face_detector = dlib.get_frontal_face_detector() - sp = dlib.shape_predictor(home + "/.deepface/weights/shape_predictor_5_face_landmarks.dat") + zipfile = bz2.BZ2File(output) + data = zipfile.read() + newfilepath = output[:-4] # discard .bz2 extension + with open(newfilepath, "wb") as f: + f.write(data) - detector = {} - detector["face_detector"] = face_detector - detector["sp"] = sp - return detector + face_detector = dlib.get_frontal_face_detector() + sp = dlib.shape_predictor(home + "/.deepface/weights/shape_predictor_5_face_landmarks.dat") + detector = {} + detector["face_detector"] = face_detector + detector["sp"] = sp + return detector -def detect_face(detector: dict, img: np.ndarray, align: bool = True) -> list: - """ - Detect and align face with dlib - Args: - face_detector (Any): dlib face detector object - img (np.ndarray): pre-loaded image - align (bool): default is true - Returns: - list of detected and aligned faces - """ - # this is not a must dependency. do not import it in the global level. - try: - import dlib - except ModuleNotFoundError as e: - raise ImportError( - "Dlib is an optional detector, ensure the library is installed." - "Please install using 'pip install dlib' " - ) from e + def detect_faces(self, img: np.ndarray, align: bool = True) -> list: + """ + Detect and align face with dlib + Args: + face_detector (Any): dlib face detector object + img (np.ndarray): pre-loaded image + align (bool): default is true + Returns: + list of detected and aligned faces + """ + # this is not a must dependency. do not import it in the global level. + try: + import dlib + except ModuleNotFoundError as e: + raise ImportError( + "Dlib is an optional detector, ensure the library is installed." + "Please install using 'pip install dlib' " + ) from e - resp = [] + resp = [] - sp = detector["sp"] + sp = self.model["sp"] - detected_face = None + detected_face = None - img_region = [0, 0, img.shape[1], img.shape[0]] + img_region = [0, 0, img.shape[1], img.shape[0]] - face_detector = detector["face_detector"] + face_detector = self.model["face_detector"] - # note that, by design, dlib's fhog face detector scores are >0 but not capped at 1 - detections, scores, _ = face_detector.run(img, 1) + # note that, by design, dlib's fhog face detector scores are >0 but not capped at 1 + detections, scores, _ = face_detector.run(img, 1) - if len(detections) > 0: + if len(detections) > 0: - for idx, d in enumerate(detections): - left = d.left() - right = d.right() - top = d.top() - bottom = d.bottom() + for idx, d in enumerate(detections): + left = d.left() + right = d.right() + top = d.top() + bottom = d.bottom() - # detected_face = img[top:bottom, left:right] - detected_face = img[ - max(0, top) : min(bottom, img.shape[0]), max(0, left) : min(right, img.shape[1]) - ] + # detected_face = img[top:bottom, left:right] + detected_face = img[ + max(0, top) : min(bottom, img.shape[0]), max(0, left) : min(right, img.shape[1]) + ] - img_region = [left, top, right - left, bottom - top] - confidence = scores[idx] + img_region = [left, top, right - left, bottom - top] + confidence = scores[idx] - if align: - img_shape = sp(img, detections[idx]) - detected_face = dlib.get_face_chip(img, img_shape, size=detected_face.shape[0]) + if align: + img_shape = sp(img, detections[idx]) + detected_face = dlib.get_face_chip(img, img_shape, size=detected_face.shape[0]) - resp.append((detected_face, img_region, confidence)) - - return resp + resp.append((detected_face, img_region, confidence)) + + return resp diff --git a/deepface/detectors/FaceDetector.py b/deepface/detectors/FaceDetector.py deleted file mode 100644 index 7dea56e3..00000000 --- a/deepface/detectors/FaceDetector.py +++ /dev/null @@ -1,149 +0,0 @@ -from typing import Any, Union -from PIL import Image -import numpy as np -from deepface.detectors import ( - OpenCvWrapper, - SsdWrapper, - DlibWrapper, - MtcnnWrapper, - RetinaFaceWrapper, - MediapipeWrapper, - YoloWrapper, - YunetWrapper, - FastMtcnnWrapper, -) - - -def build_model(detector_backend: str) -> Any: - """ - Build a face detector model - Args: - detector_backend (str): backend detector name - Returns: - built detector (Any) - """ - global face_detector_obj # singleton design pattern - - backends = { - "opencv": OpenCvWrapper.build_model, - "ssd": SsdWrapper.build_model, - "dlib": DlibWrapper.build_model, - "mtcnn": MtcnnWrapper.build_model, - "retinaface": RetinaFaceWrapper.build_model, - "mediapipe": MediapipeWrapper.build_model, - "yolov8": YoloWrapper.build_model, - "yunet": YunetWrapper.build_model, - "fastmtcnn": FastMtcnnWrapper.build_model, - } - - if not "face_detector_obj" in globals(): - face_detector_obj = {} - - built_models = list(face_detector_obj.keys()) - if detector_backend not in built_models: - face_detector = backends.get(detector_backend) - - if face_detector: - face_detector = face_detector() - face_detector_obj[detector_backend] = face_detector - else: - raise ValueError("invalid detector_backend passed - " + detector_backend) - - return face_detector_obj[detector_backend] - - -def detect_face( - face_detector: Any, detector_backend: str, img: np.ndarray, align: bool = True -) -> tuple: - """ - Detect a single face from a given image - Args: - face_detector (Any): pre-built face detector object - detector_backend (str): detector name - img (np.ndarray): pre-loaded image - alig (bool): enable or disable alignment after detection - Returns - result (tuple): tuple of face (np.ndarray), face region (list) - , confidence score (float) - """ - obj = detect_faces(face_detector, detector_backend, img, align) - - if len(obj) > 0: - face, region, confidence = obj[0] # discard multiple faces - - # If no face is detected, set face to None, - # image region to full image, and confidence to 0. - else: # len(obj) == 0 - face = None - region = [0, 0, img.shape[1], img.shape[0]] - confidence = 0 - - return face, region, confidence - - -def detect_faces( - face_detector: Any, detector_backend: str, img: np.ndarray, align: bool = True -) -> list: - """ - Detect face(s) from a given image - Args: - face_detector (Any): pre-built face detector object - detector_backend (str): detector name - img (np.ndarray): pre-loaded image - alig (bool): enable or disable alignment after detection - Returns - result (list): tuple of face (np.ndarray), face region (list) - , confidence score (float) - """ - backends = { - "opencv": OpenCvWrapper.detect_face, - "ssd": SsdWrapper.detect_face, - "dlib": DlibWrapper.detect_face, - "mtcnn": MtcnnWrapper.detect_face, - "retinaface": RetinaFaceWrapper.detect_face, - "mediapipe": MediapipeWrapper.detect_face, - "yolov8": YoloWrapper.detect_face, - "yunet": YunetWrapper.detect_face, - "fastmtcnn": FastMtcnnWrapper.detect_face, - } - - detect_face_fn = backends.get(detector_backend) - - if detect_face_fn: # pylint: disable=no-else-return - obj = detect_face_fn(face_detector, img, align) - # obj stores list of (detected_face, region, confidence) - return obj - else: - raise ValueError("invalid detector_backend passed - " + detector_backend) - - -def get_alignment_angle_arctan2( - left_eye: Union[list, tuple], right_eye: Union[list, tuple] -) -> float: - """ - Find the angle between eyes - Args: - left_eye: coordinates of left eye with respect to the you - right_eye: coordinates of right eye with respect to the you - Returns: - angle (float) - """ - return float(np.degrees(np.arctan2(right_eye[1] - left_eye[1], right_eye[0] - left_eye[0]))) - - -def alignment_procedure( - img: np.ndarray, left_eye: Union[list, tuple], right_eye: Union[list, tuple] -) -> np.ndarray: - """ - Rotate given image until eyes are on a horizontal line - Args: - img (np.ndarray): pre-loaded image - left_eye: coordinates of left eye with respect to the you - right_eye: coordinates of right eye with respect to the you - Returns: - result (np.ndarray): aligned face - """ - angle = get_alignment_angle_arctan2(left_eye, right_eye) - img = Image.fromarray(img) - img = np.array(img.rotate(angle)) - return img diff --git a/deepface/detectors/FastMtcnnWrapper.py b/deepface/detectors/FastMtcnnWrapper.py index be50db9f..3f38fa07 100644 --- a/deepface/detectors/FastMtcnnWrapper.py +++ b/deepface/detectors/FastMtcnnWrapper.py @@ -1,79 +1,83 @@ from typing import Any, Union import cv2 import numpy as np -from deepface.detectors import FaceDetector +from deepface.models.Detector import Detector # Link -> https://github.com/timesler/facenet-pytorch # Examples https://www.kaggle.com/timesler/guide-to-mtcnn-in-facenet-pytorch -def build_model() -> Any: - """ - Build a fast mtcnn face detector model - Returns: - model (Any) - """ - # this is not a must dependency. do not import it in the global level. - try: - from facenet_pytorch import MTCNN as fast_mtcnn - except ModuleNotFoundError as e: - raise ImportError( - "FastMtcnn is an optional detector, ensure the library is installed." - "Please install using 'pip install facenet-pytorch' " - ) from e +class FastMtCnn(Detector): + def __init__(self): + self.model = self.build_model() - face_detector = fast_mtcnn( - image_size=160, - thresholds=[0.6, 0.7, 0.7], # MTCNN thresholds - post_process=True, - device="cpu", - select_largest=False, # return result in descending order - ) - return face_detector + def detect_faces(self, img: np.ndarray, align: bool = True) -> list: + """ + Detect and align face with mtcnn + Args: + img (np.ndarray): pre-loaded image + align (bool): default is true + Returns: + list of detected and aligned faces + """ + resp = [] + detected_face = None + img_region = [0, 0, img.shape[1], img.shape[0]] -def xyxy_to_xywh(xyxy: Union[list, tuple]) -> list: - """ - Convert xyxy format to xywh format. - """ - x, y = xyxy[0], xyxy[1] - w = xyxy[2] - x + 1 - h = xyxy[3] - y + 1 - return [x, y, w, h] + img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # mtcnn expects RGB but OpenCV read BGR + detections = self.model.detect( + img_rgb, landmarks=True + ) # returns boundingbox, prob, landmark + if len(detections[0]) > 0: + for detection in zip(*detections): + x, y, w, h = xyxy_to_xywh(detection[0]) + detected_face = img[int(y) : int(y + h), int(x) : int(x + w)] + img_region = [x, y, w, h] + confidence = detection[1] -def detect_face(face_detector: Any, img: np.ndarray, align: bool = True) -> list: - """ - Detect and align face with mtcnn - Args: - face_detector (Any): mtcnn face detector object - img (np.ndarray): pre-loaded image - align (bool): default is true - Returns: - list of detected and aligned faces - """ - resp = [] + if align: + left_eye = detection[2][0] + right_eye = detection[2][1] + detected_face = self.align_face( + img=detected_face, left_eye=left_eye, right_eye=right_eye + ) - detected_face = None - img_region = [0, 0, img.shape[1], img.shape[0]] + resp.append((detected_face, img_region, confidence)) - img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # mtcnn expects RGB but OpenCV read BGR - detections = face_detector.detect( - img_rgb, landmarks=True - ) # returns boundingbox, prob, landmark - if len(detections[0]) > 0: + return resp - for detection in zip(*detections): - x, y, w, h = xyxy_to_xywh(detection[0]) - detected_face = img[int(y) : int(y + h), int(x) : int(x + w)] - img_region = [x, y, w, h] - confidence = detection[1] + def build_model(self) -> Any: + """ + Build a fast mtcnn face detector model + Returns: + model (Any) + """ + # this is not a must dependency. do not import it in the global level. + try: + from facenet_pytorch import MTCNN as fast_mtcnn + except ModuleNotFoundError as e: + raise ImportError( + "FastMtcnn is an optional detector, ensure the library is installed." + "Please install using 'pip install facenet-pytorch' " + ) from e - if align: - left_eye = detection[2][0] - right_eye = detection[2][1] - detected_face = FaceDetector.alignment_procedure(detected_face, left_eye, right_eye) + face_detector = fast_mtcnn( + image_size=160, + thresholds=[0.6, 0.7, 0.7], # MTCNN thresholds + post_process=True, + device="cpu", + select_largest=False, # return result in descending order + ) + return face_detector - resp.append((detected_face, img_region, confidence)) - return resp +def xyxy_to_xywh(xyxy: Union[list, tuple]) -> list: + """ + Convert xyxy format to xywh format. + """ + x, y = xyxy[0], xyxy[1] + w = xyxy[2] - x + 1 + h = xyxy[3] - y + 1 + return [x, y, w, h] diff --git a/deepface/detectors/MediapipeWrapper.py b/deepface/detectors/MediapipeWrapper.py index bf9e0d0a..56e439ec 100644 --- a/deepface/detectors/MediapipeWrapper.py +++ b/deepface/detectors/MediapipeWrapper.py @@ -1,78 +1,82 @@ from typing import Any import numpy as np -from deepface.detectors import FaceDetector +from deepface.models.Detector import Detector # Link - https://google.github.io/mediapipe/solutions/face_detection -def build_model() -> Any: - """ - Build a mediapipe face detector model - Returns: - model (Any) - """ - # this is not a must dependency. do not import it in the global level. - try: - import mediapipe as mp - except ModuleNotFoundError as e: - raise ImportError( - "MediaPipe is an optional detector, ensure the library is installed." - "Please install using 'pip install mediapipe' " - ) from e - - mp_face_detection = mp.solutions.face_detection - face_detection = mp_face_detection.FaceDetection(min_detection_confidence=0.7) - return face_detection - - -def detect_face(face_detector: Any, img: np.ndarray, align: bool = True) -> list: - """ - Detect and align face with mediapipe - Args: - face_detector (Any): mediapipe face detector object - img (np.ndarray): pre-loaded image - align (bool): default is true - Returns: - list of detected and aligned faces - """ - resp = [] - - img_width = img.shape[1] - img_height = img.shape[0] - - results = face_detector.process(img) - - # If no face has been detected, return an empty list - if results.detections is None: - return resp - - # Extract the bounding box, the landmarks and the confidence score - for detection in results.detections: - (confidence,) = detection.score - - bounding_box = detection.location_data.relative_bounding_box - landmarks = detection.location_data.relative_keypoints - - x = int(bounding_box.xmin * img_width) - w = int(bounding_box.width * img_width) - y = int(bounding_box.ymin * img_height) - h = int(bounding_box.height * img_height) - - # Extract landmarks - left_eye = (int(landmarks[0].x * img_width), int(landmarks[0].y * img_height)) - right_eye = (int(landmarks[1].x * img_width), int(landmarks[1].y * img_height)) - # nose = (int(landmarks[2].x * img_width), int(landmarks[2].y * img_height)) - # mouth = (int(landmarks[3].x * img_width), int(landmarks[3].y * img_height)) - # right_ear = (int(landmarks[4].x * img_width), int(landmarks[4].y * img_height)) - # left_ear = (int(landmarks[5].x * img_width), int(landmarks[5].y * img_height)) +class MediaPipe(Detector): + def __init__(self): + self.model = self.build_model() + + def build_model(self) -> Any: + """ + Build a mediapipe face detector model + Returns: + model (Any) + """ + # this is not a must dependency. do not import it in the global level. + try: + import mediapipe as mp + except ModuleNotFoundError as e: + raise ImportError( + "MediaPipe is an optional detector, ensure the library is installed." + "Please install using 'pip install mediapipe' " + ) from e + + mp_face_detection = mp.solutions.face_detection + face_detection = mp_face_detection.FaceDetection(min_detection_confidence=0.7) + return face_detection + + def detect_faces(self, img: np.ndarray, align: bool = True) -> list: + """ + Detect and align face with mediapipe + Args: + img (np.ndarray): pre-loaded image + align (bool): default is true + Returns: + list of detected and aligned faces + """ + resp = [] + + img_width = img.shape[1] + img_height = img.shape[0] + + results = self.model.process(img) + + # If no face has been detected, return an empty list + if results.detections is None: + return resp + + # Extract the bounding box, the landmarks and the confidence score + for detection in results.detections: + (confidence,) = detection.score + + bounding_box = detection.location_data.relative_bounding_box + landmarks = detection.location_data.relative_keypoints + + x = int(bounding_box.xmin * img_width) + w = int(bounding_box.width * img_width) + y = int(bounding_box.ymin * img_height) + h = int(bounding_box.height * img_height) + + # Extract landmarks + left_eye = (int(landmarks[0].x * img_width), int(landmarks[0].y * img_height)) + right_eye = (int(landmarks[1].x * img_width), int(landmarks[1].y * img_height)) + # nose = (int(landmarks[2].x * img_width), int(landmarks[2].y * img_height)) + # mouth = (int(landmarks[3].x * img_width), int(landmarks[3].y * img_height)) + # right_ear = (int(landmarks[4].x * img_width), int(landmarks[4].y * img_height)) + # left_ear = (int(landmarks[5].x * img_width), int(landmarks[5].y * img_height)) + + if x > 0 and y > 0: + detected_face = img[y : y + h, x : x + w] + img_region = [x, y, w, h] + + if align: + detected_face = self.align_face( + img=detected_face, left_eye=left_eye, right_eye=right_eye + ) + + resp.append((detected_face, img_region, confidence)) - if x > 0 and y > 0: - detected_face = img[y : y + h, x : x + w] - img_region = [x, y, w, h] - - if align: - detected_face = FaceDetector.alignment_procedure(detected_face, left_eye, right_eye) - - resp.append((detected_face, img_region, confidence)) - - return resp + return resp diff --git a/deepface/detectors/MtcnnWrapper.py b/deepface/detectors/MtcnnWrapper.py index f7465415..14c59f52 100644 --- a/deepface/detectors/MtcnnWrapper.py +++ b/deepface/detectors/MtcnnWrapper.py @@ -1,54 +1,51 @@ -from typing import Any import cv2 import numpy as np -from deepface.detectors import FaceDetector +from mtcnn import MTCNN +from deepface.models.Detector import Detector -def build_model() -> Any: +class MtCnn(Detector): """ - Build a mtcnn face detector model - Returns: - model (Any) + Class to cover common face detection functionalitiy for MtCnn backend """ - from mtcnn import MTCNN - face_detector = MTCNN() - return face_detector + def __init__(self): + self.model = MTCNN() + def detect_faces(self, img: np.ndarray, align: bool = True) -> list: + """ + Detect and align face with mtcnn + Args: + img (np.ndarray): pre-loaded image + align (bool): default is true + Returns: + list of detected and aligned faces + """ -def detect_face(face_detector: Any, img: np.ndarray, align: bool = True) -> list: - """ - Detect and align face with mtcnn - Args: - face_detector (mtcnn.MTCNN): mtcnn face detector object - img (np.ndarray): pre-loaded image - align (bool): default is true - Returns: - list of detected and aligned faces - """ - - resp = [] + resp = [] - detected_face = None - img_region = [0, 0, img.shape[1], img.shape[0]] + detected_face = None + img_region = [0, 0, img.shape[1], img.shape[0]] - img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # mtcnn expects RGB but OpenCV read BGR - detections = face_detector.detect_faces(img_rgb) + img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # mtcnn expects RGB but OpenCV read BGR + detections = self.model.detect_faces(img_rgb) - if len(detections) > 0: + if len(detections) > 0: - for detection in detections: - x, y, w, h = detection["box"] - detected_face = img[int(y) : int(y + h), int(x) : int(x + w)] - img_region = [x, y, w, h] - confidence = detection["confidence"] + for detection in detections: + x, y, w, h = detection["box"] + detected_face = img[int(y) : int(y + h), int(x) : int(x + w)] + img_region = [x, y, w, h] + confidence = detection["confidence"] - if align: - keypoints = detection["keypoints"] - left_eye = keypoints["left_eye"] - right_eye = keypoints["right_eye"] - detected_face = FaceDetector.alignment_procedure(detected_face, left_eye, right_eye) + if align: + keypoints = detection["keypoints"] + left_eye = keypoints["left_eye"] + right_eye = keypoints["right_eye"] + detected_face = self.align_face( + img=detected_face, left_eye=left_eye, right_eye=right_eye + ) - resp.append((detected_face, img_region, confidence)) + resp.append((detected_face, img_region, confidence)) - return resp + return resp diff --git a/deepface/detectors/OpenCvWrapper.py b/deepface/detectors/OpenCvWrapper.py index 6316c334..515fdb09 100644 --- a/deepface/detectors/OpenCvWrapper.py +++ b/deepface/detectors/OpenCvWrapper.py @@ -2,157 +2,164 @@ from typing import Any import cv2 import numpy as np -from deepface.detectors import FaceDetector +from deepface.models.Detector import Detector -def build_model() -> dict: +class OpenCv(Detector): """ - Build a opencv face&eye detector models - Returns: - model (Any) + Class to cover common face detection functionalitiy for OpenCv backend """ - detector = {} - detector["face_detector"] = build_cascade("haarcascade") - detector["eye_detector"] = build_cascade("haarcascade_eye") - return detector - -def build_cascade(model_name="haarcascade") -> Any: - """ - Build a opencv face&eye detector models - Returns: - model (Any) - """ - opencv_path = get_opencv_path() - if model_name == "haarcascade": - face_detector_path = opencv_path + "haarcascade_frontalface_default.xml" - if os.path.isfile(face_detector_path) != True: - raise ValueError( - "Confirm that opencv is installed on your environment! Expected path ", - face_detector_path, - " violated.", + def __init__(self): + self.model = self.build_model() + + def build_model(self): + """ + Build opencv's face and eye detector models + Returns: + model (dict): including face_detector and eye_detector keys + """ + detector = {} + detector["face_detector"] = self.__build_cascade("haarcascade") + detector["eye_detector"] = self.__build_cascade("haarcascade_eye") + return detector + + def detect_faces(self, img: np.ndarray, align: bool = True) -> list: + """ + Detect and align face with opencv + Args: + face_detector (Any): opencv face detector object + img (np.ndarray): pre-loaded image + align (bool): default is true + Returns: + list of detected and aligned faces + """ + resp = [] + + detected_face = None + img_region = [0, 0, img.shape[1], img.shape[0]] + + faces = [] + try: + # faces = detector["face_detector"].detectMultiScale(img, 1.3, 5) + + # note that, by design, opencv's haarcascade scores are >0 but not capped at 1 + faces, _, scores = self.model["face_detector"].detectMultiScale3( + img, 1.1, 10, outputRejectLevels=True ) - detector = cv2.CascadeClassifier(face_detector_path) - - elif model_name == "haarcascade_eye": - eye_detector_path = opencv_path + "haarcascade_eye.xml" - if os.path.isfile(eye_detector_path) != True: - raise ValueError( - "Confirm that opencv is installed on your environment! Expected path ", - eye_detector_path, - " violated.", - ) - detector = cv2.CascadeClassifier(eye_detector_path) - - else: - raise ValueError(f"unimplemented model_name for build_cascade - {model_name}") - - return detector - - -def detect_face(detector: dict, img: np.ndarray, align: bool = True) -> list: - """ - Detect and align face with opencv - Args: - face_detector (Any): opencv face detector object - img (np.ndarray): pre-loaded image - align (bool): default is true - Returns: - list of detected and aligned faces - """ - resp = [] - - detected_face = None - img_region = [0, 0, img.shape[1], img.shape[0]] - - faces = [] - try: - # faces = detector["face_detector"].detectMultiScale(img, 1.3, 5) + except: + pass - # note that, by design, opencv's haarcascade scores are >0 but not capped at 1 - faces, _, scores = detector["face_detector"].detectMultiScale3( - img, 1.1, 10, outputRejectLevels=True - ) - except: - pass + if len(faces) > 0: + for (x, y, w, h), confidence in zip(faces, scores): + detected_face = img[int(y) : int(y + h), int(x) : int(x + w)] - if len(faces) > 0: - for (x, y, w, h), confidence in zip(faces, scores): - detected_face = img[int(y) : int(y + h), int(x) : int(x + w)] + if align: + left_eye, right_eye = self.find_eyes(img=detected_face) + detected_face = self.align_face(detected_face, left_eye, right_eye) - if align: - detected_face = align_face(detector["eye_detector"], detected_face) + img_region = [x, y, w, h] - img_region = [x, y, w, h] + resp.append((detected_face, img_region, confidence)) - resp.append((detected_face, img_region, confidence)) + return resp - return resp + def find_eyes(self, img: np.ndarray) -> tuple: + """ + Find the left and right eye coordinates of given image + Args: + img (np.ndarray): given image + Returns: + left and right eye (tuple) + """ + left_eye = None + right_eye = None + # if image has unexpectedly 0 dimension then skip alignment + if img.shape[0] == 0 or img.shape[1] == 0: + return left_eye, right_eye -def align_face(eye_detector: Any, img: np.ndarray) -> np.ndarray: - """ - Align a given image with the pre-built eye_detector - Args: - eye_detector (Any): cascade classifier object - img (np.ndarray): given image - Returns: - aligned_img (np.ndarray) - """ - # if image has unexpectedly 0 dimension then skip alignment - if img.shape[0] == 0 or img.shape[1] == 0: - return img + detected_face_gray = cv2.cvtColor( + img, cv2.COLOR_BGR2GRAY + ) # eye detector expects gray scale image - detected_face_gray = cv2.cvtColor( - img, cv2.COLOR_BGR2GRAY - ) # eye detector expects gray scale image + eyes = self.model["eye_detector"].detectMultiScale(detected_face_gray, 1.1, 10) - # eyes = eye_detector.detectMultiScale(detected_face_gray, 1.3, 5) - eyes = eye_detector.detectMultiScale(detected_face_gray, 1.1, 10) + # ---------------------------------------------------------------- - # ---------------------------------------------------------------- + # opencv eye detection module is not strong. it might find more than 2 eyes! + # besides, it returns eyes with different order in each call (issue 435) + # this is an important issue because opencv is the default detector and ssd also uses this + # find the largest 2 eye. Thanks to @thelostpeace - # opencv eye detectin module is not strong. it might find more than 2 eyes! - # besides, it returns eyes with different order in each call (issue 435) - # this is an important issue because opencv is the default detector and ssd also uses this - # find the largest 2 eye. Thanks to @thelostpeace + eyes = sorted(eyes, key=lambda v: abs(v[2] * v[3]), reverse=True) - eyes = sorted(eyes, key=lambda v: abs(v[2] * v[3]), reverse=True) + # ---------------------------------------------------------------- + if len(eyes) >= 2: + # decide left and right eye - # ---------------------------------------------------------------- + eye_1 = eyes[0] + eye_2 = eyes[1] - if len(eyes) >= 2: - # decide left and right eye + if eye_1[0] < eye_2[0]: + left_eye = eye_1 + right_eye = eye_2 + else: + left_eye = eye_2 + right_eye = eye_1 - eye_1 = eyes[0] - eye_2 = eyes[1] + # ----------------------- + # find center of eyes + left_eye = (int(left_eye[0] + (left_eye[2] / 2)), int(left_eye[1] + (left_eye[3] / 2))) + right_eye = ( + int(right_eye[0] + (right_eye[2] / 2)), + int(right_eye[1] + (right_eye[3] / 2)), + ) + return left_eye, right_eye + + def __build_cascade(self, model_name="haarcascade") -> Any: + """ + Build a opencv face&eye detector models + Returns: + model (Any) + """ + opencv_path = self.__get_opencv_path() + if model_name == "haarcascade": + face_detector_path = opencv_path + "haarcascade_frontalface_default.xml" + if os.path.isfile(face_detector_path) != True: + raise ValueError( + "Confirm that opencv is installed on your environment! Expected path ", + face_detector_path, + " violated.", + ) + detector = cv2.CascadeClassifier(face_detector_path) + + elif model_name == "haarcascade_eye": + eye_detector_path = opencv_path + "haarcascade_eye.xml" + if os.path.isfile(eye_detector_path) != True: + raise ValueError( + "Confirm that opencv is installed on your environment! Expected path ", + eye_detector_path, + " violated.", + ) + detector = cv2.CascadeClassifier(eye_detector_path) - if eye_1[0] < eye_2[0]: - left_eye = eye_1 - right_eye = eye_2 else: - left_eye = eye_2 - right_eye = eye_1 - - # ----------------------- - # find center of eyes - left_eye = (int(left_eye[0] + (left_eye[2] / 2)), int(left_eye[1] + (left_eye[3] / 2))) - right_eye = (int(right_eye[0] + (right_eye[2] / 2)), int(right_eye[1] + (right_eye[3] / 2))) - img = FaceDetector.alignment_procedure(img, left_eye, right_eye) - return img # return img anyway + raise ValueError(f"unimplemented model_name for build_cascade - {model_name}") + return detector -def get_opencv_path() -> str: - """ - Returns where opencv installed - Returns: - installation_path (str) - """ - opencv_home = cv2.__file__ - folders = opencv_home.split(os.path.sep)[0:-1] + def __get_opencv_path(self) -> str: + """ + Returns where opencv installed + Returns: + installation_path (str) + """ + opencv_home = cv2.__file__ + folders = opencv_home.split(os.path.sep)[0:-1] - path = folders[0] - for folder in folders[1:]: - path = path + "/" + folder + path = folders[0] + for folder in folders[1:]: + path = path + "/" + folder - return path + "/data/" + return path + "/data/" diff --git a/deepface/detectors/RetinaFaceWrapper.py b/deepface/detectors/RetinaFaceWrapper.py index dc470264..6986ce5b 100644 --- a/deepface/detectors/RetinaFaceWrapper.py +++ b/deepface/detectors/RetinaFaceWrapper.py @@ -1,60 +1,55 @@ -from typing import Any import numpy as np -from retinaface import RetinaFace +from retinaface import RetinaFace as rf from retinaface.commons import postprocess - - -def build_model() -> Any: - """ - Build a retinaface detector model - Returns: - model (Any) - """ - face_detector = RetinaFace.build_model() - return face_detector - - -def detect_face(face_detector: Any, img: np.ndarray, align: bool = True) -> list: - """ - Detect and align face with retinaface - Args: - face_detector (Any): retinaface face detector object - img (np.ndarray): pre-loaded image - align (bool): default is true - Returns: - list of detected and aligned faces - """ - resp = [] - - obj = RetinaFace.detect_faces(img, model=face_detector, threshold=0.9) - - if isinstance(obj, dict): - for face_idx in obj.keys(): - identity = obj[face_idx] - facial_area = identity["facial_area"] - - y = facial_area[1] - h = facial_area[3] - y - x = facial_area[0] - w = facial_area[2] - x - img_region = [x, y, w, h] - confidence = identity["score"] - - # detected_face = img[int(y):int(y+h), int(x):int(x+w)] #opencv - detected_face = img[facial_area[1] : facial_area[3], facial_area[0] : facial_area[2]] - - if align: - landmarks = identity["landmarks"] - left_eye = landmarks["left_eye"] - right_eye = landmarks["right_eye"] - nose = landmarks["nose"] - # mouth_right = landmarks["mouth_right"] - # mouth_left = landmarks["mouth_left"] - - detected_face = postprocess.alignment_procedure( - detected_face, right_eye, left_eye, nose - ) - - resp.append((detected_face, img_region, confidence)) - - return resp +from deepface.models.Detector import Detector + + +class RetinaFace(Detector): + def __init__(self): + self.model = rf.build_model() + + def detect_faces(self, img: np.ndarray, align: bool = True) -> list: + """ + Detect and align face with retinaface + Args: + img (np.ndarray): pre-loaded image + align (bool): default is true + Returns: + list of detected and aligned faces + """ + resp = [] + + obj = rf.detect_faces(img, model=self.model, threshold=0.9) + + if isinstance(obj, dict): + for face_idx in obj.keys(): + identity = obj[face_idx] + facial_area = identity["facial_area"] + + y = facial_area[1] + h = facial_area[3] - y + x = facial_area[0] + w = facial_area[2] - x + img_region = [x, y, w, h] + confidence = identity["score"] + + # detected_face = img[int(y):int(y+h), int(x):int(x+w)] #opencv + detected_face = img[ + facial_area[1] : facial_area[3], facial_area[0] : facial_area[2] + ] + + if align: + landmarks = identity["landmarks"] + left_eye = landmarks["left_eye"] + right_eye = landmarks["right_eye"] + nose = landmarks["nose"] + # mouth_right = landmarks["mouth_right"] + # mouth_left = landmarks["mouth_left"] + + detected_face = postprocess.alignment_procedure( + detected_face, right_eye, left_eye, nose + ) + + resp.append((detected_face, img_region, confidence)) + + return resp diff --git a/deepface/detectors/SsdWrapper.py b/deepface/detectors/SsdWrapper.py index 9f8c5171..f88eea3d 100644 --- a/deepface/detectors/SsdWrapper.py +++ b/deepface/detectors/SsdWrapper.py @@ -5,6 +5,7 @@ import numpy as np from deepface.detectors import OpenCvWrapper from deepface.commons import functions +from deepface.models.Detector import Detector from deepface.commons.logger import Logger logger = Logger(module="detectors.SsdWrapper") @@ -12,126 +13,132 @@ # pylint: disable=line-too-long -def build_model() -> dict: - """ - Build a ssd detector model - Returns: - model (Any) - """ +class Ssd(Detector): + def __init__(self): + self.model = self.build_model() - home = functions.get_deepface_home() + def build_model(self) -> dict: + """ + Build a ssd detector model + Returns: + model (dict) + """ - # model structure - if os.path.isfile(home + "/.deepface/weights/deploy.prototxt") != True: + home = functions.get_deepface_home() - logger.info("deploy.prototxt will be downloaded...") + # model structure + if os.path.isfile(home + "/.deepface/weights/deploy.prototxt") != True: - url = "https://github.com/opencv/opencv/raw/3.4.0/samples/dnn/face_detector/deploy.prototxt" + logger.info("deploy.prototxt will be downloaded...") - output = home + "/.deepface/weights/deploy.prototxt" + url = "https://github.com/opencv/opencv/raw/3.4.0/samples/dnn/face_detector/deploy.prototxt" - gdown.download(url, output, quiet=False) + output = home + "/.deepface/weights/deploy.prototxt" - # pre-trained weights - if os.path.isfile(home + "/.deepface/weights/res10_300x300_ssd_iter_140000.caffemodel") != True: + gdown.download(url, output, quiet=False) - logger.info("res10_300x300_ssd_iter_140000.caffemodel will be downloaded...") + # pre-trained weights + if ( + os.path.isfile(home + "/.deepface/weights/res10_300x300_ssd_iter_140000.caffemodel") + != True + ): - url = "https://github.com/opencv/opencv_3rdparty/raw/dnn_samples_face_detector_20170830/res10_300x300_ssd_iter_140000.caffemodel" + logger.info("res10_300x300_ssd_iter_140000.caffemodel will be downloaded...") - output = home + "/.deepface/weights/res10_300x300_ssd_iter_140000.caffemodel" + url = "https://github.com/opencv/opencv_3rdparty/raw/dnn_samples_face_detector_20170830/res10_300x300_ssd_iter_140000.caffemodel" - gdown.download(url, output, quiet=False) + output = home + "/.deepface/weights/res10_300x300_ssd_iter_140000.caffemodel" - try: - face_detector = cv2.dnn.readNetFromCaffe( - home + "/.deepface/weights/deploy.prototxt", - home + "/.deepface/weights/res10_300x300_ssd_iter_140000.caffemodel", - ) - except Exception as err: - raise ValueError( - "Exception while calling opencv.dnn module." - + "This is an optional dependency." - + "You can install it as pip install opencv-contrib-python." - ) from err + gdown.download(url, output, quiet=False) - eye_detector = OpenCvWrapper.build_cascade("haarcascade_eye") + try: + face_detector = cv2.dnn.readNetFromCaffe( + home + "/.deepface/weights/deploy.prototxt", + home + "/.deepface/weights/res10_300x300_ssd_iter_140000.caffemodel", + ) + except Exception as err: + raise ValueError( + "Exception while calling opencv.dnn module." + + "This is an optional dependency." + + "You can install it as pip install opencv-contrib-python." + ) from err - detector = {} - detector["face_detector"] = face_detector - detector["eye_detector"] = eye_detector + detector = {} + detector["face_detector"] = face_detector + detector["opencv_module"] = OpenCvWrapper.OpenCv() - return detector + return detector + def detect_faces(self, img: np.ndarray, align: bool = True) -> list: + """ + Detect and align face with ssd + Args: + img (np.ndarray): pre-loaded image + align (bool): default is true + Returns: + list of detected and aligned faces + """ + resp = [] -def detect_face(detector: dict, img: np.ndarray, align: bool = True) -> list: - """ - Detect and align face with ssd - Args: - face_detector (Any): ssd face detector object - img (np.ndarray): pre-loaded image - align (bool): default is true - Returns: - list of detected and aligned faces - """ - resp = [] + detected_face = None + img_region = [0, 0, img.shape[1], img.shape[0]] - detected_face = None - img_region = [0, 0, img.shape[1], img.shape[0]] + ssd_labels = ["img_id", "is_face", "confidence", "left", "top", "right", "bottom"] - ssd_labels = ["img_id", "is_face", "confidence", "left", "top", "right", "bottom"] + target_size = (300, 300) - target_size = (300, 300) + base_img = img.copy() # we will restore base_img to img later - base_img = img.copy() # we will restore base_img to img later + original_size = img.shape - original_size = img.shape + img = cv2.resize(img, target_size) - img = cv2.resize(img, target_size) + aspect_ratio_x = original_size[1] / target_size[1] + aspect_ratio_y = original_size[0] / target_size[0] - aspect_ratio_x = original_size[1] / target_size[1] - aspect_ratio_y = original_size[0] / target_size[0] + imageBlob = cv2.dnn.blobFromImage(image=img) - imageBlob = cv2.dnn.blobFromImage(image=img) + face_detector = self.model["face_detector"] + face_detector.setInput(imageBlob) + detections = face_detector.forward() - face_detector = detector["face_detector"] - face_detector.setInput(imageBlob) - detections = face_detector.forward() + detections_df = pd.DataFrame(detections[0][0], columns=ssd_labels) - detections_df = pd.DataFrame(detections[0][0], columns=ssd_labels) + detections_df = detections_df[detections_df["is_face"] == 1] # 0: background, 1: face + detections_df = detections_df[detections_df["confidence"] >= 0.90] - detections_df = detections_df[detections_df["is_face"] == 1] # 0: background, 1: face - detections_df = detections_df[detections_df["confidence"] >= 0.90] + detections_df["left"] = (detections_df["left"] * 300).astype(int) + detections_df["bottom"] = (detections_df["bottom"] * 300).astype(int) + detections_df["right"] = (detections_df["right"] * 300).astype(int) + detections_df["top"] = (detections_df["top"] * 300).astype(int) - detections_df["left"] = (detections_df["left"] * 300).astype(int) - detections_df["bottom"] = (detections_df["bottom"] * 300).astype(int) - detections_df["right"] = (detections_df["right"] * 300).astype(int) - detections_df["top"] = (detections_df["top"] * 300).astype(int) + if detections_df.shape[0] > 0: - if detections_df.shape[0] > 0: + for _, instance in detections_df.iterrows(): - for _, instance in detections_df.iterrows(): + left = instance["left"] + right = instance["right"] + bottom = instance["bottom"] + top = instance["top"] - left = instance["left"] - right = instance["right"] - bottom = instance["bottom"] - top = instance["top"] + detected_face = base_img[ + int(top * aspect_ratio_y) : int(bottom * aspect_ratio_y), + int(left * aspect_ratio_x) : int(right * aspect_ratio_x), + ] + img_region = [ + int(left * aspect_ratio_x), + int(top * aspect_ratio_y), + int(right * aspect_ratio_x) - int(left * aspect_ratio_x), + int(bottom * aspect_ratio_y) - int(top * aspect_ratio_y), + ] + confidence = instance["confidence"] - detected_face = base_img[ - int(top * aspect_ratio_y) : int(bottom * aspect_ratio_y), - int(left * aspect_ratio_x) : int(right * aspect_ratio_x), - ] - img_region = [ - int(left * aspect_ratio_x), - int(top * aspect_ratio_y), - int(right * aspect_ratio_x) - int(left * aspect_ratio_x), - int(bottom * aspect_ratio_y) - int(top * aspect_ratio_y), - ] - confidence = instance["confidence"] + if align: + opencv_module: OpenCvWrapper.OpenCv = self.model["opencv_module"] + left_eye, right_eye = opencv_module.find_eyes(detected_face) + detected_face = self.align_face( + img=detected_face, left_eye=left_eye, right_eye=right_eye + ) - if align: - detected_face = OpenCvWrapper.align_face(detector["eye_detector"], detected_face) - - resp.append((detected_face, img_region, confidence)) - - return resp + resp.append((detected_face, img_region, confidence)) + return resp diff --git a/deepface/detectors/YoloWrapper.py b/deepface/detectors/YoloWrapper.py index 0786cc15..a6666c99 100644 --- a/deepface/detectors/YoloWrapper.py +++ b/deepface/detectors/YoloWrapper.py @@ -1,6 +1,6 @@ from typing import Any import numpy as np -from deepface.detectors import FaceDetector +from deepface.models.Detector import Detector from deepface.commons.logger import Logger logger = Logger() @@ -16,75 +16,78 @@ LANDMARKS_CONFIDENCE_THRESHOLD = 0.5 -def build_model() -> Any: - """ - Build a yolo detector model - Returns: - model (Any) - """ - import gdown - import os - - # Import the Ultralytics YOLO model - try: - from ultralytics import YOLO - except ModuleNotFoundError as e: - raise ImportError( - "Yolo is an optional detector, ensure the library is installed. \ - Please install using 'pip install ultralytics' " - ) from e - - from deepface.commons.functions import get_deepface_home - - weight_path = f"{get_deepface_home()}{PATH}" - - # Download the model's weights if they don't exist - if not os.path.isfile(weight_path): - gdown.download(WEIGHT_URL, weight_path, quiet=False) - logger.info(f"Downloaded YOLO model {os.path.basename(weight_path)}") - - # Return face_detector - return YOLO(weight_path) - - -def detect_face(face_detector: Any, img: np.ndarray, align: bool = False) -> list: - """ - Detect and align face with yolo - Args: - face_detector (Any): yolo face detector object - img (np.ndarray): pre-loaded image - align (bool): default is true - Returns: - list of detected and aligned faces - """ - resp = [] - - # Detect faces - results = face_detector.predict(img, verbose=False, show=False, conf=0.25)[0] - - # For each face, extract the bounding box, the landmarks and confidence - for result in results: - # Extract the bounding box and the confidence - x, y, w, h = result.boxes.xywh.tolist()[0] - confidence = result.boxes.conf.tolist()[0] - - x, y, w, h = int(x - w / 2), int(y - h / 2), int(w), int(h) - detected_face = img[y : y + h, x : x + w].copy() - - if align: - # Tuple of x,y and confidence for left eye - left_eye = result.keypoints.xy[0][0], result.keypoints.conf[0][0] - # Tuple of x,y and confidence for right eye - right_eye = result.keypoints.xy[0][1], result.keypoints.conf[0][1] - - # Check the landmarks confidence before alignment - if ( - left_eye[1] > LANDMARKS_CONFIDENCE_THRESHOLD - and right_eye[1] > LANDMARKS_CONFIDENCE_THRESHOLD - ): - detected_face = FaceDetector.alignment_procedure( - detected_face, left_eye[0].cpu(), right_eye[0].cpu() - ) - resp.append((detected_face, [x, y, w, h], confidence)) - - return resp +class Yolo(Detector): + def __init__(self): + self.model = self.build_model() + + def build_model(self) -> Any: + """ + Build a yolo detector model + Returns: + model (Any) + """ + import gdown + import os + + # Import the Ultralytics YOLO model + try: + from ultralytics import YOLO + except ModuleNotFoundError as e: + raise ImportError( + "Yolo is an optional detector, ensure the library is installed. \ + Please install using 'pip install ultralytics' " + ) from e + + from deepface.commons.functions import get_deepface_home + + weight_path = f"{get_deepface_home()}{PATH}" + + # Download the model's weights if they don't exist + if not os.path.isfile(weight_path): + gdown.download(WEIGHT_URL, weight_path, quiet=False) + logger.info(f"Downloaded YOLO model {os.path.basename(weight_path)}") + + # Return face_detector + return YOLO(weight_path) + + def detect_faces(self, img: np.ndarray, align: bool = False) -> list: + """ + Detect and align face with yolo + Args: + face_detector (Any): yolo face detector object + img (np.ndarray): pre-loaded image + align (bool): default is true + Returns: + list of detected and aligned faces + """ + resp = [] + + # Detect faces + results = self.model.predict(img, verbose=False, show=False, conf=0.25)[0] + + # For each face, extract the bounding box, the landmarks and confidence + for result in results: + # Extract the bounding box and the confidence + x, y, w, h = result.boxes.xywh.tolist()[0] + confidence = result.boxes.conf.tolist()[0] + + x, y, w, h = int(x - w / 2), int(y - h / 2), int(w), int(h) + detected_face = img[y : y + h, x : x + w].copy() + + if align: + # Tuple of x,y and confidence for left eye + left_eye = result.keypoints.xy[0][0], result.keypoints.conf[0][0] + # Tuple of x,y and confidence for right eye + right_eye = result.keypoints.xy[0][1], result.keypoints.conf[0][1] + + # Check the landmarks confidence before alignment + if ( + left_eye[1] > LANDMARKS_CONFIDENCE_THRESHOLD + and right_eye[1] > LANDMARKS_CONFIDENCE_THRESHOLD + ): + detected_face = self.align_face( + img=detected_face, left_eye=left_eye[0].cpu(), right_eye=right_eye[0].cpu() + ) + resp.append((detected_face, [x, y, w, h], confidence)) + + return resp diff --git a/deepface/detectors/YunetWrapper.py b/deepface/detectors/YunetWrapper.py index 005fa6dc..544bd5b9 100644 --- a/deepface/detectors/YunetWrapper.py +++ b/deepface/detectors/YunetWrapper.py @@ -3,112 +3,110 @@ import cv2 import numpy as np import gdown -from deepface.detectors import FaceDetector from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.Detector import Detector logger = Logger(module="detectors.YunetWrapper") -def build_model() -> Any: - """ - Build a yunet detector model - Returns: - model (Any) - """ - # pylint: disable=C0301 - url = "https://github.com/opencv/opencv_zoo/raw/main/models/face_detection_yunet/face_detection_yunet_2023mar.onnx" - file_name = "face_detection_yunet_2023mar.onnx" - home = functions.get_deepface_home() - if os.path.isfile(home + f"/.deepface/weights/{file_name}") is False: - logger.info(f"{file_name} will be downloaded...") - output = home + f"/.deepface/weights/{file_name}" - gdown.download(url, output, quiet=False) +class YuNet(Detector): + def __init__(self): + self.model = self.build_model() - try: - face_detector = cv2.FaceDetectorYN_create( - home + f"/.deepface/weights/{file_name}", "", (0, 0) - ) - except Exception as err: - raise ValueError( - "Exception while calling opencv.FaceDetectorYN_create module." - + "This is an optional dependency." - + "You can install it as pip install opencv-contrib-python." - ) from err - return face_detector + def build_model(self) -> Any: + """ + Build a yunet detector model + Returns: + model (Any) + """ + # pylint: disable=C0301 + url = "https://github.com/opencv/opencv_zoo/raw/main/models/face_detection_yunet/face_detection_yunet_2023mar.onnx" + file_name = "face_detection_yunet_2023mar.onnx" + home = functions.get_deepface_home() + if os.path.isfile(home + f"/.deepface/weights/{file_name}") is False: + logger.info(f"{file_name} will be downloaded...") + output = home + f"/.deepface/weights/{file_name}" + gdown.download(url, output, quiet=False) + try: + face_detector = cv2.FaceDetectorYN_create( + home + f"/.deepface/weights/{file_name}", "", (0, 0) + ) + except Exception as err: + raise ValueError( + "Exception while calling opencv.FaceDetectorYN_create module." + + "This is an optional dependency." + + "You can install it as pip install opencv-contrib-python." + ) from err + return face_detector -def detect_face( - detector: Any, image: np.ndarray, align: bool = True, score_threshold: float = 0.9 -) -> list: - """ - Detect and align face with yunet - Args: - face_detector (Any): yunet face detector object - img (np.ndarray): pre-loaded image - align (bool): default is true - Returns: - list of detected and aligned faces - """ - # FaceDetector.detect_faces does not support score_threshold parameter. - # We can set it via environment variable. - score_threshold = os.environ.get("yunet_score_threshold", score_threshold) - resp = [] - detected_face = None - img_region = [0, 0, image.shape[1], image.shape[0]] - faces = [] - height, width = image.shape[0], image.shape[1] - # resize image if it is too large (Yunet fails to detect faces on large input sometimes) - # I picked 640 as a threshold because it is the default value of max_size in Yunet. - resized = False - if height > 640 or width > 640: - r = 640.0 / max(height, width) - original_image = image.copy() - image = cv2.resize(image, (int(width * r), int(height * r))) - height, width = image.shape[0], image.shape[1] - resized = True - detector.setInputSize((width, height)) - detector.setScoreThreshold(score_threshold) - _, faces = detector.detect(image) - if faces is None: - return resp - for face in faces: - # pylint: disable=W0105 + def detect_faces(self, img: np.ndarray, align: bool = True) -> list: """ - The detection output faces is a two-dimension array of type CV_32F, - whose rows are the detected face instances, columns are the location - of a face and 5 facial landmarks. - The format of each row is as follows: - x1, y1, w, h, x_re, y_re, x_le, y_le, x_nt, y_nt, - x_rcm, y_rcm, x_lcm, y_lcm, - where x1, y1, w, h are the top-left coordinates, width and height of - the face bounding box, - {x, y}_{re, le, nt, rcm, lcm} stands for the coordinates of right eye, - left eye, nose tip, the right corner and left corner of the mouth respectively. + Detect and align face with yunet + Args: + img (np.ndarray): pre-loaded image + align (bool): default is true + Returns: + list of detected and aligned faces """ - (x, y, w, h, x_re, y_re, x_le, y_le) = list(map(int, face[:8])) + # FaceDetector.detect_faces does not support score_threshold parameter. + # We can set it via environment variable. + score_threshold = os.environ.get("yunet_score_threshold", "0.9") + resp = [] + detected_face = None + img_region = [0, 0, img.shape[1], img.shape[0]] + faces = [] + height, width = img.shape[0], img.shape[1] + # resize image if it is too large (Yunet fails to detect faces on large input sometimes) + # I picked 640 as a threshold because it is the default value of max_size in Yunet. + resized = False + if height > 640 or width > 640: + r = 640.0 / max(height, width) + original_image = img.copy() + img = cv2.resize(img, (int(width * r), int(height * r))) + height, width = img.shape[0], img.shape[1] + resized = True + self.model.setInputSize((width, height)) + self.model.setScoreThreshold(score_threshold) + _, faces = self.model.detect(img) + if faces is None: + return resp + for face in faces: + # pylint: disable=W0105 + """ + The detection output faces is a two-dimension array of type CV_32F, + whose rows are the detected face instances, columns are the location + of a face and 5 facial landmarks. + The format of each row is as follows: + x1, y1, w, h, x_re, y_re, x_le, y_le, x_nt, y_nt, + x_rcm, y_rcm, x_lcm, y_lcm, + where x1, y1, w, h are the top-left coordinates, width and height of + the face bounding box, + {x, y}_{re, le, nt, rcm, lcm} stands for the coordinates of right eye, + left eye, nose tip, the right corner and left corner of the mouth respectively. + """ + (x, y, w, h, x_re, y_re, x_le, y_le) = list(map(int, face[:8])) - # Yunet returns negative coordinates if it thinks part of - # the detected face is outside the frame. - # We set the coordinate to 0 if they are negative. - x = max(x, 0) - y = max(y, 0) - if resized: - image = original_image - x, y, w, h = int(x / r), int(y / r), int(w / r), int(h / r) - x_re, y_re, x_le, y_le = ( - int(x_re / r), - int(y_re / r), - int(x_le / r), - int(y_le / r), - ) - confidence = face[-1] - confidence = f"{confidence:.2f}" - detected_face = image[int(y) : int(y + h), int(x) : int(x + w)] - img_region = [x, y, w, h] - if align: - detected_face = FaceDetector.alignment_procedure( - detected_face, (x_re, y_re), (x_le, y_le) - ) - resp.append((detected_face, img_region, confidence)) - return resp + # Yunet returns negative coordinates if it thinks part of + # the detected face is outside the frame. + # We set the coordinate to 0 if they are negative. + x = max(x, 0) + y = max(y, 0) + if resized: + img = original_image + x, y, w, h = int(x / r), int(y / r), int(w / r), int(h / r) + x_re, y_re, x_le, y_le = ( + int(x_re / r), + int(y_re / r), + int(x_le / r), + int(y_le / r), + ) + confidence = face[-1] + confidence = f"{confidence:.2f}" + detected_face = img[int(y) : int(y + h), int(x) : int(x + w)] + img_region = [x, y, w, h] + if align: + detected_face = self.align_face(detected_face, (x_re, y_re), (x_le, y_le)) + resp.append((detected_face, img_region, confidence)) + return resp diff --git a/deepface/extendedmodels/Age.py b/deepface/extendedmodels/Age.py index 73ca6cc2..3c03ee56 100644 --- a/deepface/extendedmodels/Age.py +++ b/deepface/extendedmodels/Age.py @@ -5,6 +5,7 @@ from deepface.basemodels import VGGFace from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.Demography import Demography logger = Logger(module="extendedmodels.Age") @@ -22,12 +23,31 @@ # ---------------------------------------- +# pylint: disable=too-few-public-methods +class ApparentAge(Demography): + """ + Age model class + """ -def loadModel( + def __init__(self): + self.model = load_model() + self.model_name = "Age" + + def predict(self, img: np.ndarray) -> np.float64: + age_predictions = self.model.predict(img, verbose=0)[0, :] + return find_apparent_age(age_predictions) + + +def load_model( url="https://github.com/serengil/deepface_models/releases/download/v1.0/age_model_weights.h5", ) -> Model: + """ + Construct age model, download its weights and load + Returns: + model (Model) + """ - model = VGGFace.baseModel() + model = VGGFace.base_model() # -------------------------- @@ -60,7 +80,14 @@ def loadModel( # -------------------------- -def findApparentAge(age_predictions) -> np.float64: +def find_apparent_age(age_predictions: np.ndarray) -> np.float64: + """ + Find apparent age prediction from a given probas of ages + Args: + age_predictions (?) + Returns: + apparent_age (float) + """ output_indexes = np.array(list(range(0, 101))) apparent_age = np.sum(age_predictions * output_indexes) return apparent_age diff --git a/deepface/extendedmodels/Emotion.py b/deepface/extendedmodels/Emotion.py index 41214a18..7cf984a9 100644 --- a/deepface/extendedmodels/Emotion.py +++ b/deepface/extendedmodels/Emotion.py @@ -1,8 +1,11 @@ import os import gdown import tensorflow as tf +import numpy as np +import cv2 from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.Demography import Demography logger = Logger(module="extendedmodels.Emotion") @@ -30,10 +33,31 @@ # Labels for the emotions that can be detected by the model. labels = ["angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"] +# pylint: disable=too-few-public-methods +class FacialExpression(Demography): + """ + Emotion model class + """ -def loadModel( + def __init__(self): + self.model = load_model() + self.model_name = "Emotion" + + def predict(self, img: np.ndarray) -> np.ndarray: + img_gray = cv2.cvtColor(img[0], cv2.COLOR_BGR2GRAY) + img_gray = cv2.resize(img_gray, (48, 48)) + img_gray = np.expand_dims(img_gray, axis=0) + + emotion_predictions = self.model.predict(img_gray, verbose=0)[0, :] + return emotion_predictions + + +def load_model( url="https://github.com/serengil/deepface_models/releases/download/v1.0/facial_expression_model_weights.h5", ) -> Sequential: + """ + Consruct emotion model, download and load weights + """ num_classes = 7 diff --git a/deepface/extendedmodels/Gender.py b/deepface/extendedmodels/Gender.py index d53f6f25..0ecad61b 100644 --- a/deepface/extendedmodels/Gender.py +++ b/deepface/extendedmodels/Gender.py @@ -1,9 +1,11 @@ import os import gdown import tensorflow as tf +import numpy as np from deepface.basemodels import VGGFace from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.Demography import Demography logger = Logger(module="extendedmodels.Gender") @@ -25,12 +27,30 @@ # Labels for the genders that can be detected by the model. labels = ["Woman", "Man"] +# pylint: disable=too-few-public-methods +class Gender(Demography): + """ + Gender model class + """ -def loadModel( + def __init__(self): + self.model = load_model() + self.model_name = "Gender" + + def predict(self, img: np.ndarray) -> np.ndarray: + return self.model.predict(img, verbose=0)[0, :] + + +def load_model( url="https://github.com/serengil/deepface_models/releases/download/v1.0/gender_model_weights.h5", ) -> Model: + """ + Construct gender model, download its weights and load + Returns: + model (Model) + """ - model = VGGFace.baseModel() + model = VGGFace.base_model() # -------------------------- diff --git a/deepface/extendedmodels/Race.py b/deepface/extendedmodels/Race.py index 50087c10..71d2d124 100644 --- a/deepface/extendedmodels/Race.py +++ b/deepface/extendedmodels/Race.py @@ -1,9 +1,11 @@ import os import gdown import tensorflow as tf +import numpy as np from deepface.basemodels import VGGFace from deepface.commons import functions from deepface.commons.logger import Logger +from deepface.models.Demography import Demography logger = Logger(module="extendedmodels.Race") @@ -23,12 +25,28 @@ # Labels for the ethnic phenotypes that can be detected by the model. labels = ["asian", "indian", "black", "white", "middle eastern", "latino hispanic"] +# pylint: disable=too-few-public-methods +class Race(Demography): + """ + Race model class + """ -def loadModel( + def __init__(self): + self.model = load_model() + self.model_name = "Race" + + def predict(self, img: np.ndarray) -> np.ndarray: + return self.model.predict(img, verbose=0)[0, :] + + +def load_model( url="https://github.com/serengil/deepface_models/releases/download/v1.0/race_model_single_batch.h5", ) -> Model: + """ + Construct race model, download its weights and load + """ - model = VGGFace.baseModel() + model = VGGFace.base_model() # -------------------------- diff --git a/deepface/models/Demography.py b/deepface/models/Demography.py new file mode 100644 index 00000000..861a9873 --- /dev/null +++ b/deepface/models/Demography.py @@ -0,0 +1,23 @@ +from typing import Union +from abc import ABC, abstractmethod +import numpy as np +import tensorflow as tf + +tf_version = int(tf.__version__.split(".", maxsplit=1)[0]) + +if tf_version == 1: + from keras.models import Model +else: + from tensorflow.keras.models import Model + +# Notice that all facial attribute analysis models must be inherited from this class + + +# pylint: disable=too-few-public-methods +class Demography(ABC): + model: Model + model_name: str + + @abstractmethod + def predict(self, img: np.ndarray) -> Union[np.ndarray, np.float64]: + pass diff --git a/deepface/models/Detector.py b/deepface/models/Detector.py new file mode 100644 index 00000000..b5c80af1 --- /dev/null +++ b/deepface/models/Detector.py @@ -0,0 +1,39 @@ +from abc import ABC, abstractmethod +from typing import Union +import numpy as np +from PIL import Image + +# Notice that all facial detector models must be inherited from this class + + +class Detector(ABC): + @abstractmethod + def detect_faces(self, img: np.ndarray, align: bool = True) -> list: + pass + + def align_face( + self, img: np.ndarray, left_eye: Union[list, tuple], right_eye: Union[list, tuple] + ) -> np.ndarray: + """ + Align a given image horizantally with respect to their left and right eye locations + Args: + img (np.ndarray): pre-loaded image with detected face + left_eye (list or tuple): coordinates of left eye with respect to the you + right_eye(list or tuple): coordinates of right eye with respect to the you + Returns: + img (np.ndarray): aligned facial image + """ + # if eye could not be detected for the given image, return image itself + if left_eye is None or right_eye is None: + return img + + # sometimes unexpectedly detected images come with nil dimensions + if img.shape[0] == 0 or img.shape[1] == 0: + return img + + angle = float( + np.degrees(np.arctan2(right_eye[1] - left_eye[1], right_eye[0] - left_eye[0])) + ) + img = Image.fromarray(img) + img = np.array(img.rotate(angle)) + return img diff --git a/deepface/models/FacialRecognition.py b/deepface/models/FacialRecognition.py new file mode 100644 index 00000000..fa70f65a --- /dev/null +++ b/deepface/models/FacialRecognition.py @@ -0,0 +1,28 @@ +from abc import ABC +from typing import Any, Union +import numpy as np +import tensorflow as tf + +tf_version = int(tf.__version__.split(".", maxsplit=1)[0]) +if tf_version == 2: + from tensorflow.keras.models import Model +else: + from keras.models import Model + +# Notice that all facial recognition models must be inherited from this class + +# pylint: disable=too-few-public-methods +class FacialRecognition(ABC): + model: Union[Model, Any] + model_name: str + + def find_embeddings(self, img: np.ndarray) -> list: + if not isinstance(self.model, Model): + raise ValueError( + "If a facial recognition model is not type of (tf.)keras.models.Model," + "Then its find_embeddings method must be implemented its own module." + f"However {self.model_name}'s model type is {type(self.model)}" + ) + # model.predict causes memory issue when it is called in a for loop + # embedding = model.predict(img, verbose=0)[0].tolist() + return self.model(img, training=False).numpy()[0].tolist() diff --git a/deepface/modules/demography.py b/deepface/modules/demography.py index 664b85ab..96a3e058 100644 --- a/deepface/modules/demography.py +++ b/deepface/modules/demography.py @@ -4,12 +4,11 @@ # 3rd party dependencies import numpy as np from tqdm import tqdm -import cv2 # project dependencies from deepface.modules import modeling from deepface.commons import functions -from deepface.extendedmodels import Age, Gender, Race, Emotion +from deepface.extendedmodels import Gender, Race, Emotion def analyze( @@ -123,18 +122,10 @@ def analyze( pbar.set_description(f"Action: {action}") if action == "emotion": - img_gray = cv2.cvtColor(img_content[0], cv2.COLOR_BGR2GRAY) - img_gray = cv2.resize(img_gray, (48, 48)) - img_gray = np.expand_dims(img_gray, axis=0) - - emotion_predictions = modeling.build_model("Emotion").predict( - img_gray, verbose=0 - )[0, :] - + emotion_predictions = modeling.build_model("Emotion").predict(img_content) sum_of_predictions = emotion_predictions.sum() obj["emotion"] = {} - for i, emotion_label in enumerate(Emotion.labels): emotion_prediction = 100 * emotion_predictions[i] / sum_of_predictions obj["emotion"][emotion_label] = emotion_prediction @@ -142,17 +133,12 @@ def analyze( obj["dominant_emotion"] = Emotion.labels[np.argmax(emotion_predictions)] elif action == "age": - age_predictions = modeling.build_model("Age").predict(img_content, verbose=0)[ - 0, : - ] - apparent_age = Age.findApparentAge(age_predictions) + apparent_age = modeling.build_model("Age").predict(img_content) # int cast is for exception - object of type 'float32' is not JSON serializable obj["age"] = int(apparent_age) elif action == "gender": - gender_predictions = modeling.build_model("Gender").predict( - img_content, verbose=0 - )[0, :] + gender_predictions = modeling.build_model("Gender").predict(img_content) obj["gender"] = {} for i, gender_label in enumerate(Gender.labels): gender_prediction = 100 * gender_predictions[i] @@ -161,9 +147,7 @@ def analyze( obj["dominant_gender"] = Gender.labels[np.argmax(gender_predictions)] elif action == "race": - race_predictions = modeling.build_model("Race").predict(img_content, verbose=0)[ - 0, : - ] + race_predictions = modeling.build_model("Race").predict(img_content) sum_of_predictions = race_predictions.sum() obj["race"] = {} diff --git a/deepface/modules/modeling.py b/deepface/modules/modeling.py index 8936dabc..f1b4bfb1 100644 --- a/deepface/modules/modeling.py +++ b/deepface/modules/modeling.py @@ -1,32 +1,21 @@ # built-in dependencies -from typing import Any, Union - -# 3rd party dependencies -import tensorflow as tf +from typing import Any # project dependencies from deepface.basemodels import ( VGGFace, OpenFace, Facenet, - Facenet512, FbDeepFace, DeepID, - DlibWrapper, + DlibResNet, ArcFace, SFace, ) from deepface.extendedmodels import Age, Gender, Race, Emotion -# conditional dependencies -tf_version = int(tf.__version__.split(".", maxsplit=1)[0]) -if tf_version == 2: - from tensorflow.keras.models import Model -else: - from keras.models import Model - -def build_model(model_name: str) -> Union[Model, Any]: +def build_model(model_name: str) -> Any: """ This function builds a deepface model Parameters: @@ -35,26 +24,26 @@ def build_model(model_name: str) -> Union[Model, Any]: Age, Gender, Emotion, Race for facial attributes Returns: - built deepface model ( (tf.)keras.models.Model ) + built model class """ # singleton design pattern global model_obj models = { - "VGG-Face": VGGFace.loadModel, - "OpenFace": OpenFace.loadModel, - "Facenet": Facenet.loadModel, - "Facenet512": Facenet512.loadModel, - "DeepFace": FbDeepFace.loadModel, - "DeepID": DeepID.loadModel, - "Dlib": DlibWrapper.loadModel, - "ArcFace": ArcFace.loadModel, - "SFace": SFace.load_model, - "Emotion": Emotion.loadModel, - "Age": Age.loadModel, - "Gender": Gender.loadModel, - "Race": Race.loadModel, + "VGG-Face": VGGFace.VggFace, + "OpenFace": OpenFace.OpenFace, + "Facenet": Facenet.FaceNet128d, + "Facenet512": Facenet.FaceNet512d, + "DeepFace": FbDeepFace.DeepFace, + "DeepID": DeepID.DeepId, + "Dlib": DlibResNet.Dlib, + "ArcFace": ArcFace.ArcFace, + "SFace": SFace.SFace, + "Emotion": Emotion.FacialExpression, + "Age": Age.ApparentAge, + "Gender": Gender.Gender, + "Race": Race.Race, } if not "model_obj" in globals(): diff --git a/deepface/modules/representation.py b/deepface/modules/representation.py index ab2e01db..d7a87673 100644 --- a/deepface/modules/representation.py +++ b/deepface/modules/representation.py @@ -4,18 +4,11 @@ # 3rd party dependencies import numpy as np import cv2 -import tensorflow as tf # project dependencies from deepface.modules import modeling from deepface.commons import functions - -# conditional dependencies -tf_version = int(tf.__version__.split(".", maxsplit=1)[0]) -if tf_version == 2: - from tensorflow.keras.models import Model -else: - from keras.models import Model +from deepface.models.FacialRecognition import FacialRecognition def represent( @@ -71,7 +64,7 @@ def represent( """ resp_objs = [] - model = modeling.build_model(model_name) + model: FacialRecognition = modeling.build_model(model_name) # --------------------------------- # we have run pre-process in verification. so, this can be skipped if it is coming from verify. @@ -107,18 +100,7 @@ def represent( # custom normalization img = functions.normalize_input(img=img, normalization=normalization) - # represent - # if "keras" in str(type(model)): - if isinstance(model, Model): - # model.predict causes memory issue when it is called in a for loop - # embedding = model.predict(img, verbose=0)[0].tolist() - embedding = model(img, training=False).numpy()[0].tolist() - # if you still get verbose logging. try call - # - `tf.keras.utils.disable_interactive_logging()` - # in your main program - else: - # SFace and Dlib are not keras models and no verbose arguments - embedding = model.predict(img)[0].tolist() + embedding = model.find_embeddings(img) resp_obj = {} resp_obj["embedding"] = embedding diff --git a/requirements.txt b/requirements.txt index 0f1dabab..a6d7634e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,4 @@ Flask>=1.1.2 mtcnn>=0.1.0 retina-face>=0.0.1 fire>=0.4.0 -gunicorn>=20.1.0 -Deprecated>=1.2.13 \ No newline at end of file +gunicorn>=20.1.0 \ No newline at end of file diff --git a/tests/visual-test.py b/tests/visual-test.py index f6bb6b62..dbdf54ff 100644 --- a/tests/visual-test.py +++ b/tests/visual-test.py @@ -4,6 +4,10 @@ logger = Logger() +# some models (e.g. Dlib) and detectors (e.g. retinaface) do not have test cases +# because they require to install huge packages +# this module is for local runs + model_names = [ "VGG-Face", "Facenet", @@ -17,6 +21,7 @@ ] detector_backends = ["opencv", "ssd", "dlib", "mtcnn", "retinaface"] + # verification for model_name in model_names: obj = DeepFace.verify( From 25a3061ae9400e992b2b4c256ff660fc2bbdeba6 Mon Sep 17 00:00:00 2001 From: Sefik Ilkin Serengil Date: Sat, 20 Jan 2024 19:26:01 +0000 Subject: [PATCH 2/3] deprecated dependency retired --- deepface/commons/functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deepface/commons/functions.py b/deepface/commons/functions.py index 6113c759..520251d8 100644 --- a/deepface/commons/functions.py +++ b/deepface/commons/functions.py @@ -9,7 +9,6 @@ import numpy as np import cv2 import tensorflow as tf -from deprecated import deprecated # package dependencies from deepface.detectors import DetectorWrapper From a50ff7e35a4ed1d9259f3f83789296742ef566c4 Mon Sep 17 00:00:00 2001 From: Sefik Ilkin Serengil Date: Sat, 20 Jan 2024 19:27:08 +0000 Subject: [PATCH 3/3] deprecated module retired --- deepface/commons/functions.py | 53 ----------------------------------- 1 file changed, 53 deletions(-) diff --git a/deepface/commons/functions.py b/deepface/commons/functions.py index 520251d8..64f8e06a 100644 --- a/deepface/commons/functions.py +++ b/deepface/commons/functions.py @@ -340,56 +340,3 @@ def find_target_size(model_name: str) -> tuple: raise ValueError(f"unimplemented model name - {model_name}") return target_size - - -# --------------------------------------------------- -# deprecated functions - - -@deprecated(version="0.0.78", reason="Use extract_faces instead of preprocess_face") -def preprocess_face( - img: Union[str, np.ndarray], - target_size=(224, 224), - detector_backend="opencv", - grayscale=False, - enforce_detection=True, - align=True, -) -> Union[np.ndarray, None]: - """ - Preprocess only one face - - Args: - img (str or numpy): the input image. - target_size (tuple, optional): the target size. Defaults to (224, 224). - detector_backend (str, optional): the detector backend. Defaults to "opencv". - grayscale (bool, optional): whether to convert to grayscale. Defaults to False. - enforce_detection (bool, optional): whether to enforce face detection. Defaults to True. - align (bool, optional): whether to align the face. Defaults to True. - - Returns: - loaded image (numpt): the preprocessed face. - - Raises: - ValueError: if face is not detected and enforce_detection is True. - - Deprecated: - 0.0.78: Use extract_faces instead of preprocess_face. - """ - logger.warn("Function preprocess_face is deprecated. Use extract_faces instead.") - result = None - img_objs = extract_faces( - img=img, - target_size=target_size, - detector_backend=detector_backend, - grayscale=grayscale, - enforce_detection=enforce_detection, - align=align, - ) - - if len(img_objs) > 0: - result, _, _ = img_objs[0] - # discard expanded dimension - if len(result.shape) == 4: - result = result[0] - - return result