Add property to get detected face confidence

README.md

@@ -22,6 +22,12 @@ This package supports cross platform, Windows, Linux and MacOSX!!
 |FaceRecognitionDotNet for CUDA 10.2|Windows|-|✓|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.CUDA102.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.CUDA102)|
 ||Linux|-|✓|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.CUDA102.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.CUDA102)|
 ||OSX|-|-|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.CUDA102.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.CUDA102)|
+|FaceRecognitionDotNet for CUDA 11.0|Windows|-|✓|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.CUDA110.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.CUDA110)|
+||Linux|-|✓|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.CUDA110.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.CUDA110)|
+||OSX|-|-|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.CUDA110.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.CUDA110)|
+|FaceRecognitionDotNet for CUDA 11.1|Windows|-|✓|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.CUDA111.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.CUDA111)|
+||Linux|-|✓|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.CUDA111.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.CUDA111)|
+||OSX|-|-|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.CUDA111.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.CUDA111)|
 |FaceRecognitionDotNet for Intel MKL|Windows|-|✓|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.MKL.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.MKL)|
 ||Linux|-|✓|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.MKL.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.MKL)|
 ||OSX|-|✓|-|-|[![NuGet version](https://img.shields.io/nuget/v/FaceRecognitionDotNet.MKL.svg)](https://www.nuget.org/packages/FaceRecognitionDotNet.MKL)|
@@ -40,7 +46,7 @@ This package supports cross platform, Windows, Linux and MacOSX!!
 |face_distance|FaceDistance||
 |face_encodings|FaceEncodings||
 |face_landmarks|FaceLandmarks|And support **Helen dataset** :warning:|
-|face_locations|FaceLocations||
+|face_locations|FaceLocations|And support to get confidence and use custom face detector|
 |load_image_file|LoadImageFile||
 |-|CropFaces|Crop image with specified locations|
 |-|EyeBlinkDetect|Detect person is blinking or not<br>Support Large model and **Helen dataset** :warning:|

examples/BlinkDetection/Program.cs

@@ -1,12 +1,10 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.InteropServices;
 using FaceRecognitionDotNet;
 using FaceRecognitionDotNet.Extensions;
 using Microsoft.Extensions.CommandLineUtils;
 using OpenCvSharp;
-using Point = FaceRecognitionDotNet.Point;
 
 namespace BlinkDetection
 {

examples/HeadPoseEstimationDemo/Program.cs

@@ -7,7 +7,6 @@
 using FaceRecognitionDotNet.Extensions;
 using Microsoft.Extensions.CommandLineUtils;
 using OpenCvSharp;
-using Size = OpenCvSharp.Size;
 
 namespace HeadPoseEstimationDemo
 {

examples/RgbBgr/Program.cs

@@ -1,10 +1,8 @@
 using System;
-using System.Collections.Generic;
 using System.Drawing;
 using System.Drawing.Imaging;
 using System.Linq;
 using FaceRecognitionDotNet;
-using Image = FaceRecognitionDotNet.Image;
 
 namespace RgbBgr
 {

nuget/CreateAllPackage.bat

@@ -6,4 +6,6 @@ pwsh CreatePackage.ps1 CUDA92
 pwsh CreatePackage.ps1 CUDA100
 pwsh CreatePackage.ps1 CUDA101
 pwsh CreatePackage.ps1 CUDA102
+pwsh CreatePackage.ps1 CUDA110
+pwsh CreatePackage.ps1 CUDA111
 pwsh CreatePackage.ps1 MKL

src/FaceRecognitionDotNet/Dlib/Python/SimpleObjectDetector.cs

@@ -122,9 +122,9 @@ public static IEnumerable<Rectangle> RunDetectorWithUpscale1(FrontalFaceDetector
             }
         }
 
-        public static IEnumerable<Rectangle> RunDetectorWithUpscale2(FrontalFaceDetector detector,
-                                                                     Image image,
-                                                                     uint upsamplingAmount)
+        public static IEnumerable<Tuple<Rectangle, double>> RunDetectorWithUpscale2(FrontalFaceDetector detector,
+                                                                                    Image image,
+                                                                                    uint upsamplingAmount)
         {
             if (detector == null)
                 throw new ArgumentNullException(nameof(detector));
@@ -138,12 +138,17 @@ public static IEnumerable<Rectangle> RunDetectorWithUpscale2(FrontalFaceDetector
             var weightIndices = new List<ulong>();
             const double adjustThreshold = 0.0;
 
-            return RunDetectorWithUpscale1(detector,
-                                           image,
-                                           upsamplingAmount,
-                                           adjustThreshold,
-                                           detectionConfidences,
-                                           weightIndices);
+            var rects = RunDetectorWithUpscale1(detector,
+                                                image,
+                                                upsamplingAmount,
+                                                adjustThreshold,
+                                                detectionConfidences,
+                                                weightIndices).ToArray();
+
+
+            var index = 0;
+            foreach(var rect in rects)
+                yield return new Tuple<Rectangle, double>(rect, detectionConfidences[index++]);
         }
 
         #region Helpers

src/FaceRecognitionDotNet/Extensions/FaceDetector.cs

@@ -0,0 +1,32 @@
+using System.Collections.Generic;
+using DlibDotNet;
+
+namespace FaceRecognitionDotNet.Extensions
+{
+
+    /// <summary>
+    /// An abstract base class that provides functionality to detect face locations from image.
+    /// </summary>
+    public abstract class FaceDetector : DisposableObject
+    {
+
+        #region Methods
+
+        internal IEnumerable<Location> Detect(Image image, int numberOfTimesToUpsample)
+        {
+            return this.RawDetect(image.Matrix, numberOfTimesToUpsample);
+        }
+
+        /// <summary>
+        /// Returns an enumerable collection of face location correspond to all faces in specified image.
+        /// </summary>
+        /// <param name="matrix">The matrix contains a face.</param>
+        /// <param name="numberOfTimesToUpsample">The number of times to up-sample the image when finding faces.</param>
+        /// <returns>An enumerable collection of face location correspond to all faces.</returns>
+        protected abstract IEnumerable<Location> RawDetect(MatrixBase matrix, int numberOfTimesToUpsample);
+
+        #endregion
+
+    }
+
+}

src/FaceRecognitionDotNet/Extensions/SimpleAgeEstimator.cs

@@ -122,6 +122,16 @@ protected override IDictionary<uint, float> RawPredictProbability(MatrixBase mat
             }
         }
 
+        /// <summary>
+        /// Releases all unmanaged resources.
+        /// </summary>
+        protected override void DisposeUnmanaged()
+        {
+            base.DisposeUnmanaged();
+
+            this._Network?.Dispose();
+        }
+
         #endregion
 
     }

src/FaceRecognitionDotNet/Extensions/SimpleFaceDetector.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using DlibDotNet;
+
+namespace FaceRecognitionDotNet.Extensions
+{
+
+    /// <summary>
+    /// The face detector which was trained by custom dataset. This class cannot be inherited.
+    /// </summary>
+    public sealed class SimpleFaceDetector : FaceDetector
+    {
+
+        #region Fields
+
+        private readonly ScanFHogPyramid<PyramidDown, DefaultFHogFeatureExtractor> _Scanner;
+
+        private readonly ObjectDetector<ScanFHogPyramid<PyramidDown, DefaultFHogFeatureExtractor>> _ObjectDetector;
+
+        #endregion
+
+        #region Constructors
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SimpleFaceDetector"/> class with the model file path that this detector uses.
+        /// </summary>
+        /// <param name="modelPath">The model file path that this detector uses.</param>
+        /// <exception cref="FileNotFoundException">The model file is not found.</exception>
+        public SimpleFaceDetector(string modelPath)
+        {
+            if (!File.Exists(modelPath))
+                throw new FileNotFoundException(modelPath);
+
+            this._Scanner = new ScanFHogPyramid<PyramidDown, DefaultFHogFeatureExtractor>(6);
+            this._ObjectDetector = new ObjectDetector<ScanFHogPyramid<PyramidDown, DefaultFHogFeatureExtractor>>(this._Scanner);
+            this._ObjectDetector.Deserialize(modelPath);
+        }
+
+        #endregion
+
+        #region Methods
+
+        /// <summary>
+        /// Returns an enumerable collection of face location correspond to all faces in specified image.
+        /// </summary>
+        /// <param name="matrix">The matrix contains a face.</param>
+        /// <param name="numberOfTimesToUpsample">The number of times to up-sample the image when finding faces.</param>
+        /// <returns>An enumerable collection of face location correspond to all faces.</returns>
+        protected override IEnumerable<Location> RawDetect(MatrixBase matrix, int numberOfTimesToUpsample)
+        {
+            if (!(matrix is Matrix<RgbPixel> mat))
+                throw new ArgumentException();
+
+            this._ObjectDetector.Operator(mat, out IEnumerable<Tuple<double, Rectangle>> tuples);
+
+            foreach (var (confidence, rect) in tuples)
+                yield return new Location(rect, confidence);
+        }
+
+        /// <summary>
+        /// Releases all unmanaged resources.
+        /// </summary>
+        protected override void DisposeUnmanaged()
+        {
+            base.DisposeUnmanaged();
+
+            this._Scanner?.Dispose();
+            this._ObjectDetector?.Dispose();
+        }
+
+        #endregion
+
+    }
+
+}

src/FaceRecognitionDotNet/Extensions/SimpleGenderEstimator.cs

@@ -118,10 +118,20 @@ protected override IDictionary<Gender, float> RawPredictProbability(MatrixBase m
                 };
             }
 
-            #endregion
+        }
 
+        /// <summary>
+        /// Releases all unmanaged resources.
+        /// </summary>
+        protected override void DisposeUnmanaged()
+        {
+            base.DisposeUnmanaged();
+
+            this._Network?.Dispose();
         }
 
+        #endregion
+
     }
 
 }

src/FaceRecognitionDotNet/FaceRecognition.cs

@@ -35,6 +35,8 @@ public sealed class FaceRecognition : DisposableObject
 
         private FaceLandmarkDetector _CustomFaceLandmarkDetector;
 
+        private FaceDetector _CustomFaceDetector;
+
         private AgeEstimator _CustomAgeEstimator;
 
         private GenderEstimator _CustomGenderEstimator;
@@ -121,6 +123,15 @@ public GenderEstimator CustomGenderEstimator
             set => this._CustomGenderEstimator = value;
         }
 
+        /// <summary>
+        /// Gets or sets the custom face detector that user defined.
+        /// </summary>
+        public FaceDetector CustomFaceDetector
+        {
+            get => this._CustomFaceDetector;
+            set => this._CustomFaceDetector = value;
+        }
+
         /// <summary>
         /// Gets or sets the custom face landmark detector that user defined.
         /// </summary>
@@ -174,8 +185,11 @@ public IEnumerable<Location[]> BatchFaceLocations(IEnumerable<Image> images, int
             var image = imagesArray[0];
             for (var index = 0; index < rawDetectionsBatched.Length; index++)
             {
-                var faces = rawDetectionsBatched[index];
-                yield return faces.Select(rect => TrimBound(rect.Rect, image.Width, image.Height)).ToArray();
+                var faces = rawDetectionsBatched[index].ToArray();
+                var locations = faces.Select(rect => new Location(TrimBound(rect.Rect, image.Width, image.Height), rect.DetectionConfidence)).ToArray();
+                foreach (var face in faces)
+                    face.Dispose();
+                yield return locations;
             }
         }
 
@@ -511,24 +525,12 @@ public IEnumerable<Location> FaceLocations(Image image, int numberOfTimesToUpsam
             image.ThrowIfDisposed();
             this.ThrowIfDisposed();
 
-            switch (model)
+            foreach (var face in this.RawFaceLocations(image, numberOfTimesToUpsample, model))
             {
-                case Model.Cnn:
-                    foreach (var face in this.RawFaceLocations(image, numberOfTimesToUpsample, Model.Cnn))
-                    {
-                        var ret = TrimBound(face.Rect, image.Width, image.Height);
-                        face.Dispose();
-                        yield return ret;
-                    }
-                    break;
-                default:
-                    foreach (var face in this.RawFaceLocations(image, numberOfTimesToUpsample, model))
-                    {
-                        var ret = TrimBound(face.Rect, image.Width, image.Height);
-                        face.Dispose();
-                        yield return ret;
-                    }
-                    break;
+                var ret = TrimBound(face.Rect, image.Width, image.Height);
+                var confidence = face.DetectionConfidence;
+                face.Dispose();
+                yield return new Location(ret, confidence);
             }
         }
 
@@ -901,21 +903,30 @@ private IEnumerable<FullObjectDetection> RawFaceLandmarks(Image faceImage,
                                                                   PredictorModel predictorModel = PredictorModel.Large,
                                                                   Model model = Model.Hog)
         {
-            IEnumerable<MModRect> tmp;
+            IEnumerable<Location> rects;
 
             if (faceLocations == null)
-                tmp = this.RawFaceLocations(faceImage, 1, model);
+            {
+                var list = new List<Location>();
+                var tmp = this.RawFaceLocations(faceImage, 1, model);
+                foreach (var rect in tmp)
+                {
+                    list.Add(new Location(rect.Rect, rect.DetectionConfidence));
+                    rect.Dispose();
+                }
+
+                rects = list;
+            }
             else
-                tmp = faceLocations.Select(l => new MModRect { Rect = new Rectangle { Bottom = l.Bottom, Left = l.Left, Top = l.Top, Right = l.Right } });
+            {
+                rects = faceLocations;
+            }
 
             if (predictorModel == PredictorModel.Custom)
             {
-                foreach (var rect in tmp)
+                foreach (var rect in rects)
                 {
-                    var r = rect.Rect;
-                    var location = new Location(r.Left, r.Top, r.Right, r.Bottom);
-                    var ret = this._CustomFaceLandmarkDetector.Detect(faceImage, location);
-                    rect.Dispose();
+                    var ret = this._CustomFaceLandmarkDetector.Detect(faceImage, rect);
                     yield return ret;
                 }
             }
@@ -929,10 +940,9 @@ private IEnumerable<FullObjectDetection> RawFaceLandmarks(Image faceImage,
                         break;
                 }
 
-                foreach (var rect in tmp)
+                foreach (var rect in rects)
                 {
-                    var ret = posePredictor.Detect(faceImage.Matrix, rect);
-                    rect.Dispose();
+                    var ret = posePredictor.Detect(faceImage.Matrix, new Rectangle(rect.Left, rect.Top, rect.Right, rect.Bottom));
                     yield return ret;
                 }
             }
@@ -942,11 +952,19 @@ private IEnumerable<MModRect> RawFaceLocations(Image faceImage, int numberOfTime
         {
             switch (model)
             {
+                case Model.Custom:
+                    if (this._CustomFaceDetector == null)
+                        throw new NotSupportedException("The custom face detector is not ready.");
+                    return this._CustomFaceDetector.Detect(faceImage, numberOfTimesToUpsample).Select(rect => new MModRect
+                    {
+                        Rect = new Rectangle(rect.Left, rect.Top, rect.Right, rect.Bottom),
+                        DetectionConfidence = rect.Confidence
+                    });
                 case Model.Cnn:
                     return CnnFaceDetectionModelV1.Detect(this._CnnFaceDetector, faceImage, numberOfTimesToUpsample);
                 default:
                     var locations = SimpleObjectDetector.RunDetectorWithUpscale2(this._FaceDetector, faceImage, (uint)numberOfTimesToUpsample);
-                    return locations.Select(rectangle => new MModRect { Rect = rectangle });
+                    return locations.Select(tuple => new MModRect { Rect = tuple.Item1, DetectionConfidence = tuple.Item2 });
             }
         }