diff --git a/README.md b/README.md index 3017bbf..4cf4add 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,36 @@ # Practical Assignment 3 -**Dealine**: 02.04.2021 +**Deadline**: 02.04.2021 -Please put your name here: -**Name:** ....... +Please put your name(s) here: +**Name:** Alex Tretyakov and Adnan Abu Ramadan ## Problem 3.1 ### Ring Buffer (Points 30) 1. Fork the current repository -2. Study the new framework-code of +2. Study the new framework-code of - main.cpp - Camera Controller.h/cpp - Face.h/cpp - Marker.h/cpp 3. Check that the code is running correctly: it should show the video stream from the web-camera of your laptop. 4. Implement the _ring buffer_ from lecture by modifying the code in Camera Controller.cpp file (places marked with MODIFY CODE HERE tag) -5. In cases when producer overwrites a frame, which was not querried by consumer, report a _drop frame_ state by printing the corresponding message to console. Test it with increasing the delay time in `waitKey()`function in the main loop of consumer. +5. In cases when producer overwrites a frame, which was not queried by consumer, report a _drop frame_ state by printing the corresponding message to console. Test it with increasing the delay time in `waitKey()`function in the main loop of consumer. ## Problem 3.2 ### Face detection (Points 20) Incorporate the face detection solution you done in the [Assignment 1](https://github.com/Jacobs-University/visir-tracker-01) into the framework in the following way: -1. Your face detector should return a vector of pointer to the `CFace` classes with detected faces: `std::vector vpFaces` -2. Implment function `CMarker::markFaces()` and use it for drawing the faces to GUI +1. Your face detector should return a vector of pointer to the `CFace` classes with detected faces: `std::vector vpFaces` +2. Implement function `CMarker::markFaces()` and use it for drawing the faces to GUI 3. Perform face detection every 10th frame. ## Problem 3.3 ### Face tracking (Points 50) Incorporate the optical flow field solution you done in the [Assignment 2](https://github.com/Jacobs-University/visir-tracker-02) into the framework in the following way: -1. Your sparse optical flow shoud return a vector of 2-d points: `std::vector` -2. Implment function `CMarker::markVecOFF()` and use it for drawing the optical flow field (feel free to modify its arguments if needed). -3. Now detect the points only in the _face area_ , whic is described in `vpFaces` variable. +1. Your sparse optical flow should return a vector of 2-d points: `std::vector` +2. Implement function `CMarker::markVecOFF()` and use it for drawing the optical flow field (feel free to modify its arguments if needed). +3. Now detect the points only in the _face area_ , which is described in `vpFaces` variable. 4. Update the detected points every 10th frame (when face detection is used). 5. Track also the points added with mouse in the mouse callback function. -6. Inbetween 10 frames (when the face detection is not applied) track the detected faces using the optical flow field. Update the pisition of faces in `vpFaces` variable variable for every frame. +6. In between 10 frames (when the face detection is not applied) track the detected faces using the optical flow field. Update the position of faces in `vpFaces` variable variable for every frame. ## Problem 3.4 ### Graphical User Interface (GUI) (Bonus Points 20) @@ -42,5 +42,5 @@ Please submit the assignment by making a pull request. **Important** : Please make sure that - No _extra files_ are submitted (except those, which were mentioned in the assignment) - The changes were made _only_ in those files where you were asked to write your code -- The Continiouse Integration system (appVeyor) can build the submitted code -- The rendered images are also submitted in the folder "renders" +- The Continuous Integration system (appVeyor) can build the submitted code +- The rendered images are also submitted in the folder "renders" diff --git a/src/CameraController.cpp b/src/CameraController.cpp index 0634dd3..b62f202 100644 --- a/src/CameraController.cpp +++ b/src/CameraController.cpp @@ -1,83 +1,93 @@ -#include "CameraController.h" - -void CCameraCantroller::init() -{ - if (!m_camera.open(0)) { - printf("Can't find a camera\n"); - exit(1); - }; -} - -void CCameraCantroller::start() -{ - m_terminate_producer = false; - m_thr_producer = std::thread([&]() { - Mat img; - for (;;) { - // ------ MODIFY CODE HERE ------- - m_mtx_FrameBuffer.lock(); - m_camera >> m_vFrameBuffer[pointer_in % m_vFrameBuffer.size()]; - pointer_in++; - m_mtx_FrameBuffer.unlock(); - m_inFrameCounter++; - - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - - if (m_terminate_producer) break; - } - }); - - const double tickFrequency = getTickFrequency(); - m_terminate_counter = false; - m_thr_counter = std::thread([&, tickFrequency]() { - for (;;) { - static int64 ticks_old = getTickCount(); - int64 ticks_new = getTickCount(); - double sec = (ticks_new - ticks_old) / tickFrequency; - ticks_old = ticks_new; - - static int64 in_fps_old = m_inFrameCounter; - int64 in_fps_new = m_inFrameCounter; - int64 in_fps = in_fps_new - in_fps_old; - in_fps_old = in_fps_new; - - static int64 out_fps_old = m_outFrameCounter; - int64 out_fps_new = m_outFrameCounter; - int64 out_fps = out_fps_new - out_fps_old; - out_fps_old = out_fps_new; - - printf("FPS: in: %.2f | out: %.2f\n", in_fps / sec, out_fps / sec); - - std::this_thread::sleep_for(std::chrono::seconds(1)); - if (m_terminate_counter) break; - } - }); -} - -Mat CCameraCantroller::getFrame() -{ - Mat res; - if (pointer_out != pointer_in) { - - // ------ MODIFY CODE HERE ------- - m_mtx_FrameBuffer.lock(); - if (!m_vFrameBuffer.empty()) { - res = m_vFrameBuffer[pointer_out % m_vFrameBuffer.size()]; - pointer_out++; - } - m_mtx_FrameBuffer.unlock(); - } - m_outFrameCounter++; - return res; -} - -void CCameraCantroller::stop() -{ - m_terminate_producer = true; - m_thr_producer.join(); - - m_terminate_counter = true; - m_thr_counter.join(); -} - - +#include "CameraController.h" + +void CCameraCantroller::init() +{ + if (!m_camera.open(0)) { + printf("Can't find a camera\n"); + exit(1); + }; +} + +void CCameraCantroller::start() +{ + m_terminate_producer = false; + m_thr_producer = std::thread([&]() { + Mat img; + int size_of_buffer = m_vFrameBuffer.size(); + for (;;) { + // ------ MODIFY CODE HERE ------- + m_mtx_FrameBuffer.lock(); + m_camera >> m_vFrameBuffer[pointer_in]; + + if(m_FrameBufferFull){ + pointer_out = (pointer_out + 1) % size_of_buffer; + std::cout << "Frame Drop \n"; + } + pointer_in = (pointer_in + 1) % size_of_buffer; + m_FrameBufferFull = (pointer_in == pointer_out) && (m_inFrameCounter != 0) && ((m_inFrameCounter - m_outFrameCounter) == size_of_buffer); + + m_mtx_FrameBuffer.unlock(); + m_inFrameCounter++; + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (m_terminate_producer) break; + } + }); + + const double tickFrequency = getTickFrequency(); + m_terminate_counter = false; + m_thr_counter = std::thread([&, tickFrequency]() { + for (;;) { + static int64 ticks_old = getTickCount(); + int64 ticks_new = getTickCount(); + double sec = (ticks_new - ticks_old) / tickFrequency; + ticks_old = ticks_new; + + static int64 in_fps_old = m_inFrameCounter; + int64 in_fps_new = m_inFrameCounter; + int64 in_fps = in_fps_new - in_fps_old; + in_fps_old = in_fps_new; + + static int64 out_fps_old = m_outFrameCounter; + int64 out_fps_new = m_outFrameCounter; + int64 out_fps = out_fps_new - out_fps_old; + out_fps_old = out_fps_new; + + printf("FPS: in: %.2f | out: %.2f\n", in_fps / sec, out_fps / sec); + + std::this_thread::sleep_for(std::chrono::seconds(1)); + if (m_terminate_counter) break; + } + }); +} + +Mat CCameraCantroller::getFrame() +{ + Mat res; + int bufferSize = m_vFrameBuffer.size(); + if (pointer_out != pointer_in) { + + // ------ MODIFY CODE HERE ------- + m_mtx_FrameBuffer.lock(); + if (!m_vFrameBuffer.empty()) { + res = m_vFrameBuffer[pointer_out]; + m_FrameBufferFull = false; + pointer_out = (pointer_out + 1) % bufferSize; + } + m_mtx_FrameBuffer.unlock(); + } + m_outFrameCounter++; + return res; +} + +void CCameraCantroller::stop() +{ + m_terminate_producer = true; + m_thr_producer.join(); + + m_terminate_counter = true; + m_thr_counter.join(); +} + + diff --git a/src/CameraController.h b/src/CameraController.h index 2c6061f..8bc17b2 100644 --- a/src/CameraController.h +++ b/src/CameraController.h @@ -4,32 +4,33 @@ class CCameraCantroller { public: - CCameraCantroller(size_t buf_len) : m_vFrameBuffer(buf_len) {} - ~CCameraCantroller(void) = default; + CCameraCantroller(size_t buf_len) : m_vFrameBuffer(buf_len) {} + ~CCameraCantroller(void) = default; - void init(void); - void start(void); - Mat getFrame(void); - void stop(void); + void init(void); + void start(void); + Mat getFrame(void); + void stop(void); private: - VideoCapture m_camera; - std::deque m_vFrameBuffer; - std::mutex m_mtx_FrameBuffer; - - size_t m_inFrameCounter = 0; - size_t m_outFrameCounter = 0; - size_t pointer_in = 0; - size_t pointer_out = 0; - - // Producer - std::thread m_thr_producer; - bool m_terminate_producer; - - // Counter - std::thread m_thr_counter; - bool m_terminate_counter; + VideoCapture m_camera; + std::deque m_vFrameBuffer; + std::mutex m_mtx_FrameBuffer; + + size_t m_inFrameCounter = 0; + size_t m_outFrameCounter = 0; + size_t pointer_in = 0; + size_t pointer_out = 0; + bool m_FrameBufferFull = 0; + + // Producer + std::thread m_thr_producer; + bool m_terminate_producer; + + // Counter + std::thread m_thr_counter; + bool m_terminate_counter; }; diff --git a/src/Face.cpp b/src/Face.cpp index c436a8e..27021d7 100644 --- a/src/Face.cpp +++ b/src/Face.cpp @@ -1,2 +1 @@ #include "Face.h" - diff --git a/src/Face.h b/src/Face.h index 179bc5a..612b3f4 100644 --- a/src/Face.h +++ b/src/Face.h @@ -1,33 +1,33 @@ -#pragma once - -#include "types.h" - -class CFace { -public: - CFace(void) : CFace(Rect()) {} - CFace(const Rect& area, int id = -1, const std::string& text = "unknown") - : m_area(area) - , m_id(id) - , m_text(text) - {} - ~CFace(void) = default; - - // Setters - void setID(int id) { m_id = id; } - void setText(const std::string& text) { m_text = text; } - void setArea(const Rect& area) { m_area = area; } - - // Getters - int getID(void) const { return m_id; } - std::string getText(void) const { return m_text; } - Rect getArea(void) const { return m_area; } - - -private: - int m_id; - std::string m_text; - Rect m_area; +#pragma once + +#include "types.h" + +class CFace { +public: + CFace(void) : CFace(Rect()) {} + CFace(const Rect& area, int id = -1, const std::string& text = "unknown") + : m_area(area) + , m_id(id) + , m_text(text) + {} + ~CFace(void) = default; + + // Setters + void setID(int id) { m_id = id; } + void setText(const std::string& text) { m_text = text; } + void setArea(const Rect& area) { m_area = area; } + + // Getters + int getID(void) const { return m_id; } + std::string getText(void) const { return m_text; } + Rect getArea(void) const { return m_area; } + + +private: + int m_id; + std::string m_text; + Rect m_area; }; - -// Pointer -using ptr_face_t = std::shared_ptr; + +// Pointer +using ptr_face_t = std::shared_ptr; diff --git a/src/Marker.cpp b/src/Marker.cpp index 36955ba..414a485 100644 --- a/src/Marker.cpp +++ b/src/Marker.cpp @@ -1,39 +1,59 @@ -#include "Marker.h" - -void CMarker::markFaces(Mat& img, const std::vector& vFaces) -{ - for (auto face : vFaces) { - // ------ PUT YOUR CODE HERE ------- - // 1. Draw all faces using face->getArea(); - // 2. Print the text using face->getText(); - // 3. Print face id using face->getId(); - } -} - -void CMarker::markPoints(Mat& img, const std::vector& vPoints, Scalar color) -{ - for (auto& point : vPoints) - circle(img, point, 3, color, 2); -} - -void CMarker::markVecOFF(Mat& img, const Mat& hFlow, const Mat& vFlow) -{ - // ------ PUT YOUR CODE HERE ------- -} - -void CMarker::markGUI(Mat& img) -{ - // ------ PUT YOUR CODE HERE ------- - // Implement yout ouw GUI - // Show fps - for (int y = 0; y < img.rows; y++) { - Vec3b* ptr = img.ptr(y); - for (int x = 0; x < img.cols; x++) { - float k = static_cast(x) / img.cols; - ptr[x] = Vec3b(k * 255, 0, 255 - k * 255); - } - } - circle(img, Point(img.cols / 2, img.rows / 2), 50, CV_RGB(100, 255, 100), 5); - GaussianBlur(img, img, Size(17, 17), 50); - putText(img, "HCI", Point(100, 100), FONT_HERSHEY_SIMPLEX, 2, CV_RGB(255, 255, 255), 5); -} +#include "Marker.h" + +using namespace cv; + +void CMarker::markFaces(Mat& img, const std::vector& vFaces) +{ + for (auto face : vFaces){ + // ------ PUT YOUR CODE HERE ------- + // 1. Draw all faces using face->getArea(); + // 2. Print the text using face->getText(); + // 3. Print face id using face->getId(); + Point center(face->getArea().x + face->getArea().width/2, face->getArea().y + face->getArea().height/2); + ellipse(img, center, Size(face->getArea().width/2, face->getArea().height/2 ), 0, 0, 360, Scalar(255, 0, 255), 4); + } +} + +void CMarker::markPoints(Mat& img, const std::vector& vPoints, Scalar color) +{ + for (auto& point : vPoints) + circle(img, point, 3, color, 2); +} + +void CMarker::markVecOFF(Mat& img, Mat& mask, const std::vector>& flowPoints) +{ + // ------ PUT YOUR CODE HERE ------- + std::vector colors; + RNG rand; + + for(int i = 0; i < 100; i++){ + int r = rand.uniform(0, 256); + int g = rand.uniform(0, 256); + int b = rand.uniform(0, 256); + colors.push_back(Scalar(r,g,b)); + } + int j = 0; + for(const auto &i : flowPoints){ + line(mask, std::get<1>(i), std::get<0>(i), colors[j], 2); + circle(img, std::get<1>(i), 5, colors[j], -2); + j++; + } + +} + +void CMarker::markGUI(Mat& img) +{ + // ------ PUT YOUR CODE HERE ------- + // Implement yout ouw GUI + // Show fps + for (int y = 0; y < img.rows; y++) { + Vec3b* ptr = img.ptr(y); + for (int x = 0; x < img.cols; x++) { + float k = static_cast(x) / img.cols; + ptr[x] = Vec3b(k * 255, 0, 255 - k * 255); + } + } + circle(img, Point(img.cols / 2, img.rows / 2), 50, CV_RGB(100, 255, 100), 5); + GaussianBlur(img, img, Size(17, 17), 50); + putText(img, "HCI", Point(100, 100), FONT_HERSHEY_SIMPLEX, 2, CV_RGB(255, 255, 255), 5); +} diff --git a/src/Marker.h b/src/Marker.h index 4584ad6..3453966 100644 --- a/src/Marker.h +++ b/src/Marker.h @@ -1,20 +1,19 @@ -#pragma once - -#include "types.h" -#include "Face.h" - -class CMarker { -public: - // draws faces into the image - static void markFaces(Mat& img, const std::vector& vFaces); - - // draws the points (e.g. from GoodFeaturesToTrack) into the image - static void markPoints(Mat& img, const std::vector& vPoints, Scalar color = CV_RGB(255, 165, 0)); - - // draws vectors of the vector Optical Flow Field () into the image - static void markVecOFF(Mat& img, const Mat& hFlow, const Mat& vFlow); - - // draws GUI - static void markGUI(Mat& img); -}; - +#pragma once + +#include "types.h" +#include "Face.h" + +class CMarker { +public: + // draws faces into the image + static void markFaces(Mat& img, const std::vector& vFaces); + + // draws the points (e.g. from GoodFeaturesToTrack) into the image + static void markPoints(Mat& img, const std::vector& vPoints, Scalar color = CV_RGB(255, 165, 0)); + + // draws vectors of the vector Optical Flow Field () into the image + static void markVecOFF(Mat& img, Mat& mask, const std::vector>& flowPoints); + + // draws GUI + static void markGUI(Mat& img); +}; diff --git a/src/main.cpp b/src/main.cpp index 8a935ca..f850ef8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2,64 +2,171 @@ #include "CameraController.h" #include "Face.h" #include "Marker.h" +#include +#include +#include +using namespace std; +using namespace cv; +std::vector detectFaces(Mat img); +std::vector featureExtraction(Mat img); +std::vector faceFeatureExtraction(Mat img, std::vector vpFaces); +std::vector> calculateOpticalFlow(Mat old_gray, Mat new_img, std::vector vPoint); int main() { - - - std::vector vPoints; - std::vector vpFaceses; - - namedWindow("Camera"); - - // mouse callback function which fills vPoints with coordinates of mouse clicks - setMouseCallback("Camera", [] (int event, int x, int y, int flags, void* userdata) { - if (userdata && event == EVENT_LBUTTONDOWN) { - std::vector *pvPoints = (std::vector *) userdata; - pvPoints->push_back(Point2f(x, y)); - } - }, (void*) &vPoints); - - CCameraCantroller controller(16); - - controller.init(); - controller.start(); - - // Main loop - Mat img, mask; - float attenuation = 0.5f; - for(;;) { - img = controller.getFrame(); - - if (!img.empty()) { - if (mask.empty()) mask = Mat(img.size(), img.type()); - mask.setTo(0); - - // ------ PUT YOUR CODE HERE ------- - // vpFaceses = detect faces(); - CMarker::markFaces(mask, vpFaceses); - - // ------ PUT YOUR CODE HERE ------- - // vPoints = good features to track () - CMarker::markPoints(mask, vPoints); - // { hFlow, vFlow } = calculate optical flow (vPoints) - // CMarker::markVecOFF(mask, hFlow, vFlow); - - CMarker::markGUI(mask); - - - add(img, attenuation * mask, img); - - imshow("Camera", img); - } - int key = waitKey(5); - if (key == 27 || key == 'q') break; - if (key == 'a') attenuation *= 1.1f; - if (key == 'z') attenuation *= 0.9f; - } - - controller.stop(); - return 0; + + + std::vector vPoints, vFacePoints; + std::vector vpFaces; + std::vector> opticalFlowPoints, faceOpticalFlowPoints; + + namedWindow("Camera"); + + // mouse callback function which fills vPoints with coordinates of mouse clicks + setMouseCallback("Camera", [] (int event, int x, int y, int flags, void* userdata) { + if (userdata && event == EVENT_LBUTTONDOWN) { + std::vector *pvPoints = (std::vector *) userdata; + pvPoints->push_back(Point2f(x, y)); + } + }, (void*) &vPoints); + + CCameraCantroller controller(16); + + controller.init(); + controller.start(); + + // Main loop + Mat img, new_img; + Mat mask, old_gray; + float attenuation = 0.5f; + int fcounter = 0; + for(;;) { + img = controller.getFrame(); + + if (!img.empty()) { + if (mask.empty()) mask = Mat(img.size(), img.type()); + mask.setTo(0); + + if (fcounter % 10 == 0){ + vpFaces = detectFaces(img); + vPoints = featureExtraction(img); + vFacePoints = faceFeatureExtraction(img, vpFaces); + } else{ + new_img = controller.getFrame(); + opticalFlowPoints = calculateOpticalFlow(img, new_img, vPoints); + faceOpticalFlowPoints = calculateOpticalFlow(img, new_img, vFacePoints); + } + + CMarker::markFaces(img, vpFaces); + CMarker::markVecOFF(img, mask, opticalFlowPoints); + CMarker::markVecOFF(img, mask, faceOpticalFlowPoints); + CMarker::markGUI(mask); + CMarker::markPoints(img, vPoints); + CMarker::markPoints(img, vFacePoints); + + add(img, attenuation * mask, img); + + imshow("Camera", img); + } + int key = waitKey(5); + if (key == 27 || key == 'q') break; + if (key == 'a') attenuation *= 1.1f; + if (key == 'z') attenuation *= 0.9f; + fcounter++; + } + + controller.stop(); + return 0; +} + + +std::vector detectFaces(Mat img) +{ + String face_cascade_name; + CascadeClassifier face_cascade; + Mat frame_gray; + std::vector vpFaces; + + face_cascade_name = samples::findFile("../opencv4/haarcascades/haarcascade_frontalface_alt.xml"); + + face_cascade.load(face_cascade_name); + cvtColor(img, frame_gray, COLOR_BGR2GRAY); + equalizeHist(frame_gray, frame_gray); + + std::vector faces; + face_cascade.detectMultiScale(frame_gray, faces); + + for (auto face : faces) + { + CFace* vpFace = new CFace(face); + vpFaces.emplace_back(vpFace); + } + + return vpFaces; +} + + + +std::vector featureExtraction(Mat img) +{ + Mat img_gray; + std::vector corners; + cvtColor(img, img_gray, COLOR_BGR2GRAY); + goodFeaturesToTrack(img_gray, corners, 100, 0.01, 7, Mat(), 7, false, 0.04); + return corners; +} + +std::vector faceFeatureExtraction(Mat img, std::vector vpFaces) +{ + Mat img_gray; + std::vector vPoints; + cvtColor(img, img_gray, COLOR_BGR2GRAY); + for(auto face: vpFaces) + { + std::vector corners; + Mat mask = Mat::zeros(img.size(), CV_8UC1); + Mat roi(mask, face->getArea()); + roi = Scalar(255, 255, 255); + goodFeaturesToTrack(img_gray, corners, 23, 0.01, 10, mask, 3, false, 0.04); + + for(auto corner: corners) + { + vPoints.push_back(corner); + } + + } + return vPoints; +} + +std::vector> calculateOpticalFlow(Mat img, Mat new_img, std::vector vPoints) +{ + Mat old_gray; + cvtColor(img, old_gray, COLOR_BGR2GRAY); + std::vector vPoints_new, vPoints_temp; + + Mat img_gray; + cvtColor(new_img, img_gray, COLOR_BGR2GRAY); + + std::vector status; + std::vector err; + TermCriteria criteria = TermCriteria((TermCriteria::COUNT) + (TermCriteria::EPS), 10, 0.03); + + if(vPoints.size() != 0) //handle the case when vPoints is empty so the code doesn't break + { + calcOpticalFlowPyrLK(old_gray, img_gray, vPoints, vPoints_new, status, err, Size(15,15), 2, criteria); + } + std::vector> good_points; + + + for(uint i = 0; i < vPoints.size(); i++) + { + if(status[i] == 1) + { + std::tuple value = std::make_tuple(vPoints[i], vPoints_new[i]); + good_points.push_back(value); + vPoints_temp.push_back(vPoints_new[i]); + } + } + return good_points; } -