The detectEllipses() will causing memory issues and crash. #3704
Description
System information (version)
- OpenCV => 4.9.0
- Operating System / Platform => Ubuntu 22.04.4 LTS x86_64
- Compiler => gcc version 11.4.0 (Ubuntu 11.4.0-1ubuntu1~22.04)
Detailed description
My task : To detect the ball in the video from a usb-camera.
When I try to use detectEllipses() to complete my task , this function always cause my program to crash. The problem is always related to double free a invalid pointer and so on, although I don't know why.
This always happened when it deal with the video stream it get from a usb-camera for about a few minutes, especially there is no target ball in the video.
Debug information:
In function void EdgeDrawingImpl::detectEllipses(OutputArray ellipses)
delete edarcs1; \\ here
it crashed on when it executes struct EDArcs destructor function
struct EDArcs {
MyArc *arcs;
int noArcs;
public:
EDArcs(int size = 10000) {
arcs = new MyArc[size];
noArcs = 0;
}
~EDArcs() {
delete[] arcs; \\ here
}
};
Sometimes it crash here:
In function void EdgeDrawingImpl::detectEllipses(OutputArray ellipses):
//----------------------------- VALIDATE CIRCLES --------------------------
noCircles2 = 0;
circles2 = new Circle[maxNoOfCircles];
GaussianBlur(srcImage, smoothImage, Size(0,0), 0.50); // calculate kernel from sigma; \ \ here
ValidateCircles(params.NFAValidation);
Sometimes it crash here:
// ------------------------------- DETECT ARCS ---------------------------------
....
edarcs1 = new EDArcs(maxNoOfCircles);
DetectArcs(); // Detect all arcs // here
I used the Valgrind to detect the error:
==285305== Invalid write of size 8
==285305== at 0x136137: cv::ximgproc::EdgeDrawingImpl::addArc(MyArc*, int&, double, double, double, double, double, double, int, int, int, int, int, int, double*, double*, int, double) (edge_drawing.cpp:4978)
==285305== by 0x12ADCA: cv::ximgproc::EdgeDrawingImpl::DetectArcs() (edge_drawing.cpp:2951)
==285305== by 0x1286B3: cv::ximgproc::EdgeDrawingImpl::detectEllipses(cv::_OutputArray const&) (edge_drawing.cpp:2599)
==285305== by 0x1108EB: FindBallServer::find_ball(int) (main.cpp:74)
==285305== by 0x10E739: main (main.cpp:336)
==285305== Address 0x13823d38 is 8 bytes after a block of size 336 alloc'd
==285305== at 0x484A703: operator new[](unsigned long) (vg_replace_malloc.c:725)
==285305== by 0x13AB7F: EDArcs::EDArcs(int) (edge_drawing_common.hpp:535)
==285305== by 0x128696: cv::ximgproc::EdgeDrawingImpl::detectEllipses(cv::_OutputArray const&) (edge_drawing.cpp:2598)
==285305== by 0x1108EB: FindBallServer::find_ball(int) (main.cpp:74)
==285305== by 0x10E739: main (main.cpp:336)
==285305==
==285305== Invalid write of size 8
==285305== at 0x136169: cv::ximgproc::EdgeDrawingImpl::addArc(MyArc*, int&, double, double, double, double, double, double, int, int, int, int, int, int, double*, double*, int, double) (edge_drawing.cpp:4979)
==285305== by 0x12ADCA: cv::ximgproc::EdgeDrawingImpl::DetectArcs() (edge_drawing.cpp:2951)
==285305== by 0x1286B3: cv::ximgproc::EdgeDrawingImpl::detectEllipses(cv::_OutputArray const&) (edge_drawing.cpp:2599)
==285305== by 0x1108EB: FindBallServer::find_ball(int) (main.cpp:74)
==285305== by 0x10E739: main (main.cpp:336)
==285305== Address 0x13823d40 is 16 bytes after a block of size 336 alloc'd
==285305== at 0x484A703: operator new[](unsigned long) (vg_replace_malloc.c:725)
==285305== by 0x13AB7F: EDArcs::EDArcs(int) (edge_drawing_common.hpp:535)
==285305== by 0x128696: cv::ximgproc::EdgeDrawingImpl::detectEllipses(cv::_OutputArray const&) (edge_drawing.cpp:2598)
==285305== by 0x1108EB: FindBallServer::find_ball(int) (main.cpp:74)
==285305== by 0x10E739: main (main.cpp:336)
==285305==
==285305== Invalid write of size 8
==285305== at 0x13619B: cv::ximgproc::EdgeDrawingImpl::addArc(MyArc*, int&, double, double, double, double, double, double, int, int, int, int, int, int, double*, double*, int, double) (edge_drawing.cpp:4980)
==285305== by 0x12ADCA: cv::ximgproc::EdgeDrawingImpl::DetectArcs() (edge_drawing.cpp:2951)
==285305== by 0x1286B3: cv::ximgproc::EdgeDrawingImpl::detectEllipses(cv::_OutputArray const&) (edge_drawing.cpp:2599)
==285305== by 0x1108EB: FindBallServer::find_ball(int) (main.cpp:74)
==285305== by 0x10E739: main (main.cpp:336)
==285305== Address 0x13823d48 is 24 bytes after a block of size 336 in arena "client"
==285305==
Steps to reproduce
Here is my whole project code:
opencv_cpptest.zip
#include <iostream>
#include <memory>
#include <opencv2/core/mat.hpp>
#include <opencv2/highgui.hpp>
#include <vector>
#include <opencv2/opencv.hpp>
#include "opencv2/ximgproc.hpp"
// #define ENABLE_THRESHOLD
enum BallType { RED = 0, PURPLE = 1, BLUE = 2 };
cv::Scalar lower_purple = cv::Scalar(181, 132, 0);
cv::Scalar upper_purple = cv::Scalar(202, 255, 255);
cv::Scalar lower_red = cv::Scalar(117, 143, 0);
cv::Scalar upper_red = cv::Scalar(133, 255, 255);
cv::Scalar lower_blue = cv::Scalar(0, 159, 0);
cv::Scalar upper_blue = cv::Scalar(16, 255, 255);
class FindBallServer
{
public:
std::shared_ptr<cv::VideoCapture> cap;
FindBallServer():lutEqual(256), lutZero(256, 0), lutRaisen(256), lutSRaisen(256, 256, CV_8UC3), flag(0),count(0)
{
std::cout << "FindBallServer created" << std::endl;
cap = std::make_shared<cv::VideoCapture>();
ed = cv::ximgproc::createEdgeDrawing();
EDParams = std::make_shared<cv::ximgproc::EdgeDrawing::Params>();
ball_result = cv::Vec3d(0, 0, 0);
ball_last_result = cv::Vec3d(0, 0, 0);
}
cv::Vec3d find_ball(int type)
{
cv::Mat color_image = usbcamera_getImage();
if (color_image.empty()) {
std::cerr << "Failed to capture frame." << std::endl;
return cv::Vec3d(0, 0, 0);
}
cv::Mat color_image_copy = color_image.clone();
cv::Mat hsv;
cv::Mat blendSRaisen;
cv::Mat mask;
cv::Mat img;
cv::Mat gray;
cv::Mat thre;
cv::Mat edge_image;
std::vector<std::vector<cv::Point>> contours;
cv::cvtColor(color_image, hsv, cv::COLOR_BGR2HSV);
cv::LUT(hsv, lutRaisen ,blendSRaisen);
cv::inRange(blendSRaisen, this->lower[type], this->upper[type], mask);
cv::bitwise_and(color_image, color_image, img, mask);
cv::medianBlur(img, img, 7);
cv::cvtColor(img, gray, cv::COLOR_BGR2GRAY);
cv::threshold(gray, thre, 0, 255, cv::THRESH_BINARY);
cv::morphologyEx(thre, thre, cv::MORPH_CLOSE, cv::getStructuringElement(cv::MORPH_RECT, cv::Size(5, 5)), cv::Point(-1, -1), 3);
cv::findContours(thre, contours, cv::RETR_EXTERNAL, cv::CHAIN_APPROX_SIMPLE);
for (auto contour : contours) {
cv::fillPoly(thre, contour, cv::Scalar(255, 255, 255));
}
//边缘检测
std::vector<cv::Vec6d> ellipses;
std::vector<cv::Vec3d> filter_ellipses;
ed->detectEdges(thre);
ed->detectEllipses(ellipses);
ed->getEdgeImage(edge_image);
/*
// create output array
std::vector<cv::Vec6f> ells;
cv::ximgproc::findEllipses(edge_image, ells, 0.4f, 0.7f, 0.02f);
// print output
for (unsigned i = 0; i < ells.size(); i++) {
cv::Vec6f ell = ells[i];
std::cout << ell << std::endl;
cv::Scalar color(0, 0, 255);
// draw ellipse on image
ellipse(
color_image_copy,
cv::Point(cvRound(ell[0]), cvRound(ell[1])),
cv::Size(cvRound(ell[2]), cvRound(ell[3])),
ell[5] * 180 / CV_PI, 0.0, 360.0, color, 3
);
}
*/
cv::imshow("color_image", color_image_copy);
cv::imshow("hsv", hsv);
cv::imshow("mask", mask);
cv::imshow("color", blendSRaisen);
cv::imshow("thre", thre);
cv::imshow("edge", edge_image);
cv::imshow("blendSRaisen", blendSRaisen);
for (size_t i=0; i<ellipses.size(); i++)
{
cv::Point center((int)ellipses[i][0], (int)ellipses[i][1]);
cv::Size axes((int)ellipses[i][2] + (int)ellipses[i][3], (int)ellipses[i][2] + (int)ellipses[i][4]);
//double angle(ellipses[i][5]);
//cv::Scalar color = ellipses[i][2] == 0 ? cv::Scalar(255, 255, 0) : cv::Scalar(0, 255, 0);
filter_ellipses.push_back(cv::Vec3d(center.x, center.y, axes.width/2.0 + axes.height/2.0));
}
// 使用 lambda 表达式进行排序 ??????????
std::sort(filter_ellipses.begin(), filter_ellipses.end(),
[](const cv::Vec3d& a, const cv::Vec3d& b) {
return a[0] > b[0]; // 按 center.x 降序排序
});
if (ellipses.size() > 0)
{
cv::Mat circle_region;
cv::Vec3d max_ellipse = filter_ellipses[0];
cv::Mat mask_ = cv::Mat::zeros(thre.size(), CV_8UC1);
cv::circle(mask_, cv::Point(int(max_ellipse[0]), int(max_ellipse[1])), int(max_ellipse[2]), cv::Scalar(255), -1);
cv::bitwise_and(thre, mask_, circle_region);
float white_pixels = cv::countNonZero(circle_region);
float total_pixels = cv::countNonZero(mask_);
if (white_pixels / total_pixels > 0.6)
{
this->ball_result = max_ellipse;
}
else {
filter_ellipses.erase(filter_ellipses.begin() + 0);
if(filter_ellipses.size()> 0)
{
this->ball_result = filter_ellipses[0];
}
}
if( filter_ellipses.size() == 0 && ball_result[0] > 161 && ball_result[0] < 468 && ball_result[1] > 333 )
{
flag = 1;
}
{
this->ball_result = filter_ellipses[0];
}
}else if (ellipses.size() == 0 && flag == 0)
{
std::vector<cv::Vec3d> filter_ellipses_ ;
std::vector<std::vector<cv::Point>> contours_;
cv::findContours(thre, contours_, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE);
for (auto contour : contours_) {
if(contour.size() > 50 && cv::contourArea(contour) >100)
{
cv::Point2f center_;
float radius_;
cv::minEnclosingCircle(contour, center_, radius_);
filter_ellipses_.push_back(cv::Vec3d(center_.x, center_.y, radius_));
}
}
// 使用 lambda 表达式进行排序 ??????????
std::sort(filter_ellipses_.begin(), filter_ellipses_.end(),
[](const cv::Vec3d& a, const cv::Vec3d& b) {
return a[0] > b[0]; // 按 center.x 降序排序
});
if(filter_ellipses_.size() > 0)
{
this->ball_result = filter_ellipses_[0];
}
}
if (ball_result == ball_last_result && ball_result != cv::Vec3d(0, 0, 0) && ball_result[0] > 161 && ball_result[0] < 468 && ball_result[1] > 333)
{
count +=1;
}
if (count > 100)
{
flag = 0;
count = 0;
}
ball_last_result = ball_result;
//std::cout << "Pose:" << ball_result[0]<<", " << ball_result[1] << ", " << ball_result[2] << std::endl;
//std::cout << "idle" << std::endl;
return ball_result;
}
bool usbcamera_init()
{
this->cap->open(0);
if (!cap->isOpened()) {
std::cerr << "Failed to open camera." << std::endl;
return false;
}
return true;
}
cv::Mat usbcamera_getImage()
{
cv::Mat frame;
this->cap->read(frame);
if (frame.empty()) {
std::cerr << "Failed to capture frame." << std::endl;
}
return frame;
}
void init()
{
lut_init();
EDParams->MinPathLength = 60;
EDParams->MinLineLength = 20;
EDParams->PFmode = false;
EDParams->NFAValidation = true;
ed->setParams(*EDParams);
}
void lut_init()
{
// Fill lutEqual
for (int i = 0; i < 256; ++i) {
lutEqual[i] = static_cast<uint8_t>(i);
}
// Fill lutRaisen
for (int i = 0; i < 256; ++i) {
lutRaisen[i] = static_cast<uint8_t>(102 + 0.6 * i);
}
// Fill lutSRaisen
for (int i = 0; i < 256; ++i) {
for (int j = 0; j < 256; ++j) {
lutSRaisen.at<cv::Vec3b>(i, j) = cv::Vec3b(lutEqual[i], lutRaisen[j], lutEqual[i]);
}
}
}
#ifdef ENABLE_THRESHOLD
std::vector<cv::Scalar> lower = {lower_red, lower_purple, lower_blue};
std::vector<cv::Scalar> upper = {upper_red, upper_purple, upper_blue};
#endif // ENABLE_THRESHOLD
private:
cv::Ptr<cv::ximgproc::EdgeDrawing> ed;
std::shared_ptr<cv::ximgproc::EdgeDrawing::Params> EDParams;
#ifndef ENABLE_THRESHOLD
std::vector<cv::Scalar> lower = {lower_red, lower_purple, lower_blue};
std::vector<cv::Scalar> upper = {upper_red, upper_purple, upper_blue};
#endif // ENABLE_THRESHOLD
std::vector<uint8_t> lutEqual;
std::vector<uint8_t> lutZero;
std::vector<uint8_t> lutRaisen;
cv::Mat lutSRaisen;
cv::Vec3d ball_result;
cv::Vec3d ball_last_result;
int flag;
int count;
};
#ifdef ENABLE_THRESHOLD
// Trackbar callback function
void onTrackbar(int, void*) {
// Nothing to do here, just for trackbar functionality
}
#endif // ENABLE_THRESHOLD
int main()
{
#ifdef ENABLE_THRESHOLD
// Global variables to hold the threshold values
int lowH = 184, lowS = 118, lowV = 150;
int highH = 194, highS = 218, highV = 255;
cv::namedWindow("Threshold Adjustments", cv::WINDOW_NORMAL);
// Create trackbars
cv::createTrackbar("Low H", "Threshold Adjustments", &lowH, 255, onTrackbar);
cv::createTrackbar("High H", "Threshold Adjustments", &highH, 255, onTrackbar);
cv::createTrackbar("Low S", "Threshold Adjustments", &lowS, 255, onTrackbar);
cv::createTrackbar("High S", "Threshold Adjustments", &highS, 255, onTrackbar);
cv::createTrackbar("Low V", "Threshold Adjustments", &lowV, 255, onTrackbar);
cv::createTrackbar("High V", "Threshold Adjustments", &highV, 255, onTrackbar);
#endif // ENABLE_THRESHOLD
cv::namedWindow("color_image", cv::WINDOW_AUTOSIZE);
cv::namedWindow("color", cv::WINDOW_AUTOSIZE);
cv::namedWindow("hsv", cv::WINDOW_AUTOSIZE);
cv::namedWindow("blendSRaisen", cv::WINDOW_AUTOSIZE);
cv::namedWindow("thre", cv::WINDOW_AUTOSIZE);
cv::namedWindow("edge", cv::WINDOW_AUTOSIZE);
cv::namedWindow("mask",cv::WINDOW_AUTOSIZE);
//std::cout << (cv::Vec3d(0,0,0) == cv::Vec3d(0,0,0)) << std::endl;
std::cout << "Hello, OpenCV!" << std::endl;
std::shared_ptr<FindBallServer> instance = std::make_shared<FindBallServer>();
std::cout << instance->usbcamera_init() << std::endl;
instance->init();
for(int i = 0; i < 20; i++)
instance->usbcamera_getImage();
cv::Vec3d ref_ = instance->find_ball(1);
//while (ref_ == cv::Vec3d(0, 0, 0))
//{
// ref_ = instance->find_ball(1);
//}
cv::Vec2f last_measurement;
cv::Vec2f current_measurement ;
cv::Vec4f last_prediction ;
cv::Vec4f current_prediction ;
std::shared_ptr<cv::KalmanFilter> Kalman = std::make_shared<cv::KalmanFilter>(4, 2);
Kalman->measurementMatrix = (cv::Mat_<float>(2, 4) << 1, 0, 0, 0, 0, 1, 0, 0);
Kalman->transitionMatrix = (cv::Mat_<float>(4, 4) << 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1);
Kalman->processNoiseCov = (cv::Mat_<float>(4, 4) << 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1);
while (true) {
#ifdef ENABLE_THRESHOLD
cv::Mat thresholdImage(100, 600, CV_8UC3, cv::Scalar(0, 0, 0));
// Draw rectangles for each trackbar
cv::rectangle(thresholdImage, cv::Rect(0, 0, 100, 100), cv::Scalar(lowH, lowS, lowV), -1);
cv::rectangle(thresholdImage, cv::Rect(100, 0, 100, 100), cv::Scalar(highH, highS, highV), -1);
cv::rectangle(thresholdImage, cv::Rect(200, 0, 100, 100), cv::Scalar(lowH, 255, 255), -1);
cv::rectangle(thresholdImage, cv::Rect(300, 0, 100, 100), cv::Scalar(highH, 255, 255), -1);
cv::rectangle(thresholdImage, cv::Rect(400, 0, 100, 100), cv::Scalar(255, lowS, 255), -1);
cv::rectangle(thresholdImage, cv::Rect(500, 0, 100, 100), cv::Scalar(255, highS, 255), -1);
cv::rectangle(thresholdImage, cv::Rect(600, 0, 100, 100), cv::Scalar(255, 255, lowV), -1);
cv::rectangle(thresholdImage, cv::Rect(700, 0, 100, 100), cv::Scalar(255, 255, highV), -1);
instance->lower[1] = cv::Scalar(lowH, lowS, lowV);
instance->upper[1] = cv::Scalar(highH, highS, highV);
// Show the image
cv::imshow("Threshold Adjustments", thresholdImage);
#endif // ENABLE_THRESHOLD
auto ref = instance->find_ball(1);
last_prediction = current_prediction;
last_measurement = current_measurement;
if (ref == cv::Vec3d(0, 0, 0))
current_measurement = last_measurement;
else
current_measurement = cv::Vec2f(ref[0], ref[1]);
Kalman->correct(cv::Mat(current_measurement));
current_prediction = Kalman->predict();
//std::cout << "current_prediction: " << current_prediction << std::endl;
auto key = cv::waitKey(1);
if(key == 27)
break;
}
instance->cap->release();
cv::destroyAllWindows();
return 0;
}
Issue submission checklist
- I report the issue, it's not a question
- I checked the problem with documentation, FAQ, open issues,
forum.opencv.org, Stack Overflow, etc and have not found any solution - I updated to the latest OpenCV version and the issue is still there
- There is reproducer code and related data files: videos, images, onnx, etc