Robust card detection/persecutive correction OpenCV

c++, edge-detection, image-processing, opencv

Solution

A more general approach would definitely be something like Rutger Nijlunsing suggested in his answer. However, in your case, at least for the provided sample images, a very simple approach like morphological opening followed by thresholding, contour processing and convexhull would yield the result you want. Use a scaled down version of the images for processing so that you don't have to use a large kernel for morphological operations. Below are the images processed this way.

    pyrDown(large, rgb0);
    pyrDown(rgb0, rgb0);
    pyrDown(rgb0, rgb0);

    Mat small;
    cvtColor(rgb0, small, CV_BGR2GRAY);

    Mat morph;
    Mat kernel = getStructuringElement(MORPH_ELLIPSE, Size(11, 11));
    morphologyEx(small, morph, MORPH_OPEN, kernel);

    Mat bw;
    threshold(morph, bw, 0, 255.0, CV_THRESH_BINARY | CV_THRESH_OTSU);

    Mat bdry;
    kernel = getStructuringElement(MORPH_ELLIPSE, Size(3, 3));
    erode(bw, bdry, kernel);
    subtract(bw, bdry, bdry);

    // do contour processing on bdry

This approach will not work in general, so I would strongly recommend something like Rutger suggested.

Problem

I currently have a method for detecting a card in an image and for the most part it works when the lighting is fairly consistent and the background is very calm. Here is the code I am using to preform this operation: ``` Mat img = inImg.clone(); outImg = Mat(inImg.size(), CV_8UC1); inImg.copyTo(outImg); Mat img_fullRes = img.clone(); pyrDown(img, img); Mat imgGray; cvtColor(img, imgGray, CV_RGB2GRAY); outImg_gray = imgGray.clone(); // Find Edges // Mat detectedEdges = imgGray.clone(); bilateralFilter(imgGray, detectedEdges, 0, 185, 3, 0); Canny( detectedEdges, detectedEdges, 20, 65, 3 ); dilate(detectedEdges, detectedEdges, Mat::ones(3,3,CV_8UC1)); Mat cdst = img.clone(); vector<Vec4i> lines; HoughLinesP(detectedEdges, lines, 1, CV_PI/180, 60, 50, 3 ); for( size_t i = 0; i < lines.size(); i++ ) { Vec4i l = lines[i]; // For debug //line( cdst, cv::Point(l[0], l[1]), cv::Point(l[2], l[3]), Scalar(0,0,255), 1); } //cdst.copyTo(inImg); // // Find points of intersection // cv::Rect imgROI; int ext = 10; imgROI.x = ext; imgROI.y = ext; imgROI.width = img.size().width - ext; imgROI.height = img.size().height - ext; int N = lines.size(); // Creating N amount of points // N == lines.size() cv::Point** poi = new cv::Point*[N]; for( int i = 0; i < N; i++ ) poi[i] = new cv::Point[N]; vector<cv::Point> poiList; for( int i = 0; i < N; i++ ) { poi[i][i] = cv::Point(-1,-1); Vec4i line1 = lines[i]; for( int j = i + 1; j < N; j++ ) { Vec4i line2 = lines[j]; cv::Point p = computeIntersect(line1, line2, imgROI); if( p.x != -1 ) { //line(cdst, p-cv::Point(2,0), p+cv::Point(2,0), Scalar(0,255,0)); //line(cdst, p-cv::Point(0,2), p+cv::Point(0,2), Scalar(0,255,0)); poiList.push_back(p); } poi[i][j] = p; poi[j][i] = p; } } cdst.copyTo(inImg); if(poiList.size()==0) { outImg = inImg.clone(); //circle(outImg, cv::Point(100,100), 50, Scalar(255,0,0), -1); return; } convexHull(poiList, poiList, false, true); for( int i=0; i<poiList.size(); i++ ) { cv::Point p = poiList[i]; //circle(cdst, p, 3, Scalar(255,0,0), 2); } //Evaluate all possible quadrilaterals cv::Point cardCorners[4]; float metric_max = 0; int Npoi = poiList.size(); for( int p1=0; p1<Npoi; p1++ ) { cv::Point pts[4]; pts[0] = poiList[p1]; for( int p2=p1+1; p2<Npoi; p2++ ) { pts[1] = poiList[p2]; if( isCloseBy(pts[1],pts[0]) ) continue; for( int p3=p2+1; p3<Npoi; p3++ ) { pts[2] = poiList[p3]; if( isCloseBy(pts[2],pts[1]) || isCloseBy(pts[2],pts[0]) ) continue; for( int p4=p3+1; p4<Npoi; p4++ ) { pts[3] = poiList[p4]; if( isCloseBy(pts[3],pts[0]) || isCloseBy(pts[3],pts[1]) || isCloseBy(pts[3],pts[2]) ) continue; // get the metrics float area = getArea(pts); cv::Point a = pts[0]-pts[1]; cv::Point b = pts[1]-pts[2]; cv::Point c = pts[2]-pts[3]; cv::Point d = pts[3]-pts[0]; float oppLenDiff = abs(a.dot(a)-c.dot(c)) + abs(b.dot(b)-d.dot(d)); float metric = area - 0.35*oppLenDiff; if( metric > metric_max ) { metric_max = metric; cardCorners[0] = pts[0]; cardCorners[1] = pts[1]; cardCorners[2] = pts[2]; cardCorners[3] = pts[3]; } } } } } // find the corners corresponding to the 4 corners of the physical card sortPointsClockwise(cardCorners); // Calculate Homography // vector<Point2f> srcPts(4); srcPts[0] = cardCorners[0]*2; srcPts[1] = cardCorners[1]*2; srcPts[2] = cardCorners[2]*2; srcPts[3] = cardCorners[3]*2; vector<Point2f> dstPts(4); cv::Size outImgSize(1400,800); dstPts[0] = Point2f(0,0); dstPts[1] = Point2f(outImgSize.width-1,0); dstPts[2] = Point2f(outImgSize.width-1,outImgSize.height-1); dstPts[3] = Point2f(0,outImgSize.height-1); Mat Homography = findHomography(srcPts, dstPts); // Apply Homography warpPerspective( img_fullRes, outImg, Homography, outImgSize, INTER_CUBIC ); outImg.copyTo(inImg); ``` Where `computeIntersect` is defined as: ``` cv::Point computeIntersect(cv::Vec4i a, cv::Vec4i b, cv::Rect ROI) { int x1 = a[0], y1 = a[1], x2 = a[2], y2 = a[3]; int x3 = b[0], y3 = b[1], x4 = b[2], y4 = b[3]; cv::Point p1 = cv::Point (x1,y1); cv::Point p2 = cv::Point (x2,y2); cv::Point p3 = cv::Point (x3,y3); cv::Point p4 = cv::Point (x4,y4); // Check to make sure all points are within the image boundrys, if not reject them. if( !ROI.contains(p1) || !ROI.contains(p2) || !ROI.contains(p3) || !ROI.contains(p4) ) return cv::Point (-1,-1); cv::Point vec1 = p1-p2; cv::Point vec2 = p3-p4; float vec1_norm2 = vec1.x*vec1.x + vec1.y*vec1.y; float vec2_norm2 = vec2.x*vec2.x + vec2.y*vec2.y; float cosTheta = (vec1.dot(vec2))/sqrt(vec1_norm2*vec2_norm2); float den = ((float)(x1-x2) * (y3-y4)) - ((y1-y2) * (x3-x4)); if(den != 0) { cv::Point2f pt; pt.x = ((x1*y2 - y1*x2) * (x3-x4) - (x1-x2) * (x3*y4 - y3*x4)) / den; pt.y = ((x1*y2 - y1*x2) * (y3-y4) - (y1-y2) * (x3*y4 - y3*x4)) / den; if( !ROI.contains(pt) ) return cv::Point (-1,-1); // no-confidence metric float d1 = MIN( dist2(p1,pt), dist2(p2,pt) )/vec1_norm2; float d2 = MIN( dist2(p3,pt), dist2(p4,pt) )/vec2_norm2; float no_confidence_metric = MAX(sqrt(d1),sqrt(d2)); // If end point ratios are greater than .5 reject if( no_confidence_metric < 0.5 && cosTheta < 0.707 ) return cv::Point (int(pt.x+0.5), int(pt.y+0.5)); } return cv::Point(-1, -1); } ``` `sortPointsClockWise` is defined as: ``` void sortPointsClockwise(cv::Point a[]) { cv::Point b[4]; cv::Point ctr = (a[0]+a[1]+a[2]+a[3]); ctr.x /= 4; ctr.y /= 4; b[0] = a[0]-ctr; b[1] = a[1]-ctr; b[2] = a[2]-ctr; b[3] = a[3]-ctr; for( int i=0; i<4; i++ ) { if( b[i].x < 0 ) { if( b[i].y < 0 ) a[0] = b[i]+ctr; else a[3] = b[i]+ctr; } else { if( b[i].y < 0 ) a[1] = b[i]+ctr; else a[2] = b[i]+ctr; } } } ``` `getArea` is defined as: ``` float getArea(cv::Point arr[]) { cv::Point diag1 = arr[0]-arr[2]; cv::Point diag2 = arr[1]-arr[3]; return 0.5*(diag1.cross(diag2)); } ``` `isCloseBy` is defined as: ``` bool isCloseBy( cv::Point p1, cv::Point p2 ) { int D = 10; // Checking that X values are within 10, same for Y values. return ( abs(p1.x-p2.x)<=D && abs(p1.y-p2.y)<=D ); } ``` And finally `dist2`: ``` float dist2( cv::Point p1, cv::Point p2 ) { return float((p1.x-p2.x)*(p1.x-p2.x) + (p1.y-p2.y)*(p1.y-p2.y)); } ``` Here are several test images and their results: Sorry for the very lengthy post, however I am hoping someone can suggest a way I can make my method for extracting the card from the image more robust. One that can better handle disruptive backgrounds along with inconsistent lighting. When a card is placed on a contrasting background with good lighting my method works nearly 90% of the time. But it is clear I need a more robust approach. Does anyone have any suggestions? Thanks. ATTEMPT of dhanushka's soloution ``` Mat gray, bw; pyrDown(inImg, inImg); cvtColor(inImg, gray, CV_RGB2GRAY); int morph_size = 3; Mat element = getStructuringElement( MORPH_ELLIPSE, cv::Size( 4*morph_size + 1, 2*morph_size+1 ), cv::Point( morph_size, morph_size ) ); morphologyEx(gray, gray, 2, element); threshold(gray, bw, 160, 255, CV_THRESH_BINARY); vector<vector<cv::Point> > contours; vector<Vec4i> hierarchy; findContours( bw, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, cv::Point(0, 0) ); int largest_area=0; int largest_contour_index=0; cv::Rect bounding_rect; for( int i = 0; i< contours.size(); i++ ) { double a=contourArea( contours[i],false); // Find the area of contour if(a>largest_area){ largest_area=a; largest_contour_index=i; //Store the index of largest contour bounding_rect=boundingRect(contours[i]); } } //Scalar color( 255,255,255); rectangle(inImg, bounding_rect, Scalar(0,255,0),1, 8,0); Mat biggestRect = inImg(bounding_rect); Mat card1 = biggestRect.clone(); ```

Original source