enhancement: build-in function "detectChe​ckerboardP​oints" unsatisfactory corners results

When I did the corner detection on a fisheye image, no matter how I adjusted the parameters and thresholds（HighDistortion，MinCornerMetric，PartialDetections）, it was difficult to detect the corner, but when I switched to Opencv, I could detect it easily and quickly, and the built-in functions were shown to be improved.

The original image of my fisheye image is attached

oriImg = imread("right.jpg");
[points,boderSize] = detectCheckerboardPoints(oriImg,HighDistortion=true,MinCornerMetric=0.10);
if isempty(points)
    disp("can't detect corners!")
end

can't detect corners!

Answer 1

[xingxingcui]

I currently use Mex C++ available relatively easy to use opencv function to find the corner point, the following is the reference program：

/**
* @file        :findCheckerBoarderCorners.cpp
* @brief       :This is a brief description
* @details     :使用opencv来找畸变图像角点，MATLAB函数效果较差，故采用mex编写。参考：Managing External Resources from MEX Functions
* @date        :2022/09/17 16:00:46
* @author      :cuixingxing([e-mail])
* @version     :1.0
*
* @copyright Copyright (c) 2022
*
*/

#include "mex.hpp"
#include "mexAdapter.hpp"

#include <iostream>
#include <string>
#include <stdio.h>
#include <stdlib.h>

#include "opencv2/opencv.hpp"

using matlab::mex::ArgumentList;
using namespace matlab::data;

class MexFunction : public matlab::mex::Function {
    // Pointer to MATLAB engine
    std::shared_ptr<matlab::engine::MATLABEngine> matlabPtr = getEngine();

    // Factory to create MATLAB data arrays
    ArrayFactory factory;

   public:
    void operator()(ArgumentList outputs, ArgumentList inputs) {
        checkArguments(outputs, inputs);
        matlab::data::TypedArray<uint8_t> fishEyeImg = std::move(inputs[0]);
        matlab::data::TypedArray<double> patternCheckerBoard = std::move(inputs[1]);  // 内点角点rows by cols

        cv::Size patternRowsByCols = cv::Size( patternCheckerBoard[1], patternCheckerBoard[0]);
        cv::Size oriImgS = cv::Size(fishEyeImg.getDimensions()[1], fishEyeImg.getDimensions()[0]);

        // step1: convert matlab matrix to opencv Mat
        cv::Mat oriImg;
        bool is3Channels = fishEyeImg.getDimensions()[2] == 3;
        if (is3Channels)
        {
            oriImg = cv::Mat::zeros(oriImgS, CV_8UC3);
            for (size_t i = 0; i < fishEyeImg.getDimensions()[0]; i++) {
                cv::Vec3b* data = oriImg.ptr<cv::Vec3b>(i);
                for (size_t j = 0; j < fishEyeImg.getDimensions()[1]; j++) {
                    data[j] = cv::Vec3b((uchar)fishEyeImg[i][j][2], (uchar)fishEyeImg[i][j][1], (uchar)fishEyeImg[i][j][0]);

                }
            }
        }
        else
        {
            oriImg = cv::Mat::zeros(oriImgS, CV_8UC1);
            for (size_t i = 0; i < fishEyeImg.getDimensions()[0]; i++) {
                uchar* data = oriImg.ptr<uchar>(i);
                for (size_t j = 0; j < fishEyeImg.getDimensions()[1]; j++) {
                    data[j] = (uchar)fishEyeImg[i][j];
                }
            }
        } 
        
        // step2:opencv function operations
        std::vector<cv::Point2f> corners; // must be "float" type instead of "double" type, https://stackoverflow.com/questions/70167318/error-215assertion-failed-count-0-in-function-cornersubpix-in-camera27
        cv::Mat gray;
        if (is3Channels)
        {
            cv::cvtColor(oriImg, gray, cv::COLOR_BGR2GRAY);
        }
        else
        {
            gray = oriImg;
        }
       
        bool isFound = cv::findChessboardCorners(gray, patternRowsByCols, corners, cv::CALIB_CB_ADAPTIVE_THRESH + cv::CALIB_CB_FAST_CHECK + cv::CALIB_CB_NORMALIZE_IMAGE);
        if (isFound)
        {
            cv::cornerSubPix(gray, corners, cv::Size(3, 3), cv::Size(-1, -1), cv::TermCriteria(cv::TermCriteria::MAX_ITER + cv::TermCriteria::EPS, 30, 0.1));
        }
        else
        {
            displayOnMATLAB("cv::findChessboardCorners can't find corners,try change your patternSize!\n");
        }
            
        // step3:convert opencv to matlab type
        matlab::data::TypedArray<float> outCorners = factory.createArray<float>({corners.size(), 2});
        for (size_t i = 0; i < corners.size(); i++) {
            outCorners[i][0] = corners[i].x;
            outCorners[i][1] = corners[i].y;
        }
        outputs[0] = std::move(outCorners);
    }

    void displayOnMATLAB(const std::ostringstream &stream) {
        matlabPtr->feval(u"fprintf", 0,
                         std::vector<Array>({factory.createScalar(stream.str())}));
    }

    void displayOnMATLAB(const std::string& str) {
        matlabPtr->feval(u"fprintf", 0,
            std::vector<Array>({ factory.createScalar(str) }));
    }

    void checkArguments(matlab::mex::ArgumentList outputs, matlab::mex::ArgumentList inputs) {
        if (inputs.size() != 2) {
            matlabPtr->feval(u"error",
                             0, std::vector<matlab::data::Array>({factory.createScalar("应输入2个参数fisheyeImg和[cornersRows, cornersCols]")}));
        }

        if (inputs[0].getType() != matlab::data::ArrayType::UINT8) {
            matlabPtr->feval(u"error",
                             0, std::vector<matlab::data::Array>({factory.createScalar("The first input is fisheye image, uint8 type")}));
        }

        if (inputs[1].getType() != matlab::data::ArrayType::DOUBLE) {
            matlabPtr->feval(u"error",
                             0, std::vector<matlab::data::Array>({factory.createScalar("cornersRC must double type")}));
        }
        if (inputs[1].getDimensions().size() != 2) {
            matlabPtr->feval(u"error",
                             0, std::vector<matlab::data::Array>({factory.createScalar("Input must be m-by-n dimension")}));
        }
        if (inputs[1].getNumberOfElements() != 2) {
            matlabPtr->feval(u"error",
                             0, std::vector<matlab::data::Array>({factory.createScalar("Input must be m-by-n dimension")}));
        }

        if (outputs.size() != 1) {
            matlabPtr->feval(u"error", 0, std::vector<matlab::data::Array>({factory.createScalar("Output argument must only one")}));
        }
    }
};

matlab mex compile code as follows:

% Notice: first use "mex -setup" to choose your c/c++ compiler
clear;

%% -------------------------------------------------------------------
%% get the architecture of this computer
is_64bit = strcmp(computer,'MACI64') || strcmp(computer,'GLNXA64') || strcmp(computer,'PCWIN64');


%% -------------------------------------------------------------------
%% the configuration of compiler
% You need to modify this configuration according to your own path of OpenCV
% 注意：你的VS OpenCV平台一定要匹配Matlab 64位的！
out_dir='./';% 当前目录
CPPFLAGS = ' -g  -ID:\opencv_4_4_0\MinGW64_v8_OpenCV4_4_Contrib_install\include -ID:\opencv_4_4_0\MinGW64_v8_OpenCV4_4_Contrib_install\include\opencv2'; % your OpenCV "include" path
LDFLAGS = ' -LD:\opencv_4_4_0\MinGW64_v8_OpenCV4_4_Contrib_install\x64\mingw\lib'; % 用OpenCV release版本的"lib"路径
% LIBS = ' -lopencv_world440'; % msvc release版本的lib，无后缀，系统会自动加上去
LIBS = ' libopencv_world440.dll.a'; % mingw64 编译器.a库文件，无"-l"参数
if is_64bit
    CPPFLAGS = [CPPFLAGS ' -largeArrayDims'];
end
%% add your files here!
compile_files = [
    % the list of your code files which need to be compiled
    ' D:\vs_files\project1\project1\findCheckerBoarderCorners.cpp'
    ];
%-------------------------------------------------------------------
%% compiling...
str = compile_files;
fprintf('compilation of: %s\n', str);
str = [str ' -outdir ' out_dir CPPFLAGS LDFLAGS LIBS];
args = regexp(str, '\s+', 'split');
mex(args{:});

fprintf('Congratulations, compilation successful!!!\n');

matlab syntax:

corners = findCheckerBoarderCorners(fishEyeImage,[innerCornersRows,innerCornersCols]); % patternSize = [innerCornersRows,innerCornersCols]

[xingxingcui]

mex file on windows refer to attachment.