#include "NumberDectect.h" #include "DoublePointerCountALGO.h" #include "AtmosphericPressureALGO.h" #include "CircularArresterCurrentALGO.h" #include "ArresterCircularZeroThreeALGO.h" using namespace std; cv::Mat NumberDectect::DoPerspectiveTransform(cv::Mat& SrcImg) { bool isShowWindow = false; vector> sideScalarCount; Mat src = SrcImg; for (int row = 0; row < src.rows; row++) { uchar* uc_pixel = src.data + row * src.step; for (int col = 0; col < src.cols; col++) { /*uc_pixel[0] = 255 - uc_pixel[0]; uc_pixel[1] = 255 - uc_pixel[1]; uc_pixel[2] = 255 - uc_pixel[2];*/ if (row == src.rows - 1 || row == 0) { //sideScalarCount.push_back(pair()) cv::Scalar sc(uc_pixel[0], uc_pixel[1], uc_pixel[2]); auto item = std::find_if(sideScalarCount.begin(), sideScalarCount.end(), [sc](pair p) { return p.first == sc; }); if (item != sideScalarCount.end()) { item->second++; } else { sideScalarCount.push_back(pair(sc, 0)); } } else { if (col == src.cols - 1 || col == 0) { cv::Scalar sc(uc_pixel[0], uc_pixel[1], uc_pixel[2]); auto item = std::find_if(sideScalarCount.begin(), sideScalarCount.end(), [sc](pair p) { return p.first == sc; }); if (item != sideScalarCount.end()) { item->second++; } else { sideScalarCount.push_back(pair(sc, 0)); } } } uc_pixel += 3; } } int maxCount = 0; cv::Scalar maxSc; for (int i = 0; i < sideScalarCount.size(); i++) { if (sideScalarCount[i].second > maxCount) { maxCount = sideScalarCount[i].second; maxSc = sideScalarCount[i].first; } } int matCols = 500; Size size(500, 420); resize(src, src, size, (float)matCols / src.cols, (float)matCols / src.cols); if (isShowWindow) if (isShowWindow) cv::imshow("input image", src); cv::Point p(50, 50); Mat largeImage(Size(600, 520), CV_8UC3, maxSc);// Mat(Size(520, 440), , CV_8UC3); Mat imageROI; imageROI = largeImage(Rect(p.x, p.y, src.cols, src.rows)); src.copyTo(imageROI); if (isShowWindow) cv::imshow("input image imageROI", largeImage); src = largeImage; //bgr 2 gray 转为灰度图像 Mat src_gray; cvtColor(src, src_gray, COLOR_BGR2GRAY); //binary 二值化 Mat binary; threshold(src_gray, binary, 0, 255, THRESH_BINARY_INV | THRESH_OTSU); //THRESH_BINARY_INV二值化后取反 //imshow("binary", binary);//因为有一些斑点存在 //形态学 闭操作:可以填充小的区域 Mat morhp_img; Mat kernel = getStructuringElement(MORPH_RECT, Size(5, 5), Point(-1, -1)); morphologyEx(binary, morhp_img, MORPH_CLOSE, kernel, Point(-1, -1), 3); //imshow("morphology", morhp_img); Mat dst; bitwise_not(morhp_img, dst);//在取反 if (isShowWindow) cv::imshow("dst", dst);// //轮廓发现 vector> contous; vector hireachy; findContours(dst, contous, hireachy, CV_RETR_TREE, CHAIN_APPROX_SIMPLE, Point()); std::cout << "contous.size:" << contous.size() << endl; //轮廓绘制 int width = src.cols; int height = src.rows; Mat drawImage = Mat::zeros(src.size(), CV_8UC3); std::cout << contous.size() << endl; for (size_t t = 0; t < contous.size(); t++) { Rect rect = boundingRect(contous[t]); if (rect.width > width / 2 && rect.height > height / 2 && rect.width < width - 5 && rect.height < height - 5) { drawContours(drawImage, contous, static_cast(t), Scalar(0, 0, 255), 2, 8, hireachy, 0, Point(0, 0)); } } if (isShowWindow) cv::imshow("contours", drawImage);//显示找到的轮廓 //直线检测 vector lines; Mat contoursImg; int accu = min(width * 0.2, height * 0.2); cvtColor(drawImage, contoursImg, COLOR_BGR2GRAY); if (isShowWindow) cv::imshow("contours", contoursImg); Mat linesImage = Mat::zeros(src.size(), CV_8UC3); HoughLinesP(contoursImg, lines, 2, CV_PI / 180.0, accu, accu, 3); for (size_t t = 0; t < lines.size(); t++) { Vec4i ln = lines[t]; line(linesImage, Point(ln[0], ln[1]), Point(ln[2], ln[3]), Scalar(0, 0, 255), 2, 8, 0);//绘制直线 } std::cout << "number of lines:" << lines.size() << endl; if (isShowWindow) cv::imshow("linesImages", linesImage); //寻找与定位上下 左右 四条直线 int deltah = 0; //高度差 int deltaw = 0; //宽度差 Vec4i topLine, bottomLine; //直线定义 Vec4i rightLine, leftLine; for (int i = 0; i < lines.size(); i++) { Vec4i ln = lines[i];//????? /* Opencv中的累计霍夫变换HoughLinesP(),输出的是一个Vector of Vec4i, Vector每个元素代表一条直线,是由一个4元浮点数构成, 前两个一组x_1,y_1,后两个一组x_2,y_2,代表了图像中直线的起始点和结束点。 */ deltah = abs(ln[3] - ln[1]); //计算高度差(y2-y1) //topLine if (ln[3] < height / 2.0 && ln[1] < height / 2.0 && deltah < accu - 1) { topLine = lines[i]; } //bottomLine if (ln[3] > height / 2.0 && ln[1] > height / 2.0 && deltah < accu - 1) { bottomLine = lines[i]; } deltaw = abs(ln[2] - ln[0]); //计算宽度差(x2-x1) //leftLine if (ln[0] < height / 2.0 && ln[2] < height / 2.0 && deltaw < accu - 1) { leftLine = lines[i]; } //rightLine if (ln[0] > width / 2.0 && ln[2] > width / 2.0 && deltaw < accu - 1) { rightLine = lines[i]; } } // 打印四条线的坐标 std::cout << "topLine : p1(x,y)= " << topLine[0] << "," << topLine[1] << "; p2(x,y)= " << topLine[2] << "," << topLine[3] << endl; std::cout << "bottomLine : p1(x,y)= " << bottomLine[0] << "," << bottomLine[1] << "; p2(x,y)= " << bottomLine[2] << "," << bottomLine[3] << endl; std::cout << "leftLine : p1(x,y)= " << leftLine[0] << "," << leftLine[1] << "; p2(x,y)= " << leftLine[2] << "," << leftLine[3] << endl; std::cout << "rightLine : p1(x,y)= " << rightLine[0] << "," << rightLine[1] << "; p2(x,y)= " << rightLine[2] << "," << rightLine[3] << endl; //拟合四条直线 float k1, k2, k3, k4, c1, c2, c3, c4; k1 = float(topLine[3] - topLine[1]) / float(topLine[2] - topLine[0]); c1 = topLine[1] - k1 * topLine[0]; k2 = float(bottomLine[3] - bottomLine[1]) / float(bottomLine[2] - bottomLine[0]); c2 = bottomLine[1] - k2 * bottomLine[0]; k3 = float(leftLine[3] - leftLine[1]) / float(leftLine[2] - leftLine[0]); c3 = leftLine[1] - k3 * leftLine[0]; k4 = float(rightLine[3] - rightLine[1]) / float(rightLine[2] - rightLine[0]); c4 = rightLine[1] - k4 * rightLine[0]; //求四个角点, Point p1;//topLine leftLine 左上角 p1.x = static_cast(c1 - c3) / k3 - k1; p1.y = k1 * p1.x + c1; Point p2;//topLine rightLine 右上角 p2.x = static_cast(c1 - c4) / k4 - k1; p2.y = k1 * p2.x + c1; Point p3;//bottomLine leftLine 左下角 p3.x = static_cast(c2 - c3) / k3 - k2; p3.y = k2 * p3.x + c2; Point p4;//bottomLine rightLine 右下角 p4.x = static_cast(c2 - c4) / k4 - k2; p4.y = k2 * p4.x + c2; std::cout << "Point p1: (" << p1.x << "," << p1.y << ")" << endl; std::cout << "Point p2: (" << p2.x << "," << p2.y << ")" << endl; std::cout << "Point p3: (" << p3.x << "," << p3.y << ")" << endl; std::cout << "Point p4: (" << p4.x << "," << p4.y << ")" << endl; if (p1.x > 0 && p1.y > 0 && p2.x > 0 && p2.y > 0 && p3.x > 0 && p3.y > 0 && p4.x > 0 && p4.y > 0) { if (p1.x < width && p2.x < width && p3.x < width && p4.x < width) { if (p1.y < height && p2.y < height && p3.y < height && p4.y < height) { //显示四个点 cv::circle(linesImage, p1, 2, Scalar(0, 255, 0), 2); cv::circle(linesImage, p2, 2, Scalar(0, 255, 0), 2); cv::circle(linesImage, p3, 2, Scalar(0, 255, 0), 2); cv::circle(linesImage, p4, 2, Scalar(0, 255, 0), 2); if (isShowWindow) cv::imshow("find four points", linesImage); //透视变换 vector src_corners(4); src_corners[0] = p1; src_corners[1] = p2; src_corners[2] = p3; src_corners[3] = p4; Mat result_images = Mat::zeros(height * 0.7, width * 0.9, CV_8UC3); vector dst_corners(4); dst_corners[0] = Point(0, 0); dst_corners[1] = Point(result_images.cols, 0); dst_corners[2] = Point(0, result_images.rows); dst_corners[3] = Point(result_images.cols, result_images.rows); Mat warpmatrix = getPerspectiveTransform(src_corners, dst_corners); //获取透视变换矩阵 //imshow("final result warpmatrix", warpmatrix); cv::warpPerspective(src, result_images, warpmatrix, result_images.size()); //透视变换 if (isShowWindow) cv::imshow("final result", result_images); //imwrite(imgpath, result_images); if (isShowWindow) cv::waitKey(5000); return result_images; } } } return Mat(); } bool NumberDectect::Init(bool isCuda) { string model_path = "models/number-sim.onnx"; try { net = cv::dnn::readNet(model_path); if (isCuda) { net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA); net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA_FP16); } //cpu else { net.setPreferableBackend(cv::dnn::DNN_BACKEND_DEFAULT); net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); } } catch (const std::exception& ex) { YunDaISASImageRecognitionService::ConsoleLog(ex.what()); return false; } //cuda return true; return false; } IDetection::DectectResult NumberDectect::GetStateResult(cv::Mat img, cv::Rect rec) { return resultValue; } IDetection::DectectResult NumberDectect::GetDigitResult(cv::Mat img, cv::Rect rec) { //resultValue.clear(); //std::cout << "test" << std::endl; //try //{ // cv::Mat ROI = img(rec); // /*imwrite("test.png", ROI); // YunDaISASImageRecognitionService::SetImage(QString::fromStdString("test.png"));*/ // Detect(ROI); //} //catch (const std::exception& ex) //{ // YunDaISASImageRecognitionService::ConsoleLog(ex.what()); //} //if (resultValue.m_confidence < 0.1) //{ // resultValue = DectectResult(0.45, 0, ""); //} return resultValue; } vector NumberDectect::GetDigitResults() { return this->resultValues; } bool NumberDectect::Detect(cv::Mat& SrcImg) { auto pecImg = DoPerspectiveTransform(SrcImg); if (pecImg.rows>0&& pecImg.cols>0 ) { SrcImg = pecImg; } cv::Mat blob; int col = SrcImg.cols; int row = SrcImg.rows; int maxLen = MAX(col, row); cv::Mat netInputImg = SrcImg.clone(); if (maxLen > 1.2 * col || maxLen > 1.2 * row) { cv::Mat resizeImg = cv::Mat::zeros(maxLen, maxLen, CV_8UC3); SrcImg.copyTo(resizeImg(cv::Rect(0, 0, col, row))); netInputImg = resizeImg; } cv::dnn::blobFromImage(netInputImg, blob, 1 / 255.0, cv::Size(netWidth, netHeight), cv::Scalar(0, 0, 0), true, false); net.setInput(blob); std::vector netOutputImg; net.forward(netOutputImg, net.getUnconnectedOutLayersNames()); std::vector classIds;//结果id数组 std::vector confidences;//结果每个id对应置信度数组 std::vector boxes;//每个id矩形框 float ratio_h = (float)netInputImg.rows / netHeight; float ratio_w = (float)netInputImg.cols / netWidth; int net_width = className.size() + 5; //输出的网络宽度是类别数+5 float* pdata = (float*)netOutputImg[0].data; for (int stride = 0; stride < strideSize; stride++) { //stride int grid_x = (int)(netWidth / netStride[stride]); int grid_y = (int)(netHeight / netStride[stride]); for (int anchor = 0; anchor < 3; anchor++) { //anchors const float anchor_w = netAnchors[stride][anchor * 2]; const float anchor_h = netAnchors[stride][anchor * 2 + 1]; for (int i = 0; i < grid_y; i++) { for (int j = 0; j < grid_x; j++) { float box_score = pdata[4]; ;//获取每一行的box框中含有某个物体的概率 if (box_score >= boxThreshold) { cv::Mat scores(1, className.size(), CV_32FC1, pdata + 5); cv::Point classIdPoint; double max_class_socre; minMaxLoc(scores, 0, &max_class_socre, 0, &classIdPoint); max_class_socre = (float)max_class_socre; if (max_class_socre >= classThreshold) { //rect [x,y,w,h] float x = pdata[0]; //x float y = pdata[1]; //y float w = pdata[2]; //w float h = pdata[3]; //h int left = (x - 0.5 * w) * ratio_w; int top = (y - 0.5 * h) * ratio_h; left = left < 0 ? 0 : left; top = top < 0 ? 0 : top; int widthBox = int(w * ratio_w); int heightBox = int(h * ratio_h); widthBox = widthBox > col ? col : widthBox; heightBox = heightBox > row ? row : heightBox; if (left < 0 || left>col || top < 0 || top>row || widthBox > col || heightBox > row || left + widthBox > col || top + heightBox > row ) { continue; } classIds.push_back(classIdPoint.x); confidences.push_back(max_class_socre * box_score); boxes.push_back(cv::Rect(left, top, widthBox, heightBox)); } } pdata += net_width;//下一行 } } } } //执行非最大抑制以消除具有较低置信度的冗余重叠框(NMS) vector nms_result; cv::dnn::NMSBoxes(boxes, confidences, nmsScoreThreshold, nmsThreshold, nms_result); float confidenceMax = 0; int confidenceMaxId = 0; output.clear(); resultValues.clear(); if (nms_result.size() > 0) { vector units; vector> numCoordinates; for (int i = 0; i < nms_result.size(); i++) { int idx = nms_result[i]; Output result(classIds[idx], confidences[idx], boxes[idx]); auto typeName = className[classIds[idx]]; auto item = std::find_if(classTypeName.begin(), classTypeName.end(), [typeName](string strValue) { return typeName == strValue; }); output.push_back(result); if (item!= classTypeName.end()) { units.push_back(typeName); } else { numCoordinates.push_back(pair(className[classIds[idx]], Point(boxes[idx].x + (boxes[idx].width / 2), boxes[idx].y + boxes[idx].height))); } YunDaISASImageRecognitionService::ConsoleLog(QString::fromStdString(className[classIds[idx]])); } vector,vector>> sameYNums; if (numCoordinates.size()>0) { for (size_t i = 0; i < numCoordinates.size(); i++) { Point yCondinate = numCoordinates[i].second; string sValue = numCoordinates[i].first; auto item = std::find_if(sameYNums.begin(), sameYNums.end(), [yCondinate](pair, vector> p) { if (p.second.size()>0) { if (cv::abs(p.first[0].y - yCondinate.y)<50) { return true; } } return false ; }); if (item != sameYNums.end()) { item->second.push_back(sValue); item->first.push_back(yCondinate); } else { auto resultPair = pair, vector>(vector{ yCondinate }, vector{sValue}); sameYNums.push_back(resultPair); /*if (sameYNums.size()>0) { if(sameYNums[sameYNums.size() - 1].first[0].y> yCondinate.y) { sameYNums.emplace(sameYNums.end(), resultPair); } } else { sameYNums.push_back(resultPair); }*/ } } } if (sameYNums.size()>0) { Point exchange ; string excStr; for (size_t k = 0; k < sameYNums.size(); k++) { for (int i = 1; i < sameYNums[k].first.size()-1; i++) //主要算法 { for (int j = 1; j <= sameYNums[k].first.size() - i; j++) { if (sameYNums[k].first[j - 1].x > sameYNums[k].first[j].x) { exchange = sameYNums[k].first[j - 1]; excStr = sameYNums[k].second[j - 1]; sameYNums[k].first[j - 1] = sameYNums[k].first[j]; sameYNums[k].second[j - 1] = sameYNums[k].second[j]; sameYNums[k].first[j] = exchange; sameYNums[k].second[j] = excStr; } } } } for (size_t i = 0; i < sameYNums.size(); i++) { //int pointPos = -1; string strValue = ""; for (size_t j = 0; j < sameYNums[i].second.size(); j++) { if (sameYNums[i].second[j] == "point") { strValue += "."; } else { //stoi(sameYNums[i].second[j]) strValue += sameYNums[i].second[j]; } } if (sameYNums.size() == units.size()) { resultValues.push_back(DectectResult(0.99, atof(strValue.c_str()), units[i])); } else if (units.size()>0) { resultValues.push_back(DectectResult(0.99, atof(strValue.c_str()), units[0])); } else { resultValues.push_back(DectectResult(0.99, atof(strValue.c_str()), "")); } } } } else { resultValue = DectectResult(confidenceMax, 0.00, ""); } return false; }