ImageRecognition/NumberDectect.cpp

#include "NumberDectect.h"
#include "DoublePointerCountALGO.h"
#include "AtmosphericPressureALGO.h"
#include "CircularArresterCurrentALGO.h"
#include "ArresterCircularZeroThreeALGO.h"

using namespace std;


cv::Mat NumberDectect::DoPerspectiveTransform(cv::Mat& SrcImg)
{
	bool isShowWindow = false;
	vector<pair<cv::Scalar, int>> sideScalarCount;
	Mat src = SrcImg;
	for (int row = 0; row < src.rows; row++) {
		uchar* uc_pixel = src.data + row * src.step;
		for (int col = 0; col < src.cols; col++) {
			/*uc_pixel[0] = 255 - uc_pixel[0];
			uc_pixel[1] = 255 - uc_pixel[1];
			uc_pixel[2] = 255 - uc_pixel[2];*/
			if (row == src.rows - 1 || row == 0)
			{
				//sideScalarCount.push_back(pair())
				cv::Scalar sc(uc_pixel[0], uc_pixel[1], uc_pixel[2]);
				auto item = std::find_if(sideScalarCount.begin(), sideScalarCount.end(), [sc](pair<cv::Scalar, int> p) { return p.first == sc; });
				if (item != sideScalarCount.end())
				{
					item->second++;
				}
				else {
					sideScalarCount.push_back(pair<cv::Scalar, int>(sc, 0));
				}
			}
			else
			{
				if (col == src.cols - 1 || col == 0)
				{
					cv::Scalar sc(uc_pixel[0], uc_pixel[1], uc_pixel[2]);
					auto item = std::find_if(sideScalarCount.begin(), sideScalarCount.end(), [sc](pair<cv::Scalar, int> p) { return p.first == sc; });
					if (item != sideScalarCount.end())
					{
						item->second++;
					}
					else {
						sideScalarCount.push_back(pair<cv::Scalar, int>(sc, 0));
					}
				}
			}
			uc_pixel += 3;
		}

	}
	int maxCount = 0;
	cv::Scalar maxSc;
	for (int i = 0; i < sideScalarCount.size(); i++) {

		if (sideScalarCount[i].second > maxCount)
		{
			maxCount = sideScalarCount[i].second;
			maxSc = sideScalarCount[i].first;
		}
	}
	int matCols = 500;
	Size size(500, 420);
	resize(src, src, size, (float)matCols / src.cols, (float)matCols / src.cols);
	if (isShowWindow) if (isShowWindow) cv::imshow("input image", src);
	cv::Point p(50, 50);
	Mat largeImage(Size(600, 520), CV_8UC3, maxSc);// Mat(Size(520, 440), , CV_8UC3);
	Mat imageROI;
	imageROI = largeImage(Rect(p.x, p.y, src.cols, src.rows));
	src.copyTo(imageROI);
	if (isShowWindow) cv::imshow("input image imageROI", largeImage);
	src = largeImage;
	//bgr 2 gray 转为灰度图像
	Mat src_gray;
	cvtColor(src, src_gray, COLOR_BGR2GRAY);

	//binary 二值化
	Mat binary;
	threshold(src_gray, binary, 0, 255, THRESH_BINARY_INV | THRESH_OTSU); //THRESH_BINARY_INV二值化后取反
	//imshow("binary", binary);//因为有一些斑点存在

	//形态学 闭操作：可以填充小的区域
	Mat morhp_img;
	Mat kernel = getStructuringElement(MORPH_RECT, Size(5, 5), Point(-1, -1));
	morphologyEx(binary, morhp_img, MORPH_CLOSE, kernel, Point(-1, -1), 3);
	//imshow("morphology", morhp_img);

	Mat dst;
	bitwise_not(morhp_img, dst);//在取反
	if (isShowWindow) cv::imshow("dst", dst);//

	//轮廓发现
	vector<vector<Point>> contous;
	vector<Vec4i> hireachy;
	findContours(dst, contous, hireachy, CV_RETR_TREE, CHAIN_APPROX_SIMPLE, Point());
	std::cout << "contous.size:" << contous.size() << endl;


	//轮廓绘制
	int width = src.cols;
	int height = src.rows;
	Mat drawImage = Mat::zeros(src.size(), CV_8UC3);
	std::cout << contous.size() << endl;
	for (size_t t = 0; t < contous.size(); t++)
	{
		Rect rect = boundingRect(contous[t]);
		if (rect.width > width / 2 && rect.height > height / 2 && rect.width < width - 5 && rect.height < height - 5)
		{
			drawContours(drawImage, contous, static_cast<int>(t), Scalar(0, 0, 255), 2, 8, hireachy, 0, Point(0, 0));
		}
	}
	if (isShowWindow) cv::imshow("contours", drawImage);//显示找到的轮廓

	//直线检测
	vector<Vec4i> lines;
	Mat contoursImg;
	int accu = min(width * 0.2, height * 0.2);
	cvtColor(drawImage, contoursImg, COLOR_BGR2GRAY);
	if (isShowWindow) cv::imshow("contours", contoursImg);

	Mat linesImage = Mat::zeros(src.size(), CV_8UC3);
	HoughLinesP(contoursImg, lines, 2, CV_PI / 180.0, accu, accu, 3);
	for (size_t t = 0; t < lines.size(); t++)
	{
		Vec4i ln = lines[t];
		line(linesImage, Point(ln[0], ln[1]), Point(ln[2], ln[3]), Scalar(0, 0, 255), 2, 8, 0);//绘制直线
	}
	std::cout << "number of lines:" << lines.size() << endl;
	if (isShowWindow) cv::imshow("linesImages", linesImage);

	//寻找与定位上下 左右 四条直线
	int deltah = 0; //高度差
	int deltaw = 0; //宽度差
	Vec4i topLine, bottomLine; //直线定义
	Vec4i rightLine, leftLine;

	for (int i = 0; i < lines.size(); i++)
	{
		Vec4i ln = lines[i];//?????
		/*
		Opencv中的累计霍夫变换HoughLinesP()，输出的是一个Vector of Vec4i，
		Vector每个元素代表一条直线，是由一个4元浮点数构成，
		前两个一组x_1,y_1，后两个一组x_2,y_2，代表了图像中直线的起始点和结束点。
		*/
		deltah = abs(ln[3] - ln[1]); //计算高度差(y2-y1)
		//topLine
		if (ln[3] < height / 2.0 && ln[1] < height / 2.0 && deltah < accu - 1)
		{
			topLine = lines[i];
		}
		//bottomLine
		if (ln[3] > height / 2.0 && ln[1] > height / 2.0 && deltah < accu - 1)
		{
			bottomLine = lines[i];
		}

		deltaw = abs(ln[2] - ln[0]); //计算宽度差(x2-x1)	
		//leftLine
		if (ln[0] < height / 2.0 && ln[2] < height / 2.0 && deltaw < accu - 1)
		{
			leftLine = lines[i];
		}
		//rightLine
		if (ln[0] > width / 2.0 && ln[2] > width / 2.0 && deltaw < accu - 1)
		{
			rightLine = lines[i];
		}
	}

	// 打印四条线的坐标
	std::cout << "topLine : p1(x,y)= " << topLine[0] << "," << topLine[1] << "; p2(x,y)= " << topLine[2] << "," << topLine[3] << endl;
	std::cout << "bottomLine : p1(x,y)= " << bottomLine[0] << "," << bottomLine[1] << "; p2(x,y)= " << bottomLine[2] << "," << bottomLine[3] << endl;
	std::cout << "leftLine : p1(x,y)= " << leftLine[0] << "," << leftLine[1] << "; p2(x,y)= " << leftLine[2] << "," << leftLine[3] << endl;
	std::cout << "rightLine : p1(x,y)= " << rightLine[0] << "," << rightLine[1] << "; p2(x,y)= " << rightLine[2] << "," << rightLine[3] << endl;

	//拟合四条直线
	float k1, k2, k3, k4, c1, c2, c3, c4;
	k1 = float(topLine[3] - topLine[1]) / float(topLine[2] - topLine[0]);
	c1 = topLine[1] - k1 * topLine[0];

	k2 = float(bottomLine[3] - bottomLine[1]) / float(bottomLine[2] - bottomLine[0]);
	c2 = bottomLine[1] - k2 * bottomLine[0];

	k3 = float(leftLine[3] - leftLine[1]) / float(leftLine[2] - leftLine[0]);
	c3 = leftLine[1] - k3 * leftLine[0];

	k4 = float(rightLine[3] - rightLine[1]) / float(rightLine[2] - rightLine[0]);
	c4 = rightLine[1] - k4 * rightLine[0];

	//求四个角点,
	Point p1;//topLine  leftLine 左上角
	p1.x = static_cast<int>(c1 - c3) / k3 - k1;
	p1.y = k1 * p1.x + c1;

	Point p2;//topLine  rightLine 右上角
	p2.x = static_cast<int>(c1 - c4) / k4 - k1;
	p2.y = k1 * p2.x + c1;

	Point p3;//bottomLine  leftLine 左下角
	p3.x = static_cast<int>(c2 - c3) / k3 - k2;
	p3.y = k2 * p3.x + c2;

	Point p4;//bottomLine  rightLine 右下角
	p4.x = static_cast<int>(c2 - c4) / k4 - k2;
	p4.y = k2 * p4.x + c2;

	std::cout << "Point p1: (" << p1.x << "," << p1.y << ")" << endl;
	std::cout << "Point p2: (" << p2.x << "," << p2.y << ")" << endl;
	std::cout << "Point p3: (" << p3.x << "," << p3.y << ")" << endl;
	std::cout << "Point p4: (" << p4.x << "," << p4.y << ")" << endl;
	if (p1.x > 0 && p1.y > 0 && p2.x > 0 && p2.y > 0 && p3.x > 0 && p3.y > 0 && p4.x > 0 && p4.y > 0)
	{
		if (p1.x < width && p2.x < width && p3.x < width && p4.x < width)
		{
			if (p1.y < height && p2.y < height && p3.y < height && p4.y < height)
			{
				//显示四个点
				cv::circle(linesImage, p1, 2, Scalar(0, 255, 0), 2);
				cv::circle(linesImage, p2, 2, Scalar(0, 255, 0), 2);
				cv::circle(linesImage, p3, 2, Scalar(0, 255, 0), 2);
				cv::circle(linesImage, p4, 2, Scalar(0, 255, 0), 2);
				if (isShowWindow) cv::imshow("find four points", linesImage);

				//透视变换
				vector<Point2f> src_corners(4);
				src_corners[0] = p1;
				src_corners[1] = p2;
				src_corners[2] = p3;
				src_corners[3] = p4;

				Mat result_images = Mat::zeros(height * 0.7, width * 0.9, CV_8UC3);
				vector<Point2f> dst_corners(4);
				dst_corners[0] = Point(0, 0);
				dst_corners[1] = Point(result_images.cols, 0);
				dst_corners[2] = Point(0, result_images.rows);
				dst_corners[3] = Point(result_images.cols, result_images.rows);

				Mat warpmatrix = getPerspectiveTransform(src_corners, dst_corners); //获取透视变换矩阵
				//imshow("final result warpmatrix", warpmatrix);
				cv::warpPerspective(src, result_images, warpmatrix, result_images.size()); //透视变换
				if (isShowWindow) cv::imshow("final result", result_images);
				//imwrite(imgpath, result_images);
				if (isShowWindow) cv::waitKey(5000);
				return result_images;
			}
		}

	}
	return Mat();
}
bool NumberDectect::Init(bool isCuda)
{
	string model_path = "models/number-sim.onnx";
	try {
		net = cv::dnn::readNet(model_path);
		if (isCuda) {
			net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA);
			net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA_FP16);
		}
		//cpu
		else {
			net.setPreferableBackend(cv::dnn::DNN_BACKEND_DEFAULT);
			net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
		}
	}
	catch (const std::exception& ex)
	{
		YunDaISASImageRecognitionService::ConsoleLog(ex.what());
		return false;
	}
	//cuda
	
	return true;
	return false;
}

IDetection::DectectResult NumberDectect::GetStateResult(cv::Mat img, cv::Rect rec)
{
	return resultValue;
}

IDetection::DectectResult NumberDectect::GetDigitResult(cv::Mat img, cv::Rect rec)
{
	//resultValue.clear();
	//std::cout << "test" << std::endl;
	//try
	//{
	//	cv::Mat ROI = img(rec);
	//	/*imwrite("test.png", ROI);
	//	YunDaISASImageRecognitionService::SetImage(QString::fromStdString("test.png"));*/
	//	Detect(ROI);
	//}
	//catch (const std::exception& ex)
	//{
	//	YunDaISASImageRecognitionService::ConsoleLog(ex.what());
	//}
	//if (resultValue.m_confidence < 0.1)
	//{
	//	resultValue = DectectResult(0.45, 0, "");
	//}
	return resultValue;
}
vector<IDetection::DectectResult> NumberDectect::GetDigitResults()
{
	return this->resultValues;
}
bool NumberDectect::Detect(cv::Mat& SrcImg)
{
	auto  pecImg = DoPerspectiveTransform(SrcImg);
	if (pecImg.rows>0&& pecImg.cols>0  )
	{
		SrcImg = pecImg;
	}
	cv::Mat blob;
	int col = SrcImg.cols;
	int row = SrcImg.rows;
	int maxLen = MAX(col, row);
	cv::Mat netInputImg = SrcImg.clone();
	if (maxLen > 1.2 * col || maxLen > 1.2 * row) {
		cv::Mat resizeImg = cv::Mat::zeros(maxLen, maxLen, CV_8UC3);
		SrcImg.copyTo(resizeImg(cv::Rect(0, 0, col, row)));
		netInputImg = resizeImg;
	}
	cv::dnn::blobFromImage(netInputImg, blob, 1 / 255.0, cv::Size(netWidth, netHeight), cv::Scalar(0, 0, 0), true, false);
	net.setInput(blob);
	std::vector<cv::Mat> netOutputImg;
	net.forward(netOutputImg, net.getUnconnectedOutLayersNames());
	std::vector<int> classIds;//结果id数组
	std::vector<float> confidences;//结果每个id对应置信度数组
	std::vector<cv::Rect> boxes;//每个id矩形框
	float ratio_h = (float)netInputImg.rows / netHeight;
	float ratio_w = (float)netInputImg.cols / netWidth;
	int net_width = className.size() + 5;  //输出的网络宽度是类别数+5
	float* pdata = (float*)netOutputImg[0].data;
	for (int stride = 0; stride < strideSize; stride++) {    //stride
		int grid_x = (int)(netWidth / netStride[stride]);
		int grid_y = (int)(netHeight / netStride[stride]);
		for (int anchor = 0; anchor < 3; anchor++) {	//anchors
			const float anchor_w = netAnchors[stride][anchor * 2];
			const float anchor_h = netAnchors[stride][anchor * 2 + 1];
			for (int i = 0; i < grid_y; i++) {
				for (int j = 0; j < grid_x; j++) {
					float box_score = pdata[4]; ;//获取每一行的box框中含有某个物体的概率
					if (box_score >= boxThreshold) {
						cv::Mat scores(1, className.size(), CV_32FC1, pdata + 5);
						cv::Point classIdPoint;
						double max_class_socre;
						minMaxLoc(scores, 0, &max_class_socre, 0, &classIdPoint);
						max_class_socre = (float)max_class_socre;
						if (max_class_socre >= classThreshold)
						{
							//rect [x,y,w,h]
							float x = pdata[0];  //x
							float y = pdata[1];  //y
							float w = pdata[2];  //w
							float h = pdata[3];  //h
							int left = (x - 0.5 * w) * ratio_w;
							int top = (y - 0.5 * h) * ratio_h;
							left = left < 0 ? 0 : left;
							top = top < 0 ? 0 : top;
							int widthBox = int(w * ratio_w);
							int heightBox = int(h * ratio_h);
							widthBox = widthBox > col ? col : widthBox;
							heightBox = heightBox > row ? row : heightBox;
							if (left < 0 || left>col || top < 0 || top>row || widthBox > col || heightBox > row
								|| left + widthBox > col || top + heightBox > row
								)
							{
								continue;
							}
							classIds.push_back(classIdPoint.x);
							confidences.push_back(max_class_socre * box_score);
							boxes.push_back(cv::Rect(left, top, widthBox, heightBox));
						}
					}
					pdata += net_width;//下一行
				}
			}
		}
	}

	//执行非最大抑制以消除具有较低置信度的冗余重叠框（NMS）
	vector<int> nms_result;
	cv::dnn::NMSBoxes(boxes, confidences, nmsScoreThreshold, nmsThreshold, nms_result);
	float confidenceMax = 0;
	int confidenceMaxId = 0;
	output.clear();
	resultValues.clear();
	if (nms_result.size() > 0)
	{
		vector<string> units;
		vector<pair<string, Point>> numCoordinates;
		for (int i = 0; i < nms_result.size(); i++) {
			int idx = nms_result[i];
			Output result(classIds[idx], confidences[idx], boxes[idx]);
			auto typeName = className[classIds[idx]];
			auto item = std::find_if(classTypeName.begin(), classTypeName.end(), [typeName](string strValue)
				{
				return typeName == strValue;
				});
			output.push_back(result);
			if (item!= classTypeName.end())
			{
				units.push_back(typeName);
			}
			else
			{
				numCoordinates.push_back(pair<string, Point>(className[classIds[idx]], Point(boxes[idx].x + (boxes[idx].width / 2), boxes[idx].y + boxes[idx].height)));
			}
			YunDaISASImageRecognitionService::ConsoleLog(QString::fromStdString(className[classIds[idx]]));
		}
		vector<pair<vector<Point>,vector<string>>> sameYNums;
		if (numCoordinates.size()>0)
		{
			for (size_t i = 0; i < numCoordinates.size(); i++)
			{
				Point yCondinate = numCoordinates[i].second;
				string sValue = numCoordinates[i].first;
				auto item = std::find_if(sameYNums.begin(), sameYNums.end(), [yCondinate](pair<vector<Point>, vector<string>> p)
					{  
						if (p.second.size()>0)
						{
							if (cv::abs(p.first[0].y - yCondinate.y)<50)
							{
								return true;
							}
						}
						return false ; 
					});
				if (item != sameYNums.end())
				{
					
					item->second.push_back(sValue);
					item->first.push_back(yCondinate);

				}
				else {
					auto resultPair = pair<vector<Point>, vector<string >>(vector<Point>{ yCondinate }, vector<string>{sValue});
					sameYNums.push_back(resultPair);

					/*if (sameYNums.size()>0)
					{
						if(sameYNums[sameYNums.size() - 1].first[0].y> yCondinate.y) {
							sameYNums.emplace(sameYNums.end(), resultPair);
						}
					}
					else
					{
						sameYNums.push_back(resultPair);
					}*/
				}
			}
		}

		if (sameYNums.size()>0)
		{
			Point exchange ;
			string excStr;
			for (size_t k = 0; k < sameYNums.size(); k++)
			{
				for (int i = 1; i < sameYNums[k].first.size()-1; i++)                 //主要算法
				{
					for (int j = 1; j <= sameYNums[k].first.size() - i; j++)
					{
						if (sameYNums[k].first[j - 1].x > sameYNums[k].first[j].x)
						{
							exchange = sameYNums[k].first[j - 1];
							excStr = sameYNums[k].second[j - 1];
							sameYNums[k].first[j - 1] = sameYNums[k].first[j];
							sameYNums[k].second[j - 1] = sameYNums[k].second[j];
							sameYNums[k].first[j] = exchange;
							sameYNums[k].second[j] = excStr;
					   }
				   }
				}
			}
			for (size_t i = 0; i < sameYNums.size(); i++)
			{
				//int pointPos = -1;
				string strValue = "";
				for (size_t j = 0; j < sameYNums[i].second.size(); j++)
				{
					if (sameYNums[i].second[j] == "point")
					{
						strValue += ".";
					}
					else
					{
						//stoi(sameYNums[i].second[j])
						strValue += sameYNums[i].second[j];
					}
				}
				if (sameYNums.size() == units.size())
				{
					resultValues.push_back(DectectResult(0.99, atof(strValue.c_str()), units[i]));
				}
				else if (units.size()>0)
				{
					resultValues.push_back(DectectResult(0.99, atof(strValue.c_str()), units[0]));
				}
				else {
					resultValues.push_back(DectectResult(0.99, atof(strValue.c_str()), ""));
				}
			}
		}
	}
	else {
		resultValue = DectectResult(confidenceMax, 0.00, "");
	}
	return false;
}