如何在Python中实现色块矩阵的精确检测器？

Question

我先用一个例子来说明这个问题。

下面是一个示例输入图像，该图像可能带有噪声并且处于其他类型的仿射变换中。

输出应该返回一个二维数组/矩阵（使用标准颜色字符来表示每个颜色块）就像

[[w, g, w, b],
 [y, o, o, y],
 [g, b, g, y],
 [b, o, y, b]]

所以这个问题的目的是检测矩阵每个输入图像的颜色块文件，方向并不重要。

解决这个问题的想法给了我直觉，解决方案就像检测和解析二维码，但我不知道具体如何解决。

谁能给我一些建议，比如

• 解决这个问题的想法/程序。
• 我应该探索和使用哪些 Python 包的哪些 API。
• 任何类似的经典问题我都应该深入研究。
• 一些正确的代码。
• ...

Answer 1

我们可以从找到黑点并“校正”图像开始

。可以通过阈值处理和寻找“圆形”斑点来找到角点。该指标可以在这里找到：https://learnopencv.com/blob-检测使用-opencv-python-c/

一旦我们有了角点，我们就会“校正”图像（图像可能会旋转和/或根据角的顺序翻转）

转换为 HSV

在“v”通道上再次蒙版以获取各个方块

然后我们计算平均色调和每个方块的饱和度以平滑噪声
（我们需要饱和度来区分白色方块）

我们可以将这些数字与预定义的颜色值进行匹配以打印出字符串：

['b', 'w', 'g', 'w']
['y', 'o', 'o', 'y']
['y', 'g', 'b', 'g']
['b', 'y', 'o', 'b']

完整代码

import cv2
import numpy as np
import math

# get average color of image within mask
# I think this can be done with some mix of numpy commands (it'd be much faster)
def maskedAverageColor(mask, img):
    # convert to hsv
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV);
    h,s,v = cv2.split(hsv);
    height,width = img.shape[:2];
    count = 0;
    hue_ave = 0;
    sat_ave = 0;

    # calculate average
    for y in range(height):
        for x in range(width):
            if mask[y,x] == 255:
                count += 1;
                hue_ave += h[y,x];
                sat_ave += s[y,x];
    hue_ave /= count;
    sat_ave /= count;
    return [hue_ave, sat_ave];

# center of contour
def getCenter(contour):
    M = cv2.moments(contour);
    cx = int(M['m10']/M['m00']);
    cy = int(M['m01']/M['m00']);
    return [cx, cy];

# predefined colors
# white
# green
# blue
# orange
# yellow
# ??? missing one color (need 6 for a cube?)
color_map = {};
color_map['g'] = 53;
color_map['y'] = 27;
color_map['o'] = 10;
color_map['b'] = 120;

# load image
img = cv2.imread("cube.png");

# show
cv2.imshow("Image", img);

# grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY);

# threshold
mask = cv2.inRange(gray, 0, 50);

# close small black speckles and dilate
kernel = np.ones((3,3),np.uint8);
mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel);
mask = cv2.dilate(mask, kernel, iterations = 1);

# find contours
contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE);

# fill in black spaces
for con in contours:
    cv2.drawContours(mask, [con], -1, 255, -1);
cv2.imshow("Mask", mask);

# get the contours again
contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE);

# filter out little contours
filtered = [];
for con in contours:
    area = cv2.contourArea(con);
    if area > 25:
        filtered.append(con);
contours = filtered;

# circularity
corners = [];
for con in contours:
    area = cv2.contourArea(con);
    perm = cv2.arcLength(con, True);
    if area != 0 and perm != 0:
        circ = (4 * math.pi * area) / (perm**2);
        if circ > 0.5:
            # get the corners
            corners.append(getCenter(con));

# sort corners
# find point in the middle of the corners
cx = 0;
cy = 0;
for corner in corners:
    x,y = corner;
    cx += x;
    cy += y;
cx /= 4;
cy /= 4;

# calculate angles
angles = []; # [angle, point]
for corner in corners:
    x, y = corner;
    dx = 1000;
    dy = y - cy;
    angle = math.atan2(dy, dx);
    angles.append([angle, corner]);

# sort by angles (counter-clockwise)
angles = sorted(angles, key = lambda x : x[0]);
corners = [p[1] for p in angles];

cv2.destroyAllWindows();

# unmorph with corners
to_rect = [
[0, 0],
[500, 0],
[500, 500],
[0, 500]
]
corners = np.array(corners, dtype=np.float32);
to_rect = np.array(to_rect, dtype=np.float32);
warp_mat = cv2.getPerspectiveTransform(corners, to_rect);
rectified = cv2.warpPerspective(img, warp_mat, (500,500));
cv2.imshow("Rect", rectified);

# hsv and mask again
hsv = cv2.cvtColor(rectified, cv2.COLOR_BGR2HSV);
h,s,v = cv2.split(hsv);
cv2.imshow("Hue", h);
cv2.imshow("Sat", s);
cv2.imshow("Val", v);
mask = cv2.inRange(v, 0, 150);

# dilate mask
mask = cv2.dilate(mask, kernel, iterations = 5);
mask = cv2.bitwise_not(mask);

# get contours (yes again)
contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE);

# get the squares
squares = [];
for con in contours:
    area = cv2.contourArea(con);
    if area < 100000:
        squares.append(con);

# get the center of squares
drawn = np.copy(rectified);
for square in squares:
    cx, cy = getCenter(square);
    cv2.circle(drawn, (int(cx), int(cy)), 4, (255,0,0), -1);

# split into rows
rows = [];
index = -1;
for y in range(4):
    row = [];
    for x in range(4):
        row.append(squares[index]);
        index -= 1;
    rows.append(row);

# draw rows
drawn = np.copy(rectified);
for row in rows:
    for square in row:
        cx, cy = getCenter(square);
        cv2.circle(drawn, (int(cx), int(cy)), 4, (255,0,0), -1);

# get color of each square
color_rows = [];
for row in rows:
    color_row = [];
    for square in row:
        # get mask
        h,w = rectified.shape[:2];
        mask = np.zeros((h,w), np.uint8);
        mask = cv2.drawContours(mask, [square], -1, 255, -1);

        # calculate average colors
        hue, sat = maskedAverageColor(mask, rectified);

        # convert to string colors
        # white check
        color_str = "NONE";
        if sat < 50:
            color_str = "w";
        else:
            margin = 5;
            for key in color_map:
                if color_map[key] - margin < hue and hue < color_map[key] + margin:
                    color_str = key;
        if color_str == "NONE":
            print("INCREASE MARGIN");
            print("HUE SAT: " + str([hue, sat]));
        color_row.append(color_str);
    color_rows.append(color_row);

# print out results
for row in color_rows:
    print(row);
cv2.waitKey(0);