xiongyi/AutoControlSystem-git
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

UPDATE Vision box：由点云坐标改为像素坐标

Browse Source
This commit is contained in:
HJW
2024-10-09 10:44:59 +08:00
parent 2fd734562d
commit 0b34eaca53
2 changed files with 74 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
Vision/camera_coordinate_dete.py
View File

@ -33,16 +33,19 @@ class Detection:
""" """
self.use_openvino_model = use_openvino_model self.use_openvino_model = use_openvino_model
self.cameraType = cameraType
if self.use_openvino_model == False: if self.use_openvino_model == False:
model_path = ''.join([os.getcwd(), '/Vision/model/pt/best.pt']) model_path = ''.join([os.getcwd(), '/Vision/model/pt/best.pt'])
device = 'cpu' device = 'cpu'
self.cameraType = cameraType
if self.cameraType == 'RVC': if self.cameraType == 'RVC':
self.camera_rvc = camera_rvc() self.camera_rvc = camera_rvc()
self.seg_distance_threshold = 0.005 self.seg_distance_threshold = 0.005
else: elif self.cameraType == 'Pe':
self.camera_rvc = camera_pe() self.camera_rvc = camera_pe()
self.seg_distance_threshold = 10 self.seg_distance_threshold = 10
else:
print('相机参数错误')
return
self.model = yolov8_segment() self.model = yolov8_segment()
self.model.load_model(model_path, device) self.model.load_model(model_path, device)
else: else:
@ -51,17 +54,20 @@ class Detection:
if self.cameraType == 'RVC': if self.cameraType == 'RVC':
self.camera_rvc = camera_rvc() self.camera_rvc = camera_rvc()
self.seg_distance_threshold = 0.005 self.seg_distance_threshold = 0.005
else: elif self.cameraType == 'Pe':
self.camera_rvc = camera_pe() self.camera_rvc = camera_pe()
self.seg_distance_threshold = 10 self.seg_distance_threshold = 10
else:
print('相机参数错误')
return
self.model = yolov8_segment_openvino(model_path, device, conf_thres=0.3, iou_thres=0.3) self.model = yolov8_segment_openvino(model_path, device, conf_thres=0.3, iou_thres=0.3)
def get_position(self, Point_isVision=False, save_img_point=False, Height_reduce = 30, width_reduce = 30): def get_position(self, Point_isVision=False, save_img_point=0, Height_reduce = 30, width_reduce = 30):
""" """
检测料袋相关信息 检测料袋相关信息
:param Point_isVision: 点云可视化 :param Point_isVision: 点云可视化
:param save_img_point: 保存点云和图片 :param save_img_point: 0不保存 ; 1保存原图 ;2保存处理后的图 ; 3保存点云和原图；4 保存点云和处理后的图
:param Height_reduce: 检测框的高内缩像素 :param Height_reduce: 检测框的高内缩像素
:param width_reduce: 检测框的宽内缩像素 :param width_reduce: 检测框的宽内缩像素
:return ret: bool 相机是否正常工作 :return ret: bool 相机是否正常工作
@ -74,15 +80,18 @@ class Detection:
ret, img, pm = self.camera_rvc.get_img_and_point_map() # 拍照，获取图像及 ret, img, pm = self.camera_rvc.get_img_and_point_map() # 拍照，获取图像及
if self.camera_rvc.caminit_isok == True: if self.camera_rvc.caminit_isok == True:
if ret == 1: if ret == 1:
if save_img_point == True: if save_img_point != 0:
if get_disk_space(path=os.getcwd()) < 15: # 内存小于15G,停止保存数据 if get_disk_space(path=os.getcwd()) < 15: # 内存小于15G,停止保存数据
save_img_point = False save_img_point = 0
print('系统内存不足，无法保存数据') print('系统内存不足，无法保存数据')
else: else:
save_path = ''.join([os.getcwd(), '/Vision/model/data/', save_path = ''.join([os.getcwd(), '/Vision/model/data/',
time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(time.time()))]) time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(time.time()))])
save_img_name = ''.join([save_path, '.png']) save_img_name = ''.join([save_path, '.png'])
save_point_name = ''.join([save_path, '.xyz']) save_point_name = ''.join([save_path, '.xyz'])
if save_img_point==1 or save_img_point==3:
cv2.imwrite(save_img_name, img)
if save_img_point==3 or save_img_point==4:
row_list = list(range(1, img.shape[0], 2)) row_list = list(range(1, img.shape[0], 2))
column_list = list(range(1, img.shape[1], 2)) column_list = list(range(1, img.shape[1], 2))
pm_save = pm.copy() pm_save = pm.copy()
@ -165,8 +174,13 @@ class Detection:
''' '''
rect = cv2.minAreaRect(max_contour) rect = cv2.minAreaRect(max_contour)
if rect[1][0]-width_reduce < 30 or rect[1][1]-Height_reduce < 30:
rect_reduce = ( rect_reduce = (
(rect[0][0], rect[0][1]), (rect[1][0] - width_reduce, rect[1][1] - Height_reduce), rect[2]) (rect[0][0], rect[0][1]), (rect[1][0] - width_reduce, rect[1][1] - Height_reduce), rect[2])
else:
rect_reduce = (
(rect[0][0], rect[0][1]), (rect[1][0], rect[1][1]), rect[2])
# cv2.boxPoints可以将轮廓点转换为四个角点坐标 # cv2.boxPoints可以将轮廓点转换为四个角点坐标
box_outside = cv2.boxPoints(rect) box_outside = cv2.boxPoints(rect)
# 这一步不影响后面的画图，但是可以保证四个角点坐标为顺时针 # 这一步不影响后面的画图，但是可以保证四个角点坐标为顺时针
@ -224,10 +238,6 @@ class Detection:
box[2][0][1], box[2][0][0] = out_bounds_dete(pm.shape[0], pm.shape[1], box[2][0][1], box[2][0][0]) box[2][0][1], box[2][0][0] = out_bounds_dete(pm.shape[0], pm.shape[1], box[2][0][1], box[2][0][0])
box[3][0][1], box[3][0][0] = out_bounds_dete(pm.shape[0], pm.shape[1], box[3][0][1], box[3][0][0]) box[3][0][1], box[3][0][0] = out_bounds_dete(pm.shape[0], pm.shape[1], box[3][0][1], box[3][0][0])
box_point_x1, box_point_y1, box_point_z1 = remove_nan_mean_value(pm, box[0][0][1], box[0][0][0])
box_point_x2, box_point_y2, box_point_z2 = remove_nan_mean_value(pm, box[1][0][1], box[1][0][0])
box_point_x3, box_point_y3, box_point_z3 = remove_nan_mean_value(pm, box[2][0][1], box[2][0][0])
box_point_x4, box_point_y4, box_point_z4 = remove_nan_mean_value(pm, box[3][0][1], box[3][0][0])
x_rotation_center = int((box[0][0][0] + box[1][0][0] + box[2][0][0] + box[3][0][0]) / 4) x_rotation_center = int((box[0][0][0] + box[1][0][0] + box[2][0][0] + box[3][0][0]) / 4)
y_rotation_center = int((box[0][0][1] + box[1][0][1] + box[2][0][1] + box[3][0][1]) / 4) y_rotation_center = int((box[0][0][1] + box[1][0][1] + box[2][0][1] + box[3][0][1]) / 4)
point_x, point_y, point_z = remove_nan_mean_value(pm, y_rotation_center, x_rotation_center) point_x, point_y, point_z = remove_nan_mean_value(pm, y_rotation_center, x_rotation_center)
@ -235,15 +245,11 @@ class Detection:
if np.isnan(point_x): # 点云值为无效值 if np.isnan(point_x): # 点云值为无效值
continue continue
else: else:
box_list.append( box_list.append(box)
[[box_point_x1, box_point_y1, box_point_z1],
[box_point_x2, box_point_y2, box_point_z2],
[box_point_x3, box_point_y3, box_point_z3],
[box_point_x4, box_point_y4, box_point_z4]])
if self.cameraType=='RVC': if self.cameraType=='RVC':
xyz.append([point_x*1000, point_y*1000, point_z*1000]) xyz.append([point_x*1000, point_y*1000, point_z*1000])
Depth_Z.append(point_z*1000) Depth_Z.append(point_z*1000)
else: elif self.cameraType=='Pe':
xyz.append([point_x, point_y, point_z]) xyz.append([point_x, point_y, point_z])
Depth_Z.append(point_z) Depth_Z.append(point_z)
nx_ny_nz.append([a, b, c]) nx_ny_nz.append([a, b, c])
@ -271,12 +277,12 @@ class Detection:
outlier_cloud = pcd.select_by_index(inliers, invert=True) outlier_cloud = pcd.select_by_index(inliers, invert=True)
outlier_cloud.paint_uniform_color([0, 0, 1]) outlier_cloud.paint_uniform_color([0, 0, 1])
o3d.visualization.draw_geometries([inlier_cloud, outlier_cloud, pcd2]) o3d.visualization.draw_geometries([inlier_cloud, outlier_cloud, pcd2])
if save_img_point == True: if save_img_point == 2 or save_img_point ==4:
save_img = cv2.resize(img, (720, 540)) save_img = cv2.resize(img, (720, 540))
cv2.imwrite(save_img_name, save_img) cv2.imwrite(save_img_name, save_img)
return 1, img, xyz[_idx], nx_ny_nz[_idx], box_list[_idx] return 1, img, xyz[_idx], nx_ny_nz[_idx], box_list[_idx]
else: else:
if save_img_point == True: if save_img_point == 2 or save_img_point ==4:
save_img = cv2.resize(img,(720, 540)) save_img = cv2.resize(img,(720, 540))
cv2.imwrite(save_img_name, save_img) cv2.imwrite(save_img_name, save_img)
return 1, img, None, None, None return 1, img, None, None, None
@ -367,8 +373,13 @@ class Detection:
''' '''
rect = cv2.minAreaRect(max_contour) rect = cv2.minAreaRect(max_contour)
if rect[1][0] - width_reduce < 30 or rect[1][1] - Height_reduce < 30:
rect_reduce = ( rect_reduce = (
(rect[0][0], rect[0][1]), (rect[1][0] - width_reduce, rect[1][1] - Height_reduce), rect[2]) (rect[0][0], rect[0][1]), (rect[1][0] - width_reduce, rect[1][1] - Height_reduce),
rect[2])
else:
rect_reduce = (
(rect[0][0], rect[0][1]), (rect[1][0], rect[1][1]), rect[2])
# cv2.boxPoints可以将轮廓点转换为四个角点坐标 # cv2.boxPoints可以将轮廓点转换为四个角点坐标
box_outside = cv2.boxPoints(rect) box_outside = cv2.boxPoints(rect)
# 这一步不影响后面的画图，但是可以保证四个角点坐标为顺时针 # 这一步不影响后面的画图，但是可以保证四个角点坐标为顺时针
@ -402,7 +413,7 @@ class Detection:
if self.cameraType == 'RVC': if self.cameraType == 'RVC':
xyz.append([point_x * 1000, point_y * 1000, point_z * 1000]) xyz.append([point_x * 1000, point_y * 1000, point_z * 1000])
Depth_Z.append(point_z * 1000) Depth_Z.append(point_z * 1000)
else: elif self.cameraType == 'Pe':
xyz.append([point_x, point_y, point_z]) xyz.append([point_x, point_y, point_z])
Depth_Z.append(point_z) Depth_Z.append(point_z)
RegionalArea.append(cv2.contourArea(max_contour)) RegionalArea.append(cv2.contourArea(max_contour))

40
Vision/camera_coordinate_dete_img.py
View File

@ -45,11 +45,11 @@ class Detection:
pass pass
def get_position(self, Point_isVision=False, save_img_point=True, seg_distance_threshold = 0.01, Height_reduce = 30, width_reduce = 30): def get_position(self, Point_isVision=False, save_img_point=0, seg_distance_threshold = 0.01, Height_reduce = 30, width_reduce = 30):
""" """
检测料袋相关信息 检测料袋相关信息
:param Point_isVision: 点云可视化 :param Point_isVision: 点云可视化
:param save_img_point: 保存点云和图片 :param save_img_point: 0不保存 ; 1保存原图 ;2保存处理后的图 ; 3保存点云和原图；4 保存点云和处理后的图
:param Height_reduce: 检测框的高内缩像素 :param Height_reduce: 检测框的高内缩像素
:param width_reduce: 检测框的宽内缩像素 :param width_reduce: 检测框的宽内缩像素
:return ret: bool 相机是否正常工作 :return ret: bool 相机是否正常工作
@ -57,21 +57,23 @@ class Detection:
:return xyz: list 目标中心点云值形如[x,y,z] :return xyz: list 目标中心点云值形如[x,y,z]
:return nx_ny_nz: list 拟合平面法向量，形如[a,b,c] :return nx_ny_nz: list 拟合平面法向量，形如[a,b,c]
:return box_list: list 内缩检测框四顶点，形如[[x1,y1],[],[],[]] :return box_list: list 内缩检测框四顶点，形如[[x1,y1],[],[],[]]
""" """
ret = 1 ret = 1
img = self.img img = self.img
pm = self.point pm = self.point
if ret == 1: if ret == 1:
if save_img_point == True: if save_img_point != 0:
if get_disk_space(path=os.getcwd())<15: # 内存小于15G,停止保存数据 if get_disk_space(path=os.getcwd())<15: # 内存小于15G,停止保存数据
save_img_point = False save_img_point = 0
print('系统内存不足，无法保存数据') print('系统内存不足，无法保存数据')
else: else:
save_path = ''.join([os.getcwd(), '/Vision/model/data/', save_path = ''.join([os.getcwd(), '/Vision/model/data/',
time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(time.time()))]) time.strftime('%Y_%m_%d_%H_%M_%S', time.localtime(time.time()))])
save_img_name = ''.join([save_path, '.png']) save_img_name = ''.join([save_path, '.png'])
save_point_name = ''.join([save_path, '.xyz']) save_point_name = ''.join([save_path, '.xyz'])
if save_img_point == 1 or save_img_point == 3:
cv2.imwrite(save_img_name, img)
if save_img_point == 3 or save_img_point == 4:
row_list = list(range(1, img.shape[0], 2)) row_list = list(range(1, img.shape[0], 2))
column_list = list(range(1, img.shape[1], 2)) column_list = list(range(1, img.shape[1], 2))
pm_save = pm.copy() pm_save = pm.copy()
@ -156,7 +158,12 @@ class Detection:
''' '''
rect = cv2.minAreaRect(max_contour) rect = cv2.minAreaRect(max_contour)
rect_reduce = ((rect[0][0], rect[0][1]), (rect[1][0] - width_reduce, rect[1][1] - Height_reduce), rect[2]) if rect[1][0] - width_reduce < 30 or rect[1][1] - Height_reduce < 30:
rect_reduce = (
(rect[0][0], rect[0][1]), (rect[1][0] - width_reduce, rect[1][1] - Height_reduce), rect[2])
else:
rect_reduce = (
(rect[0][0], rect[0][1]), (rect[1][0], rect[1][1]), rect[2])
# cv2.boxPoints可以将轮廓点转换为四个角点坐标 # cv2.boxPoints可以将轮廓点转换为四个角点坐标
box_outside = cv2.boxPoints(rect) box_outside = cv2.boxPoints(rect)
# 这一步不影响后面的画图，但是可以保证四个角点坐标为顺时针 # 这一步不影响后面的画图，但是可以保证四个角点坐标为顺时针
@ -217,11 +224,6 @@ class Detection:
box[2][0][1], box[2][0][0] = out_bounds_dete(pm.shape[0], pm.shape[1], box[2][0][1], box[2][0][0]) box[2][0][1], box[2][0][0] = out_bounds_dete(pm.shape[0], pm.shape[1], box[2][0][1], box[2][0][0])
box[3][0][1], box[3][0][0] = out_bounds_dete(pm.shape[0], pm.shape[1], box[3][0][1], box[3][0][0]) box[3][0][1], box[3][0][0] = out_bounds_dete(pm.shape[0], pm.shape[1], box[3][0][1], box[3][0][0])
box_point_x1, box_point_y1, box_point_z1 = remove_nan_mean_value(pm, box[0][0][1], box[0][0][0])
box_point_x2, box_point_y2, box_point_z2 = remove_nan_mean_value(pm, box[1][0][1], box[1][0][0])
box_point_x3, box_point_y3, box_point_z3 = remove_nan_mean_value(pm, box[2][0][1], box[2][0][0])
box_point_x4, box_point_y4, box_point_z4 = remove_nan_mean_value(pm, box[3][0][1], box[3][0][0])
x_rotation_center = int((box[0][0][0] + box[1][0][0] + box[2][0][0] + box[3][0][0]) / 4) x_rotation_center = int((box[0][0][0] + box[1][0][0] + box[2][0][0] + box[3][0][0]) / 4)
y_rotation_center = int((box[0][0][1] + box[1][0][1] + box[2][0][1] + box[3][0][1]) / 4) y_rotation_center = int((box[0][0][1] + box[1][0][1] + box[2][0][1] + box[3][0][1]) / 4)
point_x, point_y, point_z = remove_nan_mean_value(pm, y_rotation_center, x_rotation_center) point_x, point_y, point_z = remove_nan_mean_value(pm, y_rotation_center, x_rotation_center)
@ -229,9 +231,7 @@ class Detection:
if np.isnan(point_x): # 点云值为无效值 if np.isnan(point_x): # 点云值为无效值
continue continue
else: else:
box_list.append([[box_point_x1, box_point_y1, box_point_z1], [box_point_x2, box_point_y2, box_point_z2], box_list.append(box)
[box_point_x3, box_point_y3, box_point_z3], [box_point_x4, box_point_y4, box_point_z4]])
xyz.append([point_x * 1000, point_y * 1000, point_z * 1000])
Depth_Z.append(point_z * 1000) Depth_Z.append(point_z * 1000)
nx_ny_nz.append([a, b, c]) nx_ny_nz.append([a, b, c])
RegionalArea.append(cv2.contourArea(max_contour)) RegionalArea.append(cv2.contourArea(max_contour))
@ -258,13 +258,13 @@ class Detection:
outlier_cloud = pcd.select_by_index(inliers, invert=True) outlier_cloud = pcd.select_by_index(inliers, invert=True)
outlier_cloud.paint_uniform_color([0, 1, 0]) outlier_cloud.paint_uniform_color([0, 1, 0])
o3d.visualization.draw_geometries([inlier_cloud, outlier_cloud, pcd2]) o3d.visualization.draw_geometries([inlier_cloud, outlier_cloud, pcd2])
if save_img_point == True: if save_img_point == 2 or save_img_point == 4:
save_img = cv2.resize(img,(720, 540)) save_img = cv2.resize(img,(720, 540))
cv2.imwrite(save_img_name, save_img) cv2.imwrite(save_img_name, save_img)
return 1, img, xyz[_idx], nx_ny_nz[_idx], box_list[_idx] return 1, img, xyz[_idx], nx_ny_nz[_idx], box_list[_idx]
else: else:
if save_img_point == True: if save_img_point == 2 or save_img_point == 4:
save_img = cv2.resize(img, (720, 540)) save_img = cv2.resize(img, (720, 540))
cv2.imwrite(save_img_name, save_img) cv2.imwrite(save_img_name, save_img)
@ -274,7 +274,7 @@ class Detection:
print("RVC X Camera capture failed!") print("RVC X Camera capture failed!")
return 0, None, None, None, None return 0, None, None, None, None
def get_take_photo_position(self, Point_isVision=False, save_img_point=False, Height_reduce = 30, width_reduce = 30): def get_take_photo_position(self, Height_reduce=30, width_reduce=30):
""" """
专用于拍照位置点查找，检测当前拍照点能否检测到料袋 专用于拍照位置点查找，检测当前拍照点能否检测到料袋
:param Height_reduce: :param Height_reduce:
@ -354,8 +354,12 @@ class Detection:
''' '''
rect = cv2.minAreaRect(max_contour) rect = cv2.minAreaRect(max_contour)
if rect[1][0] - width_reduce < 30 or rect[1][1] - Height_reduce < 30:
rect_reduce = ( rect_reduce = (
(rect[0][0], rect[0][1]), (rect[1][0] - width_reduce, rect[1][1] - Height_reduce), rect[2]) (rect[0][0], rect[0][1]), (rect[1][0] - width_reduce, rect[1][1] - Height_reduce), rect[2])
else:
rect_reduce = (
(rect[0][0], rect[0][1]), (rect[1][0], rect[1][1]), rect[2])
# cv2.boxPoints可以将轮廓点转换为四个角点坐标 # cv2.boxPoints可以将轮廓点转换为四个角点坐标
box_outside = cv2.boxPoints(rect) box_outside = cv2.boxPoints(rect)
# 这一步不影响后面的画图，但是可以保证四个角点坐标为顺时针 # 这一步不影响后面的画图，但是可以保证四个角点坐标为顺时针
@ -389,7 +393,7 @@ class Detection:
if self.cameraType == 'RVC': if self.cameraType == 'RVC':
xyz.append([point_x * 1000, point_y * 1000, point_z * 1000]) xyz.append([point_x * 1000, point_y * 1000, point_z * 1000])
Depth_Z.append(point_z * 1000) Depth_Z.append(point_z * 1000)
else: elif self.cameraType == 'Pe':
xyz.append([point_x, point_y, point_z]) xyz.append([point_x, point_y, point_z])
Depth_Z.append(point_z) Depth_Z.append(point_z)
RegionalArea.append(cv2.contourArea(max_contour)) RegionalArea.append(cv2.contourArea(max_contour))