From 9593fa1b902c429bbbb3226470f53414a26c9620 Mon Sep 17 00:00:00 2001 From: HJW <1576345902@qq.com> Date: Tue, 10 Sep 2024 16:09:11 +0800 Subject: [PATCH] UPDATE Vision --- Vision/camera_coordinate_dete.py | 20 ++++++---- Vision/camera_coordinate_dete_img.py | 54 ++++++++++++++------------- Vision/camera_coordinate_dete_test.py | 2 +- requirements | 2 + 4 files changed, 45 insertions(+), 33 deletions(-) diff --git a/Vision/camera_coordinate_dete.py b/Vision/camera_coordinate_dete.py index b613267..6f8cf7e 100644 --- a/Vision/camera_coordinate_dete.py +++ b/Vision/camera_coordinate_dete.py @@ -24,15 +24,18 @@ class Detection: self.camera_rvc = camera() self.model = yolov8_segment() self.model.load_model(model_path, device) - if self.camera_rvc.caminit_isok == True: - print("RVC X Camera is opened!") - else: - print("RVC X Camera is not opened!") + # if self.camera_rvc.caminit_isok == True: + # print("RVC X Camera is opened!") + # else: + # print("RVC X Camera is not opened!") - def get_position(self, Point_isVision=True): + def get_position(self, Point_isVision=False, first_depth=False): "" ''' - :param api: None + :param api: + Point_isVision： 点云可视化 + first_depth： 按Z轴深度返回坐标及向量 + :return: ret , img, (x,y,z), (nx, ny, nz) ''' ret, img, pm = self.camera_rvc.get_img_and_point_map() # 拍照，获取图像及 @@ -110,6 +113,9 @@ class Detection: pm_seg = select_point.reshape(-1, 3) pm_seg = pm_seg[~np.isnan(pm_seg).all(axis=-1), :] # 剔除 nan # cv2.imshow('result', piont_result) + if pm_seg.size < 300: + print("分割出的点云数量较少，无法拟合平面！") + continue ''' 拟合平面，计算法向量 @@ -157,7 +163,7 @@ class Detection: seg_point.append(pm_seg) cv2.polylines(img, [box], True, (0, 255, 0), 2) - _idx = find_position(Depth_Z, RegionalArea, 100,True) + _idx = find_position(Depth_Z, RegionalArea, 100, first_depth) if _idx == None: return 1, img, None, None diff --git a/Vision/camera_coordinate_dete_img.py b/Vision/camera_coordinate_dete_img.py index ba446fb..4e1cb01 100644 --- a/Vision/camera_coordinate_dete_img.py +++ b/Vision/camera_coordinate_dete_img.py @@ -13,15 +13,15 @@ import cv2 import open3d as o3d from Vision.tool.CameraRVC import camera from Vision.yolo.yolov8_pt_seg import yolov8_segment - +from Vision.tool.utils import find_position class Detection: def __init__(self): model_path = './pt_model/best.pt' device = 'cpu' - img_path = './pt_model/test0824.png' - point_path = './pt_model/test0824.xyz' + img_path = './pt_model/test0910.png' + point_path = './pt_model/test0910.xyz' self.model = yolov8_segment() self.model.load_model(model_path, device) self.img = cv2.imread(img_path) @@ -39,6 +39,7 @@ class Detection: ret = 1 img = self.img pm = self.point + if ret == 1: flag, det_cpu, dst_img, masks, category_names = self.model.model_inference(img, 0) if flag == 1: @@ -55,7 +56,7 @@ class Detection: pm2[:, 2] = pm2[:, 2] + 0.25 pcd2 = o3d.geometry.PointCloud() pcd2.points = o3d.utility.Vector3dVector(pm2) - #o3d.visualization.draw_geometries([pcd2]) + # o3d.visualization.draw_geometries([pcd2]) for i, item in enumerate(det_cpu): @@ -87,7 +88,7 @@ class Detection: ret, binary = cv2.threshold(imgray, 10, 255, 0) # 阈值 二进制图像 # cv2.imshow('bin',binary) - # cv2.waitKey(0) + # cv2.waitKey(1) contours, hierarchy = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) # all_piont_list = contours_in(contours) # print(len(all_piont_list)) @@ -106,15 +107,15 @@ class Detection: pixel_point2 = cv2.findNonZero(mask_inside) # result = np.zeros_like(color_image) select_point = [] - - for t in range(pixel_point2.shape[0]): - select_point.append(pm[pixel_point2[t][0][1] + box_y1, pixel_point2[t][0][0]+ box_x1]) - + for i in range(pixel_point2.shape[0]): + select_point.append(pm[pixel_point2[i][0][1] + box_y1, pixel_point2[i][0][0] + box_x1]) select_point = np.array(select_point) pm_seg = select_point.reshape(-1, 3) pm_seg = pm_seg[~np.isnan(pm_seg).all(axis=-1), :] # 剔除 nan # cv2.imshow('result', piont_result) - + if pm_seg.size < 300: + print("分割出的点云数量较少，无法拟合平面！") + continue ''' 拟合平面，计算法向量 ''' @@ -161,23 +162,26 @@ class Detection: seg_point.append(pm_seg) cv2.polylines(img, [box], True, (0, 255, 0), 2) - min_value = min(Depth_Z) # 求深度最小值 - min_idx = Depth_Z.index(min_value) # 求最小值对应索引 - cv2.circle(img, (uv[min_idx][0], uv[min_idx][1]), 30, (0, 0, 255), 20) # 标出中心点 + _idx = find_position(Depth_Z, RegionalArea, 100, True) - if Point_isVision == True: - pcd = o3d.geometry.PointCloud() - pcd.points = o3d.utility.Vector3dVector(seg_point[min_idx]) - plane_model, inliers = pcd.segment_plane(distance_threshold=0.01, - ransac_n=5, - num_iterations=5000) - inlier_cloud = pcd.select_by_index(inliers) # 点云可视化 - inlier_cloud.paint_uniform_color([1.0, 0, 0]) - outlier_cloud = pcd.select_by_index(inliers, invert=True) - outlier_cloud.paint_uniform_color([0, 1, 0]) - o3d.visualization.draw_geometries([inlier_cloud, outlier_cloud, pcd2]) + if _idx == None: + return 1, img, None, None + else: + cv2.circle(img, (uv[_idx][0], uv[_idx][1]), 30, (255, 0, 0), 20) # 标出中心点 - return 1, img, xyz[min_idx], nx_ny_nz[min_idx] + if Point_isVision == True: + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(seg_point[_idx]) + plane_model, inliers = pcd.segment_plane(distance_threshold=0.01, + ransac_n=5, + num_iterations=5000) + inlier_cloud = pcd.select_by_index(inliers) # 点云可视化 + inlier_cloud.paint_uniform_color([1.0, 0, 0]) + outlier_cloud = pcd.select_by_index(inliers, invert=True) + outlier_cloud.paint_uniform_color([0, 1, 0]) + o3d.visualization.draw_geometries([inlier_cloud, outlier_cloud, pcd2]) + + return 1, img, xyz[_idx], nx_ny_nz[_idx] else: return 1, img, None, None diff --git a/Vision/camera_coordinate_dete_test.py b/Vision/camera_coordinate_dete_test.py index b9fd9e2..59ea15c 100644 --- a/Vision/camera_coordinate_dete_test.py +++ b/Vision/camera_coordinate_dete_test.py @@ -5,7 +5,7 @@ # @Author : hjw # @File : camera_coordinate_dete_test.py.py ''' -from camera_coordinate_dete import Detection +from camera_coordinate_dete_img import Detection import cv2 detection = Detection() diff --git a/requirements b/requirements index dde7305..7a5275d 100644 --- a/requirements +++ b/requirements @@ -22,3 +22,5 @@ PyRVC=1.10.0 torch=1.13.1 torchvision=0.14.1 ultralytics=8.2.86 +openvino=2024.0.0 +lapx=0.5.10