UPDATE Vision

Vision/camera_coordinate_dete.py

@@ -24,15 +24,18 @@ class Detection:
         self.camera_rvc = camera()
         self.model = yolov8_segment()
         self.model.load_model(model_path, device)
-        if self.camera_rvc.caminit_isok == True:
+        # if self.camera_rvc.caminit_isok == True:
-            print("RVC X Camera is opened!")
+        #     print("RVC X Camera is opened!")
-        else:
+        # else:
-            print("RVC X Camera is not opened!")
+        #     print("RVC X Camera is not opened!")
 
-    def get_position(self, Point_isVision=True):
+    def get_position(self, Point_isVision=False, first_depth=False):
         ""
         '''
-        :param api: None
+        :param api: 
+            Point_isVision：   点云可视化
+            first_depth：      按Z轴深度返回坐标及向量
+        
         :return: ret , img, (x,y,z), (nx, ny, nz)
         '''
         ret, img, pm = self.camera_rvc.get_img_and_point_map()            # 拍照，获取图像及
@@ -110,6 +113,9 @@ class Detection:
                         pm_seg = select_point.reshape(-1, 3)
                         pm_seg = pm_seg[~np.isnan(pm_seg).all(axis=-1), :]      # 剔除 nan
                         # cv2.imshow('result', piont_result)
+                        if pm_seg.size < 300:
+                            print("分割出的点云数量较少，无法拟合平面！")
+                            continue
 
                         '''
                         拟合平面，计算法向量
@@ -157,7 +163,7 @@ class Detection:
                             seg_point.append(pm_seg)
                         cv2.polylines(img, [box], True, (0, 255, 0), 2)
 
-                    _idx = find_position(Depth_Z, RegionalArea, 100,True)
+                    _idx = find_position(Depth_Z, RegionalArea, 100, first_depth)
 
                     if _idx == None:
                         return 1, img, None, None

Vision/camera_coordinate_dete_img.py

@@ -13,15 +13,15 @@ import cv2
 import open3d as o3d
 from Vision.tool.CameraRVC import camera
 from Vision.yolo.yolov8_pt_seg import yolov8_segment
-
+from Vision.tool.utils import find_position
 
 class Detection:
 
     def __init__(self):
         model_path = './pt_model/best.pt'
         device = 'cpu'
-        img_path = './pt_model/test0824.png'
+        img_path = './pt_model/test0910.png'
-        point_path = './pt_model/test0824.xyz'
+        point_path = './pt_model/test0910.xyz'
         self.model = yolov8_segment()
         self.model.load_model(model_path, device)
         self.img = cv2.imread(img_path)
@@ -39,6 +39,7 @@ class Detection:
         ret = 1
         img = self.img
         pm = self.point
+
         if ret == 1:
             flag, det_cpu, dst_img, masks, category_names = self.model.model_inference(img, 0)
             if flag == 1:
@@ -55,7 +56,7 @@ class Detection:
                     pm2[:, 2] = pm2[:, 2] + 0.25
                     pcd2 = o3d.geometry.PointCloud()
                     pcd2.points = o3d.utility.Vector3dVector(pm2)
-                    #o3d.visualization.draw_geometries([pcd2])
+                    # o3d.visualization.draw_geometries([pcd2])
 
                 for i, item in enumerate(det_cpu):
 
@@ -87,7 +88,7 @@ class Detection:
                     ret, binary = cv2.threshold(imgray, 10, 255, 0)
                     # 阈值 二进制图像
                     # cv2.imshow('bin',binary)
-                    # cv2.waitKey(0)
+                    # cv2.waitKey(1)
                     contours, hierarchy = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
                     # all_piont_list = contours_in(contours)
                     # print(len(all_piont_list))
@@ -106,15 +107,15 @@ class Detection:
                     pixel_point2 = cv2.findNonZero(mask_inside)
                     # result = np.zeros_like(color_image)
                     select_point = []
-
+                    for i in range(pixel_point2.shape[0]):
-                    for t in range(pixel_point2.shape[0]):
+                        select_point.append(pm[pixel_point2[i][0][1] + box_y1, pixel_point2[i][0][0] + box_x1])
-                        select_point.append(pm[pixel_point2[t][0][1] + box_y1, pixel_point2[t][0][0]+ box_x1])
-
                     select_point = np.array(select_point)
                     pm_seg = select_point.reshape(-1, 3)
                     pm_seg = pm_seg[~np.isnan(pm_seg).all(axis=-1), :]  # 剔除 nan
                     # cv2.imshow('result', piont_result)
-
+                    if pm_seg.size < 300:
+                        print("分割出的点云数量较少，无法拟合平面！")
+                        continue
                     '''
                     拟合平面，计算法向量
                     '''
@@ -161,23 +162,26 @@ class Detection:
                         seg_point.append(pm_seg)
                     cv2.polylines(img, [box], True, (0, 255, 0), 2)
 
-                min_value = min(Depth_Z)  # 求深度最小值
+                _idx = find_position(Depth_Z, RegionalArea, 100, True)
-                min_idx = Depth_Z.index(min_value)  # 求最小值对应索引
-                cv2.circle(img, (uv[min_idx][0], uv[min_idx][1]), 30, (0, 0, 255), 20)  # 标出中心点
 
-                if Point_isVision == True:
+                if _idx == None:
-                    pcd = o3d.geometry.PointCloud()
+                    return 1, img, None, None
-                    pcd.points = o3d.utility.Vector3dVector(seg_point[min_idx])
+                else:
-                    plane_model, inliers = pcd.segment_plane(distance_threshold=0.01,
+                    cv2.circle(img, (uv[_idx][0], uv[_idx][1]), 30, (255, 0, 0), 20)  # 标出中心点
-                                                             ransac_n=5,
-                                                             num_iterations=5000)
-                    inlier_cloud = pcd.select_by_index(inliers)  # 点云可视化
-                    inlier_cloud.paint_uniform_color([1.0, 0, 0])
-                    outlier_cloud = pcd.select_by_index(inliers, invert=True)
-                    outlier_cloud.paint_uniform_color([0, 1, 0])
-                    o3d.visualization.draw_geometries([inlier_cloud, outlier_cloud, pcd2])
 
-                return 1, img, xyz[min_idx], nx_ny_nz[min_idx]
+                    if Point_isVision == True:
+                        pcd = o3d.geometry.PointCloud()
+                        pcd.points = o3d.utility.Vector3dVector(seg_point[_idx])
+                        plane_model, inliers = pcd.segment_plane(distance_threshold=0.01,
+                                                                 ransac_n=5,
+                                                                 num_iterations=5000)
+                        inlier_cloud = pcd.select_by_index(inliers)  # 点云可视化
+                        inlier_cloud.paint_uniform_color([1.0, 0, 0])
+                        outlier_cloud = pcd.select_by_index(inliers, invert=True)
+                        outlier_cloud.paint_uniform_color([0, 1, 0])
+                        o3d.visualization.draw_geometries([inlier_cloud, outlier_cloud, pcd2])
+
+                    return 1, img, xyz[_idx], nx_ny_nz[_idx]
             else:
                 return 1, img, None, None
 

Vision/camera_coordinate_dete_test.py

@@ -5,7 +5,7 @@
 # @Author  : hjw
 # @File    : camera_coordinate_dete_test.py.py
 '''
-from camera_coordinate_dete import Detection
+from camera_coordinate_dete_img import Detection
 import cv2
 
 detection = Detection()

requirements

@@ -22,3 +22,5 @@ PyRVC=1.10.0
 torch=1.13.1
 torchvision=0.14.1
 ultralytics=8.2.86
+openvino=2024.0.0
+lapx=0.5.10