1

src/src/main/java/com/my/graphiteDigesterBg/MyApplication.java

@@ -1,9 +1,118 @@
 package com.my.graphiteDigesterBg;
+import org.opencv.core.*;
+import org.opencv.highgui.HighGui;
+import org.opencv.imgcodecs.Imgcodecs;
+import org.opencv.imgproc.Imgproc;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+import java.util.ArrayList;
+import java.util.List;
+
 @SpringBootApplication
 public class MyApplication {
     public static void main(String[] args) {
         SpringApplication.run(MyApplication.class, args);
     }
+
+    private static void test() {
+        System.load("D:/ProgramFiles/OpenCV/opencv/build/java/x64/opencv_java490.dll");
+        var srcImg = Imgcodecs.imread("D:/image.png");
+        HighGui.imshow("preview", srcImg);
+        HighGui.waitKey(100);
+
+        // resize to 800x800
+        var size = new Size(800, 800);
+        var resizedImg = new Mat(800, 800, CvType.CV_8UC3);
+        Imgproc.resize(srcImg, resizedImg, size);
+        var resizedSrcImg = resizedImg.clone();
+
+        // slots
+        var leftOffset = 100;
+        var topOffset = 100;
+        var xDistance = 200;
+        var yDistance = 200;
+        int radius = 80;
+        int thickness = 5;
+        List<MatOfPoint> slotContours = new ArrayList<>();
+        Scalar color = new Scalar(0, 0, 255);
+        for ( var i=0; i<16; i++) {
+            var centerX = leftOffset + (i%4)*xDistance;
+            var centerY = topOffset + (i/4)*yDistance;
+            Point center = new Point(centerX, centerY);
+            Imgproc.circle(resizedImg, center, radius, color, thickness);
+
+            List<Point> pointsList = new ArrayList<>();
+            for (int angle = 0; angle <= 360; angle += 10) {
+                double x = center.x + radius * Math.cos(Math.toRadians(angle));
+                double y = center.y + radius * Math.sin(Math.toRadians(angle));
+                pointsList.add(new Point(x, y));
+            }
+
+            // Convert List<Point> to MatOfPoint
+            MatOfPoint matOfPoint = new MatOfPoint();
+            matOfPoint.fromList(pointsList);
+            slotContours.add(matOfPoint);
+        }
+        HighGui.imshow("preview", resizedImg);
+        HighGui.waitKey(100);
+
+        var grayImg = new Mat();
+        Imgproc.cvtColor(resizedSrcImg, grayImg, Imgproc.COLOR_BGR2GRAY);
+        HighGui.imshow("preview", grayImg);
+        HighGui.waitKey(100);
+
+        var blurImg = new Mat();
+        Imgproc.blur(grayImg, blurImg, new org.opencv.core.Size(30, 30));
+        HighGui.imshow("preview", blurImg);
+        HighGui.waitKey(100);
+
+        var wbImg = new Mat();
+        Imgproc.threshold(blurImg, wbImg, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
+        HighGui.imshow("preview", wbImg);
+        HighGui.waitKey(100);
+
+        List<MatOfPoint> contours = new ArrayList<>();
+        Imgproc.findContours(wbImg, contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
+
+        var resultImg = new Mat(800, 800, CvType.CV_8UC3);
+        // 绘制轮廓
+        var contourColor = new Scalar(255, 255, 255);
+        for ( var contour : contours ) {
+            Imgproc.drawContours(resultImg, List.of(contour), -1, contourColor, 3);
+            HighGui.imshow("preview", resultImg);
+            HighGui.waitKey(100);
+        }
+
+        List<Integer> errorSlotIndexList = new ArrayList<>();
+        var slotContourColor = new Scalar(0, 0, 255);
+        var slotContoursErrorColor = new Scalar(0, 255, 255);
+        for ( var index=0; index<16; index++ ) {
+            var slotContour = slotContours.get(index);
+            var isErrorSlot = false;
+            Rect slotRect = Imgproc.boundingRect(slotContour);
+
+            for ( var contour : contours ) {
+                Rect contourRect = Imgproc.boundingRect(contour);
+                if ( slotRect.contains(contourRect.tl()) && slotRect.contains(contourRect.br()) ) {
+                    errorSlotIndexList.add(index);
+                    isErrorSlot = true;
+                    break;
+                }
+            }
+
+            if ( isErrorSlot ) {
+                Imgproc.drawContours(resultImg, List.of(slotContour), -1, slotContoursErrorColor, 3);
+            } else {
+                Imgproc.drawContours(resultImg, List.of(slotContour), -1, slotContourColor, 3);
+            }
+            HighGui.imshow("preview", resultImg);
+            HighGui.waitKey(100);
+        }
+//        System.out.println("error slot index list: " + errorSlotIndexList);
+
+        HighGui.imshow("preview", resultImg);
+        HighGui.waitKey(100);
+    }
 }

src/src/main/java/com/my/graphiteDigesterBg/MyApplicationRunner.java

@@ -1,2 +1,17 @@
-package com.my.graphiteDigesterBg;public class MyApplicationRunner {
+package com.my.graphiteDigesterBg;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
+import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
+@Order(1)
+@Component
+public class MyApplicationRunner implements ApplicationRunner {
+    @Value("${opencv.library-path}")
+    private String opencvLibraryPath;
+
+    @Override
+    public void run(ApplicationArguments args) throws Exception {
+        System.load(this.opencvLibraryPath);
+    }
 }

src/src/main/java/com/my/graphiteDigesterBg/api/ApiCamera.java

@@ -1,6 +1,9 @@
 package com.my.graphiteDigesterBg.api;
 import com.my.graphiteDigesterBg.diframe.DiApiControllerBase;
 import com.my.graphiteDigesterBg.diframe.DiApiResponse;
+import com.my.graphiteDigesterBg.diframe.DiDevice;
+import com.my.graphiteDigesterBg.move.MoveDetectErrorSlots;
+import jakarta.annotation.Resource;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PostMapping;
@@ -17,6 +20,9 @@ public class ApiCamera extends DiApiControllerBase {
     @Value("${camera.image-path}")
     private String imagePath;
 
+    @Resource
+    private DiDevice device;
+
     @ResponseBody
     @PostMapping("/api/camera/image")
     public DiApiResponse getImage() {
@@ -39,4 +45,18 @@ public class ApiCamera extends DiApiControllerBase {
         String base64Data = "data:image/image/png;base64," + base64Image;
         return this.success(Map.of("data",base64Data));
     }
+
+    @ResponseBody
+    @PostMapping("/api/camera/error-detect")
+    public DiApiResponse errorDetect() {
+        var detect = new MoveDetectErrorSlots();
+        detect.setDevice(this.device);
+        detect.run();
+        var result = detect.getErrorSlotIndexList();
+        return this.success(Map.of(
+            "list", result,
+            "img", detect.getResultImageInBase64()
+        ));
+    }
+
 }

src/src/main/java/com/my/graphiteDigesterBg/move/MoveDetectErrorSlots.java

@@ -1,2 +1,120 @@
-package com.my.graphiteDigesterBg.move;public class MoveDetectErrorSlots {
+package com.my.graphiteDigesterBg.move;
+import com.my.graphiteDigesterBg.diframe.DiTaskMoveBase;
+import org.opencv.core.*;
+import org.opencv.imgcodecs.Imgcodecs;
+import org.opencv.imgproc.Imgproc;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+public class MoveDetectErrorSlots extends DiTaskMoveBase {
+    // white and black image
+    private Mat wbImage;
+    // error slot index list
+    private List<Integer> errorSlotIndexList;
+    // result image
+    private Mat resultImage;
+
+    @Override
+    public void run() {
+        this.setupImage();
+        this.detect();
+    }
+
+    // get error slot index list
+    public List<Integer> getErrorSlotIndexList() {
+        return this.errorSlotIndexList;
+    }
+
+    // get result image in base64
+    public String getResultImageInBase64() {
+        MatOfByte matOfByte = new MatOfByte();
+        Imgcodecs.imencode(".png", this.resultImage, matOfByte);
+        byte[] byteArray = matOfByte.toArray();
+        return "data:image/png;base64," + new String(Base64.getEncoder().encode(byteArray));
+    }
+
+    // detect error slots
+    private void detect() {
+        List<MatOfPoint> contours = new ArrayList<>();
+        Imgproc.findContours(this.wbImage, contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
+
+        List<MatOfPoint> slotContours = this.getSlotContours();
+        this.errorSlotIndexList = new ArrayList<>();
+        var slotContoursErrorColor = new Scalar(0, 0, 255);
+        for ( var index=0; index<16; index++ ) {
+            var slotContour = slotContours.get(index);
+            Rect slotRect = Imgproc.boundingRect(slotContour);
+            var isErrorSlot = false;
+            for ( var contour : contours ) {
+                Rect contourRect = Imgproc.boundingRect(contour);
+                if ( slotRect.contains(contourRect.tl()) && slotRect.contains(contourRect.br()) ) {
+                    errorSlotIndexList.add(index);
+                    isErrorSlot = true;
+                    break;
+                }
+            }
+            if ( isErrorSlot ) {
+                Imgproc.drawContours(this.resultImage, List.of(slotContour), -1, slotContoursErrorColor, 10);
+            }
+        }
+    }
+
+    // get slot contours
+    private List<MatOfPoint> getSlotContours() {
+        var leftOffset = 100;
+        var topOffset = 100;
+        var xDistance = 200;
+        var yDistance = 200;
+        int radius = 80;
+        var color = new Scalar(0, 255, 0);
+        int thickness = 5;
+        List<MatOfPoint> slotContours = new ArrayList<>();
+        for ( var i=0; i<16; i++) {
+            var centerX = leftOffset + (i%4)*xDistance;
+            var centerY = topOffset + (i/4)*yDistance;
+            Point center = new Point(centerX, centerY);
+            Imgproc.circle(this.resultImage, center, radius, color, thickness);
+
+            List<Point> pointsList = new ArrayList<>();
+            for (int angle = 0; angle <= 360; angle += 10) {
+                double x = center.x + radius * Math.cos(Math.toRadians(angle));
+                double y = center.y + radius * Math.sin(Math.toRadians(angle));
+                pointsList.add(new Point(x, y));
+            }
+
+            // Convert List<Point> to MatOfPoint
+            MatOfPoint matOfPoint = new MatOfPoint();
+            matOfPoint.fromList(pointsList);
+            slotContours.add(matOfPoint);
+        }
+
+        return slotContours;
+    }
+
+    // setup image
+    private void setupImage() {
+        var options = this.getDevice().getOption();
+        var srcImg = Imgcodecs.imread("D:/image.png");
+
+        // resize image
+        Integer imgWidth = options.getInteger("ErrorDetectImgSizeWidth");
+        Integer imgHeight = options.getInteger("ErrorDetectImgSizeHeight");
+        var detectSize = new Size(imgWidth, imgHeight);
+        var resizedImg = new Mat(imgHeight, imgWidth, CvType.CV_8UC3);
+        Imgproc.resize(srcImg, resizedImg, detectSize);
+
+        // convert to gray
+        var grayImg = new Mat();
+        Imgproc.cvtColor(resizedImg, grayImg, Imgproc.COLOR_BGR2GRAY);
+
+        // blur
+        var blurImg = new Mat();
+        Imgproc.blur(grayImg, blurImg, new org.opencv.core.Size(30, 30));
+
+        // white and black
+        var wbImg = new Mat();
+        Imgproc.threshold(blurImg, wbImg, 0, 255, Imgproc.THRESH_BINARY | Imgproc.THRESH_OTSU);
+        this.wbImage = wbImg;
+        this.resultImage = resizedImg.clone();
+    }
 }

src/src/main/java/com/my/graphiteDigesterBg/resource/ResHeatingTubeRackSlotManager.java

@@ -41,11 +41,11 @@ public class ResHeatingTubeRackSlotManager extends DiResourceManagerBase {
 
     // refresh slot temperature
     public void refreshSlotTemperature() {
-        var registers = this.getDevice().getRegisters();
-        for (ResHeatingTubeRackSlot slot : this.slots) {
-            slot.temperature = registers.getValue("HeatingPlateSlotTemperature" + slot.index);
-            LOG.info("[Heating Slot {}] temperature: {}", slot.index, slot.temperature);
-        }
+//        var registers = this.getDevice().getRegisters();
+//        for (ResHeatingTubeRackSlot slot : this.slots) {
+//            slot.temperature = registers.getValue("HeatingPlateSlotTemperature" + slot.index);
+//            LOG.info("[Heating Slot {}] temperature: {}", slot.index, slot.temperature);
+//        }
     }
 
     // get resource data

src/src/main/resources/application-dev.yml

@@ -11,6 +11,9 @@ server:
 camera:
   image-path: D:/image.png
 
+opencv:
+  library-path: D:/ProgramFiles/OpenCV/opencv/build/java/x64/opencv_java490.dll
+
 #mybatis:
 #  configuration:
 #    log-impl: org.apache.ibatis.logging.stdout.StdOutImpl

src/web/src/pages/main/contents/OperationCamera.vue

@@ -34,9 +34,14 @@
             ><ExperimentOutlined /></a-button>
           </a-col>
         </a-row>
-        <div class="text-right">
+        <div class="flex flex-row justify-between">
+          <div>
+            <a-button @click="actionErrorSlotDetect">异常识别</a-button>
+          </div>
+          <div>
             <a-button @click="actionTubeMoveToErrorSlot">移动至异常区</a-button>
           </div>
+        </div>
       </a-col>
     </a-row>
   </a-modal>
@@ -118,6 +123,13 @@ async function actionTubeMoveToErrorSlot() {
 async function actionTakeShotCancel() {
     await client.taskAppend('SampleMoveToHeatPlate');
 }
+
+// 异常识别
+async function actionErrorSlotDetect() {
+    let response = await client.cameraErrorDetect();
+    imageShotData.value = response.img;
+    errorTubes.value = response.list;
+}
 </script>
 <style>
 .camera {background: linear-gradient(180deg, rgba(5, 10, 39, 0.5) 0%, rgba(4, 10, 52, 0.5) 97%);backdrop-filter: blur(194px);}

src/web/src/utils/ApiClient.js

@@ -131,4 +131,9 @@ export default class ApiClient {
     async operationLogList(params) {
         return await this.call('operation-log/list', params);
     }
+
+    // camera error detect
+    async cameraErrorDetect() {
+        return await this.call('camera/error-detect');
+    }
 }