feat: add ai object detection extension (#24)

* feat: implement Arduino object detection extension and integrate video processing

* fix: update enableVideo method to capture and emit video frames in base64 format

* fix: streamline base64 conversion in enableVideo method for object detection

* fix: improve formatting and add detection result logging in Arduino object detection extension

* fix: add detection results to response in object detection callback

* fix: enhance detection result logging and add model labels to Arduino object detection extension

* fix: implement drawing and clearing of bounding boxes for detected objects in Arduino object detection extension

* fix: implement toggle functionality for bounding boxes and enhance detection result handling in Arduino object detection extension

* fix: update bounding box visibility logic and enhance detection handling in Arduino object detection extension

* fix: update WebSocket server URL to use a static IP address in Arduino extensions

* fix: implement Detection class for object properties and enhance bounding box handling in Arduino object detection extension

* fix: update clearBoundingBoxes command text and improve getColorByConfidence return type in Arduino object detection extension

* fix: refactor bounding box handling and improve visibility logic in Arduino object detection extension

* feat: add model labels constants and update model labels menu in Arduino object detection extension

* refactor: move model labels and Detection class to a separate file for better organization in Arduino object detection extension

* fix: correct class name casing for ArduinoObjectDetection and update prototype references in Arduino object detection extension

* fix: update color handling in _getColorByConfidence method for improved confidence visualization

* fix: update wsServerURL to use dynamic protocol and hostname for better compatibility

* fix: correct spacing in wsServerURL declaration for consistency

* fix: restore asset copying in app:build task for complete build process

* fix: add person detection functionality and update bounding box commands

* fix: update wsServerURL to use dynamic protocol and hostname for better compatibility; clean up argument declarations and improve person detection logic

* fix: add "when person detected" block and corresponding functionality to ArduinoObjectDetection

* fix: correct indentation in whenPersonDetected method for consistency

* fix: update object detection to decode image data directly from base64

* fix: refactor person detection logic and rename related methods

* feat: implement automatic detection loop and bounding box controls

* feat: implement start and stop methods for automatic detection loop

* fix: improve code formatting and consistency in object detection extension

* fix: remove unused imports from main.py

* fix: handle missing image data in object detection callback

Author: dido18

app.yaml

@@ -4,4 +4,5 @@ ports:
   - 7000
 bricks:
   - arduino:web_ui
+  - arduino:object_detection
 icon: 🐱

python/main.py

@@ -1,9 +1,10 @@
 from arduino.app_utils import App, Bridge
 from arduino.app_bricks.web_ui import WebUI
+from arduino.app_bricks.object_detection import ObjectDetection
 import time
+import base64
 
-ui = WebUI()
-ui.on_connect(lambda sid: (print(f"Client connected: {sid} "),))
+object_detection = ObjectDetection()
 
 
 def on_matrix_draw(_, data):
@@ -45,8 +46,40 @@ def on_set_led_rgb(_, data):
     Bridge.call("set_led_rgb", led, r_digital, g_digital, b_digital)
 
 
+def on_detect_objects(client_id, data):
+    """Callback function to handle object detection requests."""
+    try:
+        image_data = data.get("image")
+        confidence = data.get("confidence", 0.5)
+        if not image_data:
+            # TODO: implement the 'detection_error` in the extension
+            ui.send_message("detection_error", {"error": "No image data"})
+            return
+
+        start_time = time.time() * 1000
+        results = object_detection.detect(base64.b64decode(image_data), confidence=confidence)
+        diff = time.time() * 1000 - start_time
+
+        if results is None:
+            ui.send_message("detection_error", {"error": "No results returned"})
+            return
+
+        response = {
+            "detection": results.get("detection", []),
+            "detection_count": len(results.get("detection", [])) if results else 0,
+            "processing_time": f"{diff:.2f} ms",
+        }
+        ui.send_message("detection_result", response)
+
+    except Exception as e:
+        ui.send_message("detection_error", {"error": str(e)})
+
+
+ui = WebUI()
+ui.on_connect(lambda sid: (print(f"Client connected: {sid} "),))
 ui.on_message("matrix_draw", on_matrix_draw)
 ui.on_message("set_led_rgb", on_set_led_rgb)
+ui.on_message("detect_objects", on_detect_objects)
 
 
 def on_modulino_button_pressed(btn):

scratch-arduino-extensions/packages/scratch-vm/src/extensions/arduino_object_detection/index.js

@@ -0,0 +1,328 @@
+const BlockType = require("../../../../../../scratch-editor/packages/scratch-vm/src/extension-support/block-type");
+const ArgumentType = require(
+  "../../../../../../scratch-editor/packages/scratch-vm/src/extension-support/argument-type",
+);
+const io = require("../socket.io.min.js");
+const Video = require("../../../../../../scratch-editor/packages/scratch-vm/src/io/video");
+const Rectangle = require("../../../../../../scratch-editor/packages/scratch-render/src/Rectangle.js");
+const StageLayering = require("../../../../../../scratch-editor/packages/scratch-vm/src/engine/stage-layering.js");
+const { Detection, MODEL_LABELS } = require("./object_detection");
+
+/**
+ * Url of icon to be displayed at the left edge of each extension block.
+ * @type {string}
+ */
+// eslint-disable-next-line max-len
+const iconURI = "";
+
+/**
+ * Url of icon to be displayed in the toolbox menu for the extension category.
+ * @type {string}
+ */
+// eslint-disable-next-line max-len
+const menuIconURI = "";
+
+const wsServerURL = `${window.location.protocol}//${window.location.hostname}:7000`;
+
+/**
+ * RGB color constants for confidence visualization
+ */
+const RGB_COLORS = {
+  RED: { r: 1.0, g: 0.0, b: 0.0 },
+  ORANGE: { r: 1.0, g: 0.5, b: 0.0 },
+  GREEN: { r: 0.0, g: 1.0, b: 0.0 },
+};
+
+class ArduinoObjectDetection {
+  constructor(runtime) {
+    this.runtime = runtime;
+
+    /** @type {Array<Detection>} */
+    this.detectedObjects = [];
+
+    this._penSkinId = null;
+
+    this._isfaceDetected = false;
+
+    /** @type {number|null} */
+    this._loopIntervalId = null;
+
+    /** @type {boolean} */
+    this._isLoopRunning = false;
+
+    /** @type {boolean} */
+    this._enableBoundingBoxes = true;
+
+    this.runtime.on("PROJECT_LOADED", () => {
+      if (!this.runtime.renderer) {
+        console.log("Renderer is NOT available in runtime.");
+        return;
+      }
+      if (!this._penSkinId) {
+        this._penSkinId = this.runtime.renderer.createPenSkin();
+        this.penDrawableId = this.runtime.renderer.createDrawable(StageLayering.PEN_LAYER);
+        this.runtime.renderer.updateDrawableSkinId(this.penDrawableId, this._penSkinId);
+      }
+    });
+
+    this.io = io(wsServerURL, {
+      path: "/socket.io",
+      transports: ["polling", "websocket"],
+      autoConnect: true,
+    });
+
+    this.io.on("detection_result", (data) => {
+      this.detectedObjects = [];
+
+      this._clearBoundingBoxes();
+
+      data.detection.forEach((detection) => {
+        const [x1, y1, x2, y2] = detection.bounding_box_xyxy;
+
+        const detectionObject = new Detection(
+          detection.class_name,
+          this._createRectangleFromBoundingBox(x1, y1, x2, y2),
+          parseFloat(detection.confidence),
+        );
+        this.detectedObjects.push(detectionObject);
+
+        console.log(
+          `Detected ${detectionObject.label} with confidence ${
+            detectionObject.confidence.toFixed(2)
+          } took ${data.processing_time}`,
+        );
+      });
+
+      const personDetected = this.detectedObjects.some(detectionObject =>
+        detectionObject.label === MODEL_LABELS.PERSON
+      );
+      this._isfaceDetected = personDetected;
+
+      if (this._enableBoundingBoxes) {
+        this._drawBoundingBoxes();
+      } else {
+        this._clearBoundingBoxes();
+      }
+    });
+  }
+}
+
+ArduinoObjectDetection.prototype.getInfo = function() {
+  return {
+    id: "ArduinoObjectDetection",
+    name: "Arduino Object Detection",
+    menuIconURI: menuIconURI,
+    blockIconURI: iconURI,
+    blocks: [
+      {
+        opcode: "whenPersonDetected",
+        blockType: BlockType.HAT,
+        text: "when person detected",
+        func: "whenPersonDetected",
+        arguments: {},
+      },
+      {
+        opcode: "startDetectionLoop",
+        blockType: BlockType.COMMAND,
+        text: "start detection",
+        func: "startDetectionLoop",
+        arguments: {},
+      },
+      {
+        opcode: "stopDetectionLoop",
+        blockType: BlockType.COMMAND,
+        text: "stop detection",
+        func: "stopDetectionLoop",
+        arguments: {},
+      },
+      {
+        opcode: "isPersonDetected",
+        blockType: BlockType.BOOLEAN,
+        text: "is person detected",
+        func: "isPersonDetected",
+        arguments: {},
+      },
+      {
+        opcode: "showBoundingBoxes",
+        blockType: BlockType.COMMAND,
+        text: "show bounding boxes",
+        func: "showBoundingBoxes",
+        arguments: {},
+      },
+      {
+        opcode: "hideBoundingBoxes",
+        blockType: BlockType.COMMAND,
+        text: "hide bounding boxes",
+        func: "hideBoundingBoxes",
+        arguments: {},
+      },
+    ],
+    menus: {
+      modelsLabels: Object.values(MODEL_LABELS).sort(),
+    },
+  };
+};
+
+ArduinoObjectDetection.prototype.startDetectionLoop = function(args) {
+  if (this._isLoopRunning) {
+    console.log("Detection loop is already running");
+    return;
+  }
+
+  this._isLoopRunning = true;
+  this.runtime.ioDevices.video.enableVideo();
+  this._loop();
+
+  this._loopIntervalId = setInterval(() => {
+    this._loop();
+  }, 1000); // 1000ms = 1s
+};
+
+ArduinoObjectDetection.prototype.stopDetectionLoop = function(args) {
+  this.runtime.ioDevices.video.disableVideo();
+  this.hideBoundingBoxes();
+
+  if (!this._isLoopRunning) {
+    console.log("Detection loop is not running");
+    return;
+  }
+
+  console.log("Stopping detection loop");
+  this._isLoopRunning = false;
+  if (this._loopIntervalId) {
+    clearInterval(this._loopIntervalId);
+    this._loopIntervalId = null;
+  }
+};
+
+ArduinoObjectDetection.prototype._loop = function() {
+  if (!this._isLoopRunning) {
+    return;
+  }
+  this._detectObjects();
+
+  // Note: The face detection state (_isfaceDetected) will be updated
+  // automatically when the detection_result event is received
+};
+
+ArduinoObjectDetection.prototype.whenPersonDetected = function(args) {
+  return this.isPersonDetected();
+};
+
+ArduinoObjectDetection.prototype.isPersonDetected = function(args) {
+  return this._isfaceDetected;
+};
+
+ArduinoObjectDetection.prototype.hideBoundingBoxes = function(args) {
+  this._enableBoundingBoxes = false;
+  this._clearBoundingBoxes();
+};
+
+ArduinoObjectDetection.prototype.showBoundingBoxes = function(args) {
+  this._enableBoundingBoxes = true;
+  this._drawBoundingBoxes();
+};
+
+ArduinoObjectDetection.prototype._detectObjects = function(args) {
+  if (!this.runtime.ioDevices) {
+    console.log("No ioDevices available.");
+    return;
+  }
+  const canvas = this.runtime.ioDevices.video.getFrame({
+    format: Video.FORMAT_CANVAS,
+    dimensions: [480, 360], // the same as the stage resolution
+  });
+  if (!canvas) {
+    console.log("No canvas available from video frame.");
+    return;
+  }
+  const dataUrl = canvas.toDataURL("image/png");
+  const base64Frame = dataUrl.split(",")[1];
+  this.io.emit("detect_objects", { image: base64Frame });
+};
+
+ArduinoObjectDetection.prototype._clearBoundingBoxes = function(args) {
+  if (!this.runtime.renderer || !this._penSkinId) {
+    console.log("Renderer or pen skin not available for clearing");
+    return;
+  }
+  const penSkin = this.runtime.renderer._allSkins[this._penSkinId];
+  if (penSkin && penSkin.clear) {
+    penSkin.clear();
+  } else {
+    console.log("Could not clear pen skin");
+  }
+};
+
+ArduinoObjectDetection.prototype._drawBoundingBoxes = function(args) {
+  this.detectedObjects.forEach(detectionObject => {
+    const { r, g, b } = this._getColorByConfidence(detectionObject.confidence);
+    const penAttributes = {
+      color4f: [r, g, b, 1.0],
+      diameter: 3,
+    };
+    this._drawRectangleWithPen(detectionObject.rectangle, penAttributes);
+  });
+};
+
+/**
+ * Get pen color based on confidence level
+ * @param {number} confidence - Confidence score (0 to 100)
+ * @returns {Object} RGB color object {r, g, b} in 0-1 range
+ */
+ArduinoObjectDetection.prototype._getColorByConfidence = function(confidence) {
+  if (confidence >= 90) {
+    return RGB_COLORS.GREEN;
+  }
+  if (confidence >= 75 && confidence < 90) {
+    return RGB_COLORS.ORANGE;
+  }
+  return RGB_COLORS.RED;
+};
+
+/**
+ * Draw a rectangle using the Rectangle class and pen system
+ * @param {Rectangle} rectangle - Rectangle object defining the bounds
+ * @param {Object} penAttributes - Pen drawing attributes (color, thickness)
+ */
+ArduinoObjectDetection.prototype._drawRectangleWithPen = function(rectangle, penAttributes) {
+  if (!this.runtime.renderer || !this._penSkinId) {
+    console.log("Renderer or pen skin not available");
+    return;
+  }
+
+  // TODO: Get the pen skin object in a better way
+  const penSkin = this.runtime.renderer._allSkins[this._penSkinId];
+  if (!penSkin) {
+    console.log("Pen skin not found");
+    return;
+  }
+
+  const left = rectangle.left;
+  const right = rectangle.right;
+  const bottom = rectangle.bottom;
+  const top = rectangle.top;
+
+  penSkin.drawLine(penAttributes, left, top, right, top);
+  penSkin.drawLine(penAttributes, right, top, right, bottom);
+  penSkin.drawLine(penAttributes, right, bottom, left, bottom);
+  penSkin.drawLine(penAttributes, left, bottom, left, top);
+};
+
+ArduinoObjectDetection.prototype._createRectangleFromBoundingBox = function(x1, y1, x2, y2) {
+  x1 = x1 - 240; // 0-480 -> -240 to +240
+  y1 = -(y1 - 180); // 0-360 -> -180 to +180
+  x2 = x2 - 240;
+  y2 = -(y2 - 180);
+
+  const left = Math.min(x1, x2);
+  const right = Math.max(x1, x2);
+  const bottom = Math.min(y1, y2);
+  const top = Math.max(y1, y2);
+
+  const rectangle = new Rectangle();
+  rectangle.initFromBounds(left, right, bottom, top);
+  return rectangle;
+};
+
+module.exports = ArduinoObjectDetection;