feat(barcode): add logic to use barcodedetector API

Author: gjulivan

packages/pluggableWidgets/barcode-scanner-web/CHANGELOG.md

@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+### Added
+
+- We added the logic to switch to use native browser BarcodeDetector API if it is available instead of using zxing library.
+- We increase ideal image resolution to improve performance on higher end devices.
+
 ## [2.4.2] - 2024-08-30
 
 ### Fixed

packages/pluggableWidgets/barcode-scanner-web/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@mendix/barcode-scanner-web",
     "widgetName": "BarcodeScanner",
-    "version": "2.4.2",
+    "version": "2.5.0",
     "description": "Displays a barcode scanner",
     "copyright": "© Mendix Technology BV 2025. All rights reserved.",
     "license": "Apache-2.0",

packages/pluggableWidgets/barcode-scanner-web/src/components/BarcodeScanner.tsx

@@ -69,7 +69,7 @@ export function BarcodeScanner({
 }: BarcodeScannerProps): ReactElement | null {
     const [errorMessage, setError] = useCustomErrorMessage();
     const canvasMiddleRef = useRef<HTMLDivElement>(null);
-    const videoRef = useReader({
+    const { ref: videoRef, useBrowserAPI } = useReader({
         onSuccess: onDetect,
         onError: setError,
         useCrop: showMask,
@@ -104,7 +104,7 @@ export function BarcodeScanner({
 
     return (
         <BarcodeScannerOverlay
-            class={className}
+            class={classNames(className, `mx-${useBrowserAPI ? "barcode" : "zxing"}-detector`)}
             showMask={showMask}
             canvasMiddleMiddleRef={canvasMiddleRef}
             {...dimensions}

packages/pluggableWidgets/barcode-scanner-web/src/helpers/barcode-detector-utils.tsx

@@ -0,0 +1,107 @@
+import { BarcodeFormatsType } from "../../typings/BarcodeScannerProps";
+import type { BarcodeDetector, BarcodeDetectorOptions, BarcodeFormat, DetectedBarcode } from "./barcode-detector";
+
+// Map Mendix barcode format types to native BarcodeDetector format strings
+const mapToNativeFormat = (format: string): string => {
+    const formatMap: Record<string, string> = {
+        UPC_A: "upc_a",
+        UPC_E: "upc_e",
+        EAN_8: "ean_8",
+        EAN_13: "ean_13",
+        CODE_39: "code_39",
+        CODE_128: "code_128",
+        ITF: "itf",
+        QR_CODE: "qr_code",
+        DATA_MATRIX: "data_matrix",
+        AZTEC: "aztec",
+        PDF_417: "pdf417"
+    };
+    return formatMap[format] || format.toLowerCase();
+};
+
+// Check if BarcodeDetector API is available
+export const isBarcodeDetectorSupported = (): boolean => {
+    return typeof globalThis !== "undefined" && "BarcodeDetector" in globalThis;
+};
+
+// Get supported formats for BarcodeDetector
+export const getBarcodeDetectorSupportedFormats = async (): Promise<string[]> => {
+    if (!isBarcodeDetectorSupported()) {
+        return [];
+    }
+    try {
+        const detector = new window.BarcodeDetector!();
+        return await detector.getSupportedFormats();
+    } catch (error) {
+        console.warn("Failed to get BarcodeDetector supported formats:", error);
+        return [];
+    }
+};
+
+// Create BarcodeDetector options from widget configuration
+export const createBarcodeDetectorOptions = (
+    useAllFormats: boolean,
+    barcodeFormats?: BarcodeFormatsType[]
+): BarcodeDetectorOptions => {
+    const options: BarcodeDetectorOptions = {};
+
+    if (!useAllFormats && barcodeFormats && barcodeFormats.length > 0) {
+        options.formats = barcodeFormats.map(format => mapToNativeFormat(format.barcodeFormat)) as Array<
+            BarcodeFormat["format"]
+        >;
+    }
+    // If useAllFormats is true or no specific formats, don't specify formats to use all supported
+
+    return options;
+};
+
+// Create BarcodeDetector instance
+export const createBarcodeDetector = (options?: BarcodeDetectorOptions): BarcodeDetector | null => {
+    if (!isBarcodeDetectorSupported()) {
+        return null;
+    }
+
+    try {
+        return new window.BarcodeDetector!(options);
+    } catch (error) {
+        console.warn("Failed to create BarcodeDetector:", error);
+        return null;
+    }
+};
+
+// Detect barcodes from video or canvas element using BarcodeDetector API
+export const detectBarcodesFromElement = async (
+    detector: BarcodeDetector,
+    element: HTMLVideoElement | HTMLCanvasElement
+): Promise<DetectedBarcode[]> => {
+    try {
+        return await detector.detect(element);
+    } catch (error) {
+        console.warn("BarcodeDetector failed to detect:", error);
+        return [];
+    }
+};
+
+// Convert video frame to canvas for processing
+export const captureVideoFrame = (video: HTMLVideoElement, canvas?: HTMLCanvasElement): HTMLCanvasElement => {
+    if (!canvas) {
+        canvas = document.createElement("canvas");
+    }
+
+    canvas.width = video.videoWidth;
+    canvas.height = video.videoHeight;
+
+    const ctx = canvas.getContext("2d");
+    if (ctx) {
+        ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
+    }
+
+    return canvas;
+};
+
+export const setupVideoElement = (video: HTMLVideoElement, stream: MediaStream): void => {
+    video.autofocus = true;
+    video.playsInline = true; // Fix error in Safari
+    video.muted = true;
+    video.srcObject = stream;
+};

packages/pluggableWidgets/barcode-scanner-web/src/helpers/barcode-detector.ts

@@ -0,0 +1,51 @@
+// TypeScript definitions for the BarcodeDetector API
+// Based on https://developer.mozilla.org/en-US/docs/Web/API/BarcodeDetector
+
+// https://developer.mozilla.org/en-US/docs/Web/API/Barcode_Detection_API#supported_barcode_formats
+export interface BarcodeFormat {
+    format:
+        | "aztec"
+        | "code_128"
+        | "code_39"
+        | "code_93"
+        | "codabar"
+        | "data_matrix"
+        | "ean_13"
+        | "ean_8"
+        | "itf"
+        | "pdf417"
+        | "qr_code"
+        | "unknown"
+        | "upc_a"
+        | "upc_e";
+}
+
+export interface DetectedBarcode {
+    boundingBox: DOMRectReadOnly;
+    cornerPoints: ReadonlyArray<{ x: number; y: number }>;
+    format: BarcodeFormat["format"];
+    rawValue: string;
+}
+
+export interface BarcodeDetectorOptions {
+    formats?: Array<BarcodeFormat["format"]>;
+}
+
+export interface BarcodeDetector {
+    detect(image: ImageBitmapSource): Promise<DetectedBarcode[]>;
+    getSupportedFormats(): Promise<Array<BarcodeFormat["format"]>>;
+}
+
+export type BarcodeDetectorConstructor = new (options?: BarcodeDetectorOptions) => BarcodeDetector;
+
+// Extend Window interface to include BarcodeDetector
+declare global {
+    interface Window {
+        BarcodeDetector?: BarcodeDetectorConstructor;
+    }
+}
+
+export interface MxBarcodeReader {
+    start(onSuccess: (data: string) => void, onError: (e: Error) => void): Promise<void>;
+    stop(): void;
+}

packages/pluggableWidgets/barcode-scanner-web/src/components/utils.tsx renamed to packages/pluggableWidgets/barcode-scanner-web/src/helpers/utils.tsx

@@ -7,8 +7,29 @@ import {
     HybridBinarizer,
     Result
 } from "@zxing/library";
+import { RefObject } from "react";
 import { BarcodeFormatsType } from "typings/BarcodeScannerProps";
 
+export const mediaStreamConstraints: MediaStreamConstraints = {
+    audio: false,
+    video: {
+        facingMode: "environment",
+        width: { min: 1280, ideal: 4096, max: 4096 },
+        height: { min: 720, ideal: 2160, max: 2160 }
+    }
+};
+
+export type ReaderProps = {
+    onSuccess?: (data: string) => void;
+    onError?: (e: Error) => void;
+    useCrop: boolean;
+    barcodeFormats?: BarcodeFormatsType[];
+    useAllFormats: boolean;
+    canvasMiddleRef: RefObject<HTMLDivElement>;
+};
+
+export type UseReaderHook = (args: ReaderProps) => RefObject<HTMLVideoElement>;
+
 export const returnVideoWidthHeight = (
     curVideoRef: HTMLVideoElement,
     canvasMiddle: HTMLDivElement

packages/pluggableWidgets/barcode-scanner-web/src/hooks/nativeReader.ts

@@ -0,0 +1,77 @@
+import { RefObject } from "react";
+import { BarcodeDetector, MxBarcodeReader } from "../helpers/barcode-detector";
+import { createBarcodeDetector, createBarcodeDetectorOptions } from "../helpers/barcode-detector-utils";
+import { mediaStreamConstraints, ReaderProps } from "../helpers/utils";
+
+export class Reader implements MxBarcodeReader {
+    private videoRef: RefObject<HTMLVideoElement>;
+    barcodeDetector: BarcodeDetector | null;
+    useCrop: boolean;
+    stopped: boolean = false;
+    canvasMiddleRef: RefObject<HTMLDivElement>;
+    stream: MediaStream | null = null;
+    decodeInterval: NodeJS.Timeout | number | null = null;
+
+    constructor(args: ReaderProps, videoRef: RefObject<HTMLVideoElement>) {
+        this.videoRef = videoRef;
+        this.useCrop = args.useCrop;
+        this.canvasMiddleRef = args.canvasMiddleRef;
+        const options = createBarcodeDetectorOptions(args.useAllFormats, args.barcodeFormats);
+        this.barcodeDetector = createBarcodeDetector(options);
+    }
+
+    start = async (onSuccess: (data: string) => void, onError: (e: Error) => void): Promise<void> => {
+        if (this.videoRef.current === null) {
+            return;
+        }
+
+        if (this.barcodeDetector === null) {
+            if (onError) {
+                onError(new Error("Failed to create barcode detector"));
+            }
+
+            return;
+        }
+
+        const stream = await navigator.mediaDevices.getUserMedia(mediaStreamConstraints);
+
+        this.videoRef.current.autofocus = true;
+        this.videoRef.current.playsInline = true; // Fix error in Safari
+        this.videoRef.current.muted = true;
+        this.videoRef.current.autoplay = true;
+        this.videoRef.current.srcObject = stream;
+        this.decodeInterval = setTimeout(this.decodeStream, 50, onSuccess, onError);
+    };
+
+    stop = (): void => {
+        if (this.decodeInterval) {
+            clearTimeout(this.decodeInterval);
+        }
+        this.stream?.getVideoTracks().forEach(track => track.stop());
+        this.barcodeDetector = null;
+    };
+
+    decodeStream = async (
+        // loop decode canvas till it finds a result
+        resolve: (value: string) => void,
+        reject: (reason?: Error) => void
+    ): Promise<void> => {
+        try {
+            if (this.videoRef.current === null) {
+                return;
+            }
+            if (this.decodeInterval) {
+                clearTimeout(this.decodeInterval);
+            }
+            const detectionCode = await this.barcodeDetector?.detect(this.videoRef.current);
+
+            if (detectionCode && detectionCode.length > 0) {
+                if (resolve) resolve(detectionCode[0].rawValue);
+            } else {
+                this.decodeInterval = setTimeout(this.decodeStream, 50, resolve, reject);
+            }
+        } catch (error) {
+            reject(error);
+        }
+    };
+}