full segmentation working well- can kill ui thread on large images

Author: seveibar

package.json

@@ -11,7 +11,7 @@
     "color-alpha": "^1.0.4",
     "get-image-data": "^3.0.1",
     "material-survey": "^1.0.34",
-    "mmgc1-cpp": "^1.0.35",
+    "mmgc1-cpp": "^1.0.44",
     "moment": "^2.23.0",
     "react-full-screen": "^0.2.4",
     "react-hotkeys": "^2.0.0",

src/ImageCanvas/index.js

@@ -260,10 +260,6 @@ export const ImageCanvas = ({
     bottomRight: mat.clone().inverse().applyToPoint(iw, ih),
   }
 
-  const classPoints = useMemo(() => {
-    return regions.filter((region) => region.type === "point")
-  }, [regions])
-
   const highlightedRegion = useMemo(() => regions.find((r) => r.highlighted), [
     regions,
   ])
@@ -379,7 +375,7 @@ export const ImageCanvas = ({
               imagePosition={imagePosition}
               regionClsList={regionClsList}
               imageSrc={imageSrc}
-              classPoints={classPoints}
+              regions={regions}
             />
           )}
           <canvas

src/ImageMask/index.js

@@ -7,7 +7,7 @@ import { useDebounce } from "react-use"
 import MMGC_INIT from "mmgc1-cpp"
 
 export const ImageMask = ({
-  classPoints,
+  regions,
   regionClsList,
   imageSrc,
   imagePosition,
@@ -49,7 +49,7 @@ export const ImageMask = ({
       if (hide) return
       if (!canvasRef) return
       if (!sampleImageData) return
-      if (classPoints.filter((cp) => cp.cls).length < 3) return
+      if (regions.filter((cp) => cp.cls).length < 3) return
       if (!mmgc.setImageSize) return
       const context = canvasRef.getContext("2d")
 
@@ -79,9 +79,11 @@ export const ImageMask = ({
         )
         return
       }
-      mmgc.clearClassPoints()
+      mmgc.clearClassElements()
+      const classPoints = regions
+        .filter((r) => r.type === "point")
+        .filter((r) => r.cls)
       for (const classPoint of classPoints) {
-        if (!classPoint.cls) continue
         if (classPoint.x < 0 || classPoint.x >= 1) continue
         if (classPoint.y < 0 || classPoint.y >= 1) continue
         const clsIndex = regionClsList.indexOf(classPoint.cls)
@@ -96,13 +98,32 @@ export const ImageMask = ({
           Math.floor(classPoint.x * sampleImageData.width)
         )
       }
+      const classPolygons = regions
+        .filter((r) => r.type === "polygon")
+        .filter((r) => r.cls)
+      for (const polygon of classPolygons) {
+        const { points } = polygon
+        const clsIndex = regionClsList.indexOf(polygon.cls)
+        const pi = mmgc.addPolygon(clsIndex)
+        const pointPairs = points.map((p, i) => [
+          p,
+          points[(i + 1) % points.length],
+        ])
+        for (const [p1, p2] of pointPairs) {
+          const ri1 = Math.round(p1[1] * sampleImageData.height)
+          const ci1 = Math.round(p1[0] * sampleImageData.width)
+          const ri2 = Math.round(p2[1] * sampleImageData.height)
+          const ci2 = Math.round(p2[0] * sampleImageData.width)
+          mmgc.addLineToPolygon(pi, ri1, ci1, ri2, ci2)
+        }
+      }
+
       mmgc.computeMasks()
       const maskAddress = mmgc.getColoredMask()
       const cppImDataUint8 = new Uint8ClampedArray(
         mmgc.HEAPU8.buffer,
         maskAddress,
         sampleImageData.data.length
-        // sampleImageData.width * sampleImageData.height * 4
       )
       const maskImageData = new ImageData(
         cppImDataUint8,
@@ -114,7 +135,7 @@ export const ImageMask = ({
       context.putImageData(maskImageData, 0, 0)
     },
     1000,
-    [canvasRef, sampleImageData, classPoints, hide]
+    [canvasRef, sampleImageData, regions, hide]
   )
 
   const style = useMemo(() => {

src/RegionLabel/styles.js

@@ -7,7 +7,7 @@ export default {
     fontSize: 12,
     cursor: "default",
     transition: "opacity 200ms",
-    opacity: 0.7,
+    opacity: 0.5,
     "&:hover": {
       opacity: 0.9,
       cursor: "pointer",

src/RegionSelectAndTransformBoxes/index.js

@@ -25,8 +25,6 @@ const arePropsEqual = (prev, next) => {
     prev.dragWithPrimary === next.dragWithPrimary &&
     prev.createWithPrimary === next.createWithPrimary &&
     prev.zoomWithPrimary === next.zoomWithPrimary &&
-    prev.layoutParams.current.iw === next.layoutParams.current.iw &&
-    prev.layoutParams.current.ih === next.layoutParams.current.ih &&
     prev.mat === next.mat
   )
 }
@@ -42,6 +40,7 @@ export const RegionSelectAndTransformBox = memo(
     onBeginMovePoint,
     onSelectRegion,
     layoutParams,
+    fullImageSegmentationMode = false,
     mat,
     onBeginBoxTransform,
     onBeginMovePolygonPoint,
@@ -53,7 +52,7 @@ export const RegionSelectAndTransformBox = memo(
     return (
       <Fragment>
         <PreventScrollToParents>
-          {showHighlightBox && (
+          {showHighlightBox && r.type !== "polygon" && (
             <HighlightBox
               region={r}
               mouseEvents={mouseEvents}
@@ -178,7 +177,7 @@ export const RegionSelectAndTransformBoxes = memo(
         return <RegionSelectAndTransformBox key={r.id} {...props} region={r} />
       })
   },
-  (n, p) => n.regions === p.regions
+  (n, p) => n.regions === p.regions && n.mat === p.mat
 )
 
 export default RegionSelectAndTransformBoxes

src/RegionShapes/index.js

@@ -31,8 +31,9 @@ const RegionComponents = {
       />
     </g>
   )),
-  polygon: memo(({ region, iw, ih }) => {
+  polygon: memo(({ region, iw, ih, fullImageSegmentationMode }) => {
     const Component = region.open ? "polyline" : "polygon"
+    const alphaBase = fullImageSegmentationMode ? 0.5 : 1
     return (
       <Component
         points={region.points
@@ -112,12 +113,20 @@ const RegionComponents = {
 }
 
 export const WrappedRegionList = memo(
-  ({ regions, iw, ih }) => {
+  ({ regions, iw, ih, fullImageSegmentationMode }) => {
     return regions
       .filter((r) => r.visible !== false)
       .map((r) => {
         const Component = RegionComponents[r.type]
-        return <Component key={r.regionId} region={r} iw={iw} ih={ih} />
+        return (
+          <Component
+            key={r.regionId}
+            region={r}
+            iw={iw}
+            ih={ih}
+            fullImageSegmentationMode={fullSegmentationMode}
+          />
+        )
       })
   },
   (n, p) => n.regions === p.regions && n.iw === p.iw && n.ih === p.ih
@@ -126,6 +135,7 @@ export const WrappedRegionList = memo(
 export const RegionShapes = ({ mat, imagePosition, regions = [] }) => {
   const iw = imagePosition.bottomRight.x - imagePosition.topLeft.x
   const ih = imagePosition.bottomRight.y - imagePosition.topLeft.y
+  if (isNaN(iw) || isNaN(ih)) return null
   return (
     <svg
       width={iw}

src/RegionTags/index.js

@@ -97,6 +97,7 @@ export const RegionTags = ({
           <div
             style={{
               position: "absolute",
+              zIndex: 20,
               left: 0,
               ...(displayOnTop ? { bottom: 0 } : { top: 0 }),
             }}

yarn.lock

@@ -10841,10 +10841,10 @@ mkdirp@0.5.1, mkdirp@0.5.x, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@
   dependencies:
     minimist "0.0.8"
 
-mmgc1-cpp@^1.0.35:
-  version "1.0.35"
-  resolved "https://registry.yarnpkg.com/mmgc1-cpp/-/mmgc1-cpp-1.0.35.tgz#654140c7e942dc32587cfe6cdc388a453ffa7d16"
-  integrity sha512-GVXtpvdQdzd6mNPJe/QqQEAuwUROA3jP5TUiHtbuK27ZfEAM77yyjp5LxzwtvfZ2vq0b0B07SkcpmphEGXsI8g==
+mmgc1-cpp@^1.0.44:
+  version "1.0.44"
+  resolved "https://registry.yarnpkg.com/mmgc1-cpp/-/mmgc1-cpp-1.0.44.tgz#b3059e9df06d1dba5899b159c454e1d435fec317"
+  integrity sha512-7kJk5zggfAR05id6a6ltXih39y8Va+zir1wevORNXBr/Glc2gtrqyc0HciXrOqyAUZ6Y93aRLW11BODsnDSemQ==
 
 moment@^2.23.0:
   version "2.24.0"