finish initial keypoint pose interface

Author: seveibar

src/Annotator/index.js

@@ -10,6 +10,7 @@ import type {
   MainLayoutState,
   Action,
 } from "../MainLayout/types"
+import type { KeypointsDefinition } from "../ImageCanvas/region-tools"
 import SettingsProvider from "../SettingsProvider"
 
 import combineReducers from "./reducers/combine-reducers.js"
@@ -41,6 +42,7 @@ type Props = {
   videoSrc?: string,
   keyframes?: Object,
   videoName?: string,
+  keypointDefinitions: KeypointsDefinition,
   fullImageSegmentationMode?: boolean,
   autoSegmentationOptions?:
     | {| type: "simple" |}
@@ -77,6 +79,7 @@ export const Annotator = ({
   onExit,
   onNextImage,
   onPrevImage,
+  keypointDefinitions,
   autoSegmentationOptions = { type: "autoseg" },
 }: Props) => {
   if (typeof selectedImage === "string") {
@@ -112,6 +115,7 @@ export const Annotator = ({
       enabledTools,
       history: [],
       videoName,
+      keypointDefinitions,
       ...(annotationType === "image"
         ? {
             selectedImage,

src/Annotator/poses.story.js

@@ -0,0 +1,76 @@
+// @flow
+
+import React, { useState } from "react"
+
+import { storiesOf } from "@storybook/react"
+import { action as actionAddon } from "@storybook/addon-actions"
+import dancingManImage from "../ImageCanvas/dancing-man.story.jpg"
+import Annotator from "./"
+
+const middlewares = [
+  (store) => (next) => (action) => {
+    actionAddon(action.type)(action)
+    return next(action)
+  },
+]
+
+storiesOf("Annotator (Poses)", module).add("Basic", () => (
+  <Annotator
+    onExit={actionAddon("onExit")}
+    middlewares={middlewares}
+    labelImages
+    enabledTools={["create-keypoints"]}
+    keypointDefinitions={{
+      human: {
+        connections: [
+          ["head", "sternum"],
+          ["sternum", "leftElbow"],
+          ["sternum", "rightElbow"],
+        ],
+        landmarks: {
+          head: {
+            label: "Head",
+            color: "#f00",
+            defaultPosition: [0, -0.05],
+          },
+          sternum: {
+            label: "Torso",
+            color: "#0f0",
+            defaultPosition: [0, 0],
+          },
+          leftElbow: {
+            label: "Left Elbow",
+            color: "#00f",
+            defaultPosition: [-0.05, 0],
+          },
+          rightElbow: {
+            label: "Right Elbow",
+            color: "#00f",
+            defaultPosition: [0.05, 0],
+          },
+        },
+      },
+    }}
+    images={[
+      {
+        src: dancingManImage,
+        name: "Dancing Man",
+        regions: [
+          {
+            type: "keypoints",
+            id: "keypoints1",
+            keypointsDefinitionId: "human",
+            highlighted: true,
+            points: {
+              head: { x: 0.54, y: 0.2 },
+              sternum: { x: 0.57, y: 0.3 },
+              leftElbow: { x: 0.4, y: 0.39 },
+              rightElbow: { x: 0.7, y: 0.32 },
+            },
+            visible: true,
+          },
+        ],
+      },
+    ]}
+  />
+))

src/Annotator/reducers/general-reducer.js

@@ -10,6 +10,7 @@ import colors from "../../colors"
 import fixTwisted from "./fix-twisted"
 import convertExpandingLineToPolygon from "./convert-expanding-line-to-polygon"
 import clamp from "clamp"
+import getLandmarksWithTransform from "../../utils/get-landmarks-with-transform"
 
 const getRandomId = () => Math.random().toString().split(".")[1]
 
@@ -206,6 +207,16 @@ export default (state: MainLayoutState, action: Action) => {
         pointIndex,
       })
     }
+    case "BEGIN_MOVE_KEYPOINT": {
+      const { region, keypointId } = action
+      state = closeEditors(state)
+      state = saveToHistory(state, "Move Keypoint")
+      return setIn(state, ["mode"], {
+        mode: "MOVE_KEYPOINT",
+        regionId: region.id,
+        keypointId,
+      })
+    }
     case "ADD_POLYGON_POINT": {
       const { polygon, point, pointIndex } = action
       const regionIndex = getRegionIndex(polygon)
@@ -240,6 +251,22 @@ export default (state: MainLayoutState, action: Action) => {
             [x, y]
           )
         }
+        case "MOVE_KEYPOINT": {
+          const { keypointId, regionId } = state.mode
+          const [region, regionIndex] = getRegion(regionId)
+          if (regionIndex === null) return state
+          return setIn(
+            state,
+            [
+              ...pathToActiveImage,
+              "regions",
+              regionIndex,
+              "points",
+              keypointId,
+            ],
+            { ...(region: any).points[keypointId], x, y }
+          )
+        }
         case "MOVE_REGION": {
           const { regionId } = state.mode
           if (regionId === "$$allowed_area") {
@@ -312,6 +339,20 @@ export default (state: MainLayoutState, action: Action) => {
             h: dh,
           })
         }
+        case "RESIZE_KEYPOINTS": {
+          const { regionId, landmarks, centerX, centerY } = state.mode
+          const distFromCenter = Math.sqrt(
+            (centerX - x) ** 2 + (centerY - y) ** 2
+          )
+          const scale = distFromCenter / 0.15
+          return modifyRegion(regionId, {
+            points: getLandmarksWithTransform({
+              landmarks,
+              center: { x: centerX, y: centerY },
+              scale,
+            }),
+          })
+        }
         case "DRAW_POLYGON": {
           const { regionId } = state.mode
           const [region, regionIndex] = getRegion(regionId)
@@ -541,6 +582,34 @@ export default (state: MainLayoutState, action: Action) => {
           })
           break
         }
+        case "create-keypoints": {
+          state = saveToHistory(state, "Create Keypoints")
+          const [
+            [keypointsDefinitionId, { landmarks, connections }],
+          ] = (Object.entries(state.keypointDefinitions): any)
+
+          newRegion = {
+            type: "keypoints",
+            keypointsDefinitionId,
+            points: getLandmarksWithTransform({
+              landmarks,
+              center: { x, y },
+              scale: 1,
+            }),
+            highlighted: true,
+            editingLabels: false,
+            id: getRandomId(),
+          }
+          state = setIn(state, ["mode"], {
+            mode: "RESIZE_KEYPOINTS",
+            landmarks,
+            centerX: x,
+            centerY: y,
+            regionId: newRegion.id,
+            isNew: true,
+          })
+          break
+        }
         default:
           break
       }
@@ -581,9 +650,13 @@ export default (state: MainLayoutState, action: Action) => {
           }
         }
         case "MOVE_REGION":
+        case "RESIZE_KEYPOINTS":
         case "MOVE_POLYGON_POINT": {
           return { ...state, mode: null }
         }
+        case "MOVE_KEYPOINT": {
+          return { ...state, mode: null }
+        }
         case "CREATE_POINT_LINE": {
           return state
         }

src/ImageCanvas/index.js

@@ -15,6 +15,7 @@ import type {
   Point,
   Polygon,
   Box,
+  Keypoints,
   KeypointsDefinition,
 } from "./region-tools.js"
 import { makeStyles } from "@material-ui/core/styles"
@@ -72,6 +73,7 @@ type Props = {
   onDeleteRegion: (Region) => any,
   onBeginBoxTransform: (Box, [number, number]) => any,
   onBeginMovePolygonPoint: (Polygon, index: number) => any,
+  onBeginMoveKeypoint: (Keypoints, index: number) => any,
   onAddPolygonPoint: (Polygon, point: [number, number], index: number) => any,
   onSelectRegion: (Region) => any,
   onBeginMovePoint: (Point) => any,
@@ -127,6 +129,7 @@ export const ImageCanvas = ({
   onBeginBoxTransform,
   onBeginMovePolygonPoint,
   onAddPolygonPoint,
+  onBeginMoveKeypoint,
   onSelectRegion,
   onBeginMovePoint,
   onDeleteRegion,
@@ -364,6 +367,7 @@ export const ImageCanvas = ({
           mat={mat}
           onBeginBoxTransform={onBeginBoxTransform}
           onBeginMovePolygonPoint={onBeginMovePolygonPoint}
+          onBeginMoveKeypoint={onBeginMoveKeypoint}
           onAddPolygonPoint={onAddPolygonPoint}
           showHighlightBox={showHighlightBox}
         />

src/ImageCanvas/index.story.js

@@ -105,6 +105,8 @@ const events = {
   onAddPolygonPoint: action("onAddPolygonPoint"),
   onClosePolygon: action("onClosePolygon"),
 
+  onBeginMoveKeypoint: action("onBeginMoveKeypoint"),
+
   onBeginMovePoint: action("onBeginMovePoint"),
   onDeleteRegion: action("onDeleteRegion"),
 }
@@ -162,17 +164,17 @@ storiesOf("ImageCanvas", module)
       keypointDefinitions={{
         human: {
           connections: [
-            ["head", "torso"],
-            ["torso", "leftElbow"],
-            ["torso", "rightElbow"],
+            ["head", "sternum"],
+            ["sternum", "leftElbow"],
+            ["sternum", "rightElbow"],
           ],
           landmarks: {
             head: {
               label: "Head",
               color: "#f00",
               defaultPosition: [0, -0.05],
             },
-            torso: {
+            sternum: {
               label: "Torso",
               color: "#0f0",
               defaultPosition: [0, 0],
@@ -194,13 +196,13 @@ storiesOf("ImageCanvas", module)
         {
           type: "keypoints",
           id: "keypoints1",
-          keypointDefinitionId: "human",
+          keypointsDefinitionId: "human",
           highlighted: true,
           points: {
-            head: [0.5, 0.4],
-            torso: [0.5, 0.5],
-            leftElbow: [0.4, 0.5],
-            rightElbow: [0.6, 0.5],
+            head: { x: 0.54, y: 0.2 },
+            sternum: { x: 0.57, y: 0.3 },
+            leftElbow: { x: 0.4, y: 0.39 },
+            rightElbow: { x: 0.7, y: 0.32 },
           },
           visible: true,
         },

src/ImageCanvas/region-tools.js

@@ -54,8 +54,9 @@ export type ExpandingLine = {|
   points: Array<{ x: number, y: number, angle: number, width: number }>,
 |}
 
-export type Keypoint = {|
+export type KeypointDefinition = {|
   label: string,
+  color: string,
   defaultPosition: [number, number],
 |}
 
@@ -65,7 +66,7 @@ export type KeypointsDefinition = {|
   [id: string]: {
     connections: Array<[KeypointId, KeypointId]>,
     landmarks: {
-      [KeypointId]: Keypoint,
+      [KeypointId]: KeypointDefinition,
     },
   },
 |}
@@ -75,7 +76,7 @@ export type Keypoints = {|
   type: "keypoints",
   keypointsDefinitionId: string,
   points: {
-    [string]: [number, number],
+    [string]: { x: number, y: number },
   },
 |}
 
@@ -100,6 +101,26 @@ export const getEnclosingBox = (region: Region) => {
       box.h = Math.max(...region.points.map(([x, y]) => y)) - box.y
       return box
     }
+    case "keypoints": {
+      const minX = Math.min(
+        ...Object.values(region.points).map(({ x, y }) => x)
+      )
+      const minY = Math.min(
+        ...Object.values(region.points).map(({ x, y }) => y)
+      )
+      const maxX = Math.max(
+        ...Object.values(region.points).map(({ x, y }) => x)
+      )
+      const maxY = Math.max(
+        ...Object.values(region.points).map(({ x, y }) => y)
+      )
+      return {
+        x: minX,
+        y: minY,
+        w: maxX - minX,
+        h: maxY - minY,
+      }
+    }
     case "expanding-line": {
       const box = {
         x: Math.min(...region.points.map(({ x, y }) => x)),

src/MainLayout/icon-dictionary.js

@@ -18,6 +18,7 @@ import {
   faEdit,
 } from "@fortawesome/free-solid-svg-icons"
 import FullscreenIcon from "@material-ui/icons/Fullscreen"
+import AccessibilityNewIcon from "@material-ui/icons/AccessibilityNew"
 
 const faStyle = { marginTop: 4, width: 16, height: 16, marginBottom: 4 }
 
@@ -67,6 +68,7 @@ export const iconDictionary = {
   "modify-allowed-area": () => (
     <FontAwesomeIcon style={faStyle} size="xs" fixedWidth icon={faEdit} />
   ),
+  "create-keypoints": AccessibilityNewIcon,
   window: FullscreenIcon,
 }