Added auto classification feedback.

yannchevalier · yannchevalier · commit a9026b2be685 · 2025-02-11T08:22:27.000Z
diff --git a/samples/offscreen-interactivity/src/main/assets/part_conf.json b/samples/offscreen-interactivity/src/main/assets/part_conf.json
@@ -0,0 +1,36 @@
+{
+  "raw-content": {
+    "configuration": {
+      "analyzer": {
+        "bundle": "raw-content2",
+        "name": "standard"
+      },
+      "math": {
+        "bundle": "math2",
+        "name": "standard"
+      }
+    },
+    "classification": {
+      "types": [ "text", "drawing", "math", "shape" ]
+    },
+    "recognition": {
+      "types": [ "text", "math", "shape" ]
+    },
+    "pen": {
+      "gestures": [ "scratch-out", "strike-through" ]
+    }
+  },
+  "export": {
+    "jiix": {
+      "text": {
+        "chars": true,
+        "words": true
+      },
+      "bounding-box": true,
+      "strokes": false,
+      "glyphs": false,
+      "primitives": false,
+      "ids": true
+    }
+  }
+}
diff --git a/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/ink/serialization/jiix/Element.kt b/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/ink/serialization/jiix/Element.kt
@@ -14,5 +14,7 @@ data class Element(
     @SerializedName("bounding-box")
     val boundingBox: BoundingBox,
     @SerializedName("words")
-    val words: List<Word>? = null
+    val words: List<Word>? = null,
+    @SerializedName("expressions")
+    val expressions: List<Expression>? = null
 )
diff --git a/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/ink/serialization/jiix/Expression.kt b/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/ink/serialization/jiix/Expression.kt
@@ -0,0 +1,10 @@
+// Copyright @ MyScript. All rights reserved.
+
+package com.myscript.iink.demo.ink.serialization.jiix
+
+import com.google.gson.annotations.SerializedName
+
+data class Expression(
+    @SerializedName("bounding-box")
+    val boundingBox: BoundingBox? = null
+)
diff --git a/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/ink/serialization/jiix/StrokeExt.kt b/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/ink/serialization/jiix/StrokeExt.kt
@@ -86,4 +86,13 @@ fun Word.toScreenCoordinates(converter: DisplayMetricsConverter?): Word {
     } else {
         this
     }
+}
+
+fun BoundingBox.toScreenCoordinates(converter: DisplayMetricsConverter?): BoundingBox {
+    return BoundingBox(
+        x = converter?.x_mm2px(this.x) ?: this.x,
+        y = converter?.y_mm2px(this.y) ?: this.y,
+        width = converter?.x_mm2px(this.width) ?: this.width,
+        height = converter?.y_mm2px(this.height) ?: this.height
+    )
 }
diff --git a/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/ui/InkViewModel.kt b/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/ui/InkViewModel.kt
@@ -24,6 +24,8 @@ import com.myscript.iink.ItemIdHelper
 import com.myscript.iink.MimeType
 import com.myscript.iink.OffscreenEditor
 import com.myscript.iink.OffscreenGestureAction
+import com.myscript.iink.demo.ink.serialization.jiix.BoundingBox
+import com.myscript.iink.demo.ink.serialization.jiix.Element
 import com.myscript.iink.demo.ink.serialization.jiix.RecognitionRoot
 import com.myscript.iink.demo.ink.serialization.jiix.Word
 import com.myscript.iink.demo.ink.serialization.jiix.convertPointerEvent
@@ -37,7 +39,6 @@ import com.myscript.iink.demo.inksample.data.InkRepository
 import com.myscript.iink.demo.inksample.data.InkRepositoryImpl
 import com.myscript.iink.demo.inksample.util.DisplayMetricsConverter
 import com.myscript.iink.demo.inksample.util.autoCloseable
-import com.myscript.iink.demo.inksample.util.iinkConfig
 import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.asCoroutineDispatcher
@@ -46,9 +47,20 @@ import kotlinx.coroutines.withContext
 import java.io.File
 import java.util.concurrent.Executors
 
+enum class BlockType {
+    TEXT,
+    MATH
+}
+
+data class RecognitionItem(
+    val text: String,
+    val type: BlockType,
+    val boundingBox: BoundingBox?
+)
+
 data class RecognitionFeedback(
     val isVisible: Boolean = false,
-    val words: List<Word> = emptyList()
+    val items: List<RecognitionItem> = emptyList()
 )
 
 enum class EditorHistoryAction {
@@ -325,12 +337,32 @@ class InkViewModel(
                     val recognitionRoot = Gson().fromJson(exportedData, RecognitionRoot::class.java)
 
                     // Filter out elements that are null or contain any null list of words
-                    recognitionRoot.elements?.flatMap { element ->
-                        element.words?.map { word ->
-                            word.toScreenCoordinates(converter)
-                        } ?: emptyList()
-                    } ?: emptyList()
+                    val recognitionResult = mutableListOf<RecognitionItem>()
+
+                    recognitionRoot.elements?.forEach { element ->
+                        if (element.expressions != null) {
+                            recognitionResult.add(RecognitionItem(
+                                text = "",
+                                type = BlockType.MATH,
+                                boundingBox = element.boundingBox.toScreenCoordinates(converter)
+                            ))
+                        }
+                        if (element.words != null) {
+                            recognitionResult.addAll(
+                                element.words.map { word ->
+                                    RecognitionItem(
+                                        text = word.label ?: "",
+                                        type = BlockType.TEXT,
+                                        boundingBox = word.boundingBox?.toScreenCoordinates(converter)
+                                    )
+                                }
+                            )
+                        }
+                    }
+
+                    recognitionResult
                 }
+
                 _iinkJIIX.value = withContext(defaultDispatcher) {
                     val engine = engine ?: return@withContext ""
 
@@ -344,7 +376,7 @@ class InkViewModel(
 
                     })
                 } ?: ""
-                _recognitionFeedback.value = _recognitionFeedback.value?.copy(words = adjustedWords)
+                _recognitionFeedback.value = _recognitionFeedback.value?.copy(items = adjustedWords)
             }
         }
 
@@ -636,7 +668,9 @@ class InkViewModel(
                 val application = checkNotNull(this[APPLICATION_KEY])
                 val engine = checkNotNull((application as InkApplication).engine)
                 val dataDir = File(application.filesDir, "data")
-                InkViewModel(InkRepositoryImpl(dataDir), engine, dataDir, iinkConfig)
+
+                val config = application.assets.open("part_conf.json").bufferedReader().use { it.readText() }
+                InkViewModel(InkRepositoryImpl(dataDir), engine, dataDir, config)
             }
         }
     }
diff --git a/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/ui/MainActivity.kt b/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/ui/MainActivity.kt
@@ -136,8 +136,8 @@ class MainActivity : AppCompatActivity() {
             inkView.alpha = if (recognitionFeedback.isVisible) .25f else 1f
 
             if (recognitionFeedback.isVisible) {
-                recognitionFeedback.words.forEach { word ->
-                    val customView = WordView(this@MainActivity, word)
+                recognitionFeedback.items.forEach { item ->
+                    val customView = RecognitionItemView(this@MainActivity, item)
                     recognitionContent.addView(customView)
                 }
             }
diff --git a/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/ui/RecognitionItemView.kt b/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/ui/RecognitionItemView.kt
@@ -0,0 +1,79 @@
+// Copyright @ MyScript. All rights reserved.
+
+package com.myscript.iink.demo.inksample.ui
+
+import android.content.Context
+import android.graphics.Canvas
+import android.graphics.Color
+import android.graphics.Paint
+import android.graphics.Typeface
+import android.text.TextPaint
+import android.view.View
+
+class RecognitionItemView(context: Context, private val item: RecognitionItem) : View(context) {
+
+    private val textPaint = TextPaint(Paint.ANTI_ALIAS_FLAG).apply {
+        color = Color.BLACK
+        textSize = 48f
+    }
+
+    private val boxPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
+        color = Color.GRAY
+        style = Paint.Style.STROKE
+        strokeWidth = 2f
+    }
+
+    private val typePaint = TextPaint(Paint.ANTI_ALIAS_FLAG).apply {
+        color = Color.GRAY
+        textSize = 32f
+        typeface = Typeface.create(Typeface.DEFAULT, Typeface.ITALIC)
+    }
+
+    override fun onDraw(canvas: Canvas) {
+        super.onDraw(canvas)
+
+        item.boundingBox?.let { box ->
+            canvas.drawRect(
+                box.x,
+                box.y,
+                box.x + box.width,
+                box.y + box.height,
+                boxPaint.apply {
+                    color = when(item.type) {
+                        BlockType.TEXT -> TEXT_COLOR
+                        BlockType.MATH -> MATH_COLOR
+                    }
+                }
+            )
+
+            if (item.text.isNotEmpty()) {
+                canvas.drawText(
+                    item.text,
+                    box.x + 5,
+                    box.y + box.height - textPaint.descent(),
+                    textPaint
+                )
+            }
+
+            canvas.drawText(
+                when(item.type) {
+                    BlockType.TEXT -> "abc"
+                    BlockType.MATH -> "Σ"
+                },
+                box.x + 5,
+                box.y + (textPaint.descent() - textPaint.ascent()) / 2,
+                typePaint.apply {
+                    color = when(item.type) {
+                        BlockType.TEXT -> TEXT_COLOR
+                        BlockType.MATH -> MATH_COLOR
+                    }
+                }
+            )
+        }
+    }
+
+    companion object {
+        private const val TEXT_COLOR = Color.RED
+        private const val MATH_COLOR = Color.BLUE
+    }
+}
diff --git a/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/ui/WordView.kt b/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/ui/WordView.kt
diff --git a/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/util/IInkConfiguration.kt b/samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/util/IInkConfiguration.kt

Original file line number	Diff line number	Diff line change
`@@ -14,5 +14,7 @@ data class Element(`
`14`	`14`	`@SerializedName("bounding-box")`
`15`	`15`	`val boundingBox: BoundingBox,`
`16`	`16`	`@SerializedName("words")`
`17`		`- val words: List<Word>? = null`
	`17`	`+ val words: List<Word>? = null,`
	`18`	`+ @SerializedName("expressions")`
	`19`	`+ val expressions: List<Expression>? = null`
`18`	`20`	`)`
Original file line number	Diff line number	Diff line change
`@@ -86,4 +86,13 @@ fun Word.toScreenCoordinates(converter: DisplayMetricsConverter?): Word {`
`86`	`86`	`} else {`
`87`	`87`	`this`
`88`	`88`	`}`
	`89`	`+}`
	`90`	`+`
	`91`	`+fun BoundingBox.toScreenCoordinates(converter: DisplayMetricsConverter?): BoundingBox {`
	`92`	`+ return BoundingBox(`
	`93`	`+ x = converter?.x_mm2px(this.x) ?: this.x,`
	`94`	`+ y = converter?.y_mm2px(this.y) ?: this.y,`
	`95`	`+ width = converter?.x_mm2px(this.width) ?: this.width,`
	`96`	`+ height = converter?.y_mm2px(this.height) ?: this.height`
	`97`	`+ )`
`89`	`98`	`}`
Original file line number	Diff line number	Diff line change
`@@ -136,8 +136,8 @@ class MainActivity : AppCompatActivity() {`
`136`	`136`	`inkView.alpha = if (recognitionFeedback.isVisible) .25f else 1f`
`137`	`137`
`138`	`138`	`if (recognitionFeedback.isVisible) {`
`139`		`- recognitionFeedback.words.forEach { word ->`
`140`		`- val customView = WordView(this@MainActivity, word)`
	`139`	`+ recognitionFeedback.items.forEach { item ->`
	`140`	`+ val customView = RecognitionItemView(this@MainActivity, item)`
`141`	`141`	`recognitionContent.addView(customView)`
`142`	`142`	`}`
`143`	`143`	`}`