Port latest updates to step reasoning

Author: Val Brodsky

libs/labelbox/src/labelbox/schema/tool_building/step_reasoning_tool.py

@@ -4,12 +4,14 @@
 from typing import Any, Dict, List, Optional
 
 from labelbox.schema.tool_building.tool_type import ToolType
-from labelbox.schema.tool_building.variant import Variant, VariantWithActions
+from labelbox.schema.tool_building.variant import VariantWithActions
 
 
 class IncorrectStepActions(Enum):
     REGENERATE_STEPS = "regenerateSteps"
     GENERATE_AND_RATE_ALTERNATIVE_STEPS = "generateAndRateAlternativeSteps"
+    REWRITE_STEP = "rewriteStep"
+    JUSTIFICATION = "justification"
 
 
 @dataclass
@@ -20,11 +22,19 @@ class StepReasoningVariants:
     NOTE: do not change the variant values
     """
 
-    correct_step: Variant = field(
-        default_factory=lambda: Variant(id=0, name="Correct")
+    correct_step: VariantWithActions = field(
+        default_factory=lambda: VariantWithActions(
+            id=0,
+            name="Correct",
+            actions=[],
+        )
     )
-    neutral_step: Variant = field(
-        default_factory=lambda: Variant(id=1, name="Neutral")
+    neutral_step: VariantWithActions = field(
+        default_factory=lambda: VariantWithActions(
+            id=1,
+            name="Neutral",
+            actions=[],
+        )
     )
 
     incorrect_step: VariantWithActions = field(
@@ -35,9 +45,8 @@ class StepReasoningVariants:
                 action.value for action in IncorrectStepActions
             },
             actions=[
-                IncorrectStepActions.REGENERATE_STEPS.value,
-                IncorrectStepActions.GENERATE_AND_RATE_ALTERNATIVE_STEPS.value,
-            ],  # action On by default
+                action.value for action in IncorrectStepActions
+            ],  # initialize to all IncorrectStepActions by default
         )
     )
 
@@ -56,9 +65,9 @@ def from_dict(cls, dictionary: List[Dict[str, Any]]):
 
         for variant in dictionary:
             if variant["id"] == 0:
-                correct_step = Variant(**variant)
+                correct_step = VariantWithActions(**variant)
             elif variant["id"] == 1:
-                neutral_step = Variant(**variant)
+                neutral_step = VariantWithActions(**variant)
             elif variant["id"] == 2:
                 incorrect_step = VariantWithActions(**variant)
 

libs/labelbox/tests/integration/test_ontology.py

@@ -379,3 +379,56 @@ def test_step_reasoning_ontology(chat_evaluation_ontology):
             ],
         },
     ]
+
+
+def test_fact_checking_ontology(chat_evaluation_ontology):
+    ontology = chat_evaluation_ontology
+    fact_checking = None
+    for tool in ontology.normalized["tools"]:
+        if tool["tool"] == "fact-checking":
+            fact_checking = tool
+            break
+    assert fact_checking is not None
+    assert fact_checking["definition"]["variants"] == [
+        {"id": 0, "name": "Accurate"},
+        {"id": 1, "name": "Inaccurate"},
+        {"id": 2, "name": "Disputed"},
+        {"id": 3, "name": "Unsupported", "actions": ["writeJustification"]},
+        {
+            "id": 4,
+            "name": "Can't confidently assess",
+            "actions": ["writeJustification"],
+        },
+        {
+            "id": 5,
+            "name": "No factual information",
+            "actions": ["writeJustification"],
+        },
+    ]
+    assert fact_checking["definition"]["version"] == 1
+    assert fact_checking["schemaNodeId"] is not None
+    assert fact_checking["featureSchemaId"] is not None
+
+    fact_checking = None
+    for tool in ontology.tools():
+        if isinstance(tool, FactCheckingTool):
+            fact_checking = tool
+            break
+    assert fact_checking is not None
+
+    assert fact_checking.definition.variants.asdict() == [
+        {"id": 0, "name": "Accurate"},
+        {"id": 1, "name": "Inaccurate"},
+        {"id": 2, "name": "Disputed"},
+        {"id": 3, "name": "Unsupported", "actions": ["writeJustification"]},
+        {
+            "id": 4,
+            "name": "Can't confidently assess",
+            "actions": ["writeJustification"],
+        },
+        {
+            "id": 5,
+            "name": "No factual information",
+            "actions": ["writeJustification"],
+        },
+    ]