Allow to update item_schema from external (#43863)

* Allow to update item_schema from external for some use cases like data synthesizing

* Enhance copy to deepcopy

* Fix linter run Black

Author: YusakuNo1

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate_aoai.py

@@ -152,14 +152,14 @@ def _begin_single_aoai_evaluation(
     grader_name_list = []
     grader_list = []
 
-    data_source = {}
-    data_source_config = {}
+    data_source: Dict[str, Any] = {}
+    data_source_config: Dict[str, Any] = {}
 
     if kwargs.get("data_source_config") is not None:
-        data_source_config = kwargs.get("data_source_config")
+        data_source_config = kwargs.get("data_source_config", {})
 
     if kwargs.get("data_source") is not None:
-        data_source = kwargs.get("data_source")
+        data_source = kwargs.get("data_source", {})
 
     # It's expected that all graders supplied for a single eval run use the same credentials
     # so grab a client from the first grader.
@@ -176,6 +176,10 @@ def _begin_single_aoai_evaluation(
 
     # Create eval group
     LOGGER.info(f"AOAI: Creating eval group with {len(grader_list)} testing criteria...")
+
+    # Combine with the item schema with generated data outside Eval SDK
+    _combine_item_schemas(data_source_config, kwargs)
+
     eval_group_info = client.evals.create(
         data_source_config=data_source_config, testing_criteria=grader_list, metadata={"is_foundry_eval": "true"}
     )
@@ -212,6 +216,20 @@ def _begin_single_aoai_evaluation(
     )
 
 
+def _combine_item_schemas(data_source_config: Dict[str, Any], kwargs: Dict[str, Any]) -> None:
+    if "item_schema" not in kwargs or "properties" not in kwargs["item_schema"]:
+        return
+
+    if "item_schema" in data_source_config:
+        item_schema = kwargs["item_schema"]["required"] if "required" in kwargs["item_schema"] else []
+        for key in kwargs["item_schema"]["properties"]:
+            if key not in data_source_config["item_schema"]["properties"]:
+                data_source_config["item_schema"]["properties"][key] = kwargs["item_schema"]["properties"][key]
+
+                if key in item_schema:
+                    data_source_config["item_schema"]["required"].append(key)
+
+
 def _get_evaluation_run_results(all_run_info: List[OAIEvalRunCreationInfo]) -> Tuple[pd.DataFrame, Dict[str, Any]]:
     """
     Get the results of an OAI evaluation run, formatted in a way that is easy for the rest of the evaluation

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_aoai.py

@@ -0,0 +1,109 @@
+import pytest
+import copy
+from azure.ai.evaluation._evaluate._evaluate_aoai import _combine_item_schemas
+
+
+@pytest.fixture
+def default_data_source_config():
+    return {
+        "type": "custom",
+        "item_schema": {
+            "type": "object",
+            "properties": {
+                "id": {"type": "string"},
+                "text": {"type": "string"},
+            },
+            "required": ["id", "text"],
+        },
+        "include_sample_schema": False,
+    }
+
+
+class TestCombineItemSchemas:
+    """Unit tests for _combine_item_schemas"""
+
+    def test_combine_item_schemas_success(self, default_data_source_config):
+        data_source_config = copy.deepcopy(default_data_source_config)
+        kwargs = {
+            "item_schema": {
+                "properties": {
+                    "metadata": {"type": "object"},
+                    "timestamp": {"type": "string", "format": "date-time"},
+                },
+                "required": ["metadata"],
+            }
+        }
+        _combine_item_schemas(data_source_config, kwargs)
+
+        expected_properties = {
+            "id": {"type": "string"},
+            "text": {"type": "string"},
+            "metadata": {"type": "object"},
+            "timestamp": {"type": "string", "format": "date-time"},
+        }
+        expected_required = ["id", "text", "metadata"]
+        assert data_source_config["item_schema"]["properties"] == expected_properties
+        assert data_source_config["item_schema"]["required"] == expected_required
+
+    def test_combine_item_schemas_without_item_schema(self, default_data_source_config):
+        data_source_config = copy.deepcopy(default_data_source_config)
+
+        expected_properties = {
+            "id": {"type": "string"},
+            "text": {"type": "string"},
+        }
+        expected_required = ["id", "text"]
+
+        # No "item_schema" in kwargs
+        kwargs = {}
+        _combine_item_schemas(data_source_config, kwargs)
+        assert data_source_config["item_schema"]["properties"] == expected_properties
+        assert data_source_config["item_schema"]["required"] == expected_required
+
+        # "item_schema" without "properties" in kwargs
+        kwargs = {"item_schema": {}}
+        _combine_item_schemas(data_source_config, kwargs)
+        assert data_source_config["item_schema"]["properties"] == expected_properties
+        assert data_source_config["item_schema"]["required"] == expected_required
+
+    def test_combine_item_schemas_with_empty_external_properties(self, default_data_source_config):
+        data_source_config = copy.deepcopy(default_data_source_config)
+        kwargs = {
+            "item_schema": {
+                "properties": {},
+                "required": [],
+            }
+        }
+        _combine_item_schemas(data_source_config, kwargs)
+
+        expected_properties = {
+            "id": {"type": "string"},
+            "text": {"type": "string"},
+        }
+        expected_required = ["id", "text"]
+
+        assert data_source_config["item_schema"]["properties"] == expected_properties
+        assert data_source_config["item_schema"]["required"] == expected_required
+
+    def test_combine_item_schemas_with_external_properties_without_required(self, default_data_source_config):
+        data_source_config = copy.deepcopy(default_data_source_config)
+        kwargs = {
+            "item_schema": {
+                "properties": {
+                    "metadata": {"type": "object"},
+                    "timestamp": {"type": "string", "format": "date-time"},
+                },
+            }
+        }
+        _combine_item_schemas(data_source_config, kwargs)
+
+        expected_properties = {
+            "id": {"type": "string"},
+            "text": {"type": "string"},
+            "metadata": {"type": "object"},
+            "timestamp": {"type": "string", "format": "date-time"},
+        }
+        expected_required = ["id", "text"]
+
+        assert data_source_config["item_schema"]["properties"] == expected_properties
+        assert data_source_config["item_schema"]["required"] == expected_required