fix relevance and prompty test

Author: Neehar Duvvuri

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_relevance/_relevance.py

@@ -177,7 +177,8 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # t
             eval_input["query"] = reformat_conversation_history(eval_input["query"], logger)
         if not isinstance(eval_input["response"], str):
             eval_input["response"] = reformat_agent_response(eval_input["response"], logger)
-        llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        result = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input)
+        llm_output = result["llm_output"]
         score = math.nan
 
         if isinstance(llm_output, dict):
@@ -191,6 +192,13 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # t
                 f"{self._result_key}_reason": reason,
                 f"{self._result_key}_result": binary_result,
                 f"{self._result_key}_threshold": self._threshold,
+                f"{self._result_key}_prompt_tokens": result.get("input_token_count", 0),
+                f"{self._result_key}_completion_tokens": result.get("output_token_count", 0),
+                f"{self._result_key}_total_tokens": result.get("total_token_count", 0),
+                f"{self._result_key}_finish_reason": result.get("finish_reason", ""),
+                f"{self._result_key}_model": result.get("model_id", ""),
+                f"{self._result_key}_sample_input": result.get("sample_input", ""),
+                f"{self._result_key}_sample_output": result.get("sample_output", ""),
             }
 
         binary_result = self._get_binary_result(score)

sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_prompty_async.py

@@ -169,7 +169,7 @@ async def test_first_match_text_json(self, prompty_config: Dict[str, Any], outpu
             # Should have only first name, and answer
             assert "lastName" not in llm_output
         else:
-            assert "lastName" in result
+            assert "lastName" in llm_output
             assert llm_output["lastName"] == "Doh"
 
     @pytest.mark.asyncio