Revert back changes to intrinsic metrics

Author: Shangyint

examples/qa/hotpot/hotpotqa_with_assertions.ipynb

@@ -52,7 +52,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -79,7 +79,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -94,7 +94,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -117,7 +117,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def all_queries_distinct(prev_queries):\n",
+    "    query_distinct = True\n",
+    "    for i, query in enumerate(prev_queries):\n",
+    "        if validate_query_distinction_local(prev_queries[:i], query) == False:\n",
+    "            query_distinct = False\n",
+    "            break\n",
+    "    return query_distinct"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -130,20 +145,30 @@
     "        self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
     "        self.max_hops = max_hops\n",
     "\n",
+    "        # for evaluating assertions only\n",
+    "        self.passed_suggestions = 0\n",
+    "\n",
     "    def forward(self, question):\n",
     "        context = []\n",
+    "        prev_queries = [question]\n",
+    "\n",
     "        for hop in range(self.max_hops):\n",
     "            query = self.generate_query[hop](context=context, question=question).query\n",
+    "            prev_queries.append(query)\n",
     "            passages = self.retrieve(query).passages\n",
     "            context = deduplicate(context + passages)\n",
+    "        \n",
+    "        if all_queries_distinct(prev_queries):\n",
+    "            self.passed_suggestions += 1\n",
+    "        \n",
     "        pred = self.generate_answer(context=context, question=question)\n",
     "        pred = dspy.Prediction(context=context, answer=pred.answer)\n",
     "        return pred"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -155,6 +180,9 @@
     "        self.generate_answer = dspy.ChainOfThought(GenerateAnswer)\n",
     "        self.max_hops = max_hops\n",
     "\n",
+    "        # for evaluating assertions only\n",
+    "        self.passed_suggestions = 0\n",
+    "\n",
     "    def forward(self, question):\n",
     "        context = []\n",
     "        prev_queries = [question]\n",
@@ -176,6 +204,9 @@
     "            prev_queries.append(query)\n",
     "            passages = self.retrieve(query).passages\n",
     "            context = deduplicate(context + passages)\n",
+    "        \n",
+    "        if all_queries_distinct(prev_queries):\n",
+    "            self.passed_suggestions += 1\n",
     "\n",
     "        pred = self.generate_answer(context=context, question=question)\n",
     "        pred = dspy.Prediction(context=context, answer=pred.answer)\n",
@@ -184,28 +215,34 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
-    "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=25, display_progress=True, display_table=False)"
+    "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=10, display_progress=True, display_table=False)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
     "def evaluate(module):\n",
+    "    module.passed_suggestions = 0\n",
+    "\n",
     "    retrieval_score = evaluate_on_hotpotqa(\n",
     "        module, metric=gold_passages_retrieved\n",
     "    )\n",
     "    \n",
+    "    suggestions_score = module.passed_suggestions / len(devset) * 100\n",
+    "\n",
     "    accuracy_score = evaluate_on_hotpotqa(\n",
     "        module, metric=dspy.evaluate.answer_exact_match\n",
     "    )\n",
     "\n",
+    "    print(f\"## Suggestions Score: {suggestions_score}\")\n",
+    "\n",
     "    print(f\"## Retrieval Score: {retrieval_score}\")\n",
     "    print(f\"## Accuracy Score: {accuracy_score}\")"
    ]
@@ -232,6 +269,15 @@
     "evaluate(baleen_with_assertions)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "max_bootstrapped_demos = 2"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -242,11 +288,11 @@
     "baleen = SimplifiedBaleen()\n",
     "teleprompter = BootstrapFewShotWithRandomSearch(\n",
     "    metric=validate_context_and_answer_and_hops,\n",
-    "    max_bootstrapped_demos=2,\n",
+    "    max_bootstrapped_demos=max_bootstrapped_demos,\n",
     "    num_candidate_programs=6,\n",
     ")\n",
     "\n",
-    "compiled_baleen = teleprompter.compile(student = baleen, teacher = baleen, trainset = trainset, valset = devset)\n",
+    "compiled_baleen = teleprompter.compile(student = SimplifiedBaleen(), teacher = SimplifiedBaleen(), trainset = trainset, valset = devset)\n",
     "evaluate(compiled_baleen)"
    ]
   },
@@ -260,7 +306,7 @@
     "baleen = SimplifiedBaleen()\n",
     "teleprompter = BootstrapFewShotWithRandomSearch(\n",
     "    metric=validate_context_and_answer_and_hops,\n",
-    "    max_bootstrapped_demos=2,\n",
+    "    max_bootstrapped_demos=max_bootstrapped_demos,\n",
     "    num_candidate_programs=6,\n",
     ")\n",
     "compiled_baleen = teleprompter.compile(\n",
@@ -270,9 +316,9 @@
     "    ),\n",
     "    teacher=baleen,\n",
     "    trainset=trainset,\n",
-    "    valset=devset[:100]\n",
+    "    valset=devset\n",
     ")\n",
-    "evaluate(compiled_baleen)"
+    "evaluate(compiled_baleen)\n"
    ]
   }
  ],
@@ -292,7 +338,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.11.5"
   }
  },
  "nbformat": 4,