stream data from huggingface dataset instead of downloading complete split

- change to stream data from dataset instead of downloading complete split

Author: FelipeAdachi

langkit/examples/LLM_to_WhyLabs.ipynb

@@ -10,6 +10,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -58,6 +59,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -229,6 +231,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -266,13 +269,14 @@
     "    from datasets import load_dataset, Dataset\n",
     "    import pandas as pd\n",
     "\n",
-    "    archived_chats = load_dataset('alespalla/chatbot_instruction_prompts', split=\"test\")\n",
+    "    archived_chats = load_dataset('alespalla/chatbot_instruction_prompts', split=\"test\",streaming=True)\n",
+    "    chats = iter(archived_chats)\n",
+    "    for _ in range(100):\n",
+    "      response = next(chats)\n",
+    "      telemetry_agent.log(response)\n",
+    "      print(response)\n",
     "    # lets output the dataset metadata from hugging face so we can see how to\n",
     "    # access some of the contained prompts and responses.\n",
-    "    print(f\"Here's the dataset structure {archived_chats}\")\n",
-    "    for column_name in archived_chats.column_names:\n",
-    "        print(f\"Profiling column: ({column_name})\")\n",
-    "        telemetry_agent.log(pd.DataFrame(archived_chats[column_name][:1000], columns=[column_name]))\n",
     "    print(\"done profiling\")"
    ]
   },
@@ -289,6 +293,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [