[MODIFY] filter for parent doc.

Author: Dongjin

genai/aws-gen-ai-kr/20_applications/02_qa_chatbot/01_preprocess_docs/05_0_load_complex_pdf_kr_opensearch.ipynb

@@ -137,7 +137,7 @@
    },
    "outputs": [],
    "source": [
-    "from langchain_community.chat_models import BedrockChat\n",
+    "from langchain_aws import ChatBedrock\n",
     "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler"
    ]
   },
@@ -150,7 +150,7 @@
    },
    "outputs": [],
    "source": [
-    "llm_text = BedrockChat(\n",
+    "llm_text = ChatBedrock(\n",
     "    model_id=bedrock_info.get_model_id(model_name=\"Claude-V3-Sonnet\"),\n",
     "    client=boto3_bedrock,\n",
     "    streaming=True,\n",
@@ -504,7 +504,7 @@
    },
    "outputs": [],
    "source": [
-    "table_by_llama_parse = True"
+    "table_by_llama_parse = False"
    ]
   },
   {
@@ -517,7 +517,7 @@
    "outputs": [],
    "source": [
     "def api_key():\n",
-    "    os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"llx-R4G3Pzu5IZIdq5AoAFILW1PPaVZxrVRN937R6f3cItBvPs1U\"\n",
+    "    os.environ[\"LLAMA_CLOUD_API_KEY\"] = \"<your key>\"\n",
     "    nest_asyncio.apply()\n",
     "    load_dotenv()"
    ]
@@ -617,7 +617,7 @@
    },
    "outputs": [],
    "source": [
-    "table_by_pymupdf = True"
+    "table_by_pymupdf = False"
    ]
   },
   {
@@ -803,32 +803,6 @@
     "# tables_camleot[0].df.to_markdown()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "825d1b1f-5208-4a1d-8ee9-971a493d7bfb",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "# for image in images:\n",
-    "    \n",
-    "#     img = cv2.imread(image) \n",
-    "#     width, height, _ = img.shape\n",
-    "#     image_token = width*height/750\n",
-    "#     print (f'image: {image}, shape: {img.shape}, image_token_for_claude3: {image_token}' )\n",
-    "    \n",
-    "#     if image_token > 1500:\n",
-    "#         resize_img = cv2.resize(img, (0, 0), fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)\n",
-    "#         print(\"   - resize_img.shape = {0}\".format(resize_img.shape))\n",
-    "#         table_image_path = image.replace(\".jpg\", \"-resize.jpg\")\n",
-    "#         cv2.imwrite(table_image_path, resize_img)\n",
-    "#         os.remove(image)\n",
-    "    \n",
-    "#     images = glob(os.path.join(image_path, \"*\"))"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "fb16fd2a-1983-462d-94d8-4c62e81ef28c",
@@ -1102,18 +1076,6 @@
     "summarize_chain = {\"table\": lambda x:x} | prompt | llm_text | StrOutputParser()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7a0bc0a5-7f5e-4bb8-9f0f-bbaf31622448",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "len(tables), len(docs_table_pymupdf), len(docs_table_llamaparse)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -1223,18 +1185,6 @@
     "#tables_preprocessed, images_preprocessed\n"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d3504e83-d62e-40a9-9cdc-d5fbea2d7029",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "tables_preprocessed[0]"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -1314,7 +1264,7 @@
    "outputs": [],
    "source": [
     "#index_name = \"kb_complex_doc\"\n",
-    "index_name = \"summit-workshop-index-unstructured-pymupdf-llama\" #summit-workshop-index-unstructured, #summit-workshop-index-unstructured-pymupdf"
+    "index_name = \"summit-workshop-index\" #summit-workshop-index-unstructured, #summit-workshop-index-unstructured-pymupdf, summit-workshop-index-unstructured-pymupdf-llama"
    ]
   },
   {