diff --git a/cachy.jsonl b/cachy.jsonl index 5e3f037..6c423cd 100644 --- a/cachy.jsonl +++ b/cachy.jsonl @@ -19,3 +19,4 @@ {"key": "d4142886", "response": "{\n \"candidates\": [\n {\n \"content\": {\n \"parts\": [\n {\n \"text\": \"Concurrency\\n\"\n }\n ],\n \"role\": \"model\"\n },\n \"finishReason\": \"STOP\",\n \"avgLogprobs\": -0.29076665639877319\n }\n ],\n \"usageMetadata\": {\n \"promptTokenCount\": 10,\n \"candidatesTokenCount\": 2,\n \"totalTokenCount\": 12,\n \"promptTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 10\n }\n ],\n \"candidatesTokensDetails\": [\n {\n \"modality\": \"TEXT\",\n \"tokenCount\": 2\n }\n ]\n },\n \"modelVersion\": \"gemini-2.0-flash\",\n \"responseId\": \"1gvIaLDAGe2kvdIPsY6--Q0\"\n}\n"} {"key": "fe23aa62", "response": "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"Concurrency\"}],\"role\": \"model\"}}],\"usageMetadata\": {\"promptTokenCount\": 12,\"totalTokenCount\": 12,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 12}]},\"modelVersion\": \"gemini-2.0-flash\",\"responseId\": \"1gvIaKSQOt3h1PIPwJO12Ac\"}\r\n\r\ndata: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"\\n\"}],\"role\": \"model\"},\"finishReason\": \"STOP\"}],\"usageMetadata\": {\"promptTokenCount\": 11,\"candidatesTokenCount\": 2,\"totalTokenCount\": 13,\"promptTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 11}],\"candidatesTokensDetails\": [{\"modality\": \"TEXT\",\"tokenCount\": 2}]},\"modelVersion\": \"gemini-2.0-flash\",\"responseId\": \"1gvIaKSQOt3h1PIPwJO12Ac\"}\r\n\r\n"} {"key": "c90feca2", "response": "{\"id\":\"msg_01HpiQTg22STqarE33JnuHdt\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-sonnet-4-20250514\",\"content\":[{\"type\":\"tool_use\",\"id\":\"toolu_0182nVBg1pTYTadKxS5qgCt4\",\"name\":\"get_current_weather\",\"input\":{\"location\":\"Reims\"}}],\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":427,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":57,\"service_tier\":\"standard\"}}"} +{"key": "79d28180", "response": "{\n \"id\": \"resp_07096c5fa7f05f3600692af5de73f0819c85e63a30484d9b65\",\n \"object\": \"response\",\n \"created_at\": 1764423134,\n \"status\": \"completed\",\n \"background\": false,\n \"billing\": {\n \"payer\": \"developer\"\n },\n \"error\": null,\n \"incomplete_details\": null,\n \"instructions\": null,\n \"max_output_tokens\": null,\n \"max_tool_calls\": null,\n \"model\": \"gpt-4.1-2025-04-14\",\n \"output\": [\n {\n \"id\": \"msg_07096c5fa7f05f3600692af5df34a8819ca053230b7e0ede57\",\n \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": [\n {\n \"type\": \"output_text\",\n \"annotations\": [],\n \"logprobs\": [],\n \"text\": \"Hello! \\ud83d\\ude0a How can I help you today?\"\n }\n ],\n \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": true,\n \"previous_response_id\": null,\n \"prompt_cache_key\": null,\n \"prompt_cache_retention\": null,\n \"reasoning\": {\n \"effort\": null,\n \"summary\": null\n },\n \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": true,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": \"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\": \"auto\",\n \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": 9,\n \"input_tokens_details\": {\n \"cached_tokens\": 0\n },\n \"output_tokens\": 11,\n \"output_tokens_details\": {\n \"reasoning_tokens\": 0\n },\n \"total_tokens\": 20\n },\n \"user\": null,\n \"metadata\": {}\n}"} diff --git a/cachy/_modidx.py b/cachy/_modidx.py index e4be19d..2837235 100644 --- a/cachy/_modidx.py +++ b/cachy/_modidx.py @@ -9,6 +9,7 @@ 'cachy.core._apply_sync_patch': ('core.html#_apply_sync_patch', 'cachy/core.py'), 'cachy.core._cache': ('core.html#_cache', 'cachy/core.py'), 'cachy.core._key': ('core.html#_key', 'cachy/core.py'), + 'cachy.core._norm_multipart': ('core.html#_norm_multipart', 'cachy/core.py'), 'cachy.core._should_cache': ('core.html#_should_cache', 'cachy/core.py'), 'cachy.core._write_cache': ('core.html#_write_cache', 'cachy/core.py'), 'cachy.core.disable_cachy': ('core.html#disable_cachy', 'cachy/core.py'), diff --git a/cachy/core.py b/cachy/core.py index ceafb86..5f535c8 100644 --- a/cachy/core.py +++ b/cachy/core.py @@ -6,7 +6,7 @@ __all__ = ['doms', 'enable_cachy', 'disable_cachy'] # %% ../nbs/00_core.ipynb -import hashlib,httpx,json +import hashlib,httpx,json,re from fastcore.utils import * # %% ../nbs/00_core.ipynb @@ -25,10 +25,18 @@ def _cache(key, cfp): def _write_cache(key, content, cfp): with open(cfp, "a") as f: f.write(json.dumps({"key":key, "response": content})+"\n") +# %% ../nbs/00_core.ipynb +def _norm_multipart(r): + ct = r.headers.get('content-type', '') + match = re.search(r'boundary=([a-f0-9]{32})', ct) + if match: return r.content.replace(match.group(1).encode(), b'BOUNDARY') + return r.content + # %% ../nbs/00_core.ipynb def _key(r, is_stream=False): "Create a unique, deterministic id from the request `r`." - return hashlib.sha256(f"{r.url.host}{is_stream}".encode() + r.content).hexdigest()[:8] + r.read() + return hashlib.sha256(f"{r.url.host}{is_stream}".encode() + _norm_multipart(r)).hexdigest()[:8] # %% ../nbs/00_core.ipynb def _apply_async_patch(cfp, doms): diff --git a/nbs/00_core.ipynb b/nbs/00_core.ipynb index 14917d0..7642a40 100644 --- a/nbs/00_core.ipynb +++ b/nbs/00_core.ipynb @@ -71,7 +71,7 @@ "outputs": [], "source": [ "#| export\n", - "import hashlib,httpx,json\n", + "import hashlib,httpx,json,re\n", "from fastcore.utils import *" ] }, @@ -407,6 +407,27 @@ "First let's include an `is_stream` bool in our hash so that a non-streamed request will generate a different key to the same request when streamed. " ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def _norm_multipart(r):\n", + " ct = r.headers.get('content-type', '')\n", + " match = re.search(r'boundary=([a-f0-9]{32})', ct)\n", + " if match: return r.content.replace(match.group(1).encode(), b'BOUNDARY')\n", + " return r.content" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, we use `httpx.Request.read()` which reads the streaming request body into memory (storing it in `_content`) so that `r.content` can be accessed, and it's safe to call multiple times since it's a no-op if already read. For multipart requests, we normalize the content by replacing httpx's randomly-generated boundary string with a fixed value, ensuring identical requests produce the same cache key." + ] + }, { "cell_type": "code", "execution_count": null, @@ -416,7 +437,8 @@ "#| exports\n", "def _key(r, is_stream=False):\n", " \"Create a unique, deterministic id from the request `r`.\"\n", - " return hashlib.sha256(f\"{r.url.host}{is_stream}\".encode() + r.content).hexdigest()[:8]" + " r.read()\n", + " return hashlib.sha256(f\"{r.url.host}{is_stream}\".encode() + _norm_multipart(r)).hexdigest()[:8]" ] }, {