From 6ba13758372b794d765404e2f45dd3bd271a979b Mon Sep 17 00:00:00 2001 From: Fady Morris Milad Date: Sun, 22 Jan 2023 01:46:28 +0200 Subject: [PATCH 1/5] feat: Add matplotlib inline magic Enable inline plotting in Jypter notebooks. --- project/starter.ipynb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/project/starter.ipynb b/project/starter.ipynb index c782396b9..713f2cc12 100644 --- a/project/starter.ipynb +++ b/project/starter.ipynb @@ -196,6 +196,15 @@ "))" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1084,9 +1093,9 @@ "metadata": { "instance_type": "ml.t3.medium", "kernelspec": { - "display_name": "Python 3 (Data Science)", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1098,7 +1107,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.9.13" } }, "nbformat": 4, From e482deeabf00d39fd823a56bb684d4050ba368ba Mon Sep 17 00:00:00 2001 From: Fady Morris Milad Date: Sun, 22 Jan 2023 01:51:42 +0200 Subject: [PATCH 2/5] fix: The second and the third lambda returned events The returned event should be consistent across the three functions. The input events are passed to output events. --- project/starter.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project/starter.ipynb b/project/starter.ipynb index 713f2cc12..71d01746a 100644 --- a/project/starter.ipynb +++ b/project/starter.ipynb @@ -845,10 +845,10 @@ " inferences = ## TODO: fill in\n", " \n", " # We return the data back to the Step Function \n", - " event[\"inferences\"] = inferences.decode('utf-8')\n", + " event[\"body\"][\"inferences\"] = inferences.decode('utf-8')\n", " return {\n", " 'statusCode': 200,\n", - " 'body': json.dumps(event)\n", + " 'body': event[\"body\"]\n", " }\n", "```\n", "\n", @@ -878,7 +878,7 @@ "\n", " return {\n", " 'statusCode': 200,\n", - " 'body': json.dumps(event)\n", + " 'body': event[\"body\"]\n", " }\n", "```\n", "Once you have tested the lambda functions, save the code for each lambda function in a python script called 'lambda.py'.\n", From 907e3bc1458397005e87d96ce0daeff8ae488d7b Mon Sep 17 00:00:00 2001 From: Fady Morris Milad Date: Sun, 22 Jan 2023 01:55:58 +0200 Subject: [PATCH 3/5] fix: Make the events have a consistent key order. The event dictionaries keys were rearranged to have the same order acoss all the three functions. --- project/starter.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/project/starter.ipynb b/project/starter.ipynb index 71d01746a..75e3cb73f 100644 --- a/project/starter.ipynb +++ b/project/starter.ipynb @@ -760,10 +760,10 @@ "\n", "```python\n", "{\n", - " \"inferences\": [], # Output of predictor.predict\n", - " \"s3_key\": \"\", # Source data S3 key\n", " \"s3_bucket\": \"\", # Source data S3 bucket\n", - " \"image_data\": \"\" # base64 encoded string containing the image data\n", + " \"s3_key\": \"\", # Source data S3 key\n", + " \"image_data\": \"\", # base64 encoded string containing the image data\n", + " \"inferences\": [] # Output of predictor.predict\n", "}\n", "```\n", "\n", @@ -771,9 +771,9 @@ "\n", "```python\n", "{\n", - " \"image_data\": \"\",\n", " \"s3_bucket\": MY_BUCKET_NAME, # Fill in with your bucket\n", - " \"s3_key\": \"test/bicycle_s_000513.png\"\n", + " \"s3_key\": \"test/bicycle_s_000513.png\",\n", + " \"image_data\": \"\"\n", "}\n", "```\n", "\n", @@ -807,9 +807,9 @@ " return {\n", " 'statusCode': 200,\n", " 'body': {\n", - " \"image_data\": image_data,\n", " \"s3_bucket\": bucket,\n", " \"s3_key\": key,\n", + " \"image_data\": image_data,\n", " \"inferences\": []\n", " }\n", " }\n", From 881614213c9ca19d9e934cd7e70ff1ee34d332d3 Mon Sep 17 00:00:00 2001 From: Fady Morris Milad Date: Sun, 22 Jan 2023 01:58:54 +0200 Subject: [PATCH 4/5] fix: Fix extraction of model monitor `data_capture` Model monitor data caputre output has changed. In a recent update each output file has its own subdirectory. So the code needs to recurseviley walk subdirectories and read the jsonl files. --- project/starter.ipynb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/project/starter.ipynb b/project/starter.ipynb index 75e3cb73f..e9cf4ec49 100644 --- a/project/starter.ipynb +++ b/project/starter.ipynb @@ -997,12 +997,15 @@ "import os\n", "\n", "# List the file names we downloaded\n", - "file_handles = os.listdir(\"./captured_data\")\n", + "file_handles = []\n", + "for root, subFolders, files in os.walk(\"./captured_data\"):\n", + " if files:\n", + " file_handles += [root + '/' + file for file in files]\n", "\n", "# Dump all the data into an array\n", "json_data = []\n", "for jsonl in file_handles:\n", - " with jsonlines.open(f\"./captured_data/{jsonl}\") as f:\n", + " with jsonlines.open(jsonl) as f:\n", " json_data.append(f.read())" ] }, From bbc646ef13b53b687f28ff6f1c2e8007ef978fcf Mon Sep 17 00:00:00 2001 From: Fady Morris Milad Date: Sat, 5 Aug 2023 07:59:56 +0300 Subject: [PATCH 5/5] fix: Wrap the event body in `json.dumps()` Wrap the event body in `json.dumps()` for the second and the third lambda functions. --- project/starter.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/project/starter.ipynb b/project/starter.ipynb index e9cf4ec49..91d41f2eb 100644 --- a/project/starter.ipynb +++ b/project/starter.ipynb @@ -848,7 +848,7 @@ " event[\"body\"][\"inferences\"] = inferences.decode('utf-8')\n", " return {\n", " 'statusCode': 200,\n", - " 'body': event[\"body\"]\n", + " 'body': json.dumps(event[\"body\"])\n", " }\n", "```\n", "\n", @@ -878,7 +878,7 @@ "\n", " return {\n", " 'statusCode': 200,\n", - " 'body': event[\"body\"]\n", + " 'body': json.dumps(event[\"body\"])\n", " }\n", "```\n", "Once you have tested the lambda functions, save the code for each lambda function in a python script called 'lambda.py'.\n", @@ -1110,7 +1110,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.7.6" } }, "nbformat": 4,