Skip to content

Commit 7ffa109

Browse files
ovalle15ovalle15
authored
[SN-120] video composite mask (#1443)
Added composite mask examples in video
1 parent 6435b77 commit 7ffa109

File tree

2 files changed

+108
-73
lines changed

2 files changed

+108
-73
lines changed

‎examples/annotation_import/image.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,11 @@
3434
"metadata": {},
3535
"source": [
3636
"# Image annotation import\n",
37-
"This notebook will provide examples of each supported annotation type for image assets. \n",
37+
"This notebook will provide examples of each supported annotation type for image assets.\n",
3838
"\n",
3939
"### [Model-assisted labeling (MAL)](https://docs.labelbox.com/docs/model-assisted-labeling)\n",
4040
"\n",
41-
"* This workflow allows you to import computer-generated predictions (or simply annotations created outside of Labelbox) as pre-labels on an asset. \n",
41+
"* This workflow allows you to import computer-generated predictions (or simply annotations created outside of Labelbox) as pre-labels on an asset.\n",
4242
"\n",
4343
"The imported annotations will be pre-populated in the labeling editor. However, in order to convert the pre-labels to real annotations, a human labeler will still need to open the Data Row in the Editor and submit it. This functionality is designed to speed up human labeling.\n",
4444
"\n",
@@ -264,7 +264,7 @@
264264
{
265265
"metadata": {},
266266
"source": [
267-
"### Classification: Free-form text "
267+
"### Classification: Free-form text"
268268
],
269269
"cell_type": "markdown"
270270
},

‎examples/annotation_import/video.ipynb‎

Lines changed: 105 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,15 @@
6767
{
6868
"metadata": {},
6969
"source": [
70-
"import labelbox as lb\n",
71-
"import labelbox.types as lb_types\n",
7270
"import uuid\n",
71+
"from PIL import Image\n",
72+
"import requests\n",
7373
"import base64\n",
74-
"import requests"
74+
"import labelbox as lb\n",
75+
"import labelbox.types as lb_types\n",
76+
"from io import BytesIO\n",
77+
"import pprint\n",
78+
"pp = pprint.PrettyPrinter(indent=4)"
7579
],
7680
"cell_type": "code",
7781
"outputs": [],
@@ -675,82 +679,109 @@
675679
],
676680
"cell_type": "markdown"
677681
},
682+
{
683+
"metadata": {},
684+
"source": [
685+
"def extract_rgb_colors_from_url(image_url):\n",
686+
" response = requests.get(image_url)\n",
687+
" img = Image.open(BytesIO(response.content))\n",
688+
"\n",
689+
" colors = set()\n",
690+
" for x in range(img.width):\n",
691+
" for y in range(img.height):\n",
692+
" pixel = img.getpixel((x, y))\n",
693+
" if pixel[:3] != (0,0,0):\n",
694+
" colors.add(pixel[:3]) # Get only the RGB values\n",
695+
"\n",
696+
" return colors"
697+
],
698+
"cell_type": "code",
699+
"outputs": [],
700+
"execution_count": null
701+
},
678702
{
679703
"metadata": {},
680704
"source": [
681705
"### Raster Segmentation (Byte string array)\n",
682-
"url = \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/color_mask.png\"\n",
683-
"response = requests.get(url)\n",
706+
"## For this example we are going to to pass all the annotations payload in a single VideoMaskAnnotation\n",
684707
"\n",
685708
"\n",
686-
"video_mask_annotation_bytes = [\n",
687-
" lb_types.VideoMaskAnnotation(\n",
688-
" frames=[\n",
689-
" lb_types.MaskFrame(\n",
690-
" index=20,\n",
691-
" im_bytes=response.content # Instead of bytes you could also pass an instance URI : instance_uri=url\n",
692-
" )\n",
693-
" ],\n",
694-
" instances=[\n",
695-
" lb_types.MaskInstance(color_rgb=(255, 255, 1), name= \"video_mask\")\n",
696-
" ]\n",
709+
"# Single mask\n",
710+
"url = \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/frame_24_composite_mask.png\"\n",
711+
"response = requests.get(url)\n",
712+
"img_bytes = base64.b64encode(response.content).decode('utf-8')\n",
713+
"\n",
714+
"# We are generating our frames and instances in this step, and will later add them to the VideoMaskAnnotation that will contain\n",
715+
"# all frames and instances\n",
716+
"frames_mask_single=[\n",
717+
" lb_types.MaskFrame(\n",
718+
" index=20,\n",
719+
" im_bytes=response.content # Instead of bytes you could also pass an instance URI : instance_uri=url\n",
697720
" )\n",
698721
"]\n",
699-
"img_bytes = base64.b64encode(response.content).decode('utf-8')\n",
700-
"# NDJSON\n",
701-
"video_mask_ndjson_bytes = {\n",
702-
" 'masks': {\n",
703-
" 'frames': [\n",
704-
" {\n",
705-
" \"index\" : 20,\n",
706-
" \"imBytes\": img_bytes,\n",
707-
" }\n",
708-
" ],\n",
709-
" 'instances': [\n",
710-
" {\n",
711-
" \"colorRGB\" : [255, 255, 1],\n",
712-
" \"name\" : \"video_mask\"\n",
713-
" }\n",
714-
" ]\n",
715-
" }\n",
716-
" }\n",
722+
"instances_mask_single=[\n",
723+
" lb_types.MaskInstance(color_rgb=(76, 104, 177), name= \"video_mask\")\n",
724+
"]\n",
717725
"\n",
718-
"# Python annotation - same mask on multiple frames (note that tracking is not supported with masks tools)\n",
719-
"video_mask_annotation_bytes_2 = [\n",
720-
" lb_types.VideoMaskAnnotation(\n",
721-
" frames=[\n",
722-
" lb_types.MaskFrame(\n",
723-
" index=23,\n",
724-
" im_bytes=response.content\n",
725-
" ),\n",
726+
"\n",
727+
"## Add multiple masks using multiple tools in different frames - Note that only once composite mask can exist per frame\n",
728+
"frames_cp_mask_url = [\n",
729+
" {\"1\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/frame_1_composite_mask.png\"},\n",
730+
" {\"24\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/frame_24_composite_mask.png\"},\n",
731+
" {\"26\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/frame_26_composite_mask.png\" }\n",
732+
"]\n",
733+
"\n",
734+
"rgb_mask_tool = [(227, 135, 126) ,(169, 248, 152),(83, 152, 103)]\n",
735+
"cp_masks = []\n",
736+
"unique_colors = set()\n",
737+
"\n",
738+
"\n",
739+
"lb_frames = []\n",
740+
"lb_instances = []\n",
741+
"counter = 0\n",
742+
"\n",
743+
"for d in frames_cp_mask_url:\n",
744+
" for frame_no, v in d.items():\n",
745+
" response = requests.get(v)\n",
746+
" colors = extract_rgb_colors_from_url(v)\n",
747+
" for color in colors:\n",
748+
" if not color in unique_colors:\n",
749+
" unique_colors.add(color)\n",
750+
" name = \"video_mask\" if color in rgb_mask_tool else \"mask_with_text_subclass\"\n",
751+
" lb_instances.append(lb_types.MaskInstance(color_rgb=color, name=name))\n",
752+
" counter += 1\n",
753+
" lb_frames.append(\n",
726754
" lb_types.MaskFrame(\n",
727-
" index=20,\n",
755+
" index=frame_no,\n",
728756
" im_bytes=response.content\n",
729757
" )\n",
730-
" ],\n",
731-
" instances=[\n",
732-
" lb_types.MaskInstance(color_rgb=(255, 1, 1), name= \"video_mask\")\n",
733-
" ]\n",
734-
" )\n",
735-
"]\n",
758+
" )\n",
759+
"cp_masks.append(lb_types.VideoMaskAnnotation(\n",
760+
" frames=lb_frames + frames_mask_single,\n",
761+
" instances=lb_instances + instances_mask_single\n",
762+
"))\n",
736763
"\n",
764+
"pp.pprint(lb_frames)\n",
765+
"pp.pprint(cp_masks)\n",
737766
"\n",
738-
"# NDJSON\n",
767+
"\n",
768+
"\n",
769+
"# NDJSON - single tool\n",
739770
"video_mask_ndjson_bytes_2 = {\n",
740771
" 'masks': {\n",
741772
" 'frames': [\n",
742773
" {\n",
743-
" \"index\" : 20,\n",
774+
" \"index\" : 31,\n",
744775
" \"imBytes\": img_bytes,\n",
745776
" },\n",
746777
" {\n",
747-
" \"index\" : 23,\n",
778+
" \"index\" : 34,\n",
748779
" \"imBytes\": img_bytes,\n",
749780
" }\n",
750781
" ],\n",
751782
" 'instances': [\n",
752783
" {\n",
753-
" \"colorRGB\" : [255, 1, 1],\n",
784+
" \"colorRGB\" : [76, 104, 177],\n",
754785
" \"name\" : \"video_mask\"\n",
755786
" }\n",
756787
" ]\n",
@@ -886,8 +917,15 @@
886917
")\n",
887918
"task = dataset.create_data_rows([asset])\n",
888919
"task.wait_till_done()\n",
889-
"print(\"Errors :\",task.errors)\n",
890-
"print(\"Failed data rows:\" ,task.failed_data_rows)"
920+
"print(f\"Failed data rows: {task.failed_data_rows}\")\n",
921+
"print(f\"Errors: {task.errors}\")\n",
922+
"\n",
923+
"if task.errors:\n",
924+
" for error in task.errors:\n",
925+
" if 'Duplicate global key' in error['message'] and dataset.row_count == 0:\n",
926+
" # If the global key already exists in the workspace the dataset will be created empty, so we can delete it.\n",
927+
" print(f\"Deleting empty dataset: {dataset}\")\n",
928+
" dataset.delete()"
891929
],
892930
"cell_type": "code",
893931
"outputs": [],
@@ -928,7 +966,15 @@
928966
" ]\n",
929967
" )\n",
930968
" ]\n",
931-
" )\n",
969+
" ),\n",
970+
" lb.Tool(tool=lb.Tool.Type.RASTER_SEGMENTATION,\n",
971+
" name=\"mask_with_text_subclass\",\n",
972+
" classifications=[\n",
973+
" lb.Classification(\n",
974+
" class_type=lb.Classification.Type.TEXT,\n",
975+
" name=\"sub_free_text\")\n",
976+
" ]\n",
977+
" )\n",
932978
" ],\n",
933979
" classifications=[\n",
934980
" lb.Classification(\n",
@@ -1088,8 +1134,7 @@
10881134
" nested_checklist_annotation,\n",
10891135
" nested_radio_annotation,\n",
10901136
" text_annotation,\n",
1091-
" video_mask_annotation_bytes,\n",
1092-
" video_mask_annotation_bytes_2\n",
1137+
" cp_masks\n",
10931138
" ]\n",
10941139
"\n",
10951140
"for annotation in annotations_list:\n",
@@ -1138,9 +1183,7 @@
11381183
" text_annotation_ndjson,\n",
11391184
" bbox_frame_annotation_ndjson,\n",
11401185
" bbox_frame_annotation_ndjson2,\n",
1141-
" video_mask_ndjson_bytes,\n",
1142-
" video_mask_ndjson_bytes_2,\n",
1143-
"\n",
1186+
" video_mask_ndjson_bytes_2\n",
11441187
"]\n",
11451188
"\n",
11461189
"for annotation in annotations_list_ndjson:\n",
@@ -1229,19 +1272,11 @@
12291272
"source": [
12301273
"# Delete Project\n",
12311274
"# project.delete()\n",
1232-
"# dataset.delete()\n",
1233-
"\n"
1275+
"#dataset.delete()\n"
12341276
],
12351277
"cell_type": "code",
12361278
"outputs": [],
12371279
"execution_count": null
1238-
},
1239-
{
1240-
"metadata": {},
1241-
"source": [],
1242-
"cell_type": "code",
1243-
"outputs": [],
1244-
"execution_count": null
12451280
}
12461281
]
12471282
}

0 commit comments

Comments
 (0)