update to ignore logic

Author: jtsodapop

labelbox/data/serialization/ndjson/converter.py

@@ -6,6 +6,8 @@
 
 logger = logging.getLogger(__name__)
 
+IGNORE_IF_NONE = ["page", "unit"]
+
 
 class NDJsonConverter:
 
@@ -39,4 +41,8 @@ def serialize(
             A generator for accessing the ndjson representation of the data
         """
         for example in NDLabel.from_common(labels):
-            yield example.dict(by_alias=True)
+            res = example.dict(by_alias=True)
+            for k, v in list(res.items()):
+                if k in IGNORE_IF_NONE and v is None:
+                    del res[k]
+            yield res

tests/data/serialization/ndjson/test_classification.py

@@ -4,18 +4,13 @@
 from labelbox.data.serialization.ndjson.converter import NDJsonConverter
 from labelbox.data.serialization.ndjson.objects import NDLine
 
-IGNORE_KEYS = ['unit', 'page']
-
 
 def test_classification():
     with open('tests/data/assets/ndjson/classification_import.json',
               'r') as file:
         data = json.load(file)
     res = NDJsonConverter.deserialize(data).as_list()
     res = list(NDJsonConverter.serialize(res))
-    for r in res:
-        for key in IGNORE_KEYS:
-            r.pop(key, None)
     assert res == data
 
 
@@ -25,7 +20,4 @@ def test_classification_with_name():
         data = json.load(file)
     res = NDJsonConverter.deserialize(data).as_list()
     res = list(NDJsonConverter.serialize(res))
-    for r in res:
-        for key in IGNORE_KEYS:
-            r.pop(key, None)
     assert res == data

tests/data/serialization/ndjson/test_export_video_objects.py

@@ -586,7 +586,7 @@ def video_serialized_bbox_label():
 
 
 #ignore uuid because we randomize if there was none
-IGNORE_KEYS = ["uuid", "page", "unit"]
+IGNORE_KEYS = ["uuid"]
 
 
 def test_serialize_video_objects():

tests/data/serialization/ndjson/test_image.py

@@ -100,6 +100,4 @@ def test_mask_from_arr():
             "png":
                 "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAAAAABWESUoAAAAHklEQVR4nGNgGAKAEYn8j00BEyETBoOCUTAKhhwAAJW+AQwvpePVAAAAAElFTkSuQmCC"
         },
-        "page": None,
-        "unit": None
     }

tests/data/serialization/ndjson/test_nested.py

@@ -2,17 +2,12 @@
 
 from labelbox.data.serialization.ndjson.converter import NDJsonConverter
 
-IGNORE_KEYS = ['unit', 'page']
-
 
 def test_nested():
     with open('tests/data/assets/ndjson/nested_import.json', 'r') as file:
         data = json.load(file)
     res = NDJsonConverter.deserialize(data).as_list()
     res = list(NDJsonConverter.serialize(res))
-    for r in res:
-        for key in IGNORE_KEYS:
-            r.pop(key, None)
     assert res == data
 
 
@@ -22,7 +17,4 @@ def test_nested_name_only():
         data = json.load(file)
     res = NDJsonConverter.deserialize(data).as_list()
     res = list(NDJsonConverter.serialize(res))
-    for r in res:
-        for key in IGNORE_KEYS:
-            r.pop(key, None)
     assert res == data

tests/data/serialization/ndjson/test_text.py

@@ -2,17 +2,12 @@
 
 from labelbox.data.serialization.ndjson.converter import NDJsonConverter
 
-IGNORE_KEYS = ['unit', 'page']
-
 
 def test_text():
     with open('tests/data/assets/ndjson/text_import.json', 'r') as file:
         data = json.load(file)
     res = NDJsonConverter.deserialize(data).as_list()
     res = list(NDJsonConverter.serialize(res))
-    for r in res:
-        for key in IGNORE_KEYS:
-            r.pop(key, None)
     assert res == data
 
 
@@ -22,7 +17,4 @@ def test_text_name_only():
         data = json.load(file)
     res = NDJsonConverter.deserialize(data).as_list()
     res = list(NDJsonConverter.serialize(res))
-    for r in res:
-        for key in IGNORE_KEYS:
-            r.pop(key, None)
     assert res == data

tests/data/serialization/ndjson/test_video.py

@@ -2,18 +2,13 @@
 
 from labelbox.data.serialization.ndjson.converter import NDJsonConverter
 
-IGNORE_KEYS = ['unit', 'page']
-
 
 def test_video():
     with open('tests/data/assets/ndjson/video_import.json', 'r') as file:
         data = json.load(file)
 
     res = NDJsonConverter.deserialize(data).as_list()
     res = list(NDJsonConverter.serialize(res))
-    for r in res:
-        for key in IGNORE_KEYS:
-            r.pop(key, None)
     assert res == [data[2], data[0], data[1], data[3], data[4], data[5]]
 
 
@@ -24,7 +19,4 @@ def test_video_name_only():
 
     res = NDJsonConverter.deserialize(data).as_list()
     res = list(NDJsonConverter.serialize(res))
-    for r in res:
-        for key in IGNORE_KEYS:
-            r.pop(key, None)
     assert res == [data[2], data[0], data[1], data[3], data[4], data[5]]