Skip to content

Commit 22b97bd

Browse files
author
Matt Sokoloff
committed
data row content under row_data
1 parent 6c845bc commit 22b97bd

File tree

2 files changed

+135
-31
lines changed

2 files changed

+135
-31
lines changed

‎labelbox/schema/dataset.py‎

Lines changed: 26 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -229,8 +229,8 @@ def _create_descriptor_file(self, items, max_attachments_per_data_row=None):
229229
>>> {DataRow.row_data:"http://my_site.com/photos/img_01.jpg"},
230230
>>> {DataRow.row_data:"/path/to/file1.jpg"},
231231
>>> "path/to/file2.jpg",
232-
>>> {"tileLayerUrl" : "http://", ...}
233-
>>> {"conversationalData" : [...], ...}
232+
>>> {DataRow.row_data: {"tileLayerUrl" : "http://", ...}}
233+
>>> {DataRow.row_data: {"type" : ..., 'version' : ..., 'messages' : [...]}}
234234
>>> ])
235235
236236
For an example showing how to upload tiled data_rows see the following notebook:
@@ -258,7 +258,7 @@ def _create_descriptor_file(self, items, max_attachments_per_data_row=None):
258258

259259
def upload_if_necessary(item):
260260
row_data = item['row_data']
261-
if os.path.exists(row_data):
261+
if isinstance(row_data, str) and os.path.exists(row_data):
262262
item_url = self.client.upload_file(row_data)
263263
item['row_data'] = item_url
264264
if 'external_id' not in item:
@@ -341,40 +341,39 @@ def validate_keys(item):
341341
"`row_data` missing when creating DataRow.")
342342

343343
invalid_keys = set(item) - {
344-
*{f.name for f in DataRow.fields()}, 'attachments'
344+
*{f.name for f in DataRow.fields()}, 'attachments', 'media_type'
345345
}
346346
if invalid_keys:
347347
raise InvalidAttributeError(DataRow, invalid_keys)
348348
return item
349349

350+
def formatLegacyConversationalData(item):
351+
messages = item.pop("conversationalData")
352+
version = item.pop("version")
353+
type = item.pop("type")
354+
if "externalId" in item:
355+
external_id = item.pop("externalId")
356+
item["external_id"] = external_id
357+
if "globalKey" in item:
358+
global_key = item.pop("globalKey")
359+
item["globalKey"] = global_key
360+
validate_conversational_data(messages)
361+
one_conversation = \
362+
{
363+
"type": type,
364+
"version": version,
365+
"messages": messages
366+
}
367+
item["row_data"] = one_conversation
368+
return item
369+
350370
def convert_item(item):
351-
# Don't make any changes to tms data
352371
if "tileLayerUrl" in item:
353372
validate_attachments(item)
354373
return item
355374

356375
if "conversationalData" in item:
357-
messages = item.pop("conversationalData")
358-
version = item.pop("version")
359-
type = item.pop("type")
360-
if "externalId" in item:
361-
external_id = item.pop("externalId")
362-
item["external_id"] = external_id
363-
if "globalKey" in item:
364-
global_key = item.pop("globalKey")
365-
item["globalKey"] = global_key
366-
validate_conversational_data(messages)
367-
one_conversation = \
368-
{
369-
"type": type,
370-
"version": version,
371-
"messages": messages
372-
}
373-
conversationUrl = self.client.upload_data(
374-
json.dumps(one_conversation),
375-
content_type="application/json",
376-
filename="conversational_data.json")
377-
item["row_data"] = conversationUrl
376+
formatLegacyConversationalData(item)
378377

379378
# Convert all payload variations into the same dict format
380379
item = format_row(item)
@@ -386,11 +385,7 @@ def convert_item(item):
386385
parse_metadata_fields(item)
387386
# Upload any local file paths
388387
item = upload_if_necessary(item)
389-
390-
return {
391-
"data" if key == "row_data" else utils.camel_case(key): value
392-
for key, value in item.items()
393-
}
388+
return item
394389

395390
if not isinstance(items, Iterable):
396391
raise ValueError(

‎tests/integration/test_data_rows.py‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from tempfile import NamedTemporaryFile
22
import uuid
33
from datetime import datetime
4+
import json
45

56
import pytest
67
import requests
@@ -695,3 +696,111 @@ def test_data_row_rulk_creation_sync_with_same_global_keys(
695696

696697
assert len(list(dataset.data_rows())) == 1
697698
assert list(dataset.data_rows())[0].global_key == global_key_1
699+
700+
701+
def test_create_conversational_text(dataset):
702+
content = {
703+
'row_data': {
704+
"messages": [{
705+
"messageId": "message-0",
706+
"timestampUsec": 1530718491,
707+
"content": "I love iphone! i just bought new iphone! 🥰 📲",
708+
"user": {
709+
"userId": "Bot 002",
710+
"name": "Bot"
711+
},
712+
"align": "left",
713+
"canLabel": False
714+
}],
715+
"version": 1,
716+
"type": "application/vnd.labelbox.conversational"
717+
}
718+
}
719+
examples = [
720+
{
721+
**content, 'media_type': 'CONVERSATIONAL_TEXT'
722+
},
723+
content,
724+
content['row_data'] # Old way to check for backwards compatibility
725+
]
726+
dataset.create_data_rows_sync(examples)
727+
data_rows = list(dataset.data_rows())
728+
assert len(data_rows) == len(examples)
729+
for data_row in data_rows:
730+
assert requests.get(data_row.row_data).json() == content['row_data']
731+
732+
733+
def test_invalid_media_type(dataset):
734+
content = {
735+
'row_data': {
736+
"messages": [{
737+
"messageId": "message-0",
738+
"timestampUsec": 1530718491,
739+
"content": "I love iphone! i just bought new iphone! 🥰 📲",
740+
"user": {
741+
"userId": "Bot 002",
742+
"name": "Bot"
743+
},
744+
"align": "left",
745+
"canLabel": False
746+
}],
747+
"version": 1,
748+
"type": "application/vnd.labelbox.conversational"
749+
}
750+
}
751+
752+
for error_message, invalid_media_type in [[
753+
"Found invalid contents for media type: 'IMAGE'", 'IMAGE'
754+
], ["Found invalid media type: 'totallyinvalid'", 'totallyinvalid']]:
755+
# TODO: What error kind should this be? It looks like for global key we are
756+
# using malformed query. But for FileUploads we use InvalidQueryError
757+
with pytest.raises(labelbox.exceptions.InvalidQueryError):
758+
dataset.create_data_rows_sync([{
759+
**content, 'media_type': invalid_media_type
760+
}])
761+
762+
task = dataset.create_data_rows([{
763+
**content, 'media_type': invalid_media_type
764+
}])
765+
task.wait_till_done()
766+
assert task.errors == {'message': error_message}
767+
768+
769+
def test_create_tiled_layer(dataset):
770+
content = {
771+
"row_data": {
772+
"tileLayerUrl":
773+
"https://s3-us-west-1.amazonaws.com/lb-tiler-layers/mexico_city/{z}/{x}/{y}.png",
774+
"bounds": [[19.405662413477728, -99.21052827588443],
775+
[19.400498983095076, -99.20534818927473]],
776+
"minZoom":
777+
12,
778+
"maxZoom":
779+
20,
780+
"epsg":
781+
"EPSG4326",
782+
"alternativeLayers": [{
783+
"tileLayerUrl":
784+
"https://api.mapbox.com/styles/v1/mapbox/satellite-streets-v11/tiles/{z}/{x}/{y}?access_token=pk.eyJ1IjoibWFwYm94IiwiYSI6ImNpejY4NXVycTA2emYycXBndHRqcmZ3N3gifQ.rJcFIG214AriISLbB6B5aw",
785+
"name":
786+
"Satellite"
787+
}, {
788+
"tileLayerUrl":
789+
"https://api.mapbox.com/styles/v1/mapbox/navigation-guidance-night-v4/tiles/{z}/{x}/{y}?access_token=pk.eyJ1IjoibWFwYm94IiwiYSI6ImNpejY4NXVycTA2emYycXBndHRqcmZ3N3gifQ.rJcFIG214AriISLbB6B5aw",
790+
"name":
791+
"Guidance"
792+
}]
793+
}
794+
}
795+
examples = [
796+
{
797+
**content, 'media_type': 'TMS_SIMPLE'
798+
},
799+
content,
800+
content['row_data'] # Old way to check for backwards compatibility
801+
]
802+
dataset.create_data_rows_sync(examples)
803+
data_rows = list(dataset.data_rows())
804+
assert len(data_rows) == len(examples)
805+
for data_row in data_rows:
806+
assert json.loads(data_row.row_data) == content['row_data']

0 commit comments

Comments
 (0)