Skip to content

Commit f3a2222

Browse files
kkim-labelboxkkim-labelbox
authored
removed NDJSON library (#1145)
2 parents b74ca54 + 19efd1d commit f3a2222

File tree

1 file changed

+38
-104
lines changed

1 file changed

+38
-104
lines changed

‎examples/basics/custom_embeddings.ipynb‎

Lines changed: 38 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -47,25 +47,21 @@
4747
"metadata": {},
4848
"source": [
4949
"# labelbox\n",
50-
"!pip3 install -q labelbox[data]\n",
51-
"import labelbox as lb\n",
52-
"#ndjson\n",
53-
"!pip3 install -q ndjson\n",
54-
"import ndjson"
50+
"!pip3 install -q \"labelbox[data]\""
5551
],
5652
"cell_type": "code",
57-
"outputs": [
58-
{
59-
"output_type": "stream",
60-
"name": "stdout",
61-
"text": [
62-
"\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m189.2/189.2 KB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
63-
"\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m52.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
64-
"\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
65-
" Building wheel for pygeotile (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
66-
]
67-
}
53+
"outputs": [],
54+
"execution_count": null
55+
},
56+
{
57+
"metadata": {},
58+
"source": [
59+
"import labelbox as lb \n",
60+
"import numpy as np\n",
61+
"import json"
6862
],
63+
"cell_type": "code",
64+
"outputs": [],
6965
"execution_count": null
7066
},
7167
{
@@ -82,19 +78,7 @@
8278
"!pip3 install -q 'git+https://github.com/Labelbox/advlib.git'"
8379
],
8480
"cell_type": "code",
85-
"outputs": [
86-
{
87-
"output_type": "stream",
88-
"name": "stdout",
89-
"text": [
90-
" Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
91-
" Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
92-
" Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
93-
"\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m62.8/62.8 KB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
94-
"\u001b[?25h Building wheel for advlib (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"
95-
]
96-
}
97-
],
81+
"outputs": [],
9882
"execution_count": null
9983
},
10084
{
@@ -151,7 +135,7 @@
151135
{
152136
"metadata": {},
153137
"source": [
154-
"import numpy as np\n",
138+
"\n",
155139
"\n",
156140
"nb_data_rows = len(data_row_ids)\n",
157141
"# generate 1000 custom embedding vectors, of dimension 2048 each\n",
@@ -180,80 +164,49 @@
180164
"metadata": {},
181165
"source": [
182166
"# convert payload to ndjson file\n",
167+
"\n",
183168
"with open('payload.ndjson', 'w') as f:\n",
184-
" ndjson.dump(payload, f)\n",
169+
" sanity_check_payload = json.dump(payload, f)\n",
170+
"\n",
185171
"\n",
186172
"# sanity check that you can read/load the file and the payload is correct\n",
187173
"with open('payload.ndjson') as f:\n",
188-
" sanity_check_payload = ndjson.load(f)\n",
174+
" sanity_check_payload = json.load(f)\n",
175+
" \n",
189176
"\n",
190177
"print(\"Nb of custom embedding vectors in sanity_check_payload: \", len(sanity_check_payload))\n",
191178
"# print(\"sanity_check_payload: \", sanity_check_payload)"
192179
],
193180
"cell_type": "code",
194-
"outputs": [
195-
{
196-
"output_type": "stream",
197-
"name": "stdout",
198-
"text": [
199-
"Nb of custom embedding vectors in sanity_check_payload: 1000\n"
200-
]
201-
}
202-
],
181+
"outputs": [],
203182
"execution_count": null
204183
},
205-
{
206-
"metadata": {},
207-
"source": [
208-
"# Pick an existing custom embedding, or create a custom embedding"
209-
],
210-
"cell_type": "markdown"
211-
},
212184
{
213185
"metadata": {},
214186
"source": [
215187
"# See all custom embeddings available\n",
216188
"!advtool embeddings list"
217189
],
218190
"cell_type": "code",
219-
"outputs": [
220-
{
221-
"output_type": "stream",
222-
"name": "stdout",
223-
"text": [
224-
"00000000-0000-0000-0000-000000000000 - Image Embedding (CLIP ViT-B/32) - dims: 512 \n",
225-
"00000000-0000-0000-0000-000000000001 - Text embedding (All-MPNet-base-v2) - dims: 768 \n",
226-
"45cafc7a-5314-462a-8afc-7a5314062a3b - my_custom_embedding_2048_dimensions - dims: 2048 \n",
227-
"7d3a6118-589d-4b6c-ba61-18589dbb6ccf - ResNet50_2048_dimensions - dims: 2048 \n"
228-
]
229-
}
230-
],
191+
"outputs": [],
231192
"execution_count": null
232193
},
233194
{
234195
"metadata": {},
235196
"source": [
236197
"# # Create a new custom embedding\n",
237-
"!advtool embeddings create my_custom_embedding_2048_dimensions_v2 2048\n",
238-
"# will return the ID of the newly created embedding, e.g. 0ddc5d5c-0963-41ad-9c5d-5c0963a1ad98"
198+
"!advtool embeddings create my_custom_embedding_2048_dimensions 2048\n",
199+
"# will return the ID of the newly created embedding, e.g. cgbjjt5ra07710005liytdf19"
239200
],
240201
"cell_type": "code",
241-
"outputs": [
242-
{
243-
"output_type": "stream",
244-
"name": "stdout",
245-
"text": [
246-
"Embedding type created id=da5d4b0f-e440-4e2e-9d4b-0fe4400e2e8d\n"
247-
]
248-
}
249-
],
202+
"outputs": [],
250203
"execution_count": null
251204
},
252205
{
253206
"metadata": {},
254207
"source": [
255208
"# # Delete a custom embedding\n",
256-
"# !advtool embeddings delete 2e122b85-7def-44fb-922b-857defe4fb8a"
209+
"# !advtool embeddings delete cj7j0ukre0771000blj4qnxgn"
257210
],
258211
"cell_type": "code",
259212
"outputs": [],
@@ -270,38 +223,27 @@
270223
"metadata": {},
271224
"source": [
272225
"# Upload the payload to Labelbox \n",
273-
"!advtool embeddings import da5d4b0f-e440-4e2e-9d4b-0fe4400e2e8d ./payload.ndjson"
226+
"!advtool embeddings import cj7j0ukre0771000blj4qnxgn ./payload.ndjson"
274227
],
275228
"cell_type": "code",
276-
"outputs": [
277-
{
278-
"output_type": "stream",
279-
"name": "stdout",
280-
"text": [
281-
"Uploading file: ./payload.ndjson \n",
282-
"Progress: 100.0%\n",
283-
"Check 'advtool embeddings count <embedding id>' for total searchable embeddings\n"
284-
]
285-
}
286-
],
229+
"outputs": [],
287230
"execution_count": null
288231
},
232+
{
233+
"metadata": {},
234+
"source": [
235+
"# Pick an existing custom embedding, or create a custom embedding"
236+
],
237+
"cell_type": "markdown"
238+
},
289239
{
290240
"metadata": {},
291241
"source": [
292242
"# count how many data rows have a specific custom embedding (This can take a couple of minutes)\n",
293-
"!advtool embeddings count da5d4b0f-e440-4e2e-9d4b-0fe4400e2e8d"
243+
"!advtool embeddings count cj7j0ukre0771000blj4qnxgn"
294244
],
295245
"cell_type": "code",
296-
"outputs": [
297-
{
298-
"output_type": "stream",
299-
"name": "stdout",
300-
"text": [
301-
"1000\n"
302-
]
303-
}
304-
],
246+
"outputs": [],
305247
"execution_count": null
306248
},
307249
{
@@ -310,16 +252,8 @@
310252
"print(len(payload))"
311253
],
312254
"cell_type": "code",
313-
"outputs": [
314-
{
315-
"output_type": "stream",
316-
"name": "stdout",
317-
"text": [
318-
"1000\n"
319-
]
320-
}
321-
],
255+
"outputs": [],
322256
"execution_count": null
323257
}
324258
]
325-
}
259+
}

0 commit comments

Comments
 (0)