restore tests

Author: Matt Sokoloff

.github/workflows/python-package.yml

@@ -67,7 +67,7 @@ jobs:
       env:
         # make sure to tell tox to use these environs in tox.ini
         #
-        # randall@labelbox.com
+        # msokoloff+prod-python@labelbox.com
         LABELBOX_TEST_API_KEY_PROD: ${{ secrets.LABELBOX_API_KEY }}
 
         # randall+staging-python@labelbox.com

Dockerfile

@@ -1,7 +1,8 @@
-FROM python:3.6
+FROM python:3.7
+
+RUN pip install pytest
 
 COPY . /usr/src/labelbox
 WORKDIR /usr/src/labelbox
 
-RUN pip install pytest
 RUN python setup.py install

Makefile

@@ -5,11 +5,11 @@ build:
 test-staging: build
 	docker run -it -v ${PWD}:/usr/src -w /usr/src \
 		-e LABELBOX_TEST_ENVIRON="staging" \
-		-e LABELBOX_TEST_API_KEY_STAGING="<REPLACE>" \
+		-e LABELBOX_TEST_API_KEY_STAGING=${LABELBOX_TEST_API_KEY_STAGING} \
 		local/labelbox-python:test pytest $(PATH_TO_TEST) -svvx
 
 test-prod: build
 	docker run -it -v ${PWD}:/usr/src -w /usr/src \
 		-e LABELBOX_TEST_ENVIRON="prod" \
-		-e LABELBOX_TEST_API_KEY_PROD="<REPLACE>" \
+		-e LABELBOX_TEST_API_KEY_PROD=${LABELBOX_TEST_API_KEY_PROD} \
 		local/labelbox-python:test pytest $(PATH_TO_TEST) -svvx

labelbox/client.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timezone
 import json
+from labelbox.schema.ontology import Ontology
 import logging
 import mimetypes
 import os
@@ -71,6 +72,7 @@ def __init__(self,
             'X-User-Agent': f'python-sdk {SDK_VERSION}'
         }
 
+    #TODO: Add exponential backoff so we don'tt overwhelm the api
     @retry.Retry(predicate=retry.if_exception_type(
         labelbox.exceptions.InternalServerError))
     def execute(self, query, params=None, timeout=10.0):
@@ -126,25 +128,23 @@ def convert_value(value):
             logger.debug("Response: %s", response.text)
         except requests.exceptions.Timeout as e:
             raise labelbox.exceptions.TimeoutError(str(e))
-
         except requests.exceptions.RequestException as e:
             logger.error("Unknown error: %s", str(e))
             raise labelbox.exceptions.NetworkError(e)
-
         except Exception as e:
             raise labelbox.exceptions.LabelboxError(
                 "Unknown error during Client.query(): " + str(e), e)
-
         try:
             r_json = response.json()
         except:
-            error_502 = '502 Bad Gateway'
-            if error_502 in response.text:
-                raise labelbox.exceptions.InternalServerError(error_502)
             if "upstream connect error or disconnect/reset before headers" \
                     in response.text:
                 raise labelbox.exceptions.InternalServerError(
                     "Connection reset")
+            elif response.status_code == 502:
+                error_502 = '502 Bad Gateway'
+                raise labelbox.exceptions.InternalServerError(error_502)
+
             raise labelbox.exceptions.LabelboxError(
                 "Failed to parse response as JSON: %s" % response.text)
 
@@ -189,6 +189,7 @@ def check_errors(keywords, *path):
 
         # Check if API limit was exceeded
         response_msg = r_json.get("message", "")
+
         if response_msg.startswith("You have exceeded"):
             raise labelbox.exceptions.ApiLimitError(response_msg)
 
@@ -292,7 +293,6 @@ def upload_data(self,
                 "1": (filename, content, content_type) if
                      (filename and content_type) else content
             })
-
         try:
             file_data = response.json().get("data", None)
         except ValueError as e:  # response is not valid JSON
@@ -415,6 +415,12 @@ def get_datasets(self, where=None):
         """
         return self._get_all(Dataset, where)
 
+    def get_ontologies(self, where=None):
+        """
+        #TODO
+        """
+        return self._get_all(Ontology, where)
+
     def get_labeling_frontends(self, where=None):
         """ Fetches all the labeling frontends.
 
@@ -473,7 +479,9 @@ def create_dataset(self, **kwargs):
         """
         return self._create(Dataset, kwargs)
 
-    def create_project(self, **kwargs):
+    def create_project(self,
+                       ontology=None,
+                       **kwargs):  #<<<<<   TODO: Do we want that signature
         """ Creates a Project object on the server.
 
         Attribute values are passed as keyword arguments.

labelbox/schema/labeling_frontend.py

@@ -35,7 +35,6 @@ class LabelingFrontendOptions(DbObject):
         organization (Relationship): `ToOne` relationship to Organization
     """
     customization_options = Field.String("customization_options")
-
     project = Relationship.ToOne("Project")
     labeling_frontend = Relationship.ToOne("LabelingFrontend")
     organization = Relationship.ToOne("Organization")

labelbox/schema/ontology.py

@@ -63,6 +63,51 @@ def from_json(cls, json_dict):
         return cls(**_dict)
 
 
+"""
+* The reason that an ontology is read only is because it is a second class citizen to labeling front end options.
+** This is because it is a more specific implementation of this.
+
+- However, we want to support ontologies as if they were labeling front ends.
+- With this special relationship we can override the default behavior to mock the appropriate changes to the labeling front end
+
+
+###Note: The only problem is that you can't just create a stand alone ontology. right? 
+# - Since you need to create a project and query the project ontology before one exists.
+
+^^^^^^^ This is the worst. Even with hackery, you can't force a DB entry without create a new proj :(
+     However, labeling front-ends cannot be created without projects either! So maybe we just copy the use cases of that.
+     Use this as the simpler interface and make it clear that this is just a limited version
+
+"""
+
+
+class OntologyRelationship(Relationship):
+
+    def __get__(self, parent):
+        if not self.parent:
+            self.parent = parent
+        return self
+
+    def __init__(self):
+        super(OntologyRelationship, self).__init__()
+        self.parent = None
+
+    def __call__(self):
+        if self.parent.setup_complete is None:
+            #As it currently stands, it creates a new ontology with no new tools and the ontology cannot be edited.
+            return None
+        return super().__call__
+
+    def connect(self, other_ontology):
+        if not isinstance(other_ontology, OntologyRelationship):
+            raise Exception("only support ")
+
+    def disconnect(self):
+        raise Exception(
+            "Disconnect is not supported for Onotlogy. Instead connect another ontology to replace the current one."
+        )
+
+
 class Ontology(DbObject):
     """An ontology specifies which tools and classifications are available
     to a project. This is read only for now.

labelbox/schema/project.py

@@ -221,6 +221,16 @@ def review_metrics(self, net_score):
         res = self.client.execute(query_str, {id_param: self.uid})
         return res["project"]["reviewMetrics"]["labelAggregate"]["count"]
 
+    def setup_from_ontology(self, ontology):
+        """
+        * An ontology is a specific kind of a labeling frontend option that is compatible with the Editor.
+        * Most customers use this interface
+        """
+        frontend = list(
+            client.get_labeling_frontends(
+                where=LabelingFrontend.name == "Editor"))[0]
+        return self.setup(frontend, ontology.normalized)
+
     def setup(self, labeling_frontend, labeling_frontend_options):
         """ Finalizes the Project setup.
 

tests/integration/test_client_errors.py

@@ -1,10 +1,9 @@
 from multiprocessing.dummy import Pool
 import os
 import time
-
 import pytest
 
-from labelbox import Project, Dataset, User
+from labelbox import Project, Dataset, User, LabelingFrontend
 import labelbox.client
 import labelbox.exceptions
 
@@ -111,22 +110,26 @@ def test_invalid_attribute_error(client, rand_gen):
     project.delete()
 
 
-@pytest.mark.slow
-# TODO improve consistency
-@pytest.mark.skip(reason="Inconsistent test")
-def test_api_limit_error(client, rand_gen):
-    project_id = client.create_project(name=rand_gen(str)).uid
+@pytest.mark.skip("timeouts cause failure before rate limit")
+def test_api_limit_error(client):
+    global limited
+    limited = False
 
     def get(arg):
         try:
-            return client.get_project(project_id)
+            return client.get_user()
         except labelbox.exceptions.ApiLimitError as e:
             return e
 
-    with Pool(300) as pool:
-        results = pool.map(get, list(range(2000)))
+    n = 1600
+    with Pool(30) as pool:
+        start = time.time()
+        results = list(pool.imap(get, range(n)), total=n)
+        elapsed = time.time() - start
 
+    assert elapsed < 60, "Didn't finish fast enough"
     assert labelbox.exceptions.ApiLimitError in {type(r) for r in results}
+    del limited
 
     # Sleep at the end of this test to allow other tests to execute.
-    time.sleep(60)
+    #time.sleep(60)

tests/integration/test_data_upload.py

@@ -2,10 +2,8 @@
 import requests
 
 
-# TODO it seems that at some point Google Storage (gs prefix) started being
-# returned, and we can't just download those with requests. Fix this
-@pytest.mark.skip
-def test_file_upload(client, rand_gen):
+def test_file_upload(client, rand_gen, dataset):
     data = rand_gen(str)
-    url = client.upload_data(data.encode())
-    assert requests.get(url).text == data
+    uri = client.upload_data(data.encode())
+    data_row = dataset.create_data_row(row_data=uri)
+    assert requests.get(data_row.row_data).text == data

tests/integration/test_label.py

@@ -1,3 +1,4 @@
+from labelbox.schema.labeling_frontend import LabelingFrontend
 import time
 
 import pytest
@@ -30,18 +31,22 @@ def test_labels(label_pack):
     assert list(data_row.labels()) == []
 
 
-# TODO check if this is supported or not
-@pytest.mark.skip
-def test_label_export(label_pack):
+def test_label_export(client, label_pack):
     project, dataset, data_row, label = label_pack
-    project.create_label(data_row=data_row, label="l2")
-
-    exported_labels_url = project.export_labels(5)
+    #Old create_label works even with projects setup using the new editor.
+    #It will appear in the export, just not in the new editor
+    project.create_label(data_row=data_row, label="export_label")
+    #Project has to be setup for export to be possible
+    editor = list(
+        client.get_labeling_frontends(
+            where=LabelingFrontend.name == "editor"))[0]
+    empty_ontology = {"tools": [], "classifications": []}
+    project.setup(editor, empty_ontology)
+    exported_labels_url = project.export_labels()
     assert exported_labels_url is not None
-    if exported_labels_url is not None:
-        exported_labels = requests.get(exported_labels_url)
-        # TODO check content
-        assert False
+    exported_labels = requests.get(exported_labels_url)
+    labels = [example['Label'] for example in exported_labels.json()]
+    assert 'export_label' in labels
 
 
 def test_label_update(label_pack):