merge with ms/annotation-updates

Author: Matt Sokoloff

.github/workflows/python-package.yml

@@ -47,13 +47,21 @@ jobs:
       uses: AlexanderMelde/yapf-action@master
       with:
         args: --verbose --recursive --parallel --style "google"
-
+    - name: dependencies
+      run: |
+        sudo apt-get -y update
+        sudo apt install -y libsm6 \
+                         libxext6 \
+                           ffmpeg \
+                   libfontconfig1 \
+                      libxrender1 \
+                  libgl1-mesa-glx
     - name: install labelbox package
       run: |
-        python setup.py install
+        python -m pip install --upgrade pip
+        python -m pip install .
     - name: mypy
       run: |
-        python -m pip install --upgrade pip
         pip install mypy==0.782
         mypy -p labelbox --pretty --show-error-codes
     - name: Install package and test dependencies

Dockerfile

@@ -1,12 +1,17 @@
 FROM python:3.7
 
 RUN pip install pytest pytest-cases
-RUN apt-get -y update && apt install -y libsm6 libxext6 ffmpeg libfontconfig1 libxrender1 libgl1-mesa-glx
+RUN apt-get -y update
+RUN apt install -y libsm6 \
+                libxext6 \
+                ffmpeg \
+                libfontconfig1 \
+                libxrender1 \
+                libgl1-mesa-glx
 
 WORKDIR /usr/src/labelbox
 COPY requirements.txt /usr/src/labelbox
 RUN pip install -r requirements.txt
 COPY . /usr/src/labelbox
 
-
 RUN python setup.py install

labelbox/data/generator.py

@@ -2,6 +2,7 @@
 import threading
 from queue import Queue
 from typing import Any, Iterable
+from concurrent.futures import ThreadPoolExecutor
 
 logger = logging.getLogger(__name__)
 
@@ -32,10 +33,7 @@ class PrefetchGenerator:
     Useful for modifying the generator results based on data from a network
     """
 
-    def __init__(self,
-                 data: Iterable[Any],
-                 prefetch_limit=20,
-                 max_concurrency=4):
+    def __init__(self, data: Iterable[Any], prefetch_limit=20, num_executors=4):
         if isinstance(data, (list, tuple)):
             self._data = (r for r in data)
         else:
@@ -44,14 +42,13 @@ def __init__(self,
         self.queue = Queue(prefetch_limit)
         self._data = ThreadSafeGen(self._data)
         self.completed_threads = 0
-        self.max_concurrency = max_concurrency
-        self.threads = [
-            threading.Thread(target=self.fill_queue)
-            for _ in range(max_concurrency)
-        ]
-        for thread in self.threads:
-            thread.daemon = True
-            thread.start()
+        # Can only iterate over once it the queue.get hangs forever.
+        self.done = False
+        self.num_executors = num_executors
+        with ThreadPoolExecutor(max_workers=num_executors) as executor:
+            self.futures = [
+                executor.submit(self.fill_queue) for _ in range(num_executors)
+            ]
 
     def _process(self, value) -> Any:
         raise NotImplementedError("Abstract method needs to be implemented")
@@ -73,10 +70,13 @@ def __iter__(self):
         return self
 
     def __next__(self) -> Any:
+        if self.done:
+            raise StopIteration
         value = self.queue.get()
         while value is None:
             self.completed_threads += 1
-            if self.completed_threads == self.max_concurrency:
+            if self.completed_threads == self.num_executors:
+                self.done = True
                 raise StopIteration
             value = self.queue.get()
         return value

labelbox/schema/ontology.py

@@ -1,23 +1,20 @@
-import abc
 from dataclasses import dataclass, field
-from enum import Enum, auto
+from enum import Enum
 import colorsys
 
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 from labelbox.schema.project import Project
-from labelbox.orm import query
-from labelbox.orm.db_object import DbObject, Updateable, BulkDeletable
-from labelbox.orm.model import Entity, Field, Relationship
-from labelbox.utils import snake_case, camel_case
+from labelbox.orm.db_object import DbObject
+from labelbox.orm.model import Field, Relationship
 from labelbox.exceptions import InconsistentOntologyException
 
 
 @dataclass
 class Option:
     """
     An option is a possible answer within a Classification object in
-    a Project's ontology. 
+    a Project's ontology.
 
     To instantiate, only the "value" parameter needs to be passed in.
 
@@ -41,13 +38,11 @@ def label(self):
 
     @classmethod
     def from_dict(cls, dictionary: Dict[str, Any]):
-        return Option(value=dictionary["value"],
-                      schema_id=dictionary.get("schemaNodeId", None),
-                      feature_schema_id=dictionary.get("featureSchemaId", None),
-                      options=[
-                          Classification.from_dict(o)
-                          for o in dictionary.get("options", [])
-                      ])
+        return cls(
+            value=dictionary["value"],
+            schema_id=dictionary.get("schemaNodeId", None),
+            feature_schema_id=dictionary.get("featureSchemaId", None),
+            options=[cls.from_dict(o) for o in dictionary.get("options", [])])
 
     def asdict(self) -> Dict[str, Any]:
         return {
@@ -69,13 +64,13 @@ def add_option(self, option: 'Classification'):
 @dataclass
 class Classification:
     """
-    A classfication to be added to a Project's ontology. The  
+    A classfication to be added to a Project's ontology. The
     classification is dependent on the Classification Type.
 
     To instantiate, the "class_type" and "instructions" parameters must
     be passed in.
 
-    The "options" parameter holds a list of Option objects. This is not 
+    The "options" parameter holds a list of Option objects. This is not
     necessary for some Classification types, such as TEXT. To see which
     types require options, look at the "_REQUIRES_OPTIONS" class variable.
 
@@ -120,16 +115,15 @@ def name(self):
 
     @classmethod
     def from_dict(cls, dictionary: Dict[str, Any]):
-        return Classification(
-            class_type=Classification.Type(dictionary["type"]),
-            instructions=dictionary["instructions"],
-            required=dictionary.get("required", False),
-            options=[Option.from_dict(o) for o in dictionary["options"]],
-            schema_id=dictionary.get("schemaNodeId", None),
-            feature_schema_id=dictionary.get("featureSchemaId", None))
+        return cls(class_type=cls.Type(dictionary["type"]),
+                   instructions=dictionary["instructions"],
+                   required=dictionary.get("required", False),
+                   options=[Option.from_dict(o) for o in dictionary["options"]],
+                   schema_id=dictionary.get("schemaNodeId", None),
+                   feature_schema_id=dictionary.get("featureSchemaId", None))
 
     def asdict(self) -> Dict[str, Any]:
-        if self.class_type in Classification._REQUIRES_OPTIONS \
+        if self.class_type in self._REQUIRES_OPTIONS \
                 and len(self.options) < 1:
             raise InconsistentOntologyException(
                 f"Classification '{self.instructions}' requires options.")
@@ -160,13 +154,13 @@ class Tool:
     To instantiate, the "tool" and "name" parameters must
     be passed in.
 
-    The "classifications" parameter holds a list of Classification objects. 
+    The "classifications" parameter holds a list of Classification objects.
     This can be used to add nested classifications to a tool.
 
     Example(s):
         tool = Tool(
             tool = Tool.Type.LINE,
-            name = "Tool example")    
+            name = "Tool example")
         classification = Classification(
             class_type = Classification.Type.TEXT,
             instructions = "Classification Example")
@@ -200,16 +194,16 @@ class Type(Enum):
 
     @classmethod
     def from_dict(cls, dictionary: Dict[str, Any]):
-        return Tool(name=dictionary['name'],
-                    schema_id=dictionary.get("schemaNodeId", None),
-                    feature_schema_id=dictionary.get("featureSchemaId", None),
-                    required=dictionary.get("required", False),
-                    tool=Tool.Type(dictionary["tool"]),
-                    classifications=[
-                        Classification.from_dict(c)
-                        for c in dictionary["classifications"]
-                    ],
-                    color=dictionary["color"])
+        return cls(name=dictionary['name'],
+                   schema_id=dictionary.get("schemaNodeId", None),
+                   feature_schema_id=dictionary.get("featureSchemaId", None),
+                   required=dictionary.get("required", False),
+                   tool=cls.Type(dictionary["tool"]),
+                   classifications=[
+                       Classification.from_dict(c)
+                       for c in dictionary["classifications"]
+                   ],
+                   color=dictionary["color"])
 
     def asdict(self) -> Dict[str, Any]:
         return {
@@ -287,9 +281,9 @@ class OntologyBuilder:
     for making Project ontologies from scratch. OntologyBuilder can also
     pull from an already existing Project's ontology.
 
-    There are no required instantiation arguments. 
+    There are no required instantiation arguments.
 
-    To create an ontology, use the asdict() method after fully building your 
+    To create an ontology, use the asdict() method after fully building your
     ontology within this class, and inserting it into project.setup() as the
     "labeling_frontend_options" parameter.
 
@@ -303,19 +297,18 @@ class OntologyBuilder:
         tools: (list)
         classifications: (list)
 
-    
+
     """
     tools: List[Tool] = field(default_factory=list)
     classifications: List[Classification] = field(default_factory=list)
 
     @classmethod
     def from_dict(cls, dictionary: Dict[str, Any]):
-        return OntologyBuilder(
-            tools=[Tool.from_dict(t) for t in dictionary["tools"]],
-            classifications=[
-                Classification.from_dict(c)
-                for c in dictionary["classifications"]
-            ])
+        return cls(tools=[Tool.from_dict(t) for t in dictionary["tools"]],
+                   classifications=[
+                       Classification.from_dict(c)
+                       for c in dictionary["classifications"]
+                   ])
 
     def asdict(self):
         self._update_colors()
@@ -337,7 +330,7 @@ def _update_colors(self):
     @classmethod
     def from_project(cls, project: Project):
         ontology = project.ontology().normalized
-        return OntologyBuilder.from_dict(ontology)
+        return cls.from_dict(ontology)
 
     def add_tool(self, tool: Tool):
         if tool.name in (t.name for t in self.tools):

mypy.ini

@@ -6,3 +6,6 @@ ignore_missing_imports = True
 
 [mypy-google.*]
 ignore_missing_imports = True
+
+[mypy-labelbox.data.*]
+ignore_errors = True

setup.py

@@ -22,11 +22,18 @@
     install_requires=[
         "backoff==1.10.0",
         "backports-datetime-fromisoformat==1.0.0; python_version < '3.7.0'",
-        "dataclasses==0.7; python_version < '3.7.0'", "ndjson==0.3.1",
-        "requests>=2.22.0", "google-api-core>=1.22.1", "pydantic>=1.8,<2.0",
-        "shapely", "geojson", "numpy", "rasterio", "PILLOW", "opencv-python",
-        "typeguard", "tqdm"
+        "dataclasses==0.7; python_version < '3.7.0'",
+        "ndjson==0.3.1",
+        "requests>=2.22.0",
+        "google-api-core>=1.22.1",
+        "pydantic>=1.8,<2.0",
     ],
+    extras_require={
+        'data': [
+            "shapely", "geojson", "numpy", "rasterio", "PILLOW",
+            "opencv-python", "typeguard", "tqdm"
+        ],
+    },
     classifiers=[
         'Development Status :: 3 - Alpha',
         'License :: OSI Approved :: Apache Software License',