Closing OPEN-4186 API for validating goal statuses

Author: whoseoyster

CHANGELOG.md

@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 * Added `protobuf<3.20` to requirements to fix compatibility issue with Tensorflow.
 * Warnings if the dependencies from the `requirement_txt_file` and current environment are inconsistent.
 * Paths to custom SSL certificates can now be modified by altering `openlayer.api.VERIFY_REQUESTS`. The value can either be True (default), False, or a path to a certificate.
+* Ability to check for goal statuses through the API.
 
 ### Changed
 

openlayer/__init__.py

@@ -33,6 +33,7 @@
 import yaml
 
 from . import api, exceptions, utils
+from .project_versions import ProjectVersion
 from .projects import Project
 from .schemas import BaselineModelSchema, DatasetSchema, ModelSchema
 from .tasks import TaskType
@@ -473,7 +474,7 @@ def add_model(
 
     def add_baseline_model(
         self,
-        project_id: int,
+        project_id: str,
         task_type: TaskType,
         model_config_file_path: Optional[str] = None,
         force: bool = False,
@@ -994,7 +995,7 @@ class probabilities. For example, for a binary classification
                 task_type=task_type,
             )
 
-    def commit(self, message: str, project_id: int, force: bool = False):
+    def commit(self, message: str, project_id: str, force: bool = False):
         """Adds a commit message to staged resources.
 
         Parameters
@@ -1081,7 +1082,7 @@ def commit(self, message: str, project_id: int, force: bool = False):
 
         print("Committed!")
 
-    def push(self, project_id: int, task_type: TaskType):
+    def push(self, project_id: str, task_type: TaskType) -> Optional[ProjectVersion]:
         """Pushes the commited resources to the platform.
 
         Notes
@@ -1124,15 +1125,17 @@ def push(self, project_id: int, task_type: TaskType):
                     f"\t - Date: {commit['date']}"
                 )
                 payload = {"commit": {"message": commit["message"]}}
-                self.api.upload(
+                response_body = self.api.upload(
                     endpoint=f"projects/{project_id}/versions",
                     file_path=tar_file_path,
                     object_name="tarfile",
                     body=payload,
                 )
+                project_version = ProjectVersion(json=response_body, client=self)
 
             self._post_push_cleanup(project_dir=project_dir)
             print("Pushed!")
+            return project_version
 
     def _ready_for_push(self, project_dir: str, task_type: TaskType) -> bool:
         """Checks if the project's staging area is ready to be pushed to the platform.
@@ -1183,7 +1186,7 @@ def _post_push_cleanup(self, project_dir: str) -> None:
         shutil.rmtree(project_dir)
         os.makedirs(project_dir, exist_ok=True)
 
-    def export(self, destination_dir: str, project_id: int, task_type: TaskType):
+    def export(self, destination_dir: str, project_id: str, task_type: TaskType):
         """Exports the commited resources as a tarfile to the location specified
         by ``destination_dir``.
 
@@ -1229,7 +1232,7 @@ def export(self, destination_dir: str, project_id: int, task_type: TaskType):
             self._post_push_cleanup(project_dir=project_dir)
             print("Exported tarfile!")
 
-    def status(self, project_id: int):
+    def status(self, project_id: str):
         """Shows the state of the staging area.
 
         Examples
@@ -1277,7 +1280,7 @@ def status(self, project_id: int):
         print(f"\t {commit['message']}")
         print("Use the `push` method to push your changes to the platform.")
 
-    def restore(self, *resource_names: str, project_id: int):
+    def restore(self, *resource_names: str, project_id: str):
         """Removes the resource specified by ``resource_name`` from the staging area.
 
         Parameters
@@ -1328,7 +1331,7 @@ def restore(self, *resource_names: str, project_id: int):
                 os.remove(f"{project_dir}/commit.yaml")
 
     def _stage_resource(
-        self, resource_name: str, resource_dir: str, project_id: int, force: bool
+        self, resource_name: str, resource_dir: str, project_id: str, force: bool
     ):
         """Adds the resource specified by `resource_name` to the project's staging directory.
 
@@ -1370,3 +1373,43 @@ def _stage_resource(
         shutil.copytree(resource_dir, project_dir + "/" + resource_name)
 
         print(f"Staged the `{resource_name}` resource!")
+
+    def load_project_version(self, version_id: str) -> Project:
+        """Loads an existing project version from the Openlayer platform. Can be used
+        to check the status of the project version and the number of passing, failing
+        and skipped goals.
+
+        Parameters
+        ----------
+        id : str
+            UUID of the project to be loaded. You can find the UUID of a project by
+            navigating to the project's page on the Openlayer platform.
+
+            .. note::
+                When you run :obj:`push`, it will return the project version object,
+                which you can use to check your goal statuses.
+
+        Returns
+        -------
+        ProjectVersion
+            An object that is used to check for upload progress and goal statuses.
+            Also contains other useful information about a project version.
+
+        Examples
+        --------
+        Instantiate the client and load the project version:
+
+        >>> import openlayer
+        >>> client = openlayer.OpenlayerClient('YOUR_API_KEY_HERE')
+        >>>
+        >>> version = client.load_project_version(id='YOUR_PROJECT_ID_HERE')
+        >>> version.wait_for_completion()
+        >>> version.print_goal_report()
+
+        With the ProjectVersion object loaded, you are able to check progress and
+        goal statuses.
+        """
+        endpoint = f"versions/{version_id}"
+        version_data = self.api.get_request(endpoint)
+        version = ProjectVersion(version_data, self)
+        return version

openlayer/project_versions.py

@@ -0,0 +1,140 @@
+"""Module for the ProjectVersion class."""
+
+import enum
+import time
+from typing import Optional
+
+import tabulate
+
+
+class TaskStatus(enum.Enum):
+    """An enum containing the possible states of a project version."""
+
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    QUEUED = "queued"
+    PAUSED = "paused"
+    UNKNOWN = "unknown"
+
+
+class ProjectVersion:
+    """An object containing information about a project version on the
+    Openlayer platform.
+    """
+
+    def __init__(self, json, client):
+        self._json = json
+        self.id = json["id"]
+        self.client = client
+
+    def __getattr__(self, name):
+        if name in self._json:
+            return self._json[name]
+        raise AttributeError(f"'{type(self).__name__}' object has no attribute {name}")
+
+    def __hash__(self):
+        return hash(self.id)
+
+    def __str__(self):
+        return f"ProjectVersion(id={self.id})"
+
+    def __repr__(self):
+        return f"ProjectVersion({self._json})"
+
+    def to_dict(self):
+        """Returns object properties as a dict.
+
+        Returns
+        -------
+        Dict with object properties.
+        """
+        return self._json
+
+    @property
+    def status(self) -> TaskStatus:
+        """Returns the current state of the project version."""
+        return TaskStatus(self._json["status"])
+
+    @property
+    def status_message(self) -> str:
+        """Returns the status message of the project version."""
+        return self._json["statusMessage"]
+
+    @property
+    def passing_goal_count(self) -> int:
+        """Returns the number of passing goals for the project version."""
+        return self._json["passingGoalCount"]
+
+    @property
+    def failing_goal_count(self) -> int:
+        """Returns the number of failing goals for the project version."""
+        return self._json["failingGoalCount"]
+
+    @property
+    def skipped_goal_count(self) -> int:
+        """Returns the number of failing goals for the project version."""
+        return (
+            self._json["totalGoalCount"]
+            - self._json["passingGoalCount"]
+            - self._json["failingGoalCount"]
+        )
+
+    @property
+    def total_goal_count(self) -> int:
+        """Returns the number of failing goals for the project version."""
+        return self._json["totalGoalCount"]
+
+    def wait_for_completion(self, timeout: Optional[int] = None):
+        """Waits for the project version to complete.
+
+        Parameters
+        ----------
+        timeout : int, optional
+            Number of seconds to wait before timing out. If None, waits
+            indefinitely.
+
+        Returns
+        -------
+        ProjectVersion
+            The project version object.
+        """
+        while self.status not in [TaskStatus.COMPLETED, TaskStatus.FAILED]:
+            self.refresh()
+            time.sleep(1)
+            if timeout:
+                timeout -= 1
+                if timeout <= 0:
+                    print(
+                        "Timeout exceeded. Visit the Openlayer dashboard to"
+                        " check the status of the project version."
+                    )
+                    break
+        if self.status == TaskStatus.FAILED:
+            print("Project version failed with message:", self.status_message)
+        elif self.status == TaskStatus.COMPLETED:
+            print("Project version processed successfully.")
+
+    def refresh(self):
+        """Refreshes the project version object with the latest
+        information from the server."""
+        self._json = self.client.load_project_version(self.id).to_dict()
+
+    def print_goal_report(self):
+        """Prints the goal results of the project version."""
+        if self.status != TaskStatus.COMPLETED:
+            print("Project version is not complete. Nothing to print.")
+            return
+        print(
+            tabulate.tabulate(
+                [
+                    ["Passed", self.passing_goal_count],
+                    ["Failed", self.failing_goal_count],
+                    ["Skipped", self.skipped_goal_count],
+                    ["Total", self.total_goal_count],
+                ],
+                headers=["Goals", "Count"],
+                tablefmt="fancy_grid",
+            ),
+            f"\nVisit {self.links['app']} to view detailed results.",
+        )

openlayer/version.py

@@ -22,4 +22,4 @@
          data=data,
       )
 """
-__version__ = "0.1.0a1"
+__version__ = "0.1.0a2"

setup.cfg

@@ -42,6 +42,7 @@ install_requires =
     pandas
     requests
     tqdm
+    tabulate
     marshmallow
     marshmallow_oneofschema
     requests_toolbelt