diff --git a/synapseclient/api/__init__.py b/synapseclient/api/__init__.py index 09c578ed2..1e02bf24c 100644 --- a/synapseclient/api/__init__.py +++ b/synapseclient/api/__init__.py @@ -64,15 +64,28 @@ update_entity_acl, ) from .evaluation_services import ( + batch_update_submission_statuses, + cancel_submission, create_or_update_evaluation, + create_submission, delete_evaluation, + delete_submission, get_all_evaluations, + get_all_submission_statuses, get_available_evaluations, get_evaluation, get_evaluation_acl, get_evaluation_permissions, + get_evaluation_submission_bundles, + get_evaluation_submissions, get_evaluations_by_project, + get_submission, + get_submission_count, + get_submission_status, + get_user_submission_bundles, + get_user_submissions, update_evaluation_acl, + update_submission_status, ) from .file_services import ( AddPartResponse, @@ -282,4 +295,18 @@ "get_evaluation_acl", "update_evaluation_acl", "get_evaluation_permissions", + # submission-related evaluation services + "create_submission", + "get_submission", + "get_evaluation_submissions", + "get_user_submissions", + "get_submission_count", + "delete_submission", + "cancel_submission", + "get_submission_status", + "update_submission_status", + "get_all_submission_statuses", + "batch_update_submission_statuses", + "get_evaluation_submission_bundles", + "get_user_submission_bundles", ] diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 8149618b3..8cd34a942 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -386,3 +386,483 @@ async def get_evaluation_permissions( uri = f"/evaluation/{evaluation_id}/permissions" return await client.rest_get_async(uri) + + +async def create_submission( + request_body: dict, etag: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Creates a Submission and sends a submission notification email to the submitter's team members. + + + + Arguments: + request_body: The request body to send to the server. + etag: The current eTag of the Entity being submitted. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = "/evaluation/submission" + + # Add etag as query parameter if provided + params = {"etag": etag} + + response = await client.rest_post_async( + uri, body=json.dumps(request_body), params=params + ) + + return response + + +async def get_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Retrieves a Submission by its ID. + + + + Arguments: + submission_id: The ID of the submission to fetch. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The requested Submission. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}" + + response = await client.rest_get_async(uri) + + return response + + +async def get_evaluation_submissions( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 20, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Retrieves all Submissions for a specified Evaluation queue. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + # TODO: Support pagination in the return type. + A response JSON containing a paginated list of submissions for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def get_user_submissions( + evaluation_id: str, + user_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Retrieves Submissions for a specified Evaluation queue and user. + If user_id is omitted, this returns the submissions of the caller. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + user_id: Optionally specify the ID of the user whose submissions will be returned. + If omitted, this returns the submissions of the caller. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of user submissions for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission" + query_params = {"limit": limit, "offset": offset} + + if user_id: + query_params["userId"] = user_id + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def get_submission_count( + evaluation_id: str, + status: Optional[str] = None, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing the submission count. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/count" + query_params = {} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def delete_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> None: + """ + Deletes a Submission and its SubmissionStatus. + + + + Arguments: + submission_id: The ID of the submission to delete. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}" + + await client.rest_delete_async(uri) + + +async def cancel_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Cancels a Submission. Only the user who created the Submission may cancel it. + + + + Arguments: + submission_id: The ID of the submission to cancel. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + The Submission response object for the canceled submission as a JSON dict. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/cancellation" + + response = await client.rest_put_async(uri) + + return response + + +async def get_submission_status( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Gets the SubmissionStatus object associated with a specified Submission. + + + + Arguments: + submission_id: The ID of the submission to get the status for. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus object as a JSON dict. + + Note: + The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. + Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION + to see all data marked as "private" in the SubmissionStatus. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/status" + + response = await client.rest_get_async(uri) + + return response + + +async def update_submission_status( + submission_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Updates a SubmissionStatus object. + + + + Arguments: + submission_id: The ID of the SubmissionStatus being updated. + request_body: The SubmissionStatus object to update as a dictionary. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus object as a JSON dict. + + Note: + Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. Each time a SubmissionStatus is updated a new etag will be + issued to the SubmissionStatus. When an update is requested, Synapse will compare + the etag of the passed SubmissionStatus with the current etag of the SubmissionStatus. + If the etags do not match, then the update will be rejected with a PRECONDITION_FAILED + (412) response. When this occurs, the caller should fetch the latest copy of the + SubmissionStatus and re-apply any changes, then re-attempt the SubmissionStatus update. + + The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/status" + print("request body") + print(request_body) + response = await client.rest_put_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_all_submission_statuses( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. + Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION + to see all data marked as "private" in the SubmissionStatuses. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/status/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def batch_update_submission_statuses( + evaluation_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + request_body: The SubmissionStatusBatch object as a dictionary containing: + - statuses: List of SubmissionStatus objects to update + - isFirstBatch: Boolean indicating if this is the first batch in the series + - isLastBatch: Boolean indicating if this is the last batch in the series + - batchToken: Token from previous batch response (required for all but first batch) + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Note: + To allow upload of more than the maximum batch size (500), the system supports + uploading a series of batches. Synapse employs optimistic concurrency on the + series in the form of a batch token. Each request (except the first) must include + the 'batch token' returned in the response to the previous batch. If another client + begins batch upload simultaneously, a PRECONDITION_FAILED (412) response will be + generated and upload must restart from the first batch. + + After the final batch is uploaded, the data for the Evaluation queue will be + mirrored to the tables which support querying. Therefore uploaded data will not + appear in Evaluation queries until after the final batch is successfully uploaded. + + It is the client's responsibility to note in each batch request: + 1. Whether it is the first batch in the series (isFirstBatch) + 2. Whether it is the last batch (isLastBatch) + + For a single batch both flags are set to 'true'. + + The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/statusBatch" + + response = await client.rest_put_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_evaluation_submission_bundles( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission bundles for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/bundle/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def get_user_submission_bundles( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of the requesting user's submission bundles for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/bundle" + query_params = {"limit": limit, "offset": offset} + + response = await client.rest_get_async(uri, params=query_params) + + return response diff --git a/synapseclient/models/__init__.py b/synapseclient/models/__init__.py index 9a5322727..700087ab5 100644 --- a/synapseclient/models/__init__.py +++ b/synapseclient/models/__init__.py @@ -25,6 +25,9 @@ from synapseclient.models.recordset import RecordSet from synapseclient.models.schema_organization import JSONSchema, SchemaOrganization from synapseclient.models.services import FailureStrategy +from synapseclient.models.submission import Submission +from synapseclient.models.submission_bundle import SubmissionBundle +from synapseclient.models.submission_status import SubmissionStatus from synapseclient.models.submissionview import SubmissionView from synapseclient.models.table import Table from synapseclient.models.table_components import ( @@ -128,6 +131,9 @@ "EntityRef", "DatasetCollection", # Submission models + "Submission", + "SubmissionBundle", + "SubmissionStatus", "SubmissionView", # JSON Schema models "SchemaOrganization", diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py new file mode 100644 index 000000000..5d7390050 --- /dev/null +++ b/synapseclient/models/submission.py @@ -0,0 +1,714 @@ +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Protocol, Union + +from typing_extensions import Self + +from synapseclient import Synapse +from synapseclient.api import evaluation_services +from synapseclient.core.async_utils import async_to_sync, otel_trace_method +from synapseclient.models.mixins.access_control import AccessControllable + + +class SubmissionSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for Submission operations.""" + + def get( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Retrieve a Submission from Synapse. + + Arguments: + include_activity: Whether to include the activity in the returned submission. + Defaults to False. Setting this to True will include the activity + record associated with this submission. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission instance retrieved from Synapse. + + Example: Retrieving a submission by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234").get() + print(submission) + ``` + """ + return self + + def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: + """ + Delete a Submission from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Example: Delete a submission. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234") + submission.delete() + print("Deleted Submission.") + ``` + """ + pass + + +@dataclass +@async_to_sync +class Submission( + SubmissionSynchronousProtocol, + AccessControllable, +): + """A `Submission` object represents a Synapse Submission, which is created when a user + submits an entity to an evaluation queue. + + + Attributes: + id: The unique ID of this Submission. + user_id: The ID of the user that submitted this Submission. + submitter_alias: The name of the user that submitted this Submission. + entity_id: The ID of the entity being submitted. + version_number: The version number of the entity at submission. + evaluation_id: The ID of the Evaluation to which this Submission belongs. + name: The name of this Submission. + created_on: The date this Submission was created. + team_id: The ID of the team that submitted this submission (if it's a team submission). + contributors: User IDs of team members who contributed to this submission (if it's a team submission). + submission_status: The status of this Submission. + entity_bundle_json: The bundled entity information at submission. This includes the entity, annotations, + file handles, and other metadata. + docker_repository_name: For Docker repositories, the repository name. + docker_digest: For Docker repositories, the digest of the submitted Docker image. + + Example: Retrieve a Submission. + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn123456").get() + print(submission) + ``` + """ + + id: Optional[str] = None + """ + The unique ID of this Submission. + """ + + user_id: Optional[str] = None + """ + The ID of the user that submitted this Submission. + """ + + submitter_alias: Optional[str] = None + """ + The name of the user that submitted this Submission. + """ + + entity_id: Optional[str] = None + """ + The ID of the entity being submitted. + """ + + version_number: Optional[int] = field(default=None, compare=False) + """ + The version number of the entity at submission. If not provided, it will be automatically retrieved from the entity. + """ + + evaluation_id: Optional[str] = None + """ + The ID of the Evaluation to which this Submission belongs. + """ + + name: Optional[str] = None + """ + The name of this Submission. + """ + + created_on: Optional[str] = field(default=None, compare=False) + """ + The date this Submission was created. + """ + + team_id: Optional[str] = None + """ + The ID of the team that submitted this submission (if it's a team submission). + """ + + contributors: List[str] = field(default_factory=list) + """ + User IDs of team members who contributed to this submission (if it's a team submission). + """ + + submission_status: Optional[Dict] = None + """ + The status of this Submission. + """ + + entity_bundle_json: Optional[str] = None + """ + The bundled entity information at submission. This includes the entity, annotations, + file handles, and other metadata. + """ + + docker_repository_name: Optional[str] = None + """ + For Docker repositories, the repository name. + """ + + docker_digest: Optional[str] = None + """ + For Docker repositories, the digest of the submitted Docker image. + """ + + etag: Optional[str] = None + """The current eTag of the Entity being submitted. If not provided, it will be automatically retrieved.""" + + def fill_from_dict( + self, synapse_submission: Dict[str, Union[bool, str, int, List]] + ) -> "Submission": + """ + Converts a response from the REST API into this dataclass. + + Arguments: + synapse_submission: The response from the REST API. + + Returns: + The Submission object. + """ + self.id = synapse_submission.get("id", None) + self.user_id = synapse_submission.get("userId", None) + self.submitter_alias = synapse_submission.get("submitterAlias", None) + self.entity_id = synapse_submission.get("entityId", None) + self.version_number = synapse_submission.get("versionNumber", None) + self.evaluation_id = synapse_submission.get("evaluationId", None) + self.name = synapse_submission.get("name", None) + self.created_on = synapse_submission.get("createdOn", None) + self.team_id = synapse_submission.get("teamId", None) + self.contributors = synapse_submission.get("contributors", []) + self.submission_status = synapse_submission.get("submissionStatus", None) + self.entity_bundle_json = synapse_submission.get("entityBundleJSON", None) + self.docker_repository_name = synapse_submission.get( + "dockerRepositoryName", None + ) + self.docker_digest = synapse_submission.get("dockerDigest", None) + + return self + + async def _fetch_latest_entity( + self, *, synapse_client: Optional[Synapse] = None + ) -> Dict: + """ + Fetch the latest entity information from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Dictionary containing entity information from the REST API. + + Raises: + ValueError: If entity_id is not set or if unable to fetch entity information. + """ + if not self.entity_id: + raise ValueError("entity_id must be set to fetch entity information") + + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + try: + entity_info = await client.rest_get_async(f"/entity/{self.entity_id}") + + # If this is a DockerRepository, fetch docker image tag & digest, and add it to the entity_info dict + if ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.docker.DockerRepository" + ): + docker_tag_response = await client.rest_get_async( + f"/entity/{self.entity_id}/dockerTag" + ) + + # Get the latest digest from the docker tag results + if "results" in docker_tag_response and docker_tag_response["results"]: + # Sort by createdOn timestamp to get the latest entry + # Convert ISO timestamp strings to datetime objects for comparison + from datetime import datetime + + latest_result = max( + docker_tag_response["results"], + key=lambda x: datetime.fromisoformat( + x["createdOn"].replace("Z", "+00:00") + ), + ) + + # Add the latest result to entity_info + entity_info.update(latest_result) + + return entity_info + except Exception as e: + raise ValueError( + f"Unable to fetch entity information for {self.entity_id}: {e}" + ) + + def to_synapse_request(self) -> Dict: + """Creates a request body expected of the Synapse REST API for the Submission model. + + Returns: + A dictionary containing the request body for creating a submission. + + Raises: + ValueError: If any required attributes are missing. + """ + # These attributes are required for creating a submission + required_attributes = ["entity_id", "evaluation_id"] + + for attribute in required_attributes: + if not getattr(self, attribute): + raise ValueError( + f"Your submission object is missing the '{attribute}' attribute. This attribute is required to create a submission" + ) + + # Build a request body for creating a submission + request_body = { + "entityId": self.entity_id, + "evaluationId": self.evaluation_id, + "versionNumber": self.version_number, + } + + # Add optional fields if they are set + if self.name is not None: + request_body["name"] = self.name + if self.team_id is not None: + request_body["teamId"] = self.team_id + if self.contributors: + request_body["contributors"] = self.contributors + if self.docker_repository_name is not None: + request_body["dockerRepositoryName"] = self.docker_repository_name + if self.docker_digest is not None: + request_body["dockerDigest"] = self.docker_digest + + return request_body + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Store: {self.id if self.id else 'new_submission'}" + ) + async def store_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Submission": + """ + Store the submission in Synapse. This creates a new submission in an evaluation queue. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission object with the ID set. + + Raises: + ValueError: If the submission is missing required fields, or if unable to fetch entity etag. + + Example: Creating a submission +   + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + async def create_submission_example(): + + submission = Submission( + entity_id="syn123456", + evaluation_id="9614543", + name="My Submission" + ) + submission = await submission.store_async() + print(submission.id) + + asyncio.run(create_submission_example()) + ``` + """ + + if self.entity_id: + entity_info = await self._fetch_latest_entity(synapse_client=synapse_client) + + self.entity_etag = entity_info.get("etag") + + if ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.FileEntity" + ): + self.version_number = entity_info.get("versionNumber") + elif ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.docker.DockerRepository" + ): + self.version_number = ( + 1 # TODO: Docker repositories do not have version numbers + ) + self.docker_repository_name = entity_info.get("repositoryName") + self.docker_digest = entity_info.get("digest") + else: + raise ValueError("entity_id is required to create a submission") + + if not self.entity_etag: + raise ValueError("Unable to fetch etag for entity") + + # Build the request body now that all the necessary dataclass attributes are set + request_body = self.to_synapse_request() + + response = await evaluation_services.create_submission( + request_body, self.entity_etag, synapse_client=synapse_client + ) + self.fill_from_dict(response) + return self + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Get: {self.id}" + ) + async def get_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Submission": + """ + Retrieve a Submission from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission instance retrieved from Synapse. + + Raises: + ValueError: If the submission does not have an ID to get. + + Example: Retrieving a submission by ID +   + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + async def get_submission_example(): + + submission = await Submission(id="9999999").get_async() + print(submission) + + asyncio.run(get_submission_example()) + ``` + """ + if not self.id: + raise ValueError("The submission must have an ID to get.") + + response = await evaluation_services.get_submission( + submission_id=self.id, synapse_client=synapse_client + ) + + self.fill_from_dict(response) + + return self + + # TODO: Have all staticmethods return generators for pagination + @staticmethod + async def get_evaluation_submissions_async( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 20, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Retrieves all Submissions for a specified Evaluation queue. + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status. + Submission status can be one of + limit: Limits the number of submissions in a single response. Defaults to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Defaults to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of submissions for the evaluation queue. + + Example: Getting submissions for an evaluation +   + Get SCORED submissions from a specific evaluation. + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + async def get_evaluation_submissions_example(): + response = await Submission.get_evaluation_submissions_async( + evaluation_id="9999999", + status="SCORED" + ) + print(f"Found {len(response['results'])} submissions") + + asyncio.run(get_evaluation_submissions_example()) + ``` + """ + return await evaluation_services.get_evaluation_submissions( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + @staticmethod + async def get_user_submissions_async( + evaluation_id: str, + user_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Retrieves Submissions for a specified Evaluation queue and user. + If user_id is omitted, this returns the submissions of the caller. + + Arguments: + evaluation_id: The ID of the evaluation queue. + user_id: Optionally specify the ID of the user whose submissions will be returned. + If omitted, this returns the submissions of the caller. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of user submissions for the evaluation queue. + + Example: Getting user submissions + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + async def get_user_submissions_example(): + response = await Submission.get_user_submissions_async( + evaluation_id="9999999", + user_id="123456", + limit=10 + ) + print(f"Found {len(response['results'])} user submissions") + + asyncio.run(get_user_submissions_example()) + ``` + """ + return await evaluation_services.get_user_submissions( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + @staticmethod + async def get_submission_count_async( + evaluation_id: str, + status: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing the submission count. + + Example: Getting submission count +   + Get the total number of SCORED submissions from a specific evaluation. + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + async def get_submission_count_example(): + response = await Submission.get_submission_count_async( + evaluation_id="9999999", + status="SCORED" + ) + print(f"Found {response} submissions") + + asyncio.run(get_submission_count_example()) + ``` + """ + return await evaluation_services.get_submission_count( + evaluation_id=evaluation_id, status=status, synapse_client=synapse_client + ) + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Delete: {self.id}" + ) + async def delete_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> None: + """ + Delete a Submission from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Raises: + ValueError: If the submission does not have an ID to delete. + + Example: Delete a submission +   + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + async def delete_submission_example(): + submission = Submission(id="9999999") + await submission.delete_async() + print("Submission deleted successfully") + + # Run the async function + asyncio.run(delete_submission_example()) + ``` + """ + if not self.id: + raise ValueError("The submission must have an ID to delete.") + + await evaluation_services.delete_submission( + submission_id=self.id, synapse_client=synapse_client + ) + + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + logger.info(f"Submission {self.id} has successfully been deleted.") + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Cancel: {self.id}" + ) + async def cancel_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Submission": + """ + Cancel a Submission. Only the user who created the Submission may cancel it. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated Submission object. + + Raises: + ValueError: If the submission does not have an ID to cancel. + + Example: Cancel a submission +   + ```python + import asyncio + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + async def cancel_submission_example(): + submission = Submission(id="syn1234") + canceled_submission = await submission.cancel_async() + print(f"Canceled submission: {canceled_submission.id}") + + # Run the async function + asyncio.run(cancel_submission_example()) + ``` + """ + if not self.id: + raise ValueError("The submission must have an ID to cancel.") + + response = await evaluation_services.cancel_submission( + submission_id=self.id, synapse_client=synapse_client + ) + + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + logger.info(f"Submission {self.id} has successfully been cancelled.") + + # Update this object with the response + self.fill_from_dict(response) + return self diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py new file mode 100644 index 000000000..ce4c736de --- /dev/null +++ b/synapseclient/models/submission_bundle.py @@ -0,0 +1,313 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, Union + +from synapseclient import Synapse +from synapseclient.api import evaluation_services +from synapseclient.core.async_utils import async_to_sync +from synapseclient.models.mixins.access_control import AccessControllable + +if TYPE_CHECKING: + from synapseclient.models.submission import Submission + from synapseclient.models.submission_status import SubmissionStatus + + +class SubmissionBundleSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for SubmissionBundle operations.""" + + @staticmethod + def get_evaluation_submission_bundles( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the submission bundles + for the evaluation queue. + + Example: Getting submission bundles for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(bundles)} submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + return [] + + @staticmethod + def get_user_submission_bundles( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the requesting user's + submission bundles for the evaluation queue. + + Example: Getting user submission bundles + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id="9614543", + limit=25 + ) + print(f"Found {len(bundles)} user submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + return [] + + +@dataclass +@async_to_sync +class SubmissionBundle( + SubmissionBundleSynchronousProtocol, + AccessControllable, +): + """A `SubmissionBundle` object represents a bundle containing a Synapse Submission + and its accompanying SubmissionStatus. This bundle provides convenient access to both + the submission data and its current status in a single object. + + + Attributes: + submission: A Submission to a Synapse Evaluation is a pointer to a versioned Entity. + Submissions are immutable, so we archive a copy of the EntityBundle at the time of submission. + submission_status: A SubmissionStatus is a secondary, mutable object associated with a Submission. + This object should be used to contain scoring data about the Submission. + + Example: Retrieve submission bundles for an evaluation. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + # Get all submission bundles for an evaluation + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id="9614543", + status="SCORED" + ) + + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + print(f"Status: {bundle.submission_status.status if bundle.submission_status else 'N/A'}") + ``` + """ + + submission: Optional["Submission"] = None + """ + A Submission to a Synapse Evaluation is a pointer to a versioned Entity. + Submissions are immutable, so we archive a copy of the EntityBundle at the time of submission. + """ + + submission_status: Optional["SubmissionStatus"] = None + """ + A SubmissionStatus is a secondary, mutable object associated with a Submission. + This object should be used to contain scoring data about the Submission. + """ + + _last_persistent_instance: Optional["SubmissionBundle"] = field( + default=None, repr=False, compare=False + ) + """The last persistent instance of this object. This is used to determine if the + object has been changed and needs to be updated in Synapse.""" + + def fill_from_dict( + self, synapse_submission_bundle: Dict[str, Union[bool, str, int, Dict]] + ) -> "SubmissionBundle": + """ + Converts a response from the REST API into this dataclass. + + Arguments: + synapse_submission_bundle: The response from the REST API. + + Returns: + The SubmissionBundle object. + """ + from synapseclient.models.submission import Submission + from synapseclient.models.submission_status import SubmissionStatus + + submission_dict = synapse_submission_bundle.get("submission", None) + if submission_dict: + self.submission = Submission().fill_from_dict(submission_dict) + else: + self.submission = None + + submission_status_dict = synapse_submission_bundle.get("submissionStatus", None) + if submission_status_dict: + self.submission_status = SubmissionStatus().fill_from_dict( + submission_status_dict + ) + else: + self.submission_status = None + + return self + + @staticmethod + async def get_evaluation_submission_bundles_async( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the submission bundles + for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. + + Example: Getting submission bundles for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(bundles)} submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + response = await evaluation_services.get_evaluation_submission_bundles( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + bundles = [] + for bundle_dict in response.get("results", []): + bundle = SubmissionBundle().fill_from_dict(bundle_dict) + bundles.append(bundle) + + return bundles + + @staticmethod + async def get_user_submission_bundles_async( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the requesting user's + submission bundles for the evaluation queue. + + Example: Getting user submission bundles + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id="9614543", + limit=25 + ) + print(f"Found {len(bundles)} user submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + response = await evaluation_services.get_user_submission_bundles( + evaluation_id=evaluation_id, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + # Convert response to list of SubmissionBundle objects + bundles = [] + for bundle_dict in response.get("results", []): + bundle = SubmissionBundle().fill_from_dict(bundle_dict) + bundles.append(bundle) + + return bundles diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py new file mode 100644 index 000000000..08815709c --- /dev/null +++ b/synapseclient/models/submission_status.py @@ -0,0 +1,686 @@ +from dataclasses import dataclass, field, replace +from datetime import date, datetime +from typing import Dict, List, Optional, Protocol, Union + +from typing_extensions import Self + +from synapseclient import Synapse +from synapseclient.annotations import to_submission_status_annotations +from synapseclient.api import evaluation_services +from synapseclient.core.async_utils import async_to_sync, otel_trace_method +from synapseclient.core.utils import merge_dataclass_entities +from synapseclient.models import Annotations +from synapseclient.models.mixins.access_control import AccessControllable + + +class SubmissionStatusSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for SubmissionStatus operations.""" + + def get( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Retrieve a SubmissionStatus from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus instance retrieved from Synapse. + + Example: Retrieving a submission status by ID. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + status = SubmissionStatus(id="syn1234").get() + print(status) + ``` + """ + return self + + def store( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Store (update) the SubmissionStatus in Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus instance. + + Example: Update a submission status. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + status = SubmissionStatus(id="syn1234").get() + status.status = "SCORED" + status = status.store() + print("Updated SubmissionStatus.") + ``` + """ + return self + + @staticmethod + def get_all_submission_statuses( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Example: Getting all submission statuses for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + response = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(response['results'])} submission statuses") + ``` + """ + return {} + + @staticmethod + def batch_update_submission_statuses( + evaluation_id: str, + statuses: List["SubmissionStatus"], + is_first_batch: bool = True, + is_last_batch: bool = True, + batch_token: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + statuses: List of SubmissionStatus objects to update. + is_first_batch: Boolean indicating if this is the first batch in the series. Default True. + is_last_batch: Boolean indicating if this is the last batch in the series. Default True. + batch_token: Token from previous batch response (required for all but first batch). + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Example: Batch update submission statuses + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Prepare list of status updates + statuses = [ + SubmissionStatus(id="syn1", status="SCORED", submission_annotations={"score": [90.0]}), + SubmissionStatus(id="syn2", status="SCORED", submission_annotations={"score": [85.0]}) + ] + + response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id="9614543", + statuses=statuses, + is_first_batch=True, + is_last_batch=True + ) + print(f"Batch update completed: {response}") + ``` + """ + return {} + + +@dataclass +@async_to_sync +class SubmissionStatus( + SubmissionStatusSynchronousProtocol, + AccessControllable, +): + """A SubmissionStatus is a secondary, mutable object associated with a Submission. + This object should be used to contain scoring data about the Submission. + + + Attributes: + id: The unique, immutable Synapse ID of the Submission. + etag: Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. The eTag changes every time a SubmissionStatus is updated; + it is used to detect when a client's copy of a SubmissionStatus is out-of-date. + modified_on: The date on which this SubmissionStatus was last modified. + status: The possible states of a Synapse Submission (e.g., RECEIVED, VALIDATED, SCORED). + score: This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + report: This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + annotations: Primary container object for Annotations on a Synapse object. + submission_annotations: Annotations are additional key-value pair metadata that are associated with an object. + entity_id: The Synapse ID of the Entity in this Submission. + version_number: The version number of the Entity in this Submission. + status_version: A version of the status, auto-generated and auto-incremented by the system and read-only to the client. + can_cancel: Can this submission be cancelled? By default, this will be set to False. Users can read this value. + Only the queue's scoring application can change this value. + cancel_requested: Has user requested to cancel this submission? By default, this will be set to False. + Submission owner can read and request to change this value. + + Example: Retrieve and update a SubmissionStatus. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Get a submission status + status = SubmissionStatus(id="syn123456").get() + + # Update the status + status.status = "SCORED" + status.submission_annotations = {"score": [85.5], "feedback": ["Good work!"]} + status = status.store() + print(status) + ``` + """ + + id: Optional[str] = None + """ + The unique, immutable Synapse ID of the Submission. + """ + + etag: Optional[str] = field(default=None, compare=False) + """ + Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. The eTag changes every time a SubmissionStatus is updated; + it is used to detect when a client's copy of a SubmissionStatus is out-of-date. + """ + + modified_on: Optional[str] = field(default=None, compare=False) + """ + The date on which this SubmissionStatus was last modified. + """ + + status: Optional[str] = None + """ + The possible states of a Synapse Submission (e.g., RECEIVED, VALIDATED, SCORED). + """ + + score: Optional[float] = None + """ + This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + """ + + report: Optional[str] = None + """ + This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + """ + + annotations: Optional[ + Dict[ + str, + Union[ + List[str], + List[bool], + List[float], + List[int], + List[date], + List[datetime], + ], + ] + ] = field(default_factory=dict, compare=False) + """Primary container object for Annotations on a Synapse object.""" + + submission_annotations: Optional[ + Dict[ + str, + Union[ + List[str], + List[bool], + List[float], + List[int], + List[date], + List[datetime], + ], + ] + ] = field(default_factory=dict, compare=False) + """Annotations are additional key-value pair metadata that are associated with an object.""" + + is_private: Optional[bool] = field(default=True, compare=False) + """Indicates whether the submission annotations are private (True) or public (False). Default is True.""" + + entity_id: Optional[str] = None + """ + The Synapse ID of the Entity in this Submission. + """ + + version_number: Optional[int] = field(default=None, compare=False) + """ + The version number of the Entity in this Submission. + """ + + status_version: Optional[int] = field(default=None, compare=False) + """ + A version of the status, auto-generated and auto-incremented by the system and read-only to the client. + """ + + can_cancel: Optional[bool] = field(default=False, compare=False) + """ + Can this submission be cancelled? By default, this will be set to False. Users can read this value. + Only the queue's scoring application can change this value. + """ + + cancel_requested: Optional[bool] = field(default=False, compare=False) + """ + Has user requested to cancel this submission? By default, this will be set to False. + Submission owner can read and request to change this value. + """ + + _last_persistent_instance: Optional["SubmissionStatus"] = field( + default=None, repr=False, compare=False + ) + """The last persistent instance of this object. This is used to determine if the + object has been changed and needs to be updated in Synapse.""" + + @property + def has_changed(self) -> bool: + """Determines if the object has been newly created OR changed since last retrieval, and needs to be updated in Synapse.""" + return ( + not self._last_persistent_instance or self._last_persistent_instance is not self + ) + + def _set_last_persistent_instance(self) -> None: + """Stash the last time this object interacted with Synapse. This is used to + determine if the object has been changed and needs to be updated in Synapse.""" + self._last_persistent_instance = replace(self) + + def fill_from_dict( + self, synapse_submission_status: Dict[str, Union[bool, str, int, float, List]] + ) -> "SubmissionStatus": + """ + Converts a response from the REST API into this dataclass. + + Arguments: + synapse_submission_status: The response from the REST API. + + Returns: + The SubmissionStatus object. + """ + self.id = synapse_submission_status.get("id", None) + self.etag = synapse_submission_status.get("etag", None) + self.modified_on = synapse_submission_status.get("modifiedOn", None) + self.status = synapse_submission_status.get("status", None) + self.score = synapse_submission_status.get("score", None) + self.report = synapse_submission_status.get("report", None) + self.entity_id = synapse_submission_status.get("entityId", None) + self.version_number = synapse_submission_status.get("versionNumber", None) + self.status_version = synapse_submission_status.get("statusVersion", None) + self.can_cancel = synapse_submission_status.get("canCancel", False) + self.cancel_requested = synapse_submission_status.get("cancelRequested", False) + + # Handle annotations + annotations_dict = synapse_submission_status.get("annotations", {}) + if annotations_dict: + self.annotations = Annotations.from_dict(annotations_dict) + + # Handle submission annotations + submission_annotations_dict = synapse_submission_status.get( + "submissionAnnotations", {} + ) + if submission_annotations_dict: + self.submission_annotations = Annotations.from_dict( + submission_annotations_dict + ) + print(self.submission_annotations) + + return self + + def to_synapse_request(self) -> Dict: + """ + Creates a request body expected by the Synapse REST API for the SubmissionStatus model. + + Returns: + A dictionary containing the request body for updating a submission status. + + Raises: + ValueError: If any required attributes are missing. + """ + # These attributes are required for updating a submission status + required_attributes = ["id", "etag", "status_version"] + + for attribute in required_attributes: + if getattr(self, attribute) is None: + raise ValueError( + f"Your submission status object is missing the '{attribute}' attribute. This attribute is required to update a submission status" + ) + + # Build request body with required fields + request_body = { + "id": self.id, + "etag": self.etag, + "statusVersion": self.status_version, + } + + # Add optional fields only if they have values + if self.status is not None: + request_body["status"] = self.status + if self.score is not None: + request_body["score"] = self.score + if self.report is not None: + request_body["report"] = self.report + if self.entity_id is not None: + request_body["entityId"] = self.entity_id + if self.version_number is not None: + request_body["versionNumber"] = self.version_number + if self.can_cancel is not None: + request_body["canCancel"] = self.can_cancel + if self.cancel_requested is not None: + request_body["cancelRequested"] = self.cancel_requested + + # Add annotations if present + if self.annotations and len(self.annotations) > 0: + # Convert annotations to the format expected by the API + request_body["annotations"] = to_submission_status_annotations( + self.annotations + ) + + # Add submission annotations if present + if self.submission_annotations and len(self.submission_annotations) > 0: + # Convert submission annotations to the format expected by the API + request_body["submissionAnnotations"] = to_submission_status_annotations( + self.submission_annotations, is_private=self.is_private + ) + + return request_body + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"SubmissionStatus_Get: {self.id}" + ) + async def get_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "SubmissionStatus": + """ + Retrieve a SubmissionStatus from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus instance retrieved from Synapse. + + Raises: + ValueError: If the submission status does not have an ID to get. + + Example: Retrieving a submission status by ID + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + status = await SubmissionStatus(id="syn1234").get_async() + print(status) + ``` + """ + if not self.id: + raise ValueError("The submission status must have an ID to get.") + + response = await evaluation_services.get_submission_status( + submission_id=self.id, synapse_client=synapse_client + ) + + self.fill_from_dict(response) + self._set_last_persistent_instance() + return self + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"SubmissionStatus_Store: {self.id if self.id else 'new_status'}" + ) + async def store_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "SubmissionStatus": + """ + Store (update) the SubmissionStatus in Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus object. + + Raises: + ValueError: If the submission status is missing required fields. + + Example: Update a submission status + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Get existing status + status = await SubmissionStatus(id="syn1234").get_async() + + # Update fields + status.status = "SCORED" + status.submission_annotations = {"score": [85.5]} + + # Store the update + status = await status.store_async() + print(f"Updated status: {status.status}") + ``` + """ + if not self.id: + raise ValueError("The submission status must have an ID to update.") + + # Get the client for logging + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + + # Check if there are changes to apply + if self._last_persistent_instance and self.has_changed: + # Merge with the last persistent instance to preserve system-managed fields + merge_dataclass_entities( + source=self._last_persistent_instance, + destination=self, + fields_to_preserve_from_source=[ + "id", + "etag", + "modified_on", + "entity_id", + "version_number", + "status_version", + ], + logger=logger, + ) + elif self._last_persistent_instance and not self.has_changed: + logger.warning( + f"SubmissionStatus (ID: {self.id}) has not changed since last 'store' or 'get' event, so it will not be updated in Synapse. Please get the submission status again if you want to refresh its state." + ) + return self + + request_body = self.to_synapse_request() + + response = await evaluation_services.update_submission_status( + submission_id=self.id, + request_body=request_body, + synapse_client=synapse_client, + ) + + self.fill_from_dict(response) + self._set_last_persistent_instance() + return self + + @staticmethod + async def get_all_submission_statuses_async( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Example: Getting all submission statuses for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + response = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(response['results'])} submission statuses") + ``` + """ + return await evaluation_services.get_all_submission_statuses( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + @staticmethod + async def batch_update_submission_statuses_async( + evaluation_id: str, + statuses: List["SubmissionStatus"], + is_first_batch: bool = True, + is_last_batch: bool = True, + batch_token: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + statuses: List of SubmissionStatus objects to update. + is_first_batch: Boolean indicating if this is the first batch in the series. Default True. + is_last_batch: Boolean indicating if this is the last batch in the series. Default True. + batch_token: Token from previous batch response (required for all but first batch). + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Example: Batch update submission statuses + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Prepare list of status updates + statuses = [ + SubmissionStatus(id="syn1", status="SCORED", submission_annotations={"score": [90.0]}), + SubmissionStatus(id="syn2", status="SCORED", submission_annotations={"score": [85.0]}) + ] + + response = await SubmissionStatus.batch_update_submission_statuses_async( + evaluation_id="9614543", + statuses=statuses, + is_first_batch=True, + is_last_batch=True + ) + print(f"Batch update completed: {response}") + ``` + """ + # Convert SubmissionStatus objects to dictionaries + status_dicts = [] + for status in statuses: + status_dict = status.to_synapse_request() + status_dicts.append(status_dict) + + # Prepare the batch request body + request_body = { + "statuses": status_dicts, + "isFirstBatch": is_first_batch, + "isLastBatch": is_last_batch, + } + + # Add batch token if provided (required for all but first batch) + if batch_token: + request_body["batchToken"] = batch_token + + return await evaluation_services.batch_update_submission_statuses( + evaluation_id=evaluation_id, + request_body=request_body, + synapse_client=synapse_client, + ) diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index 2b856d4e5..745a60960 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -1,199 +1,652 @@ -def test_create_submission_async(): - # WHEN an evaluation is retrieved - evaluation = Evaluation(id=evaluation_id).get() - - # AND an entity is retrieved - file = File(name="test.txt", parentId=project.id).get() - - # THEN the entity can be submitted to the evaluation - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1 - ).store() - -def test_get_submission_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1 - ).store() - - # WHEN the submission is retrieved by ID - retrieved_submission = Submission(id=submission.id).get() - - # THEN the retrieved submission matches the created one - assert retrieved_submission.id == submission.id - assert retrieved_submission.name == submission.name - - # AND the user_id matches the current user - current_user = syn.getUserProfile()().id - assert retrieved_submission.user_id == current_user - -def test_get_evaluation_submissions_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN submissions are retrieved for the evaluation - submissions = Submission.get_evaluation_submissions(evaluation.id) - - # THEN the submissions list is not empty - assert len(submissions) > 0 - - # AND each submission belongs to the evaluation - for submission in submissions: - assert submission.evaluation_id == evaluation.id - -def test_get_user_submissions_async(): - # GIVEN a user has made submissions - current_user = syn.getUserProfile()().id - - # WHEN submissions are retrieved for the user - submissions = Submission.get_user_submissions(current_user) - - # THEN the submissions list is not empty - assert len(submissions) > 0 - - # AND each submission belongs to the user - for submission in submissions: - assert submission.user_id == current_user - -def test_get_submission_count_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN the submission count is retrieved for the evaluation - count = Submission.get_submission_count(evaluation.id) - - # THEN the count is greater than zero - assert count > 0 - -def test_delete_submission_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1 - ).store() - - # WHEN the submission is deleted - submission.delete() - - # THEN retrieving the submission should raise an error - try: - Submission(id=submission.id).get() - assert False, "Expected an error when retrieving a deleted submission" - except SynapseError as e: - assert e.response.status_code == 404 - -def test_cancel_submission_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1 - ).store() - - # WHEN the submission is canceled - submission.cancel() - - # THEN the submission status should be 'CANCELED' - updated_submission = Submission(id=submission.id).get() - assert updated_submission.status == 'CANCELED' - -def test_get_submission_status_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1 - ).store() - - # WHEN the submission status is retrieved - status = submission.get_status() - - # THEN the status should be 'RECEIVED' - assert status == 'RECEIVED' - -def test_update_submission_status_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1 - ).store() - - # WHEN the submission status is retrieved - status = submission.get_status() - assert status != 'SCORED' - - # AND the submission status is updated to 'SCORED' - submission.update_status('SCORED') - - # THEN the submission status should be 'SCORED' - updated_submission = Submission(id=submission.id).get() - assert updated_submission.status == 'SCORED' - -def test_get_evaluation_submission_statuses_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN the submission statuses are retrieved for the evaluation - statuses = Submission.get_evaluation_submission_statuses(evaluation.id) - - # THEN the statuses list is not empty - assert len(statuses) > 0 - -def test_batch_update_statuses_async(): - # GIVEN multiple submissions have been created - submission1 = Submission( - name="Test Submission 1", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1 - ).store() - submission2 = Submission( - name="Test Submission 2", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1 - ).store() - - # WHEN the statuses of the submissions are batch updated to 'SCORED' - Submission.batch_update_statuses( - [submission1.id, submission2.id], - 'SCORED' - ) - - # THEN each submission status should be 'SCORED' - updated_submission1 = Submission(id=submission1.id).get() - updated_submission2 = Submission(id=submission2.id).get() - assert updated_submission1.status == 'SCORED' - assert updated_submission2.status == 'SCORED' - -def test_get_evaluation_submission_bundles_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN the submission bundles are retrieved for the evaluation - bundles = Submission.get_evaluation_submission_bundles(evaluation.id) - - # THEN the bundles list is not empty - assert len(bundles) > 0 - -def test_get_user_submission_bundles_async(): - # GIVEN a user has made submissions - current_user = syn.getUserProfile()().id - - # WHEN the submission bundles are retrieved for the user - bundles = Submission.get_user_submission_bundles(current_user) - - # THEN the bundles list is not empty - assert len(bundles) > 0 +"""Async integration tests for the synapseclient.models.Submission class.""" + +import uuid +from typing import Callable + +import pytest +import pytest_asyncio + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission + + +class TestSubmissionCreationAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission tests.""" + import os + import tempfile + + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + async def test_store_submission_successfully_async( + self, test_evaluation: Evaluation, test_file: File + ): + # WHEN I create a submission with valid data using async method + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=self.syn) + self.schedule_for_cleanup(created_submission.id) + + # THEN the submission should be created successfully + assert created_submission.id is not None + assert created_submission.entity_id == test_file.id + assert created_submission.evaluation_id == test_evaluation.id + assert created_submission.name == submission.name + assert created_submission.user_id is not None + assert created_submission.created_on is not None + assert created_submission.version_number is not None + + async def test_store_submission_without_entity_id_async( + self, test_evaluation: Evaluation + ): + # WHEN I try to create a submission without entity_id using async method + submission = Submission( + evaluation_id=test_evaluation.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises( + ValueError, match="entity_id is required to create a submission" + ): + await submission.store_async(synapse_client=self.syn) + + async def test_store_submission_without_evaluation_id_async(self, test_file: File): + # WHEN I try to create a submission without evaluation_id using async method + submission = Submission( + entity_id=test_file.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + await submission.store_async(synapse_client=self.syn) + + # async def test_store_submission_with_docker_repository_async( + # self, test_evaluation: Evaluation + # ): + # # GIVEN we would need a Docker repository entity (mocked for this test) + # # This test demonstrates the expected behavior for Docker repository submissions + + # # WHEN I create a submission for a Docker repository entity using async method + # # TODO: This would require a real Docker repository entity in a full integration test + # submission = Submission( + # entity_id="syn123456789", # Would be a Docker repository ID + # evaluation_id=test_evaluation.id, + # name=f"Docker Submission {uuid.uuid4()}", + # ) + + # # THEN the submission should handle Docker-specific attributes + # # (This test would need to be expanded with actual Docker repository setup) + # assert submission.entity_id == "syn123456789" + # assert submission.evaluation_id == test_evaluation.id + + +class TestSubmissionRetrievalAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission tests.""" + import os + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + @pytest_asyncio.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for retrieval tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_get_submission_by_id_async( + self, test_submission: Submission, test_evaluation: Evaluation, test_file: File + ): + # WHEN I get a submission by ID using async method + retrieved_submission = await Submission(id=test_submission.id).get_async( + synapse_client=self.syn + ) + + # THEN the submission should be retrieved correctly + assert retrieved_submission.id == test_submission.id + assert retrieved_submission.entity_id == test_file.id + assert retrieved_submission.evaluation_id == test_evaluation.id + assert retrieved_submission.name == test_submission.name + assert retrieved_submission.user_id is not None + assert retrieved_submission.created_on is not None + + async def test_get_evaluation_submissions_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get all submissions for an evaluation using async method + response = await Submission.get_evaluation_submissions_async( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with submissions + assert "results" in response + assert len(response["results"]) > 0 + + # AND the submission should be in the results + submission_ids = [sub.get("id") for sub in response["results"]] + assert test_submission.id in submission_ids + + async def test_get_evaluation_submissions_with_status_filter_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get submissions filtered by status using async method + response = await Submission.get_evaluation_submissions_async( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN I should get submissions with the specified status + assert "results" in response + for submission in response["results"]: + if submission.get("id") == test_submission.id: + # The submission should be in RECEIVED status initially + break + else: + pytest.fail("Test submission not found in filtered results") + + async def test_get_evaluation_submissions_with_pagination_async( + self, test_evaluation: Evaluation + ): + # WHEN I get submissions with pagination parameters using async method + response = await Submission.get_evaluation_submissions_async( + evaluation_id=test_evaluation.id, + limit=5, + offset=0, + synapse_client=self.syn, + ) + + # THEN the response should respect pagination + assert "results" in response + assert len(response["results"]) <= 5 + + async def test_get_user_submissions_async(self, test_evaluation: Evaluation): + # WHEN I get submissions for the current user using async method + response = await Submission.get_user_submissions_async( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with user submissions + assert "results" in response + # Note: Could be empty if user hasn't made submissions to this evaluation + + async def test_get_submission_count_async(self, test_evaluation: Evaluation): + # WHEN I get the submission count for an evaluation using async method + response = await Submission.get_submission_count_async( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a count response + assert isinstance(response, int) + + +class TestSubmissionDeletionAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission tests.""" + import os + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + async def test_delete_submission_successfully_async( + self, test_evaluation: Evaluation, test_file: File + ): + # GIVEN a submission created with async method + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission for Deletion {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=self.syn) + + # WHEN I delete the submission using async method + await created_submission.delete_async(synapse_client=self.syn) + + # THEN attempting to retrieve it should raise an error + with pytest.raises(SynapseHTTPError): + await Submission(id=created_submission.id).get_async( + synapse_client=self.syn + ) + + async def test_delete_submission_without_id_async(self): + # WHEN I try to delete a submission without an ID using async method + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to delete"): + await submission.delete_async(synapse_client=self.syn) + + +class TestSubmissionCancelAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission tests.""" + import os + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + # async def test_cancel_submission_successfully_async( + # self, test_evaluation: Evaluation, test_file: File + # ): + # # GIVEN a submission created with async method + # submission = Submission( + # entity_id=test_file.id, + # evaluation_id=test_evaluation.id, + # name=f"Test Submission for Cancellation {uuid.uuid4()}", + # ) + # created_submission = await submission.store_async(synapse_client=self.syn) + # self.schedule_for_cleanup(created_submission.id) + + # # WHEN I cancel the submission using async method + # cancelled_submission = await created_submission.cancel_async(synapse_client=self.syn) + + # # THEN the submission should be cancelled + # assert cancelled_submission.id == created_submission.id + + async def test_cancel_submission_without_id_async(self): + # WHEN I try to cancel a submission without an ID using async method + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to cancel"): + await submission.cancel_async(synapse_client=self.syn) + + +class TestSubmissionValidationAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def test_get_submission_without_id_async(self): + # WHEN I try to get a submission without an ID using async method + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to get"): + await submission.get_async(synapse_client=self.syn) + + async def test_to_synapse_request_missing_entity_id_async(self): + # WHEN I try to create a request without entity_id + submission = Submission(evaluation_id="456", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'entity_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_missing_evaluation_id_async(self): + # WHEN I try to create a request without evaluation_id + submission = Submission(entity_id="syn123", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_valid_data_async(self): + # WHEN I create a request with valid required data + submission = Submission( + entity_id="syn123456", + evaluation_id="789", + name="Test Submission", + team_id="team123", + contributors=["user1", "user2"], + docker_repository_name="test/repo", + docker_digest="sha256:abc123", + ) + + request_body = submission.to_synapse_request() + + # THEN it should create a valid request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert request_body["name"] == "Test Submission" + assert request_body["teamId"] == "team123" + assert request_body["contributors"] == ["user1", "user2"] + assert request_body["dockerRepositoryName"] == "test/repo" + assert request_body["dockerDigest"] == "sha256:abc123" + + async def test_to_synapse_request_minimal_data_async(self): + # WHEN I create a request with only required data + submission = Submission(entity_id="syn123456", evaluation_id="789") + + request_body = submission.to_synapse_request() + + # THEN it should create a minimal request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body + + async def test_fetch_latest_entity_success_async(self): + # GIVEN a submission with a valid entity_id + submission = Submission(entity_id="syn123456", evaluation_id="789") + + # Note: This test would need a real entity ID to work in practice + # For now, we test the validation logic + with pytest.raises(ValueError, match="Unable to fetch entity information"): + await submission._fetch_latest_entity(synapse_client=self.syn) + + async def test_fetch_latest_entity_without_entity_id_async(self): + # GIVEN a submission without entity_id + submission = Submission(evaluation_id="789") + + # WHEN I try to fetch entity information + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="entity_id must be set"): + await submission._fetch_latest_entity(synapse_client=self.syn) + + +class TestSubmissionDataMappingAsync: + async def test_fill_from_dict_complete_data_async(self): + # GIVEN a complete submission response from the REST API + api_response = { + "id": "123456", + "userId": "user123", + "submitterAlias": "testuser", + "entityId": "syn789", + "versionNumber": 1, + "evaluationId": "eval456", + "name": "Test Submission", + "createdOn": "2023-01-01T10:00:00.000Z", + "teamId": "team123", + "contributors": ["user1", "user2"], + "submissionStatus": {"status": "RECEIVED"}, + "entityBundleJSON": '{"entity": {"id": "syn789"}}', + "dockerRepositoryName": "test/repo", + "dockerDigest": "sha256:abc123", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN all fields should be mapped correctly + assert submission.id == "123456" + assert submission.user_id == "user123" + assert submission.submitter_alias == "testuser" + assert submission.entity_id == "syn789" + assert submission.version_number == 1 + assert submission.evaluation_id == "eval456" + assert submission.name == "Test Submission" + assert submission.created_on == "2023-01-01T10:00:00.000Z" + assert submission.team_id == "team123" + assert submission.contributors == ["user1", "user2"] + assert submission.submission_status == {"status": "RECEIVED"} + assert submission.entity_bundle_json == '{"entity": {"id": "syn789"}}' + assert submission.docker_repository_name == "test/repo" + assert submission.docker_digest == "sha256:abc123" + + async def test_fill_from_dict_minimal_data_async(self): + # GIVEN a minimal submission response from the REST API + api_response = { + "id": "123456", + "entityId": "syn789", + "evaluationId": "eval456", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN required fields should be set and optional fields should have defaults + assert submission.id == "123456" + assert submission.entity_id == "syn789" + assert submission.evaluation_id == "eval456" + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.version_number is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py new file mode 100644 index 000000000..e6943a006 --- /dev/null +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -0,0 +1,626 @@ +"""Integration tests for the synapseclient.models.Submission class.""" + +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission + + +class TestSubmissionCreation: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission tests.""" + import os + import tempfile + + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + async def test_store_submission_successfully( + self, test_evaluation: Evaluation, test_file: File + ): + # WHEN I create a submission with valid data + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=self.syn) + self.schedule_for_cleanup(created_submission.id) + + # THEN the submission should be created successfully + assert created_submission.id is not None + assert created_submission.entity_id == test_file.id + assert created_submission.evaluation_id == test_evaluation.id + assert created_submission.name == submission.name + assert created_submission.user_id is not None + assert created_submission.created_on is not None + assert created_submission.version_number is not None + + async def test_store_submission_without_entity_id( + self, test_evaluation: Evaluation + ): + # WHEN I try to create a submission without entity_id + submission = Submission( + evaluation_id=test_evaluation.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="entity_id is required"): + submission.store(synapse_client=self.syn) + + async def test_store_submission_without_evaluation_id(self, test_file: File): + # WHEN I try to create a submission without evaluation_id + submission = Submission( + entity_id=test_file.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.store(synapse_client=self.syn) + + async def test_store_submission_with_docker_repository( + self, test_evaluation: Evaluation + ): + # GIVEN we would need a Docker repository entity (mocked for this test) + # This test demonstrates the expected behavior for Docker repository submissions + + # WHEN I create a submission for a Docker repository entity + # TODO: This would require a real Docker repository entity in a full integration test + submission = Submission( + entity_id="syn123456789", # Would be a Docker repository ID + evaluation_id=test_evaluation.id, + name=f"Docker Submission {uuid.uuid4()}", + ) + + # THEN the submission should handle Docker-specific attributes + # (This test would need to be expanded with actual Docker repository setup) + assert submission.entity_id == "syn123456789" + assert submission.evaluation_id == test_evaluation.id + + +class TestSubmissionRetrieval: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission tests.""" + import os + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for retrieval tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_get_submission_by_id( + self, test_submission: Submission, test_evaluation: Evaluation, test_file: File + ): + # WHEN I get a submission by ID + retrieved_submission = Submission(id=test_submission.id).get( + synapse_client=self.syn + ) + + # THEN the submission should be retrieved correctly + assert retrieved_submission.id == test_submission.id + assert retrieved_submission.entity_id == test_file.id + assert retrieved_submission.evaluation_id == test_evaluation.id + assert retrieved_submission.name == test_submission.name + assert retrieved_submission.user_id is not None + assert retrieved_submission.created_on is not None + + async def test_get_evaluation_submissions( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get all submissions for an evaluation + response = Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with submissions + assert "results" in response + assert len(response["results"]) > 0 + + # AND the submission should be in the results + submission_ids = [sub.get("id") for sub in response["results"]] + assert test_submission.id in submission_ids + + async def test_get_evaluation_submissions_with_status_filter( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get submissions filtered by status + response = Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN I should get submissions with the specified status + assert "results" in response + for submission in response["results"]: + if submission.get("id") == test_submission.id: + # The submission should be in RECEIVED status initially + break + else: + pytest.fail("Test submission not found in filtered results") + + async def test_get_evaluation_submissions_with_pagination( + self, test_evaluation: Evaluation + ): + # WHEN I get submissions with pagination parameters + response = Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, + limit=5, + offset=0, + synapse_client=self.syn, + ) + + # THEN the response should respect pagination + assert "results" in response + assert len(response["results"]) <= 5 + + async def test_get_user_submissions(self, test_evaluation: Evaluation): + # WHEN I get submissions for the current user + response = Submission.get_user_submissions( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with user submissions + assert "results" in response + # Note: Could be empty if user hasn't made submissions to this evaluation + + async def test_get_submission_count(self, test_evaluation: Evaluation): + # WHEN I get the submission count for an evaluation + response = Submission.get_submission_count( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a count response + assert isinstance(response, int) + assert response >= 0 + + +class TestSubmissionDeletion: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission tests.""" + import os + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + async def test_delete_submission_successfully( + self, test_evaluation: Evaluation, test_file: File + ): + # GIVEN a submission + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission for Deletion {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=self.syn) + + # WHEN I delete the submission + created_submission.delete(synapse_client=self.syn) + + # THEN attempting to retrieve it should raise an error + with pytest.raises(SynapseHTTPError): + Submission(id=created_submission.id).get(synapse_client=self.syn) + + async def test_delete_submission_without_id(self): + # WHEN I try to delete a submission without an ID + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to delete"): + submission.delete(synapse_client=self.syn) + + +class TestSubmissionCancel: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission tests.""" + import os + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + # TODO: Add with SubmissionStatus model tests + # async def test_cancel_submission_successfully( + # self, test_evaluation: Evaluation, test_file: File + # ): + # # GIVEN a submission + # submission = Submission( + # entity_id=test_file.id, + # evaluation_id=test_evaluation.id, + # name=f"Test Submission for Cancellation {uuid.uuid4()}", + # ) + # created_submission = submission.store(synapse_client=self.syn) + # self.schedule_for_cleanup(created_submission.id) + + # # WHEN I cancel the submission + # cancelled_submission = created_submission.cancel(synapse_client=self.syn) + + # # THEN the submission should be cancelled + # assert cancelled_submission.id == created_submission.id + + async def test_cancel_submission_without_id(self): + # WHEN I try to cancel a submission without an ID + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to cancel"): + submission.cancel(synapse_client=self.syn) + + +class TestSubmissionValidation: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def test_get_submission_without_id(self): + # WHEN I try to get a submission without an ID + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to get"): + submission.get(synapse_client=self.syn) + + async def test_to_synapse_request_missing_entity_id(self): + # WHEN I try to create a request without entity_id + submission = Submission(evaluation_id="456", name="Test") + + # THEN it should raise a ValueError + with pytest.raises( + ValueError, + match="Your submission object is missing the 'entity_id' attribute", + ): + submission.to_synapse_request() + + async def test_to_synapse_request_missing_evaluation_id(self): + # WHEN I try to create a request without evaluation_id + submission = Submission(entity_id="syn123", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_valid_data(self): + # WHEN I create a request with valid required data + submission = Submission( + entity_id="syn123456", + evaluation_id="789", + name="Test Submission", + team_id="team123", + contributors=["user1", "user2"], + docker_repository_name="test/repo", + docker_digest="sha256:abc123", + ) + + request_body = submission.to_synapse_request() + + # THEN it should create a valid request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert request_body["name"] == "Test Submission" + assert request_body["teamId"] == "team123" + assert request_body["contributors"] == ["user1", "user2"] + assert request_body["dockerRepositoryName"] == "test/repo" + assert request_body["dockerDigest"] == "sha256:abc123" + + async def test_to_synapse_request_minimal_data(self): + # WHEN I create a request with only required data + submission = Submission(entity_id="syn123456", evaluation_id="789") + + request_body = submission.to_synapse_request() + + # THEN it should create a minimal request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body + + +class TestSubmissionDataMapping: + async def test_fill_from_dict_complete_data(self): + # GIVEN a complete submission response from the REST API + api_response = { + "id": "123456", + "userId": "user123", + "submitterAlias": "testuser", + "entityId": "syn789", + "versionNumber": 1, + "evaluationId": "eval456", + "name": "Test Submission", + "createdOn": "2023-01-01T10:00:00.000Z", + "teamId": "team123", + "contributors": ["user1", "user2"], + "submissionStatus": {"status": "RECEIVED"}, + "entityBundleJSON": '{"entity": {"id": "syn789"}}', + "dockerRepositoryName": "test/repo", + "dockerDigest": "sha256:abc123", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN all fields should be mapped correctly + assert submission.id == "123456" + assert submission.user_id == "user123" + assert submission.submitter_alias == "testuser" + assert submission.entity_id == "syn789" + assert submission.version_number == 1 + assert submission.evaluation_id == "eval456" + assert submission.name == "Test Submission" + assert submission.created_on == "2023-01-01T10:00:00.000Z" + assert submission.team_id == "team123" + assert submission.contributors == ["user1", "user2"] + assert submission.submission_status == {"status": "RECEIVED"} + assert submission.entity_bundle_json == '{"entity": {"id": "syn789"}}' + assert submission.docker_repository_name == "test/repo" + assert submission.docker_digest == "sha256:abc123" + + async def test_fill_from_dict_minimal_data(self): + # GIVEN a minimal submission response from the REST API + api_response = { + "id": "123456", + "entityId": "syn789", + "evaluationId": "eval456", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN required fields should be set and optional fields should have defaults + assert submission.id == "123456" + assert submission.entity_id == "syn789" + assert submission.evaluation_id == "eval456" + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.version_number is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py new file mode 100644 index 000000000..8a0ca25ac --- /dev/null +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -0,0 +1,601 @@ +"""Async unit tests for the synapseclient.models.Submission class.""" +import uuid +from typing import Dict, List, Union +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Submission + +SUBMISSION_ID = "9614543" +USER_ID = "123456" +SUBMITTER_ALIAS = "test_user" +ENTITY_ID = "syn789012" +VERSION_NUMBER = 1 +EVALUATION_ID = "9999999" +SUBMISSION_NAME = "Test Submission" +CREATED_ON = "2023-01-01T10:00:00.000Z" +TEAM_ID = "team123" +CONTRIBUTORS = ["user1", "user2", "user3"] +SUBMISSION_STATUS = {"status": "RECEIVED", "score": 85.5} +ENTITY_BUNDLE_JSON = '{"entity": {"id": "syn789012", "name": "test_entity"}}' +DOCKER_REPOSITORY_NAME = "test/repository" +DOCKER_DIGEST = "sha256:abc123def456" +ETAG = "etag_value" + + +class TestSubmissionAsync: + """Async tests for the synapseclient.models.Submission class.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_response(self) -> Dict[str, Union[str, int, List, Dict]]: + """Get a complete example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "userId": USER_ID, + "submitterAlias": SUBMITTER_ALIAS, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "evaluationId": EVALUATION_ID, + "name": SUBMISSION_NAME, + "createdOn": CREATED_ON, + "teamId": TEAM_ID, + "contributors": CONTRIBUTORS, + "submissionStatus": SUBMISSION_STATUS, + "entityBundleJSON": ENTITY_BUNDLE_JSON, + "dockerRepositoryName": DOCKER_REPOSITORY_NAME, + "dockerDigest": DOCKER_DIGEST, + } + + def get_minimal_submission_response(self) -> Dict[str, str]: + """Get a minimal example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + } + + def get_example_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example entity response for testing entity fetching.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "versionNumber": VERSION_NUMBER, + "name": "test_entity", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + } + + def get_example_docker_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example Docker repository entity response for testing.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "name": "test_docker_repo", + "concreteType": "org.sagebionetworks.repo.model.docker.DockerRepository", + "repositoryName": "test/repository", + } + + def get_example_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get an example Docker tag response for testing.""" + return { + "totalNumberOfResults": 2, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:older123def456", + "createdOn": "2024-01-01T10:00:00.000Z", + }, + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z", + }, + ], + } + + def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get a more complex Docker tag response with multiple versions to test sorting.""" + return { + "totalNumberOfResults": 4, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:version1", + "createdOn": "2024-01-01T10:00:00.000Z", + }, + { + "tag": "v3.0", + "digest": "sha256:version3", + "createdOn": "2024-08-15T12:00:00.000Z", # This should be selected (latest) + }, + { + "tag": "v2.0", + "digest": "sha256:version2", + "createdOn": "2024-06-01T15:30:00.000Z", + }, + { + "tag": "v1.5", + "digest": "sha256:version1_5", + "createdOn": "2024-03-15T08:45:00.000Z", + }, + ], + } + + def test_fill_from_dict_complete_data_async(self) -> None: + # GIVEN a complete submission response from the REST API + # WHEN I call fill_from_dict with the example submission response + submission = Submission().fill_from_dict(self.get_example_submission_response()) + + # THEN the Submission object should be filled with all the data + assert submission.id == SUBMISSION_ID + assert submission.user_id == USER_ID + assert submission.submitter_alias == SUBMITTER_ALIAS + assert submission.entity_id == ENTITY_ID + assert submission.version_number == VERSION_NUMBER + assert submission.evaluation_id == EVALUATION_ID + assert submission.name == SUBMISSION_NAME + assert submission.created_on == CREATED_ON + assert submission.team_id == TEAM_ID + assert submission.contributors == CONTRIBUTORS + assert submission.submission_status == SUBMISSION_STATUS + assert submission.entity_bundle_json == ENTITY_BUNDLE_JSON + assert submission.docker_repository_name == DOCKER_REPOSITORY_NAME + assert submission.docker_digest == DOCKER_DIGEST + + def test_to_synapse_request_complete_data_async(self) -> None: + # GIVEN a submission with all optional fields set + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + docker_repository_name=DOCKER_REPOSITORY_NAME, + docker_digest=DOCKER_DIGEST, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain all fields in the correct format + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert request_body["name"] == SUBMISSION_NAME + assert request_body["teamId"] == TEAM_ID + assert request_body["contributors"] == CONTRIBUTORS + assert request_body["dockerRepositoryName"] == DOCKER_REPOSITORY_NAME + assert request_body["dockerDigest"] == DOCKER_DIGEST + + @pytest.mark.asyncio + async def test_fetch_latest_entity_success_async(self) -> None: + # GIVEN a submission with an entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with a mocked successful response + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_rest_get: + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["versionNumber"] == VERSION_NUMBER + mock_rest_get.assert_called_once_with(f"/entity/{ENTITY_ID}") + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_repository_async(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with mocked Docker repository responses + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return different responses for different URLs + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_example_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information with latest docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should have the latest tag information (v2.0 based on createdOn date) + assert entity_info["tag"] == "v2.0" + assert entity_info["digest"] == "sha256:latest456abc789" + assert entity_info["createdOn"] == "2024-06-01T15:30:00.000Z" + + # Verify both API calls were made + expected_calls = [ + call(f"/entity/{ENTITY_ID}"), + call(f"/entity/{ENTITY_ID}/dockerTag"), + ] + mock_rest_get.assert_has_calls(expected_calls) + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_empty_results_async(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with empty docker tag results + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return empty docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return {"totalNumberOfResults": 0, "results": []} + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information without docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should not have docker tag fields since results were empty + assert "tag" not in entity_info + assert "digest" not in entity_info + assert "createdOn" not in entity_info + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_complex_tag_selection_async(self) -> None: + # GIVEN a submission with a Docker repository with multiple tags + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with multiple docker tags with different dates + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return complex docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_complex_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should select the tag with the latest createdOn timestamp (v3.0) + assert entity_info["tag"] == "v3.0" + assert entity_info["digest"] == "sha256:version3" + assert entity_info["createdOn"] == "2024-08-15T12:00:00.000Z" + + @pytest.mark.asyncio + async def test_store_async_success(self) -> None: + # GIVEN a submission with valid data + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should fetch entity information, create the submission, and fill the object + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify the submission is filled with response data + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_store_async_docker_repository_success(self) -> None: + # GIVEN a submission with valid data for Docker repository + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked Docker repository entity + docker_entity_with_tag = self.get_example_docker_entity_response() + docker_entity_with_tag.update( + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z", + } + ) + + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=docker_entity_with_tag, + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should handle Docker repository specific logic + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify Docker repository attributes are set correctly + assert stored_submission.version_number == 1 # Docker repos get version 1 + assert stored_submission.docker_repository_name == "test/repository" + assert stored_submission.docker_digest == DOCKER_DIGEST + + @pytest.mark.asyncio + async def test_store_async_with_team_data_success(self) -> None: + # GIVEN a submission with team information + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should preserve team information in the stored submission + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify team data is preserved + assert stored_submission.team_id == TEAM_ID + assert stored_submission.contributors == CONTRIBUTORS + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_get_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call get_async with a mocked successful response + with patch( + "synapseclient.api.evaluation_services.get_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_get_submission: + retrieved_submission = await submission.get_async(synapse_client=self.syn) + + # THEN it should call the API and fill the object + mock_get_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + assert retrieved_submission.id == SUBMISSION_ID + assert retrieved_submission.entity_id == ENTITY_ID + assert retrieved_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_delete_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call delete_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.delete_submission", + new_callable=AsyncMock, + ) as mock_delete_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + await submission.delete_async(synapse_client=self.syn) + + # THEN it should call the API and log the deletion + mock_delete_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been deleted." + ) + + @pytest.mark.asyncio + async def test_cancel_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call cancel_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.cancel_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_cancel_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + cancelled_submission = await submission.cancel_async( + synapse_client=self.syn + ) + + # THEN it should call the API, log the cancellation, and update the object + mock_cancel_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been cancelled." + ) + assert cancelled_submission.id == SUBMISSION_ID + assert cancelled_submission.entity_id == ENTITY_ID + assert cancelled_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_get_evaluation_submissions_async(self) -> None: + # GIVEN evaluation parameters + evaluation_id = EVALUATION_ID + status = "SCORED" + limit = 10 + offset = 5 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_evaluation_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_submissions: + response = await Submission.get_evaluation_submissions_async( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_user_submissions_async(self) -> None: + # GIVEN user submission parameters + evaluation_id = EVALUATION_ID + user_id = USER_ID + limit = 15 + offset = 0 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_user_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_user_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_user_submissions: + response = await Submission.get_user_submissions_async( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_user_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_submission_count_async(self) -> None: + # GIVEN submission count parameters + evaluation_id = EVALUATION_ID + status = "VALID" + + expected_response = 42 + + # WHEN I call get_submission_count_async + with patch( + "synapseclient.api.evaluation_services.get_submission_count", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_count: + response = await Submission.get_submission_count_async( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_count.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + assert response == expected_response + + def test_to_synapse_request_minimal_data_async(self) -> None: + # GIVEN a submission with only required fields + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain only required fields + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py new file mode 100644 index 000000000..038045f11 --- /dev/null +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -0,0 +1,808 @@ +"""Unit tests for the synapseclient.models.Submission class.""" +import uuid +from typing import Dict, List, Union +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Submission + +SUBMISSION_ID = "9614543" +USER_ID = "123456" +SUBMITTER_ALIAS = "test_user" +ENTITY_ID = "syn789012" +VERSION_NUMBER = 1 +EVALUATION_ID = "9999999" +SUBMISSION_NAME = "Test Submission" +CREATED_ON = "2023-01-01T10:00:00.000Z" +TEAM_ID = "team123" +CONTRIBUTORS = ["user1", "user2", "user3"] +SUBMISSION_STATUS = {"status": "RECEIVED", "score": 85.5} +ENTITY_BUNDLE_JSON = '{"entity": {"id": "syn789012", "name": "test_entity"}}' +DOCKER_REPOSITORY_NAME = "test/repository" +DOCKER_DIGEST = "sha256:abc123def456" +ETAG = "etag_value" + + +class TestSubmission: + """Tests for the synapseclient.models.Submission class.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_response(self) -> Dict[str, Union[str, int, List, Dict]]: + """Get a complete example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "userId": USER_ID, + "submitterAlias": SUBMITTER_ALIAS, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "evaluationId": EVALUATION_ID, + "name": SUBMISSION_NAME, + "createdOn": CREATED_ON, + "teamId": TEAM_ID, + "contributors": CONTRIBUTORS, + "submissionStatus": SUBMISSION_STATUS, + "entityBundleJSON": ENTITY_BUNDLE_JSON, + "dockerRepositoryName": DOCKER_REPOSITORY_NAME, + "dockerDigest": DOCKER_DIGEST, + } + + def get_minimal_submission_response(self) -> Dict[str, str]: + """Get a minimal example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + } + + def get_example_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example entity response for testing entity fetching.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "versionNumber": VERSION_NUMBER, + "name": "test_entity", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + } + + def get_example_docker_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example Docker repository entity response for testing.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "name": "test_docker_repo", + "concreteType": "org.sagebionetworks.repo.model.docker.DockerRepository", + "repositoryName": "test/repository", + } + + def get_example_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get an example Docker tag response for testing.""" + return { + "totalNumberOfResults": 2, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:older123def456", + "createdOn": "2024-01-01T10:00:00.000Z", + }, + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z", + }, + ], + } + + def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get a more complex Docker tag response with multiple versions to test sorting.""" + return { + "totalNumberOfResults": 4, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:version1", + "createdOn": "2024-01-01T10:00:00.000Z", + }, + { + "tag": "v3.0", + "digest": "sha256:version3", + "createdOn": "2024-08-15T12:00:00.000Z", # This should be selected (latest) + }, + { + "tag": "v2.0", + "digest": "sha256:version2", + "createdOn": "2024-06-01T15:30:00.000Z", + }, + { + "tag": "v1.5", + "digest": "sha256:version1_5", + "createdOn": "2024-03-15T08:45:00.000Z", + }, + ], + } + + def test_fill_from_dict_complete_data(self) -> None: + # GIVEN a complete submission response from the REST API + # WHEN I call fill_from_dict with the example submission response + submission = Submission().fill_from_dict(self.get_example_submission_response()) + + # THEN the Submission object should be filled with all the data + assert submission.id == SUBMISSION_ID + assert submission.user_id == USER_ID + assert submission.submitter_alias == SUBMITTER_ALIAS + assert submission.entity_id == ENTITY_ID + assert submission.version_number == VERSION_NUMBER + assert submission.evaluation_id == EVALUATION_ID + assert submission.name == SUBMISSION_NAME + assert submission.created_on == CREATED_ON + assert submission.team_id == TEAM_ID + assert submission.contributors == CONTRIBUTORS + assert submission.submission_status == SUBMISSION_STATUS + assert submission.entity_bundle_json == ENTITY_BUNDLE_JSON + assert submission.docker_repository_name == DOCKER_REPOSITORY_NAME + assert submission.docker_digest == DOCKER_DIGEST + + def test_fill_from_dict_minimal_data(self) -> None: + # GIVEN a minimal submission response from the REST API + # WHEN I call fill_from_dict with the minimal submission response + submission = Submission().fill_from_dict(self.get_minimal_submission_response()) + + # THEN the Submission object should be filled with required data and defaults for optional data + assert submission.id == SUBMISSION_ID + assert submission.entity_id == ENTITY_ID + assert submission.evaluation_id == EVALUATION_ID + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.version_number is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None + + def test_to_synapse_request_complete_data(self) -> None: + # GIVEN a submission with all optional fields set + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + docker_repository_name=DOCKER_REPOSITORY_NAME, + docker_digest=DOCKER_DIGEST, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain all fields in the correct format + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert request_body["name"] == SUBMISSION_NAME + assert request_body["teamId"] == TEAM_ID + assert request_body["contributors"] == CONTRIBUTORS + assert request_body["dockerRepositoryName"] == DOCKER_REPOSITORY_NAME + assert request_body["dockerDigest"] == DOCKER_DIGEST + + def test_to_synapse_request_minimal_data(self) -> None: + # GIVEN a submission with only required fields + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain only required fields + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body + + def test_to_synapse_request_missing_entity_id(self) -> None: + # GIVEN a submission without entity_id + submission = Submission(evaluation_id=EVALUATION_ID) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'entity_id' attribute"): + submission.to_synapse_request() + + def test_to_synapse_request_missing_evaluation_id(self) -> None: + # GIVEN a submission without evaluation_id + submission = Submission(entity_id=ENTITY_ID) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.to_synapse_request() + + @pytest.mark.asyncio + async def test_fetch_latest_entity_success(self) -> None: + # GIVEN a submission with an entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with a mocked successful response + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_rest_get: + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["versionNumber"] == VERSION_NUMBER + mock_rest_get.assert_called_once_with(f"/entity/{ENTITY_ID}") + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_repository(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with mocked Docker repository responses + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return different responses for different URLs + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_example_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information with latest docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should have the latest tag information (v2.0 based on createdOn date) + assert entity_info["tag"] == "v2.0" + assert entity_info["digest"] == "sha256:latest456abc789" + assert entity_info["createdOn"] == "2024-06-01T15:30:00.000Z" + + # Verify both API calls were made + expected_calls = [ + call(f"/entity/{ENTITY_ID}"), + call(f"/entity/{ENTITY_ID}/dockerTag"), + ] + mock_rest_get.assert_has_calls(expected_calls) + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_empty_results(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with empty docker tag results + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return empty docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return {"totalNumberOfResults": 0, "results": []} + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information without docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should not have docker tag fields since results were empty + assert "tag" not in entity_info + assert "digest" not in entity_info + assert "createdOn" not in entity_info + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_complex_tag_selection(self) -> None: + # GIVEN a submission with a Docker repository with multiple tags + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with multiple docker tags with different dates + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return complex docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_complex_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should select the tag with the latest createdOn timestamp (v3.0) + assert entity_info["tag"] == "v3.0" + assert entity_info["digest"] == "sha256:version3" + assert entity_info["createdOn"] == "2024-08-15T12:00:00.000Z" + + @pytest.mark.asyncio + async def test_fetch_latest_entity_without_entity_id(self) -> None: + # GIVEN a submission without entity_id + submission = Submission(evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity + # THEN it should raise a ValueError + with pytest.raises( + ValueError, match="entity_id must be set to fetch entity information" + ): + await submission._fetch_latest_entity(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_fetch_latest_entity_api_error(self) -> None: + # GIVEN a submission with an entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity and the API returns an error + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + side_effect=SynapseHTTPError("Entity not found"), + ) as mock_rest_get: + # THEN it should raise a ValueError with context about the original error + with pytest.raises( + ValueError, match=f"Unable to fetch entity information for {ENTITY_ID}" + ): + await submission._fetch_latest_entity(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_store_async_success(self) -> None: + # GIVEN a submission with valid data + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should fetch entity information, create the submission, and fill the object + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Check the call arguments to create_submission + call_args = mock_create_submission.call_args + request_body = call_args[0][0] + etag = call_args[0][1] + + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["name"] == SUBMISSION_NAME + assert request_body["versionNumber"] == VERSION_NUMBER + assert etag == ETAG + + # Verify the submission is filled with response data + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_store_async_docker_repository_success(self) -> None: + # GIVEN a submission with valid data for Docker repository + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked Docker repository entity + docker_entity_with_tag = self.get_example_docker_entity_response() + docker_entity_with_tag.update( + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z", + } + ) + + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=docker_entity_with_tag, + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should handle Docker repository specific logic + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify Docker repository attributes are set correctly + assert submission.version_number == 1 # Docker repos get version 1 + assert submission.docker_repository_name == "test/repository" + assert stored_submission.docker_digest == DOCKER_DIGEST + + @pytest.mark.asyncio + async def test_store_async_with_team_data_success(self) -> None: + # GIVEN a submission with team information + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should preserve team information in the stored submission + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify team data is preserved + assert stored_submission.team_id == TEAM_ID + assert stored_submission.contributors == CONTRIBUTORS + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_store_async_missing_entity_id(self) -> None: + # GIVEN a submission without entity_id + submission = Submission(evaluation_id=EVALUATION_ID, name=SUBMISSION_NAME) + + # WHEN I call store_async + # THEN it should raise a ValueError during to_synapse_request + with pytest.raises( + ValueError, match="entity_id is required to create a submission" + ): + await submission.store_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_store_async_entity_fetch_failure(self) -> None: + # GIVEN a submission with valid data but entity fetch fails + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async and entity fetching fails + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + side_effect=ValueError("Unable to fetch entity information"), + ) as mock_fetch_entity: + # THEN it should propagate the ValueError + with pytest.raises(ValueError, match="Unable to fetch entity information"): + await submission.store_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_get_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call get_async with a mocked successful response + with patch( + "synapseclient.api.evaluation_services.get_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_get_submission: + retrieved_submission = await submission.get_async(synapse_client=self.syn) + + # THEN it should call the API and fill the object + mock_get_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + assert retrieved_submission.id == SUBMISSION_ID + assert retrieved_submission.entity_id == ENTITY_ID + assert retrieved_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_get_async_without_id(self) -> None: + # GIVEN a submission without an ID + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call get_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to get"): + await submission.get_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_delete_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call delete_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.delete_submission", + new_callable=AsyncMock, + ) as mock_delete_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + await submission.delete_async(synapse_client=self.syn) + + # THEN it should call the API and log the deletion + mock_delete_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been deleted." + ) + + @pytest.mark.asyncio + async def test_delete_async_without_id(self) -> None: + # GIVEN a submission without an ID + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call delete_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to delete"): + await submission.delete_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_cancel_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call cancel_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.cancel_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_cancel_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + cancelled_submission = await submission.cancel_async( + synapse_client=self.syn + ) + + # THEN it should call the API, log the cancellation, and update the object + mock_cancel_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been cancelled." + ) + assert cancelled_submission.id == SUBMISSION_ID + assert cancelled_submission.entity_id == ENTITY_ID + assert cancelled_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_cancel_async_without_id(self) -> None: + # GIVEN a submission without an ID + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call cancel_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to cancel"): + await submission.cancel_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_get_evaluation_submissions_async(self) -> None: + # GIVEN evaluation parameters + evaluation_id = EVALUATION_ID + status = "SCORED" + limit = 10 + offset = 5 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_evaluation_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_submissions: + response = await Submission.get_evaluation_submissions_async( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_user_submissions_async(self) -> None: + # GIVEN user submission parameters + evaluation_id = EVALUATION_ID + user_id = USER_ID + limit = 15 + offset = 0 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_user_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_user_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_user_submissions: + response = await Submission.get_user_submissions_async( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_user_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_submission_count_async(self) -> None: + # GIVEN submission count parameters + evaluation_id = EVALUATION_ID + status = "VALID" + + expected_response = 42 + + # WHEN I call get_submission_count_async + with patch( + "synapseclient.api.evaluation_services.get_submission_count", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_count: + response = await Submission.get_submission_count_async( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_count.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + assert response == expected_response + + def test_default_values(self) -> None: + # GIVEN a new Submission object with no parameters + submission = Submission() + + # THEN all attributes should have their default values + assert submission.id is None + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.entity_id is None + assert submission.version_number is None + assert submission.evaluation_id is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None + assert submission.etag is None + + def test_constructor_with_values(self) -> None: + # GIVEN specific values for submission attributes + # WHEN I create a Submission object with those values + submission = Submission( + id=SUBMISSION_ID, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + docker_repository_name=DOCKER_REPOSITORY_NAME, + docker_digest=DOCKER_DIGEST, + ) + + # THEN the object should be initialized with those values + assert submission.id == SUBMISSION_ID + assert submission.entity_id == ENTITY_ID + assert submission.evaluation_id == EVALUATION_ID + assert submission.name == SUBMISSION_NAME + assert submission.team_id == TEAM_ID + assert submission.contributors == CONTRIBUTORS + assert submission.docker_repository_name == DOCKER_REPOSITORY_NAME + assert submission.docker_digest == DOCKER_DIGEST + + def test_to_synapse_request_with_none_values(self) -> None: + # GIVEN a submission with some None values for optional fields + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=None, # Explicitly None + team_id=None, # Explicitly None + contributors=[], # Empty list (falsy) + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN None and empty values should not be included + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body