Support $ref in responses (#1207)

Updates and closes #1148

---------

Co-authored-by: Eli Bishop <eli.bishop@benchling.com>
Co-authored-by: Dylan Anthony <dbanty@users.noreply.github.com>

Author: dbanty

.changeset/support_ref_in_responses.md

@@ -0,0 +1,12 @@
+---
+default: major
+---
+
+# Support `$ref` in responses
+
+Previously, using a `$ref` to define a response was ignored, the code to call the endpoint was still generated, but 
+the response would not be parsed. Now, responses defined with `$ref` will be used to generate the response model, which 
+will parse the response at runtime.
+
+If a `$ref` is incorrect or uses a feature that is not supported by the generator, these endpoints will start failing to 
+generate.

end_to_end_tests/baseline_openapi_3.0.json

@@ -991,6 +991,20 @@
         }
       }
     },
+    "/responses/reference": {
+      "get": {
+        "tags": [
+          "responses"
+        ],
+        "summary": "Endpoint using predefined response",
+        "operationId": "reference_response",
+        "responses": {
+          "200": {
+            "$ref": "#/components/responses/AResponse"
+          }
+        }
+      }
+    },
     "/auth/token_with_cookie": {
       "get": {
         "tags": [
@@ -2969,6 +2983,18 @@
           }
         }
       }
+    },
+    "responses": {
+      "AResponse": {
+        "description": "OK",
+        "content": {
+          "application/json": {
+            "schema": {
+              "$ref": "#/components/schemas/AModel"
+            }
+          }
+        }
+      }
     }
   }
 }

end_to_end_tests/baseline_openapi_3.1.yaml

@@ -983,6 +983,20 @@ info:
       }
     }
   },
+  "/responses/reference": {
+    "get": {
+      "tags": [
+        "responses"
+      ],
+      "summary": "Endpoint using predefined response",
+      "operationId": "reference_response",
+      "responses": {
+        "200": {
+          "$ref": "#/components/responses/AResponse"
+        }
+      }
+    }
+  },
   "/auth/token_with_cookie": {
     "get": {
       "tags": [
@@ -2960,3 +2974,10 @@ info:
         "application/json":
           "schema":
             "$ref": "#/components/schemas/AModel"
+  responses:
+    AResponse:
+      description: OK
+      content:
+        "application/json":
+          "schema":
+            "$ref": "#/components/schemas/AModel"

end_to_end_tests/custom-templates-golden-record/my_test_api_client/api/responses/__init__.py

@@ -2,7 +2,7 @@
 
 import types
 
-from . import post_responses_unions_simple_before_complex, text_response
+from . import post_responses_unions_simple_before_complex, reference_response, text_response
 
 
 class ResponsesEndpoints:
@@ -19,3 +19,10 @@ def text_response(cls) -> types.ModuleType:
         Text Response
         """
         return text_response
+
+    @classmethod
+    def reference_response(cls) -> types.ModuleType:
+        """
+        Endpoint using predefined response
+        """
+        return reference_response

end_to_end_tests/golden-record/my_test_api_client/api/responses/reference_response.py

@@ -0,0 +1,122 @@
+from http import HTTPStatus
+from typing import Any, Optional, Union
+
+import httpx
+
+from ... import errors
+from ...client import AuthenticatedClient, Client
+from ...models.a_model import AModel
+from ...types import Response
+
+
+def _get_kwargs() -> dict[str, Any]:
+    _kwargs: dict[str, Any] = {
+        "method": "get",
+        "url": "/responses/reference",
+    }
+
+    return _kwargs
+
+
+def _parse_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Optional[AModel]:
+    if response.status_code == 200:
+        response_200 = AModel.from_dict(response.json())
+
+        return response_200
+    if client.raise_on_unexpected_status:
+        raise errors.UnexpectedStatus(response.status_code, response.content)
+    else:
+        return None
+
+
+def _build_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Response[AModel]:
+    return Response(
+        status_code=HTTPStatus(response.status_code),
+        content=response.content,
+        headers=response.headers,
+        parsed=_parse_response(client=client, response=response),
+    )
+
+
+def sync_detailed(
+    *,
+    client: Union[AuthenticatedClient, Client],
+) -> Response[AModel]:
+    """Endpoint using predefined response
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[AModel]
+    """
+
+    kwargs = _get_kwargs()
+
+    response = client.get_httpx_client().request(
+        **kwargs,
+    )
+
+    return _build_response(client=client, response=response)
+
+
+def sync(
+    *,
+    client: Union[AuthenticatedClient, Client],
+) -> Optional[AModel]:
+    """Endpoint using predefined response
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        AModel
+    """
+
+    return sync_detailed(
+        client=client,
+    ).parsed
+
+
+async def asyncio_detailed(
+    *,
+    client: Union[AuthenticatedClient, Client],
+) -> Response[AModel]:
+    """Endpoint using predefined response
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[AModel]
+    """
+
+    kwargs = _get_kwargs()
+
+    response = await client.get_async_httpx_client().request(**kwargs)
+
+    return _build_response(client=client, response=response)
+
+
+async def asyncio(
+    *,
+    client: Union[AuthenticatedClient, Client],
+) -> Optional[AModel]:
+    """Endpoint using predefined response
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        AModel
+    """
+
+    return (
+        await asyncio_detailed(
+            client=client,
+        )
+    ).parsed

openapi_python_client/parser/bodies.py

@@ -9,6 +9,7 @@
     Schemas,
     property_from_data,
 )
+from openapi_python_client.parser.properties.schemas import get_reference_simple_name
 
 from .. import schema as oai
 from ..config import Config
@@ -138,7 +139,7 @@ def _resolve_reference(
     references_seen = []
     while isinstance(body, oai.Reference) and body.ref not in references_seen:
         references_seen.append(body.ref)
-        body = request_bodies.get(body.ref.split("/")[-1])
+        body = request_bodies.get(get_reference_simple_name(body.ref))
     if isinstance(body, oai.Reference):
         return ParseError(detail="Circular $ref in request body", data=body)
     if body is None and references_seen:

openapi_python_client/parser/openapi.py

@@ -51,6 +51,7 @@ def from_data(
         schemas: Schemas,
         parameters: Parameters,
         request_bodies: dict[str, Union[oai.RequestBody, oai.Reference]],
+        responses: dict[str, Union[oai.Response, oai.Reference]],
         config: Config,
     ) -> tuple[dict[utils.PythonIdentifier, "EndpointCollection"], Schemas, Parameters]:
         """Parse the openapi paths data to get EndpointCollections by tag"""
@@ -78,6 +79,7 @@ def from_data(
                     schemas=schemas,
                     parameters=parameters,
                     request_bodies=request_bodies,
+                    responses=responses,
                     config=config,
                 )
                 # Add `PathItem` parameters
@@ -151,7 +153,12 @@ class Endpoint:
 
     @staticmethod
     def _add_responses(
-        *, endpoint: "Endpoint", data: oai.Responses, schemas: Schemas, config: Config
+        *,
+        endpoint: "Endpoint",
+        data: oai.Responses,
+        schemas: Schemas,
+        responses: dict[str, Union[oai.Response, oai.Reference]],
+        config: Config,
     ) -> tuple["Endpoint", Schemas]:
         endpoint = deepcopy(endpoint)
         for code, response_data in data.items():
@@ -174,6 +181,7 @@ def _add_responses(
                 status_code=status_code,
                 data=response_data,
                 schemas=schemas,
+                responses=responses,
                 parent_name=endpoint.name,
                 config=config,
             )
@@ -403,6 +411,7 @@ def from_data(
         schemas: Schemas,
         parameters: Parameters,
         request_bodies: dict[str, Union[oai.RequestBody, oai.Reference]],
+        responses: dict[str, Union[oai.Response, oai.Reference]],
         config: Config,
     ) -> tuple[Union["Endpoint", ParseError], Schemas, Parameters]:
         """Construct an endpoint from the OpenAPI data"""
@@ -431,7 +440,13 @@ def from_data(
         )
         if isinstance(result, ParseError):
             return result, schemas, parameters
-        result, schemas = Endpoint._add_responses(endpoint=result, data=data.responses, schemas=schemas, config=config)
+        result, schemas = Endpoint._add_responses(
+            endpoint=result,
+            data=data.responses,
+            schemas=schemas,
+            responses=responses,
+            config=config,
+        )
         if isinstance(result, ParseError):
             return result, schemas, parameters
         bodies, schemas = body_from_data(
@@ -521,8 +536,14 @@ def from_dict(data: dict[str, Any], *, config: Config) -> Union["GeneratorData",
                 config=config,
             )
         request_bodies = (openapi.components and openapi.components.requestBodies) or {}
+        responses = (openapi.components and openapi.components.responses) or {}
         endpoint_collections_by_tag, schemas, parameters = EndpointCollection.from_data(
-            data=openapi.paths, schemas=schemas, parameters=parameters, request_bodies=request_bodies, config=config
+            data=openapi.paths,
+            schemas=schemas,
+            parameters=parameters,
+            request_bodies=request_bodies,
+            responses=responses,
+            config=config,
         )
 
         enums = (

openapi_python_client/parser/properties/schemas.py

@@ -46,6 +46,13 @@ def parse_reference_path(ref_path_raw: str) -> Union[ReferencePath, ParseError]:
     return cast(ReferencePath, parsed.fragment)
 
 
+def get_reference_simple_name(ref_path: str) -> str:
+    """
+    Takes a path like `/components/schemas/NameOfThing` and returns a string like `NameOfThing`.
+    """
+    return ref_path.split("/")[-1]
+
+
 @define
 class Class:
     """Represents Python class which will be generated from an OpenAPI schema"""
@@ -56,7 +63,7 @@ class Class:
     @staticmethod
     def from_string(*, string: str, config: Config) -> "Class":
         """Get a Class from an arbitrary string"""
-        class_name = string.split("/")[-1]  # Get rid of ref path stuff
+        class_name = get_reference_simple_name(string)  # Get rid of ref path stuff
         class_name = ClassName(class_name, config.field_prefix)
         override = config.class_overrides.get(class_name)
 

openapi_python_client/parser/responses.py

@@ -6,6 +6,7 @@
 from attrs import define
 
 from openapi_python_client import utils
+from openapi_python_client.parser.properties.schemas import get_reference_simple_name, parse_reference_path
 
 from .. import Config
 from .. import schema as oai
@@ -79,27 +80,30 @@ def empty_response(
     )
 
 
-def response_from_data(
+def response_from_data(  # noqa: PLR0911
     *,
     status_code: HTTPStatus,
     data: Union[oai.Response, oai.Reference],
     schemas: Schemas,
+    responses: dict[str, Union[oai.Response, oai.Reference]],
     parent_name: str,
     config: Config,
 ) -> tuple[Union[Response, ParseError], Schemas]:
     """Generate a Response from the OpenAPI dictionary representation of it"""
 
     response_name = f"response_{status_code}"
     if isinstance(data, oai.Reference):
-        return (
-            empty_response(
-                status_code=status_code,
-                response_name=response_name,
-                config=config,
-                data=data,
-            ),
-            schemas,
-        )
+        ref_path = parse_reference_path(data.ref)
+        if isinstance(ref_path, ParseError):
+            return ref_path, schemas
+        if not ref_path.startswith("/components/responses/"):
+            return ParseError(data=data, detail=f"$ref to {data.ref} not allowed in responses"), schemas
+        resp_data = responses.get(get_reference_simple_name(ref_path), None)
+        if not resp_data:
+            return ParseError(data=data, detail=f"Could not find reference: {data.ref}"), schemas
+        if not isinstance(resp_data, oai.Response):
+            return ParseError(data=data, detail="Top-level $ref inside components/responses is not supported"), schemas
+        data = resp_data
 
     content = data.content
     if not content: