feat: Added support for text/yaml in OpenAPI to be used in benchling-api-client BNCH-37249 (#118)

* feat: Added support for text/yaml in OpenAPI to be used in benchling-api-client BNCH-37249

* Fixed all tests

* Added yaml parse

* Added yaml body to mimic json object

* Added logic for proper parsing

* renamed yaml to be a json object on return package

* Updated test for yaml parsing changes

* Uses ruamel.yaml instead of yaml for parsing

* Updated tests

Author: miggyst

end_to_end_tests/golden-record-custom/pyproject.toml

@@ -14,8 +14,8 @@ include = ["CHANGELOG.md", "custom_e2e/py.typed"]
 
 [tool.poetry.dependencies]
 python = "^3.6"
-httpx = "^0.15.0"
-attrs = "^20.1.0"
+httpx = ">=0.15.0, <=0.22.0"
+attrs = ">=20.1.0, <22.0"
 python-dateutil = "^2.8.0"
 
 [tool.black]

end_to_end_tests/golden-record/pyproject.toml

@@ -14,8 +14,8 @@ include = ["CHANGELOG.md", "my_test_api_client/py.typed"]
 
 [tool.poetry.dependencies]
 python = "^3.6"
-httpx = "^0.15.0"
-attrs = "^20.1.0"
+httpx = ">=0.15.0, <=0.22.0"
+attrs = ">=20.1.0, <22.0"
 python-dateutil = "^2.8.0"
 
 [tool.black]

openapi_python_client/parser/openapi.py

@@ -94,11 +94,29 @@ class Endpoint:
     path_parameters: List[Property] = field(default_factory=list)
     header_parameters: List[Property] = field(default_factory=list)
     responses: List[Response] = field(default_factory=list)
+    yaml_body: Optional[Property] = None
     form_body_reference: Optional[Reference] = None
     json_body: Optional[Property] = None
     multipart_body_reference: Optional[Reference] = None
     errors: List[ParseError] = field(default_factory=list)
 
+    @staticmethod
+    def parse_request_yaml_body(
+            *, body: oai.RequestBody, schemas: Schemas, parent_name: str
+    ) -> Tuple[Union[Property, PropertyError, None], Schemas]:
+        """ Return yaml_body """
+        body_content = body.content
+        yaml_body = body_content.get("text/yaml")
+        if yaml_body is not None and yaml_body.media_type_schema is not None:
+            return property_from_data(
+                name="yaml_body",
+                required=True,
+                data=yaml_body.media_type_schema,
+                schemas=schemas,
+                parent_name=parent_name,
+            )
+        return None, schemas
+
     @staticmethod
     def parse_request_form_body(body: oai.RequestBody) -> Optional[Reference]:
         """ Return form_body_reference """
@@ -110,7 +128,7 @@ def parse_request_form_body(body: oai.RequestBody) -> Optional[Reference]:
 
     @staticmethod
     def parse_multipart_body(body: oai.RequestBody) -> Optional[Reference]:
-        """ Return form_body_reference """
+        """ Return multipart_body_reference """
         body_content = body.content
         json_body = body_content.get("multipart/form-data")
         if json_body is not None and isinstance(json_body.media_type_schema, oai.Reference):
@@ -150,6 +168,12 @@ def _add_body(
         if isinstance(json_body, ParseError):
             return ParseError(detail=f"cannot parse body of endpoint {endpoint.name}", data=json_body.data), schemas
 
+        yaml_body, schemas = Endpoint.parse_request_yaml_body(
+            body=data.requestBody, schemas=schemas, parent_name=endpoint.name
+        )
+        if isinstance(yaml_body, ParseError):
+            return ParseError(detail=f"cannot parse body of endpoint {endpoint.name}", data=yaml_body.data), schemas
+
         endpoint.multipart_body_reference = Endpoint.parse_multipart_body(data.requestBody)
 
         if endpoint.form_body_reference:
@@ -163,6 +187,10 @@ def _add_body(
         if json_body is not None:
             endpoint.json_body = json_body
             endpoint.relative_imports.update(endpoint.json_body.get_imports(prefix="..."))
+        if yaml_body is not None:
+            endpoint.yaml_body = yaml_body
+            endpoint.relative_imports.update(endpoint.yaml_body.get_imports(prefix="..."))
+
         return endpoint, schemas
 
     @staticmethod
@@ -177,7 +205,7 @@ def _add_responses(*, endpoint: "Endpoint", data: oai.Responses, schemas: Schema
                     ParseError(
                         detail=(
                             f"Cannot parse response for status code {code}, "
-                            f"response will be ommitted from generated client"
+                            f"response will be omitted from generated client"
                         ),
                         data=response.data,
                     )

openapi_python_client/parser/responses.py

@@ -22,6 +22,7 @@ class Response:
     "application/json": "response.json()",
     "application/octet-stream": "response.content",
     "text/html": "response.text",
+    "text/yaml": "response.yaml", # Only used as an identifier, not the actual source
 }
 
 

openapi_python_client/templates/endpoint_macros.pyi

@@ -56,6 +56,17 @@ if {% if not property.required %}not isinstance({{ property_name }}, Unset) and
 {% endif %}
 {% endmacro %}
 
+{% macro yaml_body(endpoint) %}
+{% if endpoint.yaml_body %}
+    {% set property = endpoint.yaml_body %}
+    {% set destination = "yaml_" + property.python_name %}
+    {% if property.template %}
+        {% from "property_templates/" + property.template import transform %}
+{{ transform(property, property.python_name, destination) }}
+    {% endif %}
+{% endif %}
+{% endmacro %}
+
 {% macro return_type(endpoint) %}
 {% if endpoint.responses | length == 0 %}
 None
@@ -83,6 +94,10 @@ client: Client,
 {% for parameter in endpoint.path_parameters %}
 {{ parameter.to_string() }},
 {% endfor %}
+{# Yaml body if any #}
+{% if endpoint.yaml_body %}
+yaml_body: {{ endpoint.yaml_body.get_type_string() }},
+{% endif %}
 {# Form data if any #}
 {% if endpoint.form_body_reference %}
 form_data: {{ endpoint.form_body_reference.class_name }},
@@ -110,6 +125,9 @@ client=client,
 {% for parameter in endpoint.path_parameters %}
 {{ parameter.python_name }}={{ parameter.python_name }},
 {% endfor %}
+{% if endpoint.yaml_body %}
+yaml_body=yaml_body,
+{% endif %}
 {% if endpoint.form_body_reference %}
 form_data=form_data,
 {% endif %}

openapi_python_client/templates/endpoint_module.pyi

@@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional, Union, cast
 
 import httpx
 from attr import asdict
+from ruamel.yaml import YAML
 
 from ...client import AuthenticatedClient, Client
 from ...types import Response
@@ -10,7 +11,7 @@ from ...types import Response
 {{ relative }}
 {% endfor %}
 
-{% from "endpoint_macros.pyi" import header_params, query_params, json_body, return_type, arguments, client, kwargs, parse_response %}
+{% from "endpoint_macros.pyi" import header_params, query_params, json_body, yaml_body, return_type, arguments, client, kwargs, parse_response %}
 
 {% set return_string = return_type(endpoint) %}
 {% set parsed_responses = (endpoint.responses | length > 0) and return_string != "None" %}
@@ -33,6 +34,8 @@ def _get_kwargs(
 
     {{ json_body(endpoint) | indent(4) }}
 
+    {{ yaml_body(endpoint) | indent(4) }}
+
     return {
         "url": url,
         "headers": headers,
@@ -46,7 +49,9 @@ def _get_kwargs(
         {% endif %}
         {% if endpoint.json_body %}
         "json": {{ "json_" + endpoint.json_body.python_name }},
-        {% endif %}
+        {%- elif endpoint.yaml_body %}
+        "json": {{ "yaml_" + endpoint.yaml_body.python_name }},
+        {%- endif %}
         {% if endpoint.query_parameters %}
         "params": params,
         {% endif %}

openapi_python_client/templates/property_templates/model_property.pyi

@@ -1,6 +1,12 @@
 {% macro construct(property, source, initial_value=None) %}
 {% if property.required and not property.nullable %}
+{% if source == "response.yaml" %}
+yaml = YAML(typ="base")
+yaml_dict = yaml.load(response.text.encode('utf-8'))
+{{ property.python_name }} = {{ property.reference.class_name }}.from_dict(yaml_dict)
+{% else %}
 {{ property.python_name }} = {{ property.reference.class_name }}.from_dict({{ source }})
+{% endif %}
 {% else %}
 {% if initial_value != None %}
 {{ property.python_name }} = {{ initial_value }}

poetry.lock

@@ -34,6 +34,7 @@ attrs = ">=20.1.0, <22.0"
 python-dateutil = "^2.8.1"
 httpx = ">=0.15.4,<=0.22.0"
 autoflake = "^1.4"
+"ruamel.yaml" = "^0.17.21"
 
 [tool.poetry.scripts]
 openapi-python-client = "openapi_python_client.cli:app"
@@ -57,7 +58,8 @@ isort .\
  && mypy openapi_python_client\
  && task unit\
 """
-unit = "pytest --cov openapi_python_client tests --cov-report=term-missing"
+code_coverage = "pytest --cov openapi_python_client tests --cov-report=term-missing"
+unit = "pytest openapi_python_client tests"
 regen = "python -m end_to_end_tests.regen_golden_record"
 regen_custom = "python -m end_to_end_tests.regen_golden_record custom"
 e2e = "pytest openapi_python_client end_to_end_tests/test_end_to_end.py"

pyproject.toml

@@ -66,6 +66,32 @@ def test_from_dict_invalid_schema(self, mocker):
 
 
 class TestEndpoint:
+    def test_parse_yaml_body(self, mocker):
+        from openapi_python_client.parser.openapi import Endpoint, Schemas
+        schema = mocker.MagicMock()
+        body = oai.RequestBody.construct(
+            content={"text/yaml": oai.MediaType.construct(media_type_schema=schema)}
+        )
+        property_from_data = mocker.patch(f"{MODULE_NAME}.property_from_data")
+        schemas = Schemas()
+
+        result = Endpoint.parse_request_yaml_body(body=body, schemas=schemas, parent_name="parent")
+
+        property_from_data.assert_called_once_with(
+            name="yaml_body", required=True, data=schema, schemas=schemas, parent_name="parent"
+        )
+        assert result == property_from_data.return_value
+
+    def test_parse_yaml_body_no_data(self):
+        from openapi_python_client.parser.openapi import Endpoint, Schemas
+
+        body = oai.RequestBody.construct(content={})
+        schemas = Schemas()
+
+        result = Endpoint.parse_request_yaml_body(body=body, schemas=schemas, parent_name="parent")
+
+        assert result == (None, schemas)
+
     def test_parse_request_form_body(self, mocker):
         ref = mocker.MagicMock()
         body = oai.RequestBody.construct(
@@ -211,6 +237,12 @@ def test_add_body_happy(self, mocker):
         parse_request_json_body = mocker.patch.object(
             Endpoint, "parse_request_json_body", return_value=(json_body, parsed_schemas)
         )
+        yaml_body = mocker.MagicMock(autospec=Property)
+        yaml_body_imports = mocker.MagicMock()
+        yaml_body.get_imports.return_value = {yaml_body_imports}
+        parse_request_yaml_body = mocker.patch.object(
+            Endpoint, "parse_request_yaml_body", return_value=(yaml_body, parsed_schemas)
+        )
         import_string_from_reference = mocker.patch(
             f"{MODULE_NAME}.import_string_from_reference", side_effect=["import_1", "import_2"]
         )
@@ -233,15 +265,18 @@ def test_add_body_happy(self, mocker):
         assert response_schemas == parsed_schemas
         parse_request_form_body.assert_called_once_with(request_body)
         parse_request_json_body.assert_called_once_with(body=request_body, schemas=initial_schemas, parent_name="name")
+        parse_request_yaml_body.assert_called_once_with(body=request_body, schemas=parsed_schemas, parent_name="name")
         parse_multipart_body.assert_called_once_with(request_body)
         import_string_from_reference.assert_has_calls(
             [
                 mocker.call(form_body_reference, prefix="...models"),
                 mocker.call(multipart_body_reference, prefix="...models"),
             ]
         )
+        yaml_body.get_imports.assert_called_once_with(prefix="...")
         json_body.get_imports.assert_called_once_with(prefix="...")
-        assert endpoint.relative_imports == {"import_1", "import_2", "import_3", json_body_imports}
+        assert endpoint.relative_imports == {"import_1", "import_2", "import_3", yaml_body_imports, json_body_imports}
+        assert endpoint.yaml_body == yaml_body
         assert endpoint.json_body == json_body
         assert endpoint.form_body_reference == form_body_reference
         assert endpoint.multipart_body_reference == multipart_body_reference
@@ -312,12 +347,12 @@ def test__add_responses(self, mocker):
         response_1 = Response(
             status_code=200,
             source="source",
-            prop=DateTimeProperty(name="datetime", required=True, nullable=False, default=None),
+            prop=DateTimeProperty(name="datetime", required=True, nullable=False, default=None, description=None),
         )
         response_2 = Response(
             status_code=404,
             source="source",
-            prop=DateProperty(name="date", required=True, nullable=False, default=None),
+            prop=DateProperty(name="date", required=True, nullable=False, default=None, description=None),
         )
         response_from_data = mocker.patch(
             f"{MODULE_NAME}.response_from_data", side_effect=[(response_1, schemas_1), (response_2, schemas_2)]