Fix pagination

Author: lsbardel

.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "PyTest",
+            "type": "python",
+            "request": "launch",
+            "cwd": "${workspaceFolder}",
+            "console": "integratedTerminal",
+            "module": "pytest",
+            "env": {},
+            "justMyCode": false,
+            "args": [
+                "-x",
+                "-vvv",
+                "tests/test_spaces.py",
+            ],
+            "debugOptions": [
+                "RedirectOutput"
+            ]
+        }
+    ]
+}

metablock/client.py

@@ -5,7 +5,7 @@
 import sys
 from typing import Any
 
-from aiohttp import ClientSession
+from aiohttp import ClientResponse, ClientSession
 from yarl import URL
 
 from .components import Callback, HttpComponent, MetablockResponseError
@@ -108,6 +108,10 @@ async def request(
         response = await self.session.request(method, url, headers=headers_, **kw)
         if callback:
             return await callback(response)
+        else:
+            return await self.handle_response(response, wrap=wrap)
+
+    async def handle_response(self, response: ClientResponse, wrap: Any = None) -> Any:
         if response.status == 204:
             return True
         if response.status >= 400:

metablock/components.py

@@ -9,6 +9,7 @@
     Awaitable,
     Callable,
     Generic,
+    Mapping,
     TypeVar,
     cast,
 )
@@ -46,11 +47,11 @@ def __str__(self) -> str:
 
 class HttpComponent(ABC):
     @abstractproperty
-    def cli(self) -> Metablock:
+    def cli(self) -> Metablock:  # pragma: no cover
         ...
 
     @abstractproperty
-    def url(self) -> str:
+    def url(self) -> str:  # pragma: no cover
         ...
 
     def __repr__(self) -> str:
@@ -146,15 +147,14 @@ def __init__(
         self.factory = factory
 
     async def paginate(self, **params: Any) -> AsyncIterator[E]:
-        url = self.list_create_url()
-        next_ = url
-        url_params = as_params(**params)
+        next_ = self.list_create_url()
+        url_params: Any = as_params(**params)
         while next_:
-            if not next_.startswith(url):
-                next_ = f'{url}?{next_.split("?")[1]}'
-            data = await self.cli.request(next_, params=url_params)
-            next_ = data.get("next")
-            for d in data["data"]:
+            next_, data = await self.cli.request(
+                next_, params=url_params, callback=self._paginated
+            )
+            url_params = None
+            for d in data:
                 yield self.entity(d)
 
     async def get_list(self, **kwargs: Any) -> list[E]:
@@ -175,7 +175,7 @@ async def get(self, id_: str, **kwargs: Any) -> E:
 
     async def has(self, id_: str, **kwargs: Any) -> bool:
         url = f"{self.url}/{id_}"
-        return cast(bool, await self.cli.get(url, callback=self.head))
+        return cast(bool, await self.cli.get(url, callback=self._head))
 
     async def create(self, callback: Callback | None = None, **params: Any) -> E:
         url = self.list_create_url()
@@ -219,14 +219,6 @@ async def delete_all(self) -> int:
             n += 1
         return n
 
-    async def head(self, response: ClientResponse) -> bool:
-        if response.status == 404:
-            return False
-        elif response.status == 200:
-            return True
-        else:  # pragma: no cover
-            raise MetablockResponseError(response)
-
     def entity(self, data: dict) -> E:
         return self.factory(self, data)
 
@@ -241,3 +233,22 @@ def update_url(self, id_name: str) -> str:
 
     def delete_url(self, id_name: str) -> str:
         return f"{self.url}/{id_name}"
+
+    # callbacks
+
+    async def _head(self, response: ClientResponse) -> bool:
+        if response.status == 404:
+            return False
+        elif response.status == 200:
+            return True
+        else:  # pragma: no cover
+            raise MetablockResponseError(response)
+
+    async def _paginated(self, response: ClientResponse) -> Any:
+        next = response.links.get("next")
+        if isinstance(next, Mapping):
+            url = next.get("url")
+        else:
+            url = None
+        data = await self.cli.handle_response(response)
+        return (url, data)

metablock/utils.py

@@ -3,16 +3,6 @@
 from multidict import MultiDict
 
 
-def as_list(key: str, data: dict) -> dict:
-    v = data.get(key)
-    if isinstance(v, str):
-        v = [v]
-    elif not isinstance(v, list):
-        v = []
-    data[key] = v
-    return data
-
-
 def as_dict(data: Any, key: str = "data") -> dict:
     return {key: data} if not isinstance(data, dict) else data
 

readme.md

@@ -3,7 +3,7 @@
 [![PyPI version](https://badge.fury.io/py/metablock.svg)](https://badge.fury.io/py/metablock)
 [![Python versions](https://img.shields.io/pypi/pyversions/metablock.svg)](https://pypi.org/project/metablock)
 [![Build](https://github.com/quantmind/metablock-py/workflows/build/badge.svg)](https://github.com/quantmind/metablock-py/actions?query=workflow%3Abuild)
-[![codecov](https://codecov.io/gh/quantmind/metablock-py/branch/master/graph/badge.svg?token=EAdSVpD0Af)](https://codecov.io/gh/quantmind/metablock-py)
+[![codecov](https://codecov.io/gh/quantmind/metablock-py/branch/main/graph/badge.svg?token=EAdSVpD0Af)](https://codecov.io/gh/quantmind/metablock-py)
 
 This is an asynchronous python client for [metablock API](https://api.metablock.io/v1/docs).
 

tests/test_spaces.py

@@ -7,6 +7,13 @@ async def test_list_spaces(cli: Metablock) -> None:
     assert spaces
 
 
+async def test_paginate_spaces(cli: Metablock) -> None:
+    spaces1 = await cli.spaces.get_full_list()
+    spaces2 = await cli.spaces.get_full_list(limit=1)
+    assert spaces1
+    assert len(spaces2) == len(spaces1)
+
+
 async def test_get_space_403(cli: Metablock, invalid_headers: dict) -> None:
     with pytest.raises(MetablockResponseError) as exc:
         await cli.spaces.get_list(headers=invalid_headers)