feat(SYPL-5699): use record limit

Author: Daniel Bush

sypht/util.py

@@ -2,18 +2,19 @@
 
 
 def fetch_all_pages(
-    name: str, fetch_page: Callable[..., List[Any]], page_limit=1000
+    name: str, fetch_page: Callable[..., List[Any]], rec_limit=20000
 ) -> Callable[..., Iterator[List[Any]]]:
     """Returns an iterator that calls fetch_page with an offset that we increment by the number of pages fetched.  Stop if page returns empty list."""
 
     def fetch_all_pages(*args, **kwargs) -> Iterator[List[Any]]:
         page_count = 0
+        recs = 0
         while True:
             page_count += 1
-            if page_count > page_limit:
+            if recs > rec_limit:
                 # Don't want to DOS ourselves...
                 raise Exception(
-                    f"fetch_all_pages({name}): fetched more than {page_limit} pages.  Consider using a date range to reduce the number of pages fetched."
+                    f"fetch_all_pages({name}): fetched more than {rec_limit} items.  Consider using adding or adjusting a filter to reduce the number of items fetched."
                 )
             try:
                 result = fetch_page(
@@ -27,6 +28,7 @@ def fetch_all_pages(*args, **kwargs) -> Iterator[List[Any]]:
                 ) from err
             if not result:
                 break
+            recs += len(result)
             yield result
 
     return fetch_all_pages

tests/tests_util.py

@@ -61,7 +61,7 @@ def never_ending(*args, **kwargs):
             results += page
 
     # assert
-    assert "fetched more than 1000 pages" in str(exc_info)
+    assert "fetched more than 20000 items" in str(exc_info)
 
 
 def test_fetch_all_pages_handle_error():