feat: Make all collectionClient.list methods return value also be asyncIterator of relevant data (#790)

### Description

- Add `ResourceCollectionClient._getIterablePagination` which extends
the return value of `ResourceCollectionClient._list` by `asyncIterator`
that can be used to iterate over individual items. (It is made in a
generic way and can be applied to various endpoints if desired.)
- Apply `_getIterablePagination` to:
  - `ActorCollectionClient.list`
  -  `ActorEnvVarCollectionClient.list`
  - `ActorVersionCollectionClient.list`
  - `BuildCollectionClient.list`
  - `RunCollectionClient.list`
  - `DatasetCollectionClient.list`
  - `KeyValueStoreCollectionClient.list`
  - `RequestQueueCollectionClient.list`
  - `ScheduleCollectionClient.list`
  - `StoreCollectionClient.list`
  -  `TaskCollectionClient.list`
  - `WebhookCollectionClient.list`
  - `WebhookDispatchCollectionClient.list`

- Add unit tests of async iteration for the single representative class
`DatasetCollectionClient.list` (it works the same way on all the
classes)

### Example usage

It can still be used the same way as before:
```
actors = await apifyClient.store().list({ limit, offset });
// Paginated response with up to 1000 items with actor details
console.log(actors.items.length);
```
Or it can be used as `asyncIterator` that can return more individual
items from more than one chunk based on the `limit`, and `offset`
options and also based on the number of items returned from the API:
```
for await (const actor of apifyClient.store().list({ limit, offset })) {
    // Single actor details
    console.log(actor);
}
```

### Issues

- Closes: #777

Author: Pijukatel

src/base/resource_collection_client.ts

@@ -1,3 +1,4 @@
+import type { PaginatedResponse, PaginationOptions } from '../utils';
 import { parseDateFields, pluckData } from '../utils';
 import { ApiClient } from './api_client';
 
@@ -18,6 +19,41 @@ export class ResourceCollectionClient extends ApiClient {
         return parseDateFields(pluckData(response.data)) as R;
     }
 
+    /**
+     * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
+     */
+    protected _listPaginated<T extends PaginationOptions, Data, R extends PaginatedResponse<Data>>(
+        options: T = {} as T,
+    ): AsyncIterable<Data> & Promise<R> {
+        const getPaginatedList = this._list.bind(this);
+        const paginatedListPromise = getPaginatedList<T, R>(options);
+
+        async function* asyncGenerator() {
+            let currentPage = await paginatedListPromise;
+            yield* currentPage.items;
+            const offset = options.offset ?? 0;
+            const limit = Math.min(options.limit || currentPage.total, currentPage.total);
+
+            let currentOffset = offset + currentPage.items.length;
+            let remainingItems = Math.min(currentPage.total - offset, limit) - currentPage.items.length;
+
+            while (
+                currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page.
+                remainingItems > 0
+            ) {
+                const newOptions = { ...options, limit: remainingItems, offset: currentOffset };
+                currentPage = await getPaginatedList<T, R>(newOptions);
+                yield* currentPage.items;
+                currentOffset += currentPage.items.length;
+                remainingItems -= currentPage.items.length;
+            }
+        }
+
+        return Object.defineProperty(paginatedListPromise, Symbol.asyncIterator, {
+            value: asyncGenerator,
+        }) as unknown as AsyncIterable<Data> & Promise<R>;
+    }
+
     protected async _create<D, R>(resource: D): Promise<R> {
         const response = await this.httpClient.call({
             url: this._url(),

src/resource_clients/actor_collection.ts

@@ -2,7 +2,7 @@ import ow from 'ow';
 
 import type { ApiClientSubResourceOptions } from '../base/api_client';
 import { ResourceCollectionClient } from '../base/resource_collection_client';
-import type { PaginatedList } from '../utils';
+import type { PaginatedIterator, PaginatedList, PaginationOptions } from '../utils';
 import type { Actor, ActorDefaultRunOptions, ActorExampleRunInput, ActorStandby } from './actor';
 import type { ActorVersion } from './actor_version';
 
@@ -19,8 +19,21 @@ export class ActorCollectionClient extends ResourceCollectionClient {
 
     /**
      * https://docs.apify.com/api/v2#/reference/actors/actor-collection/get-list-of-actors
+     *
+     * Awaiting the return value (as you would with a Promise) will result in a single API call. The amount of fetched
+     * items in a single API call is limited.
+     * ```javascript
+     * const paginatedList = await client.list(options);
+     *```
+     *
+     * Asynchronous iteration is also supported. This will fetch additional pages if needed until all items are
+     * retrieved.
+     *
+     * ```javascript
+     * for await (const singleItem of client.list(options)) {...}
+     * ```
      */
-    async list(options: ActorCollectionListOptions = {}): Promise<ActorCollectionListResult> {
+    list(options: ActorCollectionListOptions = {}): PaginatedIterator<ActorCollectionListItem> {
         ow(
             options,
             ow.object.exactShape({
@@ -32,7 +45,7 @@ export class ActorCollectionClient extends ResourceCollectionClient {
             }),
         );
 
-        return this._list(options);
+        return this._listPaginated(options);
     }
 
     /**
@@ -50,11 +63,8 @@ export enum ActorListSortBy {
     LAST_RUN_STARTED_AT = 'stats.lastRunStartedAt',
 }
 
-export interface ActorCollectionListOptions {
+export interface ActorCollectionListOptions extends PaginationOptions {
     my?: boolean;
-    limit?: number;
-    offset?: number;
-    desc?: boolean;
     sortBy?: ActorListSortBy;
 }
 

src/resource_clients/actor_env_var_collection.ts

@@ -2,7 +2,7 @@ import ow from 'ow';
 
 import type { ApiClientSubResourceOptions } from '../base/api_client';
 import { ResourceCollectionClient } from '../base/resource_collection_client';
-import type { PaginatedList } from '../utils';
+import type { PaginatedList, PaginationOptions } from '../utils';
 import type { ActorEnvironmentVariable } from './actor_version';
 
 export class ActorEnvVarCollectionClient extends ResourceCollectionClient {
@@ -18,8 +18,23 @@ export class ActorEnvVarCollectionClient extends ResourceCollectionClient {
 
     /**
      * https://docs.apify.com/api/v2#/reference/actors/environment-variable-collection/get-list-of-environment-variables
+     *
+     * Awaiting the return value (as you would with a Promise) will result in a single API call. The amount of fetched
+     * items in a single API call is limited.
+     * ```javascript
+     * const paginatedList = await client.list(options);
+     *```
+     *
+     * Asynchronous iteration is also supported. This will fetch additional pages if needed until all items are
+     * retrieved.
+     *
+     * ```javascript
+     * for await (const singleItem of client.list(options)) {...}
+     * ```
      */
-    async list(options: ActorEnvVarCollectionListOptions = {}): Promise<ActorEnvVarListResult> {
+    list(
+        options: ActorEnvVarCollectionListOptions = {},
+    ): Promise<ActorEnvVarListResult> & AsyncIterable<ActorEnvironmentVariable> {
         ow(
             options,
             ow.object.exactShape({
@@ -28,7 +43,7 @@ export class ActorEnvVarCollectionClient extends ResourceCollectionClient {
                 desc: ow.optional.boolean,
             }),
         );
-        return this._list(options);
+        return this._listPaginated(options);
     }
 
     /**
@@ -40,9 +55,7 @@ export class ActorEnvVarCollectionClient extends ResourceCollectionClient {
     }
 }
 
-export interface ActorEnvVarCollectionListOptions {
-    limit?: number;
-    offset?: number;
+export interface ActorEnvVarCollectionListOptions extends PaginationOptions {
     desc?: boolean;
 }
 

src/resource_clients/actor_version_collection.ts

@@ -2,7 +2,7 @@ import ow from 'ow';
 
 import type { ApiClientSubResourceOptions } from '../base/api_client';
 import { ResourceCollectionClient } from '../base/resource_collection_client';
-import type { PaginatedList } from '../utils';
+import type { PaginatedList, PaginationOptions } from '../utils';
 import type { ActorVersion, FinalActorVersion } from './actor_version';
 
 export class ActorVersionCollectionClient extends ResourceCollectionClient {
@@ -18,8 +18,23 @@ export class ActorVersionCollectionClient extends ResourceCollectionClient {
 
     /**
      * https://docs.apify.com/api/v2#/reference/actors/version-collection/get-list-of-versions
+     *
+     * Awaiting the return value (as you would with a Promise) will result in a single API call. The amount of fetched
+     * items in a single API call is limited.
+     * ```javascript
+     * const paginatedList = await client.list(options);
+     *```
+     *
+     * Asynchronous iteration is also supported. This will fetch additional pages if needed until all items are
+     * retrieved.
+     *
+     * ```javascript
+     * for await (const singleItem of client.list(options)) {...}
+     * ```
      */
-    async list(options: ActorVersionCollectionListOptions = {}): Promise<ActorVersionListResult> {
+    list(
+        options: ActorVersionCollectionListOptions = {},
+    ): Promise<ActorVersionListResult> & AsyncIterable<FinalActorVersion> {
         ow(
             options,
             ow.object.exactShape({
@@ -29,7 +44,7 @@ export class ActorVersionCollectionClient extends ResourceCollectionClient {
             }),
         );
 
-        return this._list(options);
+        return this._listPaginated(options);
     }
 
     /**
@@ -42,9 +57,7 @@ export class ActorVersionCollectionClient extends ResourceCollectionClient {
     }
 }
 
-export interface ActorVersionCollectionListOptions {
-    limit?: number;
-    offset?: number;
+export interface ActorVersionCollectionListOptions extends PaginationOptions {
     desc?: boolean;
 }
 

src/resource_clients/build_collection.ts

@@ -2,7 +2,7 @@ import ow from 'ow';
 
 import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client';
 import { ResourceCollectionClient } from '../base/resource_collection_client';
-import type { PaginatedList } from '../utils';
+import type { PaginatedIterator, PaginatedList, PaginationOptions } from '../utils';
 import type { Build } from './build';
 
 export class BuildCollectionClient extends ResourceCollectionClient {
@@ -18,8 +18,21 @@ export class BuildCollectionClient extends ResourceCollectionClient {
 
     /**
      * https://docs.apify.com/api/v2#/reference/actors/build-collection/get-list-of-builds
+     *
+     * Awaiting the return value (as you would with a Promise) will result in a single API call. The amount of fetched
+     * items in a single API call is limited.
+     * ```javascript
+     * const paginatedList = await client.list(options);
+     *```
+     *
+     * Asynchronous iteration is also supported. This will fetch additional pages if needed until all items are
+     * retrieved.
+     *
+     * ```javascript
+     * for await (const singleItem of client.list(options)) {...}
+     * ```
      */
-    async list(options: BuildCollectionClientListOptions = {}): Promise<BuildCollectionClientListResult> {
+    list(options: BuildCollectionClientListOptions = {}): PaginatedIterator<BuildCollectionClientListItem> {
         ow(
             options,
             ow.object.exactShape({
@@ -29,13 +42,11 @@ export class BuildCollectionClient extends ResourceCollectionClient {
             }),
         );
 
-        return this._list(options);
+        return this._listPaginated(options);
     }
 }
 
-export interface BuildCollectionClientListOptions {
-    limit?: number;
-    offset?: number;
+export interface BuildCollectionClientListOptions extends PaginationOptions {
     desc?: boolean;
 }
 

src/resource_clients/dataset_collection.ts

@@ -2,7 +2,7 @@ import ow from 'ow';
 
 import type { ApiClientSubResourceOptions } from '../base/api_client';
 import { ResourceCollectionClient } from '../base/resource_collection_client';
-import type { PaginatedList } from '../utils';
+import type { PaginatedIterator, PaginatedList, PaginationOptions } from '../utils';
 import type { Dataset } from './dataset';
 
 export class DatasetCollectionClient extends ResourceCollectionClient {
@@ -18,8 +18,21 @@ export class DatasetCollectionClient extends ResourceCollectionClient {
 
     /**
      * https://docs.apify.com/api/v2#/reference/datasets/dataset-collection/get-list-of-datasets
+     *
+     * Awaiting the return value (as you would with a Promise) will result in a single API call. The amount of fetched
+     * items in a single API call is limited.
+     * ```javascript
+     * const paginatedList = await client.list(options);
+     *```
+     *
+     * Asynchronous iteration is also supported. This will fetch additional pages if needed until all items are
+     * retrieved.
+     *
+     * ```javascript
+     * for await (const singleItem of client.list(options)) {...}
+     * ```
      */
-    async list(options: DatasetCollectionClientListOptions = {}): Promise<DatasetCollectionClientListResult> {
+    list(options: DatasetCollectionClientListOptions = {}): PaginatedIterator<Dataset> {
         ow(
             options,
             ow.object.exactShape({
@@ -30,7 +43,7 @@ export class DatasetCollectionClient extends ResourceCollectionClient {
             }),
         );
 
-        return this._list(options);
+        return this._listPaginated(options);
     }
 
     /**
@@ -44,10 +57,8 @@ export class DatasetCollectionClient extends ResourceCollectionClient {
     }
 }
 
-export interface DatasetCollectionClientListOptions {
+export interface DatasetCollectionClientListOptions extends PaginationOptions {
     unnamed?: boolean;
-    limit?: number;
-    offset?: number;
     desc?: boolean;
 }
 

src/resource_clients/key_value_store_collection.ts

@@ -2,7 +2,7 @@ import ow from 'ow';
 
 import type { ApiClientSubResourceOptions } from '../base/api_client';
 import { ResourceCollectionClient } from '../base/resource_collection_client';
-import type { PaginatedList } from '../utils';
+import type { PaginatedList, PaginationOptions } from '../utils';
 import type { KeyValueStore } from './key_value_store';
 
 export class KeyValueStoreCollectionClient extends ResourceCollectionClient {
@@ -18,10 +18,23 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient {
 
     /**
      * https://docs.apify.com/api/v2#/reference/key-value-stores/store-collection/get-list-of-key-value-stores
+     *
+     * Awaiting the return value (as you would with a Promise) will result in a single API call. The amount of fetched
+     * items in a single API call is limited.
+     * ```javascript
+     * const paginatedList = await client.list(options);
+     *```
+     *
+     * Asynchronous iteration is also supported. This will fetch additional pages if needed until all items are
+     * retrieved.
+     *
+     * ```javascript
+     * for await (const singleItem of client.list(options)) {...}
+     * ```
      */
-    async list(
+    list(
         options: KeyValueStoreCollectionClientListOptions = {},
-    ): Promise<PaginatedList<KeyValueStoreCollectionListResult>> {
+    ): Promise<PaginatedList<KeyValueStoreCollectionListResult>> & AsyncIterable<KeyValueStore> {
         ow(
             options,
             ow.object.exactShape({
@@ -32,7 +45,7 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient {
             }),
         );
 
-        return this._list(options);
+        return this._listPaginated(options);
     }
 
     /**
@@ -49,10 +62,8 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient {
     }
 }
 
-export interface KeyValueStoreCollectionClientListOptions {
+export interface KeyValueStoreCollectionClientListOptions extends PaginationOptions {
     unnamed?: boolean;
-    limit?: number;
-    offset?: number;
     desc?: boolean;
 }