Make kwarg-only, update docs

changelog

Author: snbianco

CHANGES.rst

@@ -52,6 +52,10 @@ mast
 
 - Raise informative error if ``MastMissions`` query radius is too large. [#3447]
 
+- Add ``batch_size`` parameter to ``MastMissions.get_product_list``, ``Observations.get_product_list``,
+  and ``utils.resolve_object`` to allow controlling the number of items sent in each batch request to the server.
+  This can help avoid timeouts or connection errors for large requests. [#3454]
+
 jplspec
 ^^^^^^^
 

astroquery/mast/missions.py

@@ -409,7 +409,7 @@ def query_object_async(self, objectname, *, radius=3*u.arcmin, limit=5000, offse
                                        select_cols=select_cols, **criteria)
 
     @class_or_instance
-    def get_product_list_async(self, datasets, batch_size=1000):
+    def get_product_list_async(self, datasets, *, batch_size=1000):
         """
         Given a dataset ID or list of dataset IDs, returns a list of associated data products.
 
@@ -434,9 +434,8 @@ def get_product_list_async(self, datasets, batch_size=1000):
         if isinstance(datasets, Table) or isinstance(datasets, Row):
             dataset_kwd = self.get_dataset_kwd()
             if not dataset_kwd:
-                error_msg = (f'Dataset keyword not found for mission "{self.mission}". '
-                             'Please input dataset IDs as a string, list of strings, or `~astropy.table.Column`.')
-                raise InvalidQueryError(error_msg)
+                raise InvalidQueryError(f'Dataset keyword not found for mission "{self.mission}". Please input '
+                                        'dataset IDs as a string, list of strings, or `~astropy.table.Column`.')
 
         # Extract dataset IDs based on input type and mission
         if isinstance(datasets, Table):
@@ -472,7 +471,7 @@ def get_product_list_async(self, datasets, batch_size=1000):
         # Return a list of responses
         return results
 
-    def get_unique_product_list(self, datasets):
+    def get_unique_product_list(self, datasets, *, batch_size=1000):
         """
         Given a dataset ID or list of dataset IDs, returns a list of associated data products with unique
         filenames.
@@ -482,13 +481,16 @@ def get_unique_product_list(self, datasets):
         datasets : str, list, `~astropy.table.Row`, `~astropy.table.Column`, `~astropy.table.Table`
             Row/Table of MastMissions query results (e.g. output from `query_object`)
             or single/list of dataset ID(s).
+        batch_size : int, optional
+            Default 1000. Number of dataset IDs to include in each batch request to the server.
+            If you experience timeouts or connection errors, consider lowering this value.
 
         Returns
         -------
         unique_products : `~astropy.table.Table`
             Table containing products with unique URIs.
         """
-        products = self.get_product_list(datasets)
+        products = self.get_product_list(datasets, batch_size=batch_size)
         unique_products = utils.remove_duplicate_products(products, 'filename')
         if len(unique_products) < len(products):
             log.info("To return all products, use `MastMissions.get_product_list`")

astroquery/mast/observations.py

@@ -504,7 +504,7 @@ def _filter_ffi_observations(self, observations):
         return obs_table[mask]
 
     @class_or_instance
-    def get_product_list_async(self, observations, batch_size=500):
+    def get_product_list_async(self, observations, *, batch_size=500):
         """
         Given a "Product Group Id" (column name obsid) returns a list of associated data products.
         Note that obsid is NOT the same as obs_id, and inputting obs_id values will result in
@@ -1048,7 +1048,7 @@ def get_cloud_uri(self, data_product, *, include_bucket=True, full_url=False):
         # Query for product URIs
         return self._cloud_connection.get_cloud_uri(data_product, include_bucket, full_url)
 
-    def get_unique_product_list(self, observations):
+    def get_unique_product_list(self, observations, *, batch_size=500):
         """
         Given a "Product Group Id" (column name obsid), returns a list of associated data products with
         unique dataURIs. Note that obsid is NOT the same as obs_id, and inputting obs_id values will result in
@@ -1060,13 +1060,16 @@ def get_unique_product_list(self, observations):
             Row/Table of MAST query results (e.g. output from `query_object`)
             or single/list of MAST Product Group Id(s) (obsid).
             See description `here <https://masttest.stsci.edu/api/v0/_c_a_o_mfields.html>`__.
+        batch_size : int, optional
+            Default 500. Number of obsids to include in each batch request to the server.
+            If you experience timeouts or connection errors, consider lowering this value.
 
         Returns
         -------
         unique_products : `~astropy.table.Table`
             Table containing products with unique dataURIs.
         """
-        products = self.get_product_list(observations)
+        products = self.get_product_list(observations, batch_size=batch_size)
         unique_products = utils.remove_duplicate_products(products, 'dataURI')
         if len(unique_products) < len(products):
             log.info("To return all products, use `Observations.get_product_list`")

docs/mast/mast_missions.rst

@@ -203,11 +203,16 @@ Each observation returned from a MAST query can have one or more associated data
 one or more datasets or dataset IDs, the `~astroquery.mast.MastMissionsClass.get_product_list` function 
 will return a `~astropy.table.Table` containing the associated data products.
 
+`~astroquery.mast.MastMissionsClass.get_product_list` also includes an optional ``batch_size`` parameter, 
+which controls how many datasets are sent to the MAST service per request. This can be useful for managing 
+memory usage or avoiding timeouts when requesting product lists for large numbers of datasets.
+If not provided, batch_size defaults to 1000.
+
 .. doctest-remote-data::
    >>> datasets = missions.query_criteria(sci_pep_id=12451,
    ...                                    sci_instrume='ACS',
    ...                                    sci_hlsp='>1')
-   >>> products = missions.get_product_list(datasets[:2])
+   >>> products = missions.get_product_list(datasets[:2], batch_size=1000)
    >>> print(products[:5])  # doctest: +IGNORE_OUTPUT
            product_key          access  dataset  ...  category     size     type 
    ---------------------------- ------ --------- ... ---------- --------- -------

docs/mast/mast_obsquery.rst

@@ -214,17 +214,22 @@ Getting Product Lists
 ---------------------
 
 Each observation returned from a MAST query can have one or more associated data products.
-Given one or more observations or MAST Product Group IDs ("obsid")
+Given one or more observations or MAST Product Group IDs ("obsid"),
 `~astroquery.mast.ObservationsClass.get_product_list` will return
 a `~astropy.table.Table` containing the associated data products.
 The product fields are documented `here <https://mast.stsci.edu/api/v0/_productsfields.html>`__.
 
+`~astroquery.mast.ObservationsClass.get_product_list` also includes an optional ``batch_size`` parameter, 
+which controls how many observations are sent to the MAST service per request. This can be useful for managing 
+memory usage or avoiding timeouts when requesting product lists for large numbers of observations. 
+If not provided, batch_size defaults to 500.
+
 .. doctest-remote-data::
 
    >>> from astroquery.mast import Observations
    ...
    >>> obs_table = Observations.query_criteria(objectname="M8", obs_collection=["K2", "IUE"])
-   >>> data_products_by_obs = Observations.get_product_list(obs_table[0:2])
+   >>> data_products_by_obs = Observations.get_product_list(obs_table[0:2], batch_size=500)
    >>> print(data_products_by_obs)  # doctest: +IGNORE_OUTPUT
    obsID  obs_collection dataproduct_type ... dataRights calib_level filters
    ------ -------------- ---------------- ... ---------- ----------- -------