BUG: Ensure headers is always defined

Allow headers to be overridden
Set default behavior when user passes a session

Author: bashtage

pandas_datareader/base.py

@@ -38,7 +38,7 @@ class _BaseReader(object):
     pause : float, default 0.1
         Time, in seconds, of the pause between retries.
     session : Session, default None
-        requests.sessions.Session instance to be used
+        requests.sessions.Session instance to be used.
     freq : {str, None}
         Frequency to use in select readers
     """
@@ -47,15 +47,15 @@ class _BaseReader(object):
     _format = "string"
 
     def __init__(
-            self,
-            symbols,
-            start=None,
-            end=None,
-            retry_count=3,
-            pause=0.1,
-            timeout=30,
-            session=None,
-            freq=None,
+        self,
+        symbols,
+        start=None,
+        end=None,
+        retry_count=3,
+        pause=0.1,
+        timeout=30,
+        session=None,
+        freq=None,
     ):
 
         self.symbols = symbols
@@ -72,6 +72,7 @@ def __init__(
         self.pause_multiplier = 1
         self.session = _init_session(session, retry_count)
         self.freq = freq
+        self.headers = None
 
     def close(self):
         """Close network session"""
@@ -148,8 +149,8 @@ def _get_response(self, url, params=None, headers=None):
             parameters passed to the URL
         """
 
-        # initial attempt + retry
-        if headers == None:
+        # Use default headers if not passes and not using a user session
+        if headers is None:
             headers = self.headers
 
         pause = self.pause
@@ -227,14 +228,14 @@ class _DailyBaseReader(_BaseReader):
     """Base class for Google / Yahoo daily reader"""
 
     def __init__(
-            self,
-            symbols=None,
-            start=None,
-            end=None,
-            retry_count=3,
-            pause=0.1,
-            session=None,
-            chunksize=25,
+        self,
+        symbols=None,
+        start=None,
+        end=None,
+        retry_count=3,
+        pause=0.1,
+        session=None,
+        chunksize=25,
     ):
         super(_DailyBaseReader, self).__init__(
             symbols=symbols,
@@ -300,7 +301,7 @@ def _in_chunks(seq, size):
     """
     Return sequence in 'chunks' of size defined by size
     """
-    return (seq[pos: pos + size] for pos in range(0, len(seq), size))
+    return (seq[pos : pos + size] for pos in range(0, len(seq), size))
 
 
 class _OptionBaseReader(_BaseReader):
@@ -334,7 +335,7 @@ def get_put_data(self, month=None, year=None, expiry=None):
         raise NotImplementedError
 
     def get_near_stock_price(
-            self, above_below=2, call=True, put=False, month=None, year=None, expiry=None
+        self, above_below=2, call=True, put=False, month=None, year=None, expiry=None
     ):
         """
         ***Experimental***
@@ -343,7 +344,7 @@ def get_near_stock_price(
         raise NotImplementedError
 
     def get_forward_data(
-            self, months, call=True, put=False, near=False, above_below=2
+        self, months, call=True, put=False, near=False, above_below=2
     ):  # pragma: no cover
         """
         ***Experimental***

pandas_datareader/yahoo/daily.py

@@ -34,7 +34,10 @@ class YahooDailyReader(_DailyBaseReader):
         Time, in seconds, to pause between consecutive queries of chunks. If
         single value given for symbol, represents the pause between retries.
     session : Session, default None
-        requests.sessions.Session instance to be used
+        requests.sessions.Session instance to be used. Passing a session
+        is an advanced usage and you must either set the required
+        headers in the session directly or explicitly override
+        using the ``headers`` argument.
     adjust_price : bool, default False
         If True, adjusts all prices in hist_data ('Open', 'High', 'Low',
         'Close') based on 'Adj Close' price. Adds 'Adj_Ratio' column and drops
@@ -50,6 +53,9 @@ class YahooDailyReader(_DailyBaseReader):
         If True, adds Dividend and Split columns to dataframe.
     adjust_dividends: bool, default true
         If True, adjusts dividends for splits.
+    headers : dict, optional
+        Headers to use when reading data. If None (the default), a
+        standard set of headers is used.
     """
 
     def __init__(
@@ -66,6 +72,7 @@ def __init__(
         interval="d",
         get_actions=False,
         adjust_dividends=True,
+        headers=None,
     ):
         super(YahooDailyReader, self).__init__(
             symbols=symbols,
@@ -80,17 +87,21 @@ def __init__(
         # Ladder up the wait time between subsequent requests to improve
         # probability of a successful retry
         self.pause_multiplier = 2.5
-
-        self.headers = {
-            "Connection": "keep-alive",
-            "Expires": str(-1),
-            "Upgrade-Insecure-Requests": str(1),
-            # Google Chrome:
-            "User-Agent": (
-                "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
-                "(KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
-            ),
-        }
+        if headers is not None:
+            self.headers = headers
+        elif session is None:
+            self.headers = {
+                "Connection": "keep-alive",
+                "Expires": str(-1),
+                "Upgrade-Insecure-Requests": str(1),
+                # Google Chrome:
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                    "(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+                ),
+            }
+        else:
+            self.headers = None
 
         self.adjust_price = adjust_price
         self.ret_index = ret_index