Merge pull request #883 from bashtage/galashour-master

BUG: Correct behavior of yahoo

Author: bashtage

pandas_datareader/base.py

@@ -38,7 +38,7 @@ class _BaseReader(object):
     pause : float, default 0.1
         Time, in seconds, of the pause between retries.
     session : Session, default None
-        requests.sessions.Session instance to be used
+        requests.sessions.Session instance to be used.
     freq : {str, None}
         Frequency to use in select readers
     """
@@ -72,6 +72,7 @@ def __init__(
         self.pause_multiplier = 1
         self.session = _init_session(session, retry_count)
         self.freq = freq
+        self.headers = None
 
     def close(self):
         """Close network session"""
@@ -148,7 +149,10 @@ def _get_response(self, url, params=None, headers=None):
             parameters passed to the URL
         """
 
-        # initial attempt + retry
+        # Use default headers if not passes and not using a user session
+        if headers is None:
+            headers = self.headers
+
         pause = self.pause
         last_response_text = ""
         for _ in range(self.retry_count + 1):

pandas_datareader/yahoo/daily.py

@@ -34,7 +34,10 @@ class YahooDailyReader(_DailyBaseReader):
         Time, in seconds, to pause between consecutive queries of chunks. If
         single value given for symbol, represents the pause between retries.
     session : Session, default None
-        requests.sessions.Session instance to be used
+        requests.sessions.Session instance to be used. Passing a session
+        is an advanced usage and you must either set the required
+        headers in the session directly or explicitly override
+        using the ``headers`` argument.
     adjust_price : bool, default False
         If True, adjusts all prices in hist_data ('Open', 'High', 'Low',
         'Close') based on 'Adj Close' price. Adds 'Adj_Ratio' column and drops
@@ -50,6 +53,9 @@ class YahooDailyReader(_DailyBaseReader):
         If True, adds Dividend and Split columns to dataframe.
     adjust_dividends: bool, default true
         If True, adjusts dividends for splits.
+    headers : dict, optional
+        Headers to use when reading data. If None (the default), a
+        standard set of headers is used.
     """
 
     def __init__(
@@ -66,6 +72,7 @@ def __init__(
         interval="d",
         get_actions=False,
         adjust_dividends=True,
+        headers=None,
     ):
         super(YahooDailyReader, self).__init__(
             symbols=symbols,
@@ -80,17 +87,21 @@ def __init__(
         # Ladder up the wait time between subsequent requests to improve
         # probability of a successful retry
         self.pause_multiplier = 2.5
-
-        self.headers = {
-            "Connection": "keep-alive",
-            "Expires": str(-1),
-            "Upgrade-Insecure-Requests": str(1),
-            # Google Chrome:
-            "User-Agent": (
-                "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
-                "(KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"
-            ),
-        }
+        if headers is not None:
+            self.headers = headers
+        elif session is None:
+            self.headers = {
+                "Connection": "keep-alive",
+                "Expires": str(-1),
+                "Upgrade-Insecure-Requests": str(1),
+                # Google Chrome:
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+                    "(KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+                ),
+            }
+        else:
+            self.headers = None
 
         self.adjust_price = adjust_price
         self.ret_index = ret_index
@@ -150,7 +161,7 @@ def _read_one_data(self, url, params):
         del params["symbol"]
         url = url.format(symbol)
 
-        resp = self._get_response(url, params=params)
+        resp = self._get_response(url, params=params, headers=self.headers)
         ptrn = r"root\.App\.main = (.*?);\n}\(this\)\);"
         try:
             j = json.loads(re.search(ptrn, resp.text, re.DOTALL).group(1))