CLN: Clean morninginstar daily

Clean and make tests pass

Author: bashtage

docs/source/whatsnew/v0.6.0.txt

@@ -24,6 +24,9 @@ Highlights include:
   have been removed.  PDR would like to restore these features, and pull
   requests are welcome.
 
+- A new connector for Morningstart Open, High, Low, Close and Volume was
+  introduced (:issue:`467`)
+
 - A new connector for IEX daily price data was introduced (:issue:`465`).
 
 - A new connector for IEX the majority of the IEX API was introduced
@@ -50,7 +53,8 @@ Enhancements
 - A new data connector for data provided by the Bank of Canada was
   introduced. (:issue:`440`)
 
-- A new data connector for stock pricing data provided by Morningstar was introduced.
+- A new data connector for stock pricing data provided by Morningstar
+  was introduced. (:issue:`467`)
 
 .. _whatsnew_060.api_breaking:
 

pandas_datareader/__init__.py

@@ -18,6 +18,6 @@
            'get_iex_book', 'get_iex_symbols', 'get_last_iex',
            'get_markets_iex', 'get_recent_iex', 'get_records_iex',
            'get_summary_iex', 'get_tops_iex',
-           'get_nasdaq_symbols', 'get_nasdaq_symbols', 'get_data_quandl', 'get_data_moex',
+           'get_nasdaq_symbols', 'get_data_quandl', 'get_data_moex',
            'get_data_fred', 'get_dailysummary_iex', 'get_data_morningstar',
-           'get_data_stooq','DataReader', 'Options']
+           'get_data_stooq', 'DataReader', 'Options']

pandas_datareader/mstar/daily.py

@@ -5,29 +5,56 @@
 import requests
 from pandas import DataFrame
 
-from pandas_datareader._utils import (SymbolWarning, _sanitize_dates)
-
-
-class MorningstarDailyReader(object):
-
-    def __init__(self, start=None, end=None, *args, **kwargs):
-        if end is None:
-            end = datetime.today().strftime("%Y-%m-%d")
-
-        self.start, self.end = _sanitize_dates(start, end)
-
-        self.retry_count = kwargs.get("retry_count", 3)
-        self.pause = kwargs.get("pause", 0.001)
-        self.timeout = kwargs.get("timeout", 30)
-        self.session = kwargs.get("session", requests.session())
-
-        self.incl_splits = kwargs.get("incl_splits", False)
-        self.incl_dividends = kwargs.get("incl_dividends", False)
-        self.incl_vol = kwargs.get("incl_volume", True)
-        self.currency = kwargs.get("currency", "usd")
-        self.interval = kwargs.get("interval", "d")
-
-        self.symbols = kwargs.get("symbols")
+from pandas_datareader._utils import SymbolWarning
+from pandas_datareader.base import _BaseReader
+
+
+class MorningstarDailyReader(_BaseReader):
+    """
+    Read daily data from Morningstar
+
+    Parameters
+    ----------
+    symbols : {str, List[str]}
+        String symbol of like of symbols
+    start : string, (defaults to '1/1/2010')
+        Starting date, timestamp. Parses many different kind of date
+        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
+    end : string, (defaults to today)
+        Ending date, timestamp. Same format as starting date.
+    retry_count : int, default 3
+        Number of times to retry query request.
+    pause : float, default 0.1
+        Time, in seconds, of the pause between retries.
+    session : Session, default None
+        requests.sessions.Session instance to be used
+    freq : {str, None}
+        Frequency to use in select readers
+    incl_splits : bool, optional
+        Include splits in data
+    incl_dividends : bool,, optional
+        Include divdends in data
+    incl_volume : bool, optional
+        Include volume in data
+    currency : str, optional
+        Currency to use for data
+    interval : str, optional
+        Sampling interval to use for downloaded data
+    """
+
+    def __init__(self, symbols, start=None, end=None, retry_count=3,
+                 pause=0.1, timeout=30, session=None, freq=None,
+                 incl_splits=False, incl_dividends=False, incl_volume=True,
+                 currency='usd', interval='d'):
+        super(MorningstarDailyReader, self).__init__(symbols, start, end,
+                                                     retry_count, pause,
+                                                     timeout, session, freq)
+
+        self.incl_splits = incl_splits
+        self.incl_dividends = incl_dividends
+        self.incl_vol = incl_volume
+        self.currency = currency
+        self.interval = interval
 
         self._symbol_data_cache = []
 
@@ -55,12 +82,15 @@ def _url_params(self):
 
         return p
 
-    def _check_dates(self, *dates):
-        if dates[0] > dates[1]:
-            raise ValueError("Invalid start & end date! Start date cannot "
-                             "be later than end date.")
-        else:
-            return dates[0], dates[1]
+    @property
+    def url(self):
+        """API URL"""
+        return "http://globalquote.morningstar.com/globalcomponent/" \
+               "RealtimeHistoricalStockData.ashx"
+
+    def _get_crumb(self, *args):
+        """Not required """
+        pass
 
     def _dl_mult_symbols(self, symbols):
         failed = []
@@ -69,11 +99,9 @@ def _dl_mult_symbols(self, symbols):
 
             params = self._url_params()
             params.update({"ticker": symbol})
-            _baseurl = "http://globalquote.morningstar.com/globalcomponent/" \
-                       "RealtimeHistoricalStockData.ashx"
 
             try:
-                resp = requests.get(_baseurl, params=params)
+                resp = requests.get(self.url, params=params)
             except Exception:
                 if symbol not in failed:
                     if self.retry_count == 0:
@@ -85,8 +113,12 @@ def _dl_mult_symbols(self, symbols):
                         failed.append(symbol)
             else:
                 if resp.status_code == requests.codes.ok:
+                    jsondata = resp.json()
+                    if jsondata is None:
+                        failed.append(symbol)
+                        continue
                     jsdata = self._restruct_json(symbol=symbol,
-                                                 jsondata=resp.json())
+                                                 jsondata=jsondata)
                     symbol_data.extend(jsdata)
                 else:
                     raise Exception("Request Error!: %s : %s" % (
@@ -95,12 +127,16 @@ def _dl_mult_symbols(self, symbols):
             time.sleep(self.pause)
 
         if len(failed) > 0 and self.retry_count > 0:
+            # TODO: This appears to do nothing since
+            # TODO: successful symbols are not added to
             self._dl_mult_symbols(symbols=failed)
             self.retry_count -= 1
         else:
             self.retry_count = 0
 
-        if self.retry_count == 0 and len(failed) > 0:
+        if not symbol_data:
+            raise ValueError('All symbols were invalid')
+        elif self.retry_count == 0 and len(failed) > 0:
             warn("The following symbols were excluded do to http "
                  "request errors: \n %s" % failed, SymbolWarning)
 
@@ -117,19 +153,9 @@ def _convert_index2date(enddate, indexvals):
             i += 1
             yield d.strftime("%Y-%m-%d")
 
-    #
-    # def _adjust_close_price(price, event_type, event_value): #noqa
-    #     if event_type is "split":
-    #         e, s = event_value.split(":")
-    #         adj=(price * int(s))/e
-    #     elif event_type is "dividend":
-    #         adj = price - float(event_value)
-    #     else:
-    #         raise ValueError("Invalid event_type")
-    #     return adj
-
     def _restruct_json(self, symbol, jsondata):
-
+        if jsondata is None:
+            return
         divdata = jsondata["DividendData"]
 
         pricedata = jsondata["PriceDataList"][0]["Datapoints"]
@@ -156,12 +182,11 @@ def _restruct_json(self, symbol, jsondata):
                     if delta.days == 0:
                         events.append(x)
                 for e in events:
-                    if (self.incl_dividends is True and
-                            e["Type"].find("Div") > -1):
+                    if self.incl_dividends and e["Type"].find("Div") > -1:
                         val = e["Desc"].replace(e["Type"], "")
                         bardict.update({"isDividend": val})
                     elif (self.incl_splits is True and
-                            e["Type"].find("Split") > -1):
+                          e["Type"].find("Split") > -1):
                         val = e["Desc"].replace(e["Type"], "")
                         bardict.update({"isSplit": val})
                     else:
@@ -175,19 +200,24 @@ def _restruct_json(self, symbol, jsondata):
         return barss
 
     def read(self):
-        if type(self.symbols) is str:
-            df = self._dl_mult_symbols(symbols=[self.symbols])
-            if len(df.Close.keys()) == 0:
-                raise IndexError("None of the provided symbols were valid")
-            else:
-                return df
-        elif hasattr(self.symbols, "__iter__"):
-            df = self._dl_mult_symbols(symbols=self.symbols)
-            if len(df.Close.keys()) == 0:
-                raise IndexError("None of the provided symbols were valid")
-            else:
-                return df
+        """Read data"""
+        if isinstance(self.symbols, str):
+            symbols = [self.symbols]
+        else:
+            symbols = self.symbols
+
+        is_str = False
+        try:
+            is_str = all(map(lambda v: isinstance(v, str), symbols))
+        except Exception:
+            pass
+
+        if not is_str:
+            raise TypeError("symbols must be iterable or string and not "
+                            "type %s" % type(self.symbols))
+
+        df = self._dl_mult_symbols(symbols=symbols)
+        if len(df.index.levels[0]) == 0:
+            raise ValueError("None of the provided symbols were valid")
         else:
-            raise TypeError(
-                "symbols must be iterable or string and not type %s" %
-                type(self.symbols))
+            return df

pandas_datareader/tests/mstar/test_daily.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+import numpy as np
 import pandas as pd
 import pytest
 import requests
@@ -12,35 +13,32 @@
 
 class TestMorningstarDaily(object):
 
-
     @skip_on_exception(RemoteDataError)
-    def invalid_date(self):
+    def test_invalid_date(self):
         with pytest.raises(ValueError):
             web.DataReader("MSFT", 'morningstar', start="1990-03-A")
         with pytest.raises(ValueError):
             web.DataReader("MSFT", 'morningstar', start="2001-02-02",
                            end="1999-03-03")
 
-    @skip_on_exception(RemoteDataError)
-    def invalid_partial_multi_symbols(self):
-        df = web.DataReader(['MSFT', "21##", ""], "morningstar")
-        assert (len(df.Close.keys()) == 1)
+    def test_invalid_partial_multi_symbols(self):
+        df = web.DataReader(['MSFT', "21##", ""], "morningstar", retry_count=0)
+        assert (len(df.index.levels[0]) == 1)
 
-    @skip_on_exception(RemoteDataError)
-    def invalid_multi_symbols(self):
-        with pytest.raises(IndexError):
-            web.DataReader(["#$@", "21122"], "morningstar")
+    def test_invalid_multi_symbols(self):
+        with pytest.raises(ValueError):
+            web.DataReader(["#$@", "21122"], "morningstar", retry_count=0)
 
-    @skip_on_exception(RemoteDataError)
-    def invalid_symbol_type(self):
+    def test_invalid_symbol_type(self):
         with pytest.raises(TypeError):
-            web.DataReader([12332], data_source='morningstar')
+            web.DataReader([12332], data_source='morningstar', retry_count=0)
 
     @skip_on_exception(RemoteDataError)
     def test_mstar(self):
         start = datetime(2014, 3, 5)
-        assert (web.DataReader('MSFT', 'morningstar', start=start)['Close'][
-                    -1] == 89.8)
+        end = datetime(2018, 1, 18)
+        df = web.DataReader('MSFT', 'morningstar', start=start, end=end)
+        assert (df['Close'][-1] == 89.8)
 
     @skip_on_exception(RemoteDataError)
     def test_get_data_single_symbol(self):
@@ -51,17 +49,17 @@ def test_get_data_single_symbol(self):
     @skip_on_exception(RemoteDataError)
     def test_get_data_interval(self):
         # daily interval data
-        pan = web.get_data_morningstar(symbol='XOM', start='2013-01-01',
+        pan = web.get_data_morningstar(symbols='XOM', start='2013-01-01',
                                        end='2013-12-31', interval='d')
         assert len(pan) == 261
 
         # weekly interval data
-        pan = web.get_data_morningstar(symbol='XOM', start='2013-01-01',
+        pan = web.get_data_morningstar(symbols='XOM', start='2013-01-01',
                                        end='2013-12-31', interval='w')
         assert len(pan) == 54
 
         # monthly interval data
-        pan = web.get_data_morningstar(symbol='XOM', start='2013-01-01',
+        pan = web.get_data_morningstar(symbols='XOM', start='2013-01-01',
                                        end='2013-12-31', interval='m')
         assert len(pan) == 13
 
@@ -80,30 +78,31 @@ def test_get_data_multiple_symbols_two_dates(self):
         df = web.get_data_morningstar(symbols=['XOM', 'MSFT'],
                                       start='2013-01-01',
                                       end='2013-03-04')
-        result = df.keys()
 
-        assert len(result) == 6
+        assert len(df.index.levels[0]) == 2
+        assert 'XOM' in df.index.levels[0]
+        assert 'MSFT' in df.index.levels[0]
 
         # sanity checking
+        assert df.dtypes['Close'] == np.float64
+        assert df.dtypes['Open'] == np.float64
+        assert df.dtypes['Low'] == np.float64
+        assert df.dtypes['High'] == np.float64
+        assert df.dtypes['Volume'] == np.int64
 
-        assert all([type(i) == float for i in df.Close.values[0:3]])
-
-    @skip_on_exception(RemoteDataError)
     def incl_dividend_column_multi(self):
         df = web.get_data_morningstar(symbols=['XOM', 'MSFT'],
                                       start='2013-01-01',
                                       end='2013-03-04', incl_dividends=True)
 
-        assert ("isDividend" in df.keys())
+        assert ("isDividend" in df)
 
-    @skip_on_exception(RemoteDataError)
     def incl_splits_column_multi(self):
         df = web.get_data_morningstar(symbols=['XOM', 'MSFT'],
                                       start='2013-01-01',
                                       end='2013-03-04', incl_dividends=True)
-        assert ("isSplit" in df.keys())
+        assert ("isSplit" in df)
 
-    @skip_on_exception(RemoteDataError)
     def excl_volume_column_multi(self):
         df = web.get_data_morningstar(symbols=["XOM", "MSFT"],
                                       start='2013-01-01',
@@ -112,15 +111,16 @@ def excl_volume_column_multi(self):
 
     @skip_on_exception(RemoteDataError)
     def test_mstar_reader_class(self):
-        df = MorningstarDailyReader(symbols="GOOG", interval="d")
-        df = df.read().Volume["2014-04-07"]
-        assert df == 4401618
+        dr = MorningstarDailyReader(symbols="GOOG", interval="d")
+        df = dr.read()
+
+        assert df.Volume[('GOOG', '2017-12-13')] == 1279659
 
         session = requests.Session()
 
-        r = MorningstarDailyReader('GOOG', session=session)
-        df = r.read()
-        assert (df.session is session)
+        dr = MorningstarDailyReader('GOOG', session=session)
+        dr.read()
+        assert dr.session is session
 
     @skip_on_exception(RemoteDataError)
     def test_mstar_DataReader_multi(self):