Added QuandlReader (plus tests and doc) (#361)

gliptak · jreback · commit a88c84e71dc1 · 2017-07-07T05:57:52.000-04:00
diff --git a/docs/source/remote_data.rst b/docs/source/remote_data.rst
@@ -26,6 +26,7 @@ Currently the following sources are supported:
     - :ref:`Yahoo! Finance<remote_data.yahoo>`
     - :ref:`Google Finance<remote_data.google>`
     - :ref:`Enigma<remote_data.enigma>`
+    - :ref:`Quandl<remote_data.quandl>`
     - :ref:`St.Louis FED (FRED)<remote_data.fred>`
     - :ref:`Kenneth French's data library<remote_data.ff>`
     - :ref:`World Bank<remote_data.wb>`
@@ -79,7 +80,7 @@ Historical dividends from Yahoo! Finance.
 .. _remote_data.yahoo_quotes:
 
 Yahoo! Finance Quotes
-----------------------
+---------------------
 ***Experimental***
 
 The YahooQuotesReader class allows to get quotes data from Yahoo! Finance.
@@ -165,7 +166,7 @@ Google Finance
 .. _remote_data.google_quotes:
 
 Google Finance Quotes
-----------------------
+---------------------
 ***Experimental***
 
 The GoogleQuotesReader class allows to get quotes data from Google Finance.
@@ -213,6 +214,33 @@ the world's largest repository of structured public data.
     df = pdr.get_data_enigma('enigma.trade.ams.toxic.2015', os.getenv('ENIGMA_API_KEY'))
     df.columns
 
+.. _remote_data.quandl:
+
+Quandl
+======
+
+Daily financial data (prices of stocks, ETFs etc.) from
+`Quandl <https://www.quandl.com/>`__.
+The symbol names consist of two parts: DB name and symbol name.
+DB names can be all the
+`free ones listed on the Quandl website <https://blog.quandl.com/free-data-on-quandl>__.
+Symbol names vary with DB name; for WIKI (US stocks), they are the common
+ticker symbols, in some other cases (such as FSE) they can be a bit strange.
+Some sources are also mapped to suitable ISO country codes in the dot suffix
+style shown above, currently available for
+`BE, CN, DE, FR, IN, JP, NL, PT, UK, US <https://www.quandl.com/search?query=>`__.
+
+As of June 2017, each DB has a different data schema,
+the coverage in terms of time range is sometimes surprisingly small, and
+the data quality is not always good.
+
+.. ipython:: python
+
+    import pandas_datareader.data as web
+    symbol = 'WIKI/AAPL'  # or 'AAPL.US'
+    df = web.DataReader(symbol, 'quandl', "2015-01-01", "2015-01-05")
+    df.loc['2015-01-02']
+
 .. _remote_data.fred:
 
 FRED
@@ -548,7 +576,7 @@ Download currency historical rate from `Oanda <https://www.oanda.com/>`__.
 .. _remote_data.nasdaq_symbols:
 
 Nasdaq Trader Symbol Definitions
-==============================
+================================
 
 Download the latest symbols from `Nasdaq <ftp://ftp.nasdaqtrader.com/SymbolDirectory/nasdaqtraded.txt>`_.
 
@@ -572,5 +600,3 @@ available. More information on the `field <http://www.nasdaqtrader.com/trader.as
         NASDAQ Symbol                                                     IBM
         NextShares                                                      False
         Name: IBM, dtype: object
-
-
diff --git a/docs/source/whatsnew/v0.5.0.txt b/docs/source/whatsnew/v0.5.0.txt
@@ -13,6 +13,13 @@ Highlights include:
     :local:
     :backlinks: none
 
+.. _whatsnew_050.enhancements:
+
+Enhancements
+~~~~~~~~~~~~
+
+- ``DataReader`` now supports Quandl, see :ref:`here<remote_data.quandl>` (:issue:`361`).
+
 .. _whatsnew_050.bug_fixes:
 
 Bug Fixes
diff --git a/pandas_datareader/data.py b/pandas_datareader/data.py
@@ -22,6 +22,7 @@
 from pandas_datareader.enigma import EnigmaReader
 from pandas_datareader.oanda import get_oanda_currency_historical_rates
 from pandas_datareader.nasdaq_trader import get_nasdaq_symbols
+from pandas_datareader.quandl import QuandlReader
 
 
 def get_data_fred(*args, **kwargs):
@@ -56,6 +57,10 @@ def get_quote_google(*args, **kwargs):
     return GoogleQuotesReader(*args, **kwargs).read()
 
 
+def get_data_quandl(*args, **kwargs):
+    return QuandlReader(*args, **kwargs).read()
+
+
 def DataReader(name, data_source=None, start=None, end=None,
                retry_count=3, pause=0.001, session=None, access_key=None):
     """
@@ -168,6 +173,10 @@ def DataReader(name, data_source=None, start=None, end=None,
             raise ValueError("Only the string 'symbols' is supported for "
                              "Nasdaq, not %r" % (name,))
         return get_nasdaq_symbols(retry_count=retry_count, pause=pause)
+    elif data_source == "quandl":
+        return QuandlReader(symbols=name, start=start, end=end,
+                            retry_count=retry_count, pause=pause,
+                            session=session).read()
     else:
         msg = "data_source=%r is not implemented" % data_source
         raise NotImplementedError(msg)
diff --git a/pandas_datareader/quandl.py b/pandas_datareader/quandl.py
@@ -0,0 +1,115 @@
+import re
+
+from pandas_datareader.base import _DailyBaseReader
+
+
+class QuandlReader(_DailyBaseReader):
+
+    """
+    Returns DataFrame of historical stock prices from symbol, over date
+    range, start to end.
+
+    .. versionadded:: 0.5.0
+
+    Parameters
+    ----------
+    symbols : string
+        Possible formats:
+        1. DB/SYM: The Quandl 'codes': DB is the database name,
+        SYM is a ticker-symbol-like Quandl abbreviation
+        for a particular security.
+        2. SYM.CC: SYM is the same symbol and CC is an ISO country code,
+        will try to map to the best single Quandl database for that country.
+        Beware of ambiguous symbols (different securities per country)!
+        Note: Cannot use more than a single string because of the inflexible
+        way the URL is composed of url and _get_params in the superclass
+    start : string
+        Starting date, timestamp. Parses many different kind of date
+        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
+    end : string, (defaults to today)
+        Ending date, timestamp. Same format as starting date.
+    retry_count : int, default 3
+        Number of times to retry query request.
+    pause : int, default 0
+        Time, in seconds, to pause between consecutive queries of chunks. If
+        single value given for symbol, represents the pause between retries.
+    chunksize : int, default 25
+        Number of symbols to download consecutively before intiating pause.
+    session : Session, default None
+        requests.sessions.Session instance to be used
+    """
+
+    _BASE_URL = "https://www.quandl.com/api/v3/datasets/"
+
+    @property
+    def url(self):
+        symbol = self.symbols if isinstance(self.symbols, str) \
+                              else self.symbols[0]
+        mm = self._fullmatch(r"([A-Z0-9]+)(([/\.])([A-Z0-9_]+))?", symbol)
+        assert mm, ("Symbol '%s' must conform to Quandl convention 'DB/SYM'" %
+                    symbol)
+        datasetname = 'WIKI'
+        if not mm.group(2):
+            # bare symbol:
+            datasetname = 'WIKI'  # default; symbol stays itself
+        elif mm.group(3) == "/":
+            # --- normal Quandl DB/SYM convention:
+            symbol = mm.group(4)
+            datasetname = mm.group(1)
+        elif mm.group(3) == ".":
+            # secondary convention SYM.CountryCode:
+            symbol = mm.group(1)
+            datasetname = self._db_from_countrycode(mm.group(4))
+        params = {
+            'start_date': self.start.strftime('%Y-%m-%d'),
+            'end_date': self.end.strftime('%Y-%m-%d'),
+            'order': "asc",
+        }
+        paramstring = '&'.join(['%s=%s' % (k, v) for k, v in params.items()])
+        return '%s%s/%s.csv?%s' % (self._BASE_URL, datasetname,
+                                   symbol, paramstring)
+
+    def _fullmatch(self, regex, string, flags=0):
+        """Emulate python-3.4 re.fullmatch()."""
+        return re.match("(?:" + regex + r")\Z", string, flags=flags)
+
+    _COUNTRYCODE_TO_DATASET = dict(
+            # https://www.quandl.com/data/EURONEXT-Euronext-Stock-Exchange
+            BE='EURONEXT',
+            # https://www.quandl.com/data/HKEX-Hong-Kong-Exchange
+            CN='HKEX',
+            # https://www.quandl.com/data/SSE-Boerse-Stuttgart
+            DE='SSE',
+            FR='EURONEXT',
+            # https://www.quandl.com/data/NSE-National-Stock-Exchange-of-India
+            IN='NSE',
+            # https://www.quandl.com/data/TSE-Tokyo-Stock-Exchange
+            JP='TSE',
+            NL='EURONEXT',
+            PT='EURONEXT',
+            # https://www.quandl.com/data/LSE-London-Stock-Exchange
+            UK='LSE',
+            # https://www.quandl.com/data/WIKI-Wiki-EOD-Stock-Prices
+            US='WIKI',
+        )
+
+    def _db_from_countrycode(self, code):
+        assert code in self._COUNTRYCODE_TO_DATASET,\
+            "No Quandl dataset known for country code '%s'" % code
+        return self._COUNTRYCODE_TO_DATASET[code]
+
+    def _get_params(self, symbol):
+        return {}
+
+    def read(self):
+        df = super(QuandlReader, self).read()
+        df.rename(columns=lambda n: n.replace(' ', '')
+                                     .replace('.', '')
+                                     .replace('/', '')
+                                     .replace('%', '')
+                                     .replace('(', '')
+                                     .replace(')', '')
+                                     .replace("'", '')
+                                     .replace('-', ''),
+                  inplace=True)
+        return df
diff --git a/pandas_datareader/tests/test_quandl.py b/pandas_datareader/tests/test_quandl.py
@@ -0,0 +1,94 @@
+import pandas_datareader.data as web
+from pandas_datareader._utils import RemoteDataError
+from pandas_datareader._testing import skip_on_exception
+
+
+class TestQuandl(object):
+    # we test data from 10 years back where possible, 2 years otherwise, or...
+    start10 = "2007-01-01"  # over ten years back
+    end10 = "2007-01-05"
+    day10 = "2007-01-04"
+    start2 = "2015-01-01"  # over two years back
+    end2 = "2015-01-05"
+    day2 = "2015-01-02"
+
+    def check_headers(self, df, expected_cols):
+        expected_cols = frozenset(expected_cols)
+        act_cols = frozenset(df.columns.tolist())
+        assert expected_cols == act_cols, "unexpected cols: " + str(act_cols)
+
+    @skip_on_exception(RemoteDataError)
+    def test_db_wiki_us(self):
+        df = web.DataReader('F', 'quandl', self.start10, self.end10)
+        self.check_headers(df, ['Open', 'High', 'Low', 'Close', 'Volume',
+                                'ExDividend', 'SplitRatio', 'AdjOpen',
+                                'AdjHigh', 'AdjLow', 'AdjClose', 'AdjVolume'])
+        assert df.Close.at[self.day10] == 7.70
+
+    @skip_on_exception(RemoteDataError)
+    def test_db_fse_frankfurt(self):
+        # ALV_X: Allianz SE
+        df = web.DataReader('FSE/ALV_X', 'quandl', self.start10, self.end10)
+        self.check_headers(df, ['Open', 'High', 'Low', 'Close', 'Change',
+                                'TradedVolume', 'Turnover',
+                                'LastPriceoftheDay', 'DailyTradedUnits',
+                                'DailyTurnover'])
+        assert df.Close.at[self.day10] == 159.45
+
+    @skip_on_exception(RemoteDataError)
+    def test_db_sse_de_stuttgart(self):
+        # ALV: Allianz SE
+        df = web.DataReader('SSE/ALV', 'quandl', self.start2, self.end2)
+        self.check_headers(df, [
+                "High", "Low", "Last", "PreviousDayPrice", "Volume"])
+        # as of 2017-06-11: PreviousDayPrice can be outside Low/High range;
+        # Volume can be NaN
+        assert df.Last.at[self.day2] == 136.47
+        df2 = web.DataReader('ALV.DE', 'quandl', self.start2, self.end2)
+        assert (df.Last == df2.Last).all()
+
+    @skip_on_exception(RemoteDataError)
+    def test_db_euronext_be_fr_nl_pt(self):
+        # FP: Total SA
+        # as of 2017-06-11, some datasets end a few months after their start,
+        # e.g. ALVD, BASD
+        df = web.DataReader('EURONEXT/FP', 'quandl', self.start2, self.end2)
+        self.check_headers(df, [
+                "Open", "High", "Low", "Last", "Turnover", "Volume"])
+        assert df.Last.at[self.day2] == 42.525
+        df2 = web.DataReader('FP.FR', 'quandl', self.start2, self.end2)
+        assert (df.Last == df2.Last).all()
+
+    @skip_on_exception(RemoteDataError)
+    def test_db_lse_uk(self):
+        # RBS: Royal Bank of Scotland
+        df = web.DataReader('LSE/RBS', 'quandl', self.start10, self.end10)
+        self.check_headers(df, ["High", "Low", "LastClose", "Price",
+                                "Volume", "Change", "Var"])
+        # as of 2017-06-11, Price == LastClose, all others are NaN
+        assert df.Price.at[self.day10] == 5950.983
+
+    @skip_on_exception(RemoteDataError)
+    def test_db_nse_in(self):
+        # TCS: Tata Consutancy Services
+        df = web.DataReader('NSE/TCS', 'quandl', self.start10, self.end10)
+        self.check_headers(df, ['Open', 'High', 'Low', 'Last', 'Close',
+                                'TotalTradeQuantity', 'TurnoverLacs'])
+        assert df.Close.at[self.day10] == 1259.05
+
+    @skip_on_exception(RemoteDataError)
+    def test_db_tse_jp(self):
+        # TSE/6758: Sony Corp.
+        df = web.DataReader('TSE/6758', 'quandl', self.start10, self.end10)
+        self.check_headers(df, ['Open', 'High', 'Low', 'Close', 'Volume'])
+        assert df.Close.at[self.day10] == 5190.0
+
+    @skip_on_exception(RemoteDataError)
+    def test_db_hkex_cn(self):
+        # HKEX/00941: China Mobile
+        df = web.DataReader('HKEX/00941', 'quandl', self.start2, self.end2)
+        self.check_headers(df,
+                           ['NominalPrice', 'NetChange', 'Change', 'Bid',
+                            'Ask', 'PEx', 'High', 'Low', 'PreviousClose',
+                            'ShareVolume000', 'Turnover000', 'LotSize'])
+        assert df.High.at[self.day2] == 91.9
