Merge pull request #615 from econdb/master

Including a parser for econdb

Author: bashtage

docs/source/readers/econdb.rst

@@ -0,0 +1,8 @@
+Econdb
+--------
+
+.. py:module:: pandas_datareader.econdb
+
+.. autoclass:: EcondbReader
+   :members:
+   :inherited-members:

docs/source/readers/index.rst

@@ -8,6 +8,7 @@ Data Readers
    fred
    famafrench
    bank-of-canada
+   econdb
    enigma
    eurostat
    iex

docs/source/remote_data.rst

@@ -200,6 +200,23 @@ performances through the top-level function ``get_sector_performance_av``.
 
 .. _remote_data.enigma:
 
+Econdb
+======
+
+`Econdb <https://www.econdb.com>`__ provides economic data from 90+
+official statistical agencies. Free API allows access to the complete
+Econdb database of time series aggregated into datasets.
+
+.. ipython:: python
+
+    import os
+    import pandas_datareader.data as web
+
+    f = web.DataReader('ticker=RGDPQNO', 'econdb')
+    f.head()
+
+.. _remote_data.econdb:
+
 Enigma
 ======
 

docs/source/whatsnew/v0.8.0.txt

@@ -5,9 +5,13 @@ v0.8.0 (TBD)
 
 Highlights include:
 
+- A new connector for Econdb was introduced.  Econdb provides
+  aggregated economic data from 90+ official statistical agencies
+  (:issue:`615`)
 - Immediate deprecation of Robinhood for quotes and historical data. Robinhood
   ended support for these endpoints in 1/2019
 
+
 .. contents:: What's new in v0.8.0
     :local:
     :backlinks: none

pandas_datareader/data.py

@@ -9,6 +9,7 @@
 from pandas_datareader.av.sector import AVSectorPerformanceReader
 from pandas_datareader.av.time_series import AVTimeSeriesReader
 from pandas_datareader.bankofcanada import BankOfCanadaReader
+from pandas_datareader.econdb import EcondbReader
 from pandas_datareader.enigma import EnigmaReader
 from pandas_datareader.eurostat import EurostatReader
 from pandas_datareader.exceptions import DEP_ERROR_MSG, \
@@ -310,7 +311,7 @@ def DataReader(name, data_source=None, start=None, end=None,
                        "tiingo", "yahoo-actions", "yahoo-dividends",
                        "av-forex", "av-daily", "av-daily-adjusted",
                        "av-weekly", "av-weekly-adjusted", "av-monthly",
-                       "av-monthly-adjusted"]
+                       "av-monthly-adjusted", "econdb"]
 
     if data_source not in expected_source:
         msg = "data_source=%r is not implemented" % data_source
@@ -466,6 +467,11 @@ def DataReader(name, data_source=None, start=None, end=None,
                                   retry_count=retry_count, pause=pause,
                                   session=session, api_key=access_key).read()
 
+    elif data_source == "econdb":
+        return EcondbReader(symbols=name, start=start, end=end,
+                            retry_count=retry_count, pause=pause,
+                            session=session).read()
+
     else:
         msg = "data_source=%r is not implemented" % data_source
         raise NotImplementedError(msg)

pandas_datareader/econdb.py

@@ -0,0 +1,49 @@
+import requests
+import pandas as pd
+import pandas.compat as compat
+
+from pandas_datareader.base import _BaseReader
+
+
+class EcondbReader(_BaseReader):
+    """Get data for the given name from Econdb."""
+
+    _URL = 'https://www.econdb.com/api/series/'
+    _format = None
+    _show = 'labels'
+
+    @property
+    def url(self):
+        """API URL"""
+        if not isinstance(self.symbols, compat.string_types):
+            raise ValueError('data name must be string')
+
+        return ('{0}?{1}&format=json&page_size=500&expand=meta'
+                .format(self._URL, self.symbols))
+
+    def read(self):
+        """ read one data from specified URL """
+        results = requests.get(self.url).json()['results']
+        df = pd.DataFrame({'dates': []}).set_index('dates')
+
+        for entry in results:
+            head = entry['additional_metadata']
+            series = (pd.DataFrame(entry['data'])[['dates', 'values']]
+                      .set_index('dates'))
+            if self._show == 'labels':
+                def show_func(x): return x.split(':')[1]
+            elif self._show == 'codes':
+                def show_func(x): return x.split(':')[0]
+
+            series.columns = pd.MultiIndex.from_tuples(
+                [[show_func(x) for x in head.values()]],
+                names=[show_func(x) for x in head.keys()])
+
+            if not df.empty:
+                df = df.join(series, how='outer')
+            else:
+                df = series
+        df.index = pd.to_datetime(df.index, errors='ignore')
+        df.index.name = 'TIME_PERIOD'
+        df = df.truncate(self.start, self.end)
+        return df

pandas_datareader/tests/test_econdb.py

@@ -0,0 +1,65 @@
+import numpy as np
+import pandas as pd
+import pandas.util.testing as tm
+import pandas_datareader.data as web
+
+
+class TestEcondb(object):
+
+    def test_get_cdh_e_fos(self):
+        # EUROSTAT
+        # Employed doctorate holders in non managerial and non professional
+        # occupations by fields of science (%)
+        df = web.DataReader(
+            'dataset=CDH_E_FOS&GEO=NO,PL,PT,RU&FOS07=FOS1&Y_GRAD=TOTAL',
+            'econdb',
+            start=pd.Timestamp('2005-01-01'),
+            end=pd.Timestamp('2010-01-01'))
+        assert isinstance(df, pd.DataFrame)
+        assert df.shape == (2, 4)
+
+        df = df['Natural sciences']['Annual'][
+                ['Norway', 'Poland', 'Portugal', 'Russia']]
+
+        exp_col = pd.MultiIndex.from_product(
+            [['Norway', 'Poland', 'Portugal', 'Russia'],
+             ['Percentage'], ['Total']],
+            names=['Geopolitical entity (reporting)', 'Unit of measure',
+                   'Year of graduation'])
+        exp_idx = pd.DatetimeIndex(['2006-01-01', '2009-01-01'],
+                                   name='TIME_PERIOD')
+
+        values = np.array([[25.49, np.nan, 39.05, np.nan],
+                           [20.38, 25.1, 27.77, 38.1]])
+        expected = pd.DataFrame(values, index=exp_idx, columns=exp_col)
+        tm.assert_frame_equal(df, expected)
+
+    def test_get_tourism(self):
+        # OECD
+        # TOURISM_INBOUND
+
+        df = web.DataReader(
+            'dataset=OE_TOURISM_INBOUND&COUNTRY=JPN,USA&'
+            'VARIABLE=INB_ARRIVALS_TOTAL', 'econdb',
+            start=pd.Timestamp('2008-01-01'), end=pd.Timestamp('2012-01-01'))
+        df = df.astype(np.float)
+        jp = np.array([8351000, 6790000, 8611000, 6219000,
+                       8368000], dtype=float)
+        us = np.array([175702309, 160507417, 164079732, 167600277,
+                       171320408], dtype=float)
+        index = pd.date_range('2008-01-01', '2012-01-01', freq='AS',
+                              name='TIME_PERIOD')
+        for label, values in [('Japan', jp), ('United States', us)]:
+            expected = pd.Series(values, index=index,
+                                 name='Total international arrivals')
+            tm.assert_series_equal(df[label]['Total international arrivals'],
+                                   expected)
+
+    def test_bls(self):
+        # BLS
+        # CPI
+        df = web.DataReader(
+            'ticker=BLS_CU.CUSR0000SA0.M.US', 'econdb',
+            start=pd.Timestamp('2010-01-01'), end=pd.Timestamp('2013-01-27'))
+
+        assert df.loc['2010-05-01'][0] == 217.3