Merge pull request #812 from GreenArt11/master

MOEX: Use data from the main board only for each ticker

Author: bashtage

docs/source/remote_data.rst

@@ -680,11 +680,19 @@ MOEX Data
 =========
 The Moscow Exchange (MOEX) provides historical data.
 
+pandas_datareader.get_data_moex(*args) is equivalent to 
+pandas_datareader.moex.MoexReader(*args).read() 
 
 .. ipython:: python
 
-   import pandas_datareader.data as web
-   f = web.DataReader('USD000UTSTOM', 'moex', start='2017-07-01', end='2017-07-31')
+   import pandas_datareader as pdr
+   f = pdr.get_data_moex(['USD000UTSTOM', 'MAGN'], '2020-07-02', '2020-07-07')
+   f.head()
+
+   f = pdr.moex.MoexReader('SBER', '2020-07-02', '2020-07-03').read()
+   f.head()
+   
+   f = pdr.moex.MoexReader('SBER', '2020-07-02', '2020-07-03').read_all_boards()
    f.head()
 
 .. _remote_data.naver:

docs/source/whatsnew.rst

@@ -18,6 +18,7 @@ What's New
 
 These are new features and improvements of note in each release.
 
+.. include:: whatsnew/v0.9.1.txt
 .. include:: whatsnew/v0.9.0.txt
 .. include:: whatsnew/v0.8.0.txt
 .. include:: whatsnew/v0.7.0.txt

docs/source/whatsnew/v.0.9.1.txt

@@ -0,0 +1,32 @@
+.. _whatsnew_091:
+
+v0.9.1 (July 26, 2020)
+---------------------------
+
+Highlights include:
+
+.. contents:: What's new in v0.9.1
+    :local:
+    :backlinks: none
+
+.. _whatsnew_091.enhancements:
+
+Enhancements
+~~~~~~~~~~~~
+
+- MOEX now returns data only from primary trading boards for 
+  each ticker (:issue:`811`, :issue:`748`)
+- Added read_all_boards() method for MOEX that returns data 
+  from every trading board (ver. 0.9.0 behaviour)
+- Docs for MOEX reedited
+
+.. _whatsnew_080.bug_fixes:
+
+Bug Fixes
+~~~~~~~~~
+
+- Fixed broken links on the github ussies on "What’s New" docs pages
+
+Contributors
+~~~~~~~~~~~~
+- Dmitry Alekseev

docs/source/whatsnew/v0.7.0.txt

@@ -73,10 +73,10 @@ Bug Fixes
   and currency pairs (:issue:`487`).
 - Add `is_list_like` to compatibility layer to avoid failure on pandas >= 0.23 (:issue:`520`).
 - Fixed Yahoo! time offset (:issue:`487`).
-- Fix Yahoo! quote reader (:issue: `540`).
-- Remove import of deprecated `tm.get_data_path` (:issue: `566`)
+- Fix Yahoo! quote reader (:issue:`540`).
+- Remove import of deprecated `tm.get_data_path` (:issue:`566`)
 - Allow full usage of stooq url parameters.
 - Removed unused requests-file and requests-ftp dependencies.
 - Fix Yahoo! actions issue where the default reporting adjusts dividends. The unadjusted
   dividends may lack precision due to accumulated numerical error when converting adjusted
-  to the original dividend amount. (:issue: `495`)
+  to the original dividend amount. (:issue:`495`)

docs/source/whatsnew/v0.8.0.txt

@@ -26,7 +26,7 @@ Enhancements
 ~~~~~~~~~~~~
 
 - Added Tiingo IEX Historical reader. (:issue:`619`)
-- Added support for Alpha Vantage intraday time series prices (:issue: `631`)
+- Added support for Alpha Vantage intraday time series prices (:issue:`631`)
 - Up to 15 years of historical prices from IEX with new platform, IEX Cloud
 - Added testing on Python 3.7 (:issue:`667`)
 - Allow IEX to read less than 1 year of data (:issue:`649`)
@@ -52,9 +52,9 @@ Bug Fixes
 ~~~~~~~~~
 
 - Fix Yahoo! actions issue where dividends are adjusted twice as a result of a
-  change to the Yahoo! API. (:issue: `583`)
+  change to the Yahoo! API. (:issue:`583`)
 - Fix AlphaVantage time series data ordering after provider switch to
-  descending order (maintains ascending order for consistency). (:issue: `662`)
+  descending order (maintains ascending order for consistency). (:issue:`662`)
 - Refactored compatibility library to be independent of pandas version.
 - Fixed quarter value handling in JSDMX and OECD. (:issue:`685`)
 - Fixed a bug in ``base`` so that the reader does not error when response.encoding

pandas_datareader/moex.py

@@ -102,6 +102,7 @@ def _get_metadata(self):
         """Get markets and engines for the given symbols"""
 
         markets_n_engines = {}
+        boards = {}
 
         for symbol in self.symbols:
             response = self._get_response(self.__url_metadata.format(symbol=symbol))
@@ -130,6 +131,11 @@ def _get_metadata(self):
                     markets_n_engines[symbol].append(
                         (fields[5], fields[7])
                     )  # market and engine
+
+                    if fields[14] == "1":  # main board for symbol
+                        symbol_U = symbol.upper()
+                        boards[symbol_U] = fields[1]
+
             if symbol not in markets_n_engines:
                 raise IOError(
                     "{} request returned no metadata: {}\n"
@@ -141,13 +147,14 @@ def _get_metadata(self):
                 )
             if symbol in markets_n_engines:
                 markets_n_engines[symbol] = list(set(markets_n_engines[symbol]))
-        return markets_n_engines
+        return markets_n_engines, boards
 
-    def read(self):
-        """Read data"""
+    def read_all_boards(self):
+        """Read all data from every board for every ticker"""
 
+        markets_n_engines, boards = self._get_metadata()
         try:
-            self.__markets_n_engines = self._get_metadata()
+            self.__markets_n_engines = markets_n_engines
 
             urls = self.url  # generate urls per symbols
             dfs = []  # an array of pandas dataframes per symbol to concatenate
@@ -198,9 +205,21 @@ def read(self):
                 "check URL or correct a date".format(self.__class__.__name__)
             )
         elif len(dfs) > 1:
-            return concat(dfs, axis=0, join="outer", sort=True)
+            b = concat(dfs, axis=0, join="outer", sort=True)
         else:
-            return dfs[0]
+            b = dfs[0]
+        return b
+
+    def read(self):
+        """Read data from the primary board for each ticker"""
+        markets_n_engines, boards = self._get_metadata()
+        b = self.read_all_boards()
+        result = pd.DataFrame()
+        for secid in list(set(b["SECID"].tolist())):
+            part = b[b["BOARDID"] == boards[secid]]
+            result = result.append(part)
+        result = result.drop_duplicates()
+        return result
 
     def _read_url_as_String(self, url, params=None):
         """Open an url (and retry)"""

pandas_datareader/tests/test_moex.py

@@ -21,6 +21,13 @@ def test_moex_stock_datareader(self):
             df = web.DataReader(
                 ["GAZP", "SIBN"], "moex", start="2019-12-26", end="2019-12-26"
             )
-            assert df.size == 740
+            assert df.size == 74
+        except HTTPError as e:
+            pytest.skip(e)
+
+    def test_moex_datareader_filter(self):
+        try:
+            df = web.DataReader("SBER", "moex", start="2020-07-14", end="2020-07-14")
+            assert len(df) == 1
         except HTTPError as e:
             pytest.skip(e)