CLN: rename internal modules. move tests. (#163)

* CLN: rename internal modules. move tests.

Per GH#154 we don't need to use underscore to show submodules are
private. To be 100% clear, I have added a disclaimer to the API
reference as well.

Also, I have split the system tests from the unit tests so that the
(fast) unit tests can be more easily run separately from the (slow)
system tests. I have followed the same directory structure as used in
the google-cloud-bigquery library.

* CLN: move _version.py back for versioneer

* CLN: run tests from tests directory in conda test run

Author: tswast

.travis.yml

@@ -45,8 +45,8 @@ script:
   - if [[ $PYTHON == '3.6' ]] && [[ "$PANDAS" == "MASTER" ]]; then nox -s test36master ; fi
   - REQ="ci/requirements-${PYTHON}-${PANDAS}" ;
     if [ -f "$REQ.conda" ]; then
-      pip install coverage pytest pytest-cov codecov ;
-      pytest -v --cov=pandas_gbq --cov-report xml:/tmp/pytest-cov.xml pandas_gbq ;
+      pip install pytest ;
+      pytest -v tests ;
     fi
   - if [[ $COVERAGE == 'true' ]]; then nox -s cover ; fi
   - if [[ $LINT == 'true' ]]; then nox -s lint ; fi

docs/source/api.rst

@@ -5,6 +5,11 @@
 API Reference
 *************
 
+.. note::
+
+  Only functions and classes which are members of the ``pandas_gbq`` module
+  are considered public. Submodules and their members are considered private.
+
 .. autosummary::
 
    read_gbq

docs/source/changelog.rst

@@ -5,6 +5,7 @@ Changelog
 -----------
 
 - Tests now use `nox` to run in multiple Python environments. (:issue:`52`)
+- Renamed internal modules. (:issue:`154`)
 
 0.4.1 / 2018-04-05
 ------------------

nox.py

@@ -19,9 +19,27 @@ def default(session):
     session.install('-e', '.')
     session.run(
         'pytest',
-        os.path.join('pandas_gbq', 'tests'),
+        os.path.join('.', 'tests', 'unit'),
+        os.path.join('.', 'tests', 'system.py'),
         '--quiet',
         '--cov=pandas_gbq',
+        '--cov=tests.unit',
+        '--cov-report',
+        'xml:/tmp/pytest-cov.xml',
+        *session.posargs
+    )
+
+
+@nox.session
+def unit(session):
+    session.install('mock', 'pytest', 'pytest-cov')
+    session.install('-e', '.')
+    session.run(
+        'pytest',
+        os.path.join('.', 'tests', 'unit'),
+        '--quiet',
+        '--cov=pandas_gbq',
+        '--cov=tests.unit',
         '--cov-report',
         'xml:/tmp/pytest-cov.xml',
         *session.posargs
@@ -69,7 +87,7 @@ def test36master(session):
 @nox.session
 def lint(session):
     session.install('flake8')
-    session.run('flake8', 'pandas_gbq', '-v')
+    session.run('flake8', 'pandas_gbq', 'tests', '-v')
 
 
 @nox.session

pandas_gbq/gbq.py

@@ -458,7 +458,7 @@ def process_http_error(ex):
     def run_query(self, query, **kwargs):
         from google.auth.exceptions import RefreshError
         from concurrent.futures import TimeoutError
-        from pandas_gbq import _query
+        import pandas_gbq.query
 
         job_config = {
             'query': {
@@ -484,7 +484,7 @@ def run_query(self, query, **kwargs):
             logger.info('Requesting query... ')
             query_reply = self.client.query(
                 query,
-                job_config=_query.query_config(
+                job_config=pandas_gbq.query.query_config(
                     job_config, BIGQUERY_INSTALLED_VERSION))
             logger.info('ok.\nQuery running...')
         except (RefreshError, ValueError):
@@ -552,13 +552,13 @@ def run_query(self, query, **kwargs):
     def load_data(
             self, dataframe, dataset_id, table_id, chunksize=None,
             schema=None):
-        from pandas_gbq import _load
+        from pandas_gbq import load
 
         total_rows = len(dataframe)
         logger.info("\n\n")
 
         try:
-            for remaining_rows in _load.load_chunks(
+            for remaining_rows in load.load_chunks(
                     self.client, dataframe, dataset_id, table_id,
                     chunksize=chunksize, schema=schema):
                 logger.info("\rLoad is {0}% Complete".format(
@@ -1000,8 +1000,8 @@ def generate_bq_schema(df, default_type='STRING'):
 
 
 def _generate_bq_schema(df, default_type='STRING'):
-    from pandas_gbq import _schema
-    return _schema.generate_bq_schema(df, default_type=default_type)
+    from pandas_gbq import schema
+    return schema.generate_bq_schema(df, default_type=default_type)
 
 
 class _Table(GbqConnector):

pandas_gbq/_load.py renamed to pandas_gbq/load.py

@@ -3,7 +3,7 @@
 import six
 from google.cloud import bigquery
 
-from pandas_gbq import _schema
+import pandas_gbq.schema
 
 
 def encode_chunk(dataframe):
@@ -51,7 +51,7 @@ def load_chunks(
     job_config.source_format = 'CSV'
 
     if schema is None:
-        schema = _schema.generate_bq_schema(dataframe)
+        schema = pandas_gbq.schema.generate_bq_schema(dataframe)
 
     # Manually create the schema objects, adding NULLABLE mode
     # as a workaround for

pandas_gbq/_query.py renamed to pandas_gbq/query.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 import os
-import re
 import sys
 from datetime import datetime
 from random import randint
@@ -13,13 +12,12 @@
 import pytz
 from pandas import DataFrame, NaT, compat
 from pandas.compat import range, u
-from pandas.compat.numpy import np_datetime64_compat
 
 from pandas_gbq import gbq
 
 try:
     import mock
-except ImportError:
+except ImportError:  # pragma: NO COVER
     from unittest import mock
 
 TABLE_ID = 'new_test'
@@ -280,233 +278,7 @@ def test_get_user_account_credentials_returns_credentials(self):
         assert isinstance(credentials, Credentials)
 
 
-class TestGBQUnit(object):
-
-    @pytest.fixture(autouse=True)
-    def mock_bigquery_client(self, monkeypatch):
-        import google.cloud.bigquery
-        import google.cloud.bigquery.table
-        mock_client = mock.create_autospec(google.cloud.bigquery.Client)
-        # Mock out SELECT 1 query results.
-        mock_query = mock.create_autospec(google.cloud.bigquery.QueryJob)
-        mock_query.state = 'DONE'
-        mock_rows = mock.create_autospec(
-            google.cloud.bigquery.table.RowIterator)
-        mock_rows.total_rows = 1
-        mock_rows.schema = [
-            google.cloud.bigquery.SchemaField('_f0', 'INTEGER')]
-        mock_rows.__iter__.return_value = [(1,)]
-        mock_query.result.return_value = mock_rows
-        mock_client.query.return_value = mock_query
-        monkeypatch.setattr(
-            gbq.GbqConnector, 'get_client', lambda _: mock_client)
-
-    @pytest.fixture(autouse=True)
-    def no_auth(self, monkeypatch):
-        import google.auth.credentials
-        mock_credentials = mock.create_autospec(
-            google.auth.credentials.Credentials)
-        monkeypatch.setattr(
-            gbq.GbqConnector,
-            'get_application_default_credentials',
-            lambda _: mock_credentials)
-        monkeypatch.setattr(
-            gbq.GbqConnector,
-            'get_user_account_credentials',
-            lambda _: mock_credentials)
-
-    def test_should_return_credentials_path_set_by_env_var(self):
-        env = {'PANDAS_GBQ_CREDENTIALS_FILE': '/tmp/dummy.dat'}
-        with mock.patch.dict('os.environ', env):
-            assert gbq._get_credentials_file() == '/tmp/dummy.dat'
-
-    @pytest.mark.parametrize(
-        ('input', 'type_', 'expected'), [
-            (1, 'INTEGER', int(1)),
-            (1, 'FLOAT', float(1)),
-            pytest.param('false', 'BOOLEAN', False, marks=pytest.mark.xfail),
-            pytest.param(
-                '0e9', 'TIMESTAMP',
-                np_datetime64_compat('1970-01-01T00:00:00Z'),
-                marks=pytest.mark.xfail),
-            ('STRING', 'STRING', 'STRING'),
-        ])
-    def test_should_return_bigquery_correctly_typed(
-            self, input, type_, expected):
-        result = gbq._parse_data(
-            dict(fields=[dict(name='x', type=type_, mode='NULLABLE')]),
-            rows=[[input]]).iloc[0, 0]
-        assert result == expected
-
-    def test_to_gbq_should_fail_if_invalid_table_name_passed(self):
-        with pytest.raises(gbq.NotFoundException):
-            gbq.to_gbq(DataFrame(), 'invalid_table_name', project_id="1234")
-
-    def test_to_gbq_with_no_project_id_given_should_fail(self):
-        with pytest.raises(TypeError):
-            gbq.to_gbq(DataFrame(), 'dataset.tablename')
-
-    def test_to_gbq_with_verbose_new_pandas_warns_deprecation(self):
-        import pkg_resources
-        min_bq_version = pkg_resources.parse_version('0.29.0')
-        pandas_version = pkg_resources.parse_version('0.23.0')
-        with pytest.warns(FutureWarning), \
-                mock.patch(
-                    'pkg_resources.Distribution.parsed_version',
-                    new_callable=mock.PropertyMock) as mock_version:
-            mock_version.side_effect = [min_bq_version, pandas_version]
-            try:
-                gbq.to_gbq(
-                    DataFrame(),
-                    'dataset.tablename',
-                    project_id='my-project',
-                    verbose=True)
-            except gbq.TableCreationError:
-                pass
-
-    def test_to_gbq_with_not_verbose_new_pandas_warns_deprecation(self):
-        import pkg_resources
-        min_bq_version = pkg_resources.parse_version('0.29.0')
-        pandas_version = pkg_resources.parse_version('0.23.0')
-        with pytest.warns(FutureWarning), \
-                mock.patch(
-                    'pkg_resources.Distribution.parsed_version',
-                    new_callable=mock.PropertyMock) as mock_version:
-            mock_version.side_effect = [min_bq_version, pandas_version]
-            try:
-                gbq.to_gbq(
-                    DataFrame(),
-                    'dataset.tablename',
-                    project_id='my-project',
-                    verbose=False)
-            except gbq.TableCreationError:
-                pass
-
-    def test_to_gbq_wo_verbose_w_new_pandas_no_warnings(self, recwarn):
-        import pkg_resources
-        min_bq_version = pkg_resources.parse_version('0.29.0')
-        pandas_version = pkg_resources.parse_version('0.23.0')
-        with mock.patch(
-                'pkg_resources.Distribution.parsed_version',
-                new_callable=mock.PropertyMock) as mock_version:
-            mock_version.side_effect = [min_bq_version, pandas_version]
-            try:
-                gbq.to_gbq(
-                    DataFrame(), 'dataset.tablename', project_id='my-project')
-            except gbq.TableCreationError:
-                pass
-            assert len(recwarn) == 0
-
-    def test_to_gbq_with_verbose_old_pandas_no_warnings(self, recwarn):
-        import pkg_resources
-        min_bq_version = pkg_resources.parse_version('0.29.0')
-        pandas_version = pkg_resources.parse_version('0.22.0')
-        with mock.patch(
-                'pkg_resources.Distribution.parsed_version',
-                new_callable=mock.PropertyMock) as mock_version:
-            mock_version.side_effect = [min_bq_version, pandas_version]
-            try:
-                gbq.to_gbq(
-                    DataFrame(),
-                    'dataset.tablename',
-                    project_id='my-project',
-                    verbose=True)
-            except gbq.TableCreationError:
-                pass
-            assert len(recwarn) == 0
-
-    def test_read_gbq_with_no_project_id_given_should_fail(self):
-        with pytest.raises(TypeError):
-            gbq.read_gbq('SELECT 1')
-
-    def test_that_parse_data_works_properly(self):
-
-        from google.cloud.bigquery.table import Row
-        test_schema = {'fields': [
-            {'mode': 'NULLABLE', 'name': 'column_x', 'type': 'STRING'}]}
-        field_to_index = {'column_x': 0}
-        values = ('row_value',)
-        test_page = [Row(values, field_to_index)]
-
-        test_output = gbq._parse_data(test_schema, test_page)
-        correct_output = DataFrame({'column_x': ['row_value']})
-        tm.assert_frame_equal(test_output, correct_output)
-
-    def test_read_gbq_with_invalid_private_key_json_should_fail(self):
-        with pytest.raises(gbq.InvalidPrivateKeyFormat):
-            gbq.read_gbq('SELECT 1', project_id='x', private_key='y')
-
-    def test_read_gbq_with_empty_private_key_json_should_fail(self):
-        with pytest.raises(gbq.InvalidPrivateKeyFormat):
-            gbq.read_gbq('SELECT 1', project_id='x', private_key='{}')
-
-    def test_read_gbq_with_private_key_json_wrong_types_should_fail(self):
-        with pytest.raises(gbq.InvalidPrivateKeyFormat):
-            gbq.read_gbq(
-                'SELECT 1', project_id='x',
-                private_key='{ "client_email" : 1, "private_key" : True }')
-
-    def test_read_gbq_with_empty_private_key_file_should_fail(self):
-        with tm.ensure_clean() as empty_file_path:
-            with pytest.raises(gbq.InvalidPrivateKeyFormat):
-                gbq.read_gbq('SELECT 1', project_id='x',
-                             private_key=empty_file_path)
-
-    def test_read_gbq_with_corrupted_private_key_json_should_fail(self):
-        _skip_if_no_private_key_contents()
-
-        with pytest.raises(gbq.InvalidPrivateKeyFormat):
-            gbq.read_gbq(
-                'SELECT 1', project_id='x',
-                private_key=re.sub('[a-z]', '9', _get_private_key_contents()))
-
-    def test_read_gbq_with_verbose_new_pandas_warns_deprecation(self):
-        import pkg_resources
-        min_bq_version = pkg_resources.parse_version('0.29.0')
-        pandas_version = pkg_resources.parse_version('0.23.0')
-        with pytest.warns(FutureWarning), \
-                mock.patch(
-                    'pkg_resources.Distribution.parsed_version',
-                    new_callable=mock.PropertyMock) as mock_version:
-            mock_version.side_effect = [min_bq_version, pandas_version]
-            gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True)
-
-    def test_read_gbq_with_not_verbose_new_pandas_warns_deprecation(self):
-        import pkg_resources
-        min_bq_version = pkg_resources.parse_version('0.29.0')
-        pandas_version = pkg_resources.parse_version('0.23.0')
-        with pytest.warns(FutureWarning), \
-                mock.patch(
-                    'pkg_resources.Distribution.parsed_version',
-                    new_callable=mock.PropertyMock) as mock_version:
-            mock_version.side_effect = [min_bq_version, pandas_version]
-            gbq.read_gbq('SELECT 1', project_id='my-project', verbose=False)
-
-    def test_read_gbq_wo_verbose_w_new_pandas_no_warnings(self, recwarn):
-        import pkg_resources
-        min_bq_version = pkg_resources.parse_version('0.29.0')
-        pandas_version = pkg_resources.parse_version('0.23.0')
-        with mock.patch(
-                'pkg_resources.Distribution.parsed_version',
-                new_callable=mock.PropertyMock) as mock_version:
-            mock_version.side_effect = [min_bq_version, pandas_version]
-            gbq.read_gbq('SELECT 1', project_id='my-project')
-            assert len(recwarn) == 0
-
-    def test_read_gbq_with_verbose_old_pandas_no_warnings(self, recwarn):
-        import pkg_resources
-        min_bq_version = pkg_resources.parse_version('0.29.0')
-        pandas_version = pkg_resources.parse_version('0.22.0')
-        with mock.patch(
-                'pkg_resources.Distribution.parsed_version',
-                new_callable=mock.PropertyMock) as mock_version:
-            mock_version.side_effect = [min_bq_version, pandas_version]
-            gbq.read_gbq('SELECT 1', project_id='my-project', verbose=True)
-            assert len(recwarn) == 0
-
-
 def test_should_read(project, credentials):
-
     query = 'SELECT "PI" AS valid_string'
     df = gbq.read_gbq(query, project_id=project, private_key=credentials)
     tm.assert_frame_equal(df, DataFrame({'valid_string': ['PI']}))