From d690e5e5c86112c9b7dd98178627993d4eeca83d Mon Sep 17 00:00:00 2001 From: Brian Kohan Date: Wed, 10 Jun 2020 00:42:51 -0700 Subject: [PATCH 1/9] fix mypy run instruction --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0d9c33..5851629 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ We use `mypy` to run type checks on our code. To use it: ```bash -mypy django_split_settings +mypy split_settings ``` This step is mandatory during the CI. From 351197f3032566f0e8df656e16bea2d00d494b2e Mon Sep 17 00:00:00 2001 From: Brian Kohan Date: Wed, 10 Jun 2020 01:21:23 -0700 Subject: [PATCH 2/9] add unit tests for includes with alternate extensions --- tests/conftest.py | 5 +++++ tests/settings/alt_ext/__init__.py | 10 ++++++++++ tests/settings/alt_ext/include | 3 +++ tests/settings/alt_ext/include.conf | 3 +++ tests/settings/alt_ext/include.double.conf | 3 +++ tests/settings/alt_ext/optional.ext | 3 +++ tests/test_split_settings.py | 8 ++++++++ 7 files changed, 35 insertions(+) create mode 100644 tests/settings/alt_ext/__init__.py create mode 100644 tests/settings/alt_ext/include create mode 100644 tests/settings/alt_ext/include.conf create mode 100644 tests/settings/alt_ext/include.double.conf create mode 100644 tests/settings/alt_ext/optional.ext diff --git a/tests/conftest.py b/tests/conftest.py index e14b3f7..2c74f2b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,11 @@ def merged(): from tests.settings import merged as _merged # noqa: WPS433 return _merged +@pytest.fixture() +def alt_ext(): + """This fixture returns basic merged settings example.""" + from tests.settings import alt_ext as _alt_ext # noqa: WPS433 + return _alt_ext @pytest.fixture() def stacked(): diff --git a/tests/settings/alt_ext/__init__.py b/tests/settings/alt_ext/__init__.py new file mode 100644 index 0000000..8c4eab6 --- /dev/null +++ b/tests/settings/alt_ext/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + +from split_settings.tools import include, optional + +# Includes files with non-standard extensions: +include( + 'include', + '*.conf', + optional('optional.ext') +) diff --git a/tests/settings/alt_ext/include b/tests/settings/alt_ext/include new file mode 100644 index 0000000..a8feda4 --- /dev/null +++ b/tests/settings/alt_ext/include @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +NO_EXT_INCLUDED = True diff --git a/tests/settings/alt_ext/include.conf b/tests/settings/alt_ext/include.conf new file mode 100644 index 0000000..adae986 --- /dev/null +++ b/tests/settings/alt_ext/include.conf @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +DOT_CONF_INCLUDED = True diff --git a/tests/settings/alt_ext/include.double.conf b/tests/settings/alt_ext/include.double.conf new file mode 100644 index 0000000..8e31368 --- /dev/null +++ b/tests/settings/alt_ext/include.double.conf @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +DOUBLE_EXT_INCLUDED = True diff --git a/tests/settings/alt_ext/optional.ext b/tests/settings/alt_ext/optional.ext new file mode 100644 index 0000000..6cecc38 --- /dev/null +++ b/tests/settings/alt_ext/optional.ext @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +OPTIONAL_INCLUDED = True diff --git a/tests/test_split_settings.py b/tests/test_split_settings.py index 42ce703..0ed6447 100644 --- a/tests/test_split_settings.py +++ b/tests/test_split_settings.py @@ -7,6 +7,14 @@ def test_merge(merged): assert merged.STATIC_ROOT +def test_alt_ext(alt_ext): + """Test that all values from settings are present.""" + assert alt_ext.NO_EXT_INCLUDED + assert alt_ext.DOT_CONF_INCLUDED + assert alt_ext.DOUBLE_EXT_INCLUDED + assert alt_ext.OPTIONAl_INCLUDED + + def test_override(merged, monkeypatch): """This setting must be overridden in the testing.py.""" monkeypatch.setenv('DJANGO_SETTINGS_MODULE', 'tests.settings.merged') From 6a8f7b4bc0a139cadaa91a5523b88399efc5499c Mon Sep 17 00:00:00 2001 From: Brian Kohan Date: Wed, 10 Jun 2020 01:28:04 -0700 Subject: [PATCH 3/9] allow alternative extensions to .py --- split_settings/tools.py | 9 ++++++--- tests/test_split_settings.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/split_settings/tools.py b/split_settings/tools.py index 307d347..52a3af8 100644 --- a/split_settings/tools.py +++ b/split_settings/tools.py @@ -11,7 +11,9 @@ import inspect import os import sys -from importlib.util import module_from_spec, spec_from_file_location +from importlib.util import module_from_spec, spec_from_loader +from importlib.machinery import SourceFileLoader + __all__ = ('optional', 'include') # noqa: WPS410 @@ -114,8 +116,9 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 rel_path[:rel_path.rfind('.')].replace('/', '.'), ) - spec = spec_from_file_location( - module_name, included_file, + spec = spec_from_loader( + module_name, + SourceFileLoader(os.path.basename(included_file).split('.')[0], included_file), ) module = module_from_spec(spec) sys.modules[module_name] = module diff --git a/tests/test_split_settings.py b/tests/test_split_settings.py index 0ed6447..18d82e6 100644 --- a/tests/test_split_settings.py +++ b/tests/test_split_settings.py @@ -12,7 +12,7 @@ def test_alt_ext(alt_ext): assert alt_ext.NO_EXT_INCLUDED assert alt_ext.DOT_CONF_INCLUDED assert alt_ext.DOUBLE_EXT_INCLUDED - assert alt_ext.OPTIONAl_INCLUDED + assert alt_ext.OPTIONAL_INCLUDED def test_override(merged, monkeypatch): From 6d84c58ca81ac4edb09b0d7815c6a766fbac6b53 Mon Sep 17 00:00:00 2001 From: Brian Kohan Date: Wed, 10 Jun 2020 01:43:26 -0700 Subject: [PATCH 4/9] add tests for package resource includes --- tests/conftest.py | 11 ++++++++++- tests/settings/resource/__init__.py | 22 ++++++++++++++++++++++ tests/settings/resource/apps_middleware | 3 +++ tests/settings/resource/base.conf | 5 +++++ tests/settings/resource/database.conf | 3 +++ tests/settings/resource/locale.conf | 4 ++++ tests/settings/resource/logging.py | 3 +++ tests/settings/resource/static.settings | 3 +++ tests/settings/resource/templates.py | 3 +++ tests/test_split_settings.py | 12 ++++++++++++ 10 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 tests/settings/resource/__init__.py create mode 100644 tests/settings/resource/apps_middleware create mode 100644 tests/settings/resource/base.conf create mode 100644 tests/settings/resource/database.conf create mode 100644 tests/settings/resource/locale.conf create mode 100644 tests/settings/resource/logging.py create mode 100644 tests/settings/resource/static.settings create mode 100644 tests/settings/resource/templates.py diff --git a/tests/conftest.py b/tests/conftest.py index 2c74f2b..c70cbd1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,12 +42,21 @@ def merged(): from tests.settings import merged as _merged # noqa: WPS433 return _merged + @pytest.fixture() def alt_ext(): - """This fixture returns basic merged settings example.""" + """This fixture returns alt_ext settings example.""" from tests.settings import alt_ext as _alt_ext # noqa: WPS433 return _alt_ext + +@pytest.fixture() +def resource(): + """This fixture returns resource settings example.""" + from tests.settings import resource as _resource # noqa: WPS433 + return _resource + + @pytest.fixture() def stacked(): """This fixture returns stacked settings example.""" diff --git a/tests/settings/resource/__init__.py b/tests/settings/resource/__init__.py new file mode 100644 index 0000000..73b0e84 --- /dev/null +++ b/tests/settings/resource/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +from split_settings.tools import include, optional +from tests.settings import resource + +include( + # Components: + ('tests.settings.alt_ext', 'base.conf'), + ('tests.settings.alt_ext', 'locale.conf'), + ('tests.settings.alt_ext', 'apps_middleware'), + (resource, 'static.settings'), + (resource, 'templates.py'), + optional((resource, 'database.conf')), + 'logging.py', + + # Missing file: + optional((resource, 'missing_file.py')), + optional(('tests.settings.alt_ext', 'missing_file.py')), + + # Scope: + scope=globals(), # noqa: WPS421 +) diff --git a/tests/settings/resource/apps_middleware b/tests/settings/resource/apps_middleware new file mode 100644 index 0000000..ea0c33b --- /dev/null +++ b/tests/settings/resource/apps_middleware @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +APPS_MIDDLEWARE_INCLUDED = True diff --git a/tests/settings/resource/base.conf b/tests/settings/resource/base.conf new file mode 100644 index 0000000..583edec --- /dev/null +++ b/tests/settings/resource/base.conf @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# Django settings for example project. + +BASE_INCLUDED = True +OVERRIDDEN = True diff --git a/tests/settings/resource/database.conf b/tests/settings/resource/database.conf new file mode 100644 index 0000000..7683e1a --- /dev/null +++ b/tests/settings/resource/database.conf @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +DATABASE_INCLUDED = True diff --git a/tests/settings/resource/locale.conf b/tests/settings/resource/locale.conf new file mode 100644 index 0000000..693617f --- /dev/null +++ b/tests/settings/resource/locale.conf @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +LOCALE_INCLUDED = True +OVERRIDDEN = False diff --git a/tests/settings/resource/logging.py b/tests/settings/resource/logging.py new file mode 100644 index 0000000..cbaf432 --- /dev/null +++ b/tests/settings/resource/logging.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +LOGGING_INCLUDED = True diff --git a/tests/settings/resource/static.settings b/tests/settings/resource/static.settings new file mode 100644 index 0000000..8728818 --- /dev/null +++ b/tests/settings/resource/static.settings @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +STATIC_SETTINGS_INCLUDED = True diff --git a/tests/settings/resource/templates.py b/tests/settings/resource/templates.py new file mode 100644 index 0000000..5d02f8d --- /dev/null +++ b/tests/settings/resource/templates.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +TEMPLATES_INCLUDED = True diff --git a/tests/test_split_settings.py b/tests/test_split_settings.py index 18d82e6..c0b1f47 100644 --- a/tests/test_split_settings.py +++ b/tests/test_split_settings.py @@ -15,6 +15,18 @@ def test_alt_ext(alt_ext): assert alt_ext.OPTIONAL_INCLUDED +def test_resource(resource): + """Test that all values from settings are present.""" + assert resource.APPS_MIDDLEWARE_INCLUDED + assert resource.BASE_INCLUDED + assert resource.DATABASE_INCLUDED + assert resource.LOCALE_INCLUDED + assert resource.LOGGING_INCLUDED + assert resource.STATIC_SETTINGS_INCLUDED + assert resource.TEMPLATES_INCLUDED + assert not resource.OVERRIDDEN + + def test_override(merged, monkeypatch): """This setting must be overridden in the testing.py.""" monkeypatch.setenv('DJANGO_SETTINGS_MODULE', 'tests.settings.merged') From a20531a50e07ed4af02d5836fdcb100eb3fe7848 Mon Sep 17 00:00:00 2001 From: Brian Kohan Date: Wed, 10 Jun 2020 02:18:04 -0700 Subject: [PATCH 5/9] address and eliminate linter complaints --- setup.cfg | 2 +- split_settings/tools.py | 8 +++++--- tests/conftest.py | 2 +- tests/settings/alt_ext/__init__.py | 2 +- tests/settings/resource/__init__.py | 8 ++++---- tests/settings/resource/base.conf | 2 +- tests/settings/resource/locale.conf | 2 +- tests/test_split_settings.py | 4 ++-- 8 files changed, 16 insertions(+), 14 deletions(-) diff --git a/setup.cfg b/setup.cfg index e3b6089..aa35bf5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,7 @@ per-file-ignores = # Tests contain examples with logic in init files: tests/*/__init__.py: WPS412 # There are multiple fixtures, `assert`s, and subprocesses in tests: - tests/*.py: S101, S105, S404, S603, S607 + tests/*.py: S101, S105, S404, S603, S607, WPS202 [isort] diff --git a/split_settings/tools.py b/split_settings/tools.py index 52a3af8..1abf428 100644 --- a/split_settings/tools.py +++ b/split_settings/tools.py @@ -11,9 +11,8 @@ import inspect import os import sys -from importlib.util import module_from_spec, spec_from_loader from importlib.machinery import SourceFileLoader - +from importlib.util import module_from_spec, spec_from_loader __all__ = ('optional', 'include') # noqa: WPS410 @@ -118,7 +117,10 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 spec = spec_from_loader( module_name, - SourceFileLoader(os.path.basename(included_file).split('.')[0], included_file), + SourceFileLoader( + os.path.basename(included_file).split('.')[0], + included_file, + ), ) module = module_from_spec(spec) sys.modules[module_name] = module diff --git a/tests/conftest.py b/tests/conftest.py index c70cbd1..57c6cfd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import pytest -class Scope(dict): # noqa: WPS600 +class Scope(dict): # noqa: WPS600, WPS202 """This class emulates `globals()`, but does not share state in tests.""" def __init__(self, *args, **kwargs): diff --git a/tests/settings/alt_ext/__init__.py b/tests/settings/alt_ext/__init__.py index 8c4eab6..3b43cfb 100644 --- a/tests/settings/alt_ext/__init__.py +++ b/tests/settings/alt_ext/__init__.py @@ -6,5 +6,5 @@ include( 'include', '*.conf', - optional('optional.ext') + optional('optional.ext'), ) diff --git a/tests/settings/resource/__init__.py b/tests/settings/resource/__init__.py index 73b0e84..1c0c89c 100644 --- a/tests/settings/resource/__init__.py +++ b/tests/settings/resource/__init__.py @@ -5,9 +5,9 @@ include( # Components: - ('tests.settings.alt_ext', 'base.conf'), - ('tests.settings.alt_ext', 'locale.conf'), - ('tests.settings.alt_ext', 'apps_middleware'), + ('tests.settings.resource', 'base.conf'), + ('tests.settings.resource', 'locale.conf'), + ('tests.settings.resource', 'apps_middleware'), (resource, 'static.settings'), (resource, 'templates.py'), optional((resource, 'database.conf')), @@ -15,7 +15,7 @@ # Missing file: optional((resource, 'missing_file.py')), - optional(('tests.settings.alt_ext', 'missing_file.py')), + optional(('tests.settings.alt_ext', 'missing_file.conf')), # Scope: scope=globals(), # noqa: WPS421 diff --git a/tests/settings/resource/base.conf b/tests/settings/resource/base.conf index 583edec..217be50 100644 --- a/tests/settings/resource/base.conf +++ b/tests/settings/resource/base.conf @@ -2,4 +2,4 @@ # Django settings for example project. BASE_INCLUDED = True -OVERRIDDEN = True +OVERRIDE_WORKS = False diff --git a/tests/settings/resource/locale.conf b/tests/settings/resource/locale.conf index 693617f..3615b7b 100644 --- a/tests/settings/resource/locale.conf +++ b/tests/settings/resource/locale.conf @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- LOCALE_INCLUDED = True -OVERRIDDEN = False +OVERRIDE_WORKS = True diff --git a/tests/test_split_settings.py b/tests/test_split_settings.py index c0b1f47..510fd2e 100644 --- a/tests/test_split_settings.py +++ b/tests/test_split_settings.py @@ -15,7 +15,7 @@ def test_alt_ext(alt_ext): assert alt_ext.OPTIONAL_INCLUDED -def test_resource(resource): +def test_resource(resource): # noqa: WPS218 """Test that all values from settings are present.""" assert resource.APPS_MIDDLEWARE_INCLUDED assert resource.BASE_INCLUDED @@ -24,7 +24,7 @@ def test_resource(resource): assert resource.LOGGING_INCLUDED assert resource.STATIC_SETTINGS_INCLUDED assert resource.TEMPLATES_INCLUDED - assert not resource.OVERRIDDEN + assert resource.OVERRIDE_WORKS def test_override(merged, monkeypatch): From 66461c4096bfe0e6a2987021e4275eee5dac6152 Mon Sep 17 00:00:00 2001 From: Brian Kohan Date: Thu, 11 Jun 2020 01:37:44 -0700 Subject: [PATCH 6/9] add resource include functionality, fix linting and type errors, update tests --- setup.cfg | 2 +- split_settings/tools.py | 79 ++++++++++++++++++- tests/conftest.py | 6 +- tests/settings/resource/__init__.py | 22 ------ tests/settings/resource/base.conf | 5 -- tests/settings/resources/__init__.py | 22 ++++++ .../{resource => resources}/apps_middleware | 0 tests/settings/resources/base.conf | 18 +++++ .../{resource => resources}/database.conf | 0 .../{resource => resources}/locale.conf | 0 .../{resource => resources}/logging.py | 0 .../{resource => resources}/static.settings | 0 .../{resource => resources}/templates.py | 0 tests/test_split_settings.py | 20 ++--- 14 files changed, 132 insertions(+), 42 deletions(-) delete mode 100644 tests/settings/resource/__init__.py delete mode 100644 tests/settings/resource/base.conf create mode 100644 tests/settings/resources/__init__.py rename tests/settings/{resource => resources}/apps_middleware (100%) create mode 100644 tests/settings/resources/base.conf rename tests/settings/{resource => resources}/database.conf (100%) rename tests/settings/{resource => resources}/locale.conf (100%) rename tests/settings/{resource => resources}/logging.py (100%) rename tests/settings/{resource => resources}/static.settings (100%) rename tests/settings/{resource => resources}/templates.py (100%) diff --git a/setup.cfg b/setup.cfg index aa35bf5..049932e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,7 @@ per-file-ignores = # Our module is complex, there's nothing we can do: split_settings/tools.py: WPS232 # Tests contain examples with logic in init files: - tests/*/__init__.py: WPS412 + tests/*/__init__.py: WPS412, WPS226 # There are multiple fixtures, `assert`s, and subprocesses in tests: tests/*.py: S101, S105, S404, S603, S607, WPS202 diff --git a/split_settings/tools.py b/split_settings/tools.py index 1abf428..674d771 100644 --- a/split_settings/tools.py +++ b/split_settings/tools.py @@ -11,10 +11,19 @@ import inspect import os import sys +import types +from importlib import import_module from importlib.machinery import SourceFileLoader from importlib.util import module_from_spec, spec_from_loader +from typing import Union -__all__ = ('optional', 'include') # noqa: WPS410 +try: + from importlib import resources as pkg_resources # noqa: WPS433 +except ImportError: + # Use backport to PY<3.7 `importlib_resources`. + import importlib_resources as pkg_resources # type: ignore # noqa: WPS433, WPS440, E501 + +__all__ = ('optional', 'include', 'resource') # noqa: WPS410 #: Special magic attribute that is sometimes set by `uwsgi` / `gunicord`. _INCLUDED_FILE = '__included_file__' @@ -45,6 +54,61 @@ class _Optional(str): # noqa: WPS600 """ +def resource(package: Union[str, types.ModuleType], filename: str) -> str: + """ + Include a packaged resource as a settings file. + + Args: + package: the package as either an imported module, or a string + filename: the filename of the resource to include. + + Returns: + New instance of :class:`_Resource`. + + """ + return _Resource(package, filename) + + +class _Resource(str): # noqa: WPS600 + """ + Wrap an included package resource as a str. + + Resource includes may also be wrapped as Optional and record if the + package was found or not. + """ + + module_not_found = False + + def __new__( + cls, + package: Union[str, types.ModuleType], + filename: str, + ) -> '_Resource': + + # the type ignores workaround a known mypy bug + # https://github.com/python/mypy/issues/1021 + + if isinstance(package, str): + try: + package = import_module(package) + except ModuleNotFoundError: + rsrc = super().__new__(cls, package) # type: ignore + rsrc.module_not_found = True + return rsrc + try: + with pkg_resources.path(package, filename) as pth: + return super().__new__(cls, str(pth)) # type: ignore + except FileNotFoundError: + # even if it doesnt exist - return what the path would be + return super().__new__( # type: ignore + cls, + os.path.join( + os.path.dirname(package.__file__), # noqa: WPS609 + filename, + ), + ) + + def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 """ Used for including Django project settings from multiple files. @@ -53,11 +117,13 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 .. code:: python - from split_settings.tools import optional, include + from split_settings.tools import optional, include, resource + from . import components include( 'components/base.py', 'components/database.py', + resource(components, settings.conf), # package resource optional('local_settings.py'), scope=globals(), # optional scope @@ -69,6 +135,7 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 Raises: IOError: if a required settings file is not found. + ModuleNotFoundError: if a required resource package is not found. """ # we are getting globals() from previous frame @@ -88,6 +155,14 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 saved_included_file = scope.get(_INCLUDED_FILE) pattern = os.path.join(conf_path, conf_file) + # check if this include is a resource with an unfound module + # before we glob it - avoid the small chance there is a file + # with the same name as the package in cwd + if isinstance(conf_file, _Resource) and conf_file.module_not_found: + raise ModuleNotFoundError( + 'No module named {0}'.format(conf_file), + ) + # find files per pattern, raise an error if not found # (unless file is optional) files_to_include = glob.glob(pattern) diff --git a/tests/conftest.py b/tests/conftest.py index 57c6cfd..74fae97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -51,10 +51,10 @@ def alt_ext(): @pytest.fixture() -def resource(): +def resources(): """This fixture returns resource settings example.""" - from tests.settings import resource as _resource # noqa: WPS433 - return _resource + from tests.settings import resources as _resources # noqa: WPS433 + return _resources @pytest.fixture() diff --git a/tests/settings/resource/__init__.py b/tests/settings/resource/__init__.py deleted file mode 100644 index 1c0c89c..0000000 --- a/tests/settings/resource/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- - -from split_settings.tools import include, optional -from tests.settings import resource - -include( - # Components: - ('tests.settings.resource', 'base.conf'), - ('tests.settings.resource', 'locale.conf'), - ('tests.settings.resource', 'apps_middleware'), - (resource, 'static.settings'), - (resource, 'templates.py'), - optional((resource, 'database.conf')), - 'logging.py', - - # Missing file: - optional((resource, 'missing_file.py')), - optional(('tests.settings.alt_ext', 'missing_file.conf')), - - # Scope: - scope=globals(), # noqa: WPS421 -) diff --git a/tests/settings/resource/base.conf b/tests/settings/resource/base.conf deleted file mode 100644 index 217be50..0000000 --- a/tests/settings/resource/base.conf +++ /dev/null @@ -1,5 +0,0 @@ -# -*- coding: utf-8 -*- -# Django settings for example project. - -BASE_INCLUDED = True -OVERRIDE_WORKS = False diff --git a/tests/settings/resources/__init__.py b/tests/settings/resources/__init__.py new file mode 100644 index 0000000..a2bcb7b --- /dev/null +++ b/tests/settings/resources/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +from split_settings.tools import include, optional, resource +from tests.settings import resources + +include( + # Components: + resource('tests.settings.resources', 'base.conf'), + resource('tests.settings.resources', 'locale.conf'), + resource('tests.settings.resources', 'apps_middleware'), + resource(resources, 'static.settings'), + resource(resources, 'templates.py'), + optional(resource(resources, 'database.conf')), + 'logging.py', + + # Missing file: + optional(resource(resources, 'missing_file.py')), + optional(resource('tests.settings.resources', 'missing_file.conf')), + + # Scope: + scope=globals(), # noqa: WPS421 +) diff --git a/tests/settings/resource/apps_middleware b/tests/settings/resources/apps_middleware similarity index 100% rename from tests/settings/resource/apps_middleware rename to tests/settings/resources/apps_middleware diff --git a/tests/settings/resources/base.conf b/tests/settings/resources/base.conf new file mode 100644 index 0000000..fd18fef --- /dev/null +++ b/tests/settings/resources/base.conf @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from split_settings.tools import include, resource + +BASE_INCLUDED = True +OVERRIDE_WORKS = False + +UNFOUND_RESOURCE_IS_IOERROR = False +UNFOUND_PACKAGE_IS_MODULE_ERROR = False +try: + include(resource('tests.settings.resources', 'does_not_exist.conf')) +except IOError: + UNFOUND_RESOURCE_IS_IOERROR = True + +try: + include(resource('does.not.exist', 'database.conf')) +except ModuleNotFoundError: + UNFOUND_PACKAGE_IS_MODULE_ERROR = True diff --git a/tests/settings/resource/database.conf b/tests/settings/resources/database.conf similarity index 100% rename from tests/settings/resource/database.conf rename to tests/settings/resources/database.conf diff --git a/tests/settings/resource/locale.conf b/tests/settings/resources/locale.conf similarity index 100% rename from tests/settings/resource/locale.conf rename to tests/settings/resources/locale.conf diff --git a/tests/settings/resource/logging.py b/tests/settings/resources/logging.py similarity index 100% rename from tests/settings/resource/logging.py rename to tests/settings/resources/logging.py diff --git a/tests/settings/resource/static.settings b/tests/settings/resources/static.settings similarity index 100% rename from tests/settings/resource/static.settings rename to tests/settings/resources/static.settings diff --git a/tests/settings/resource/templates.py b/tests/settings/resources/templates.py similarity index 100% rename from tests/settings/resource/templates.py rename to tests/settings/resources/templates.py diff --git a/tests/test_split_settings.py b/tests/test_split_settings.py index 510fd2e..561bcfa 100644 --- a/tests/test_split_settings.py +++ b/tests/test_split_settings.py @@ -15,16 +15,18 @@ def test_alt_ext(alt_ext): assert alt_ext.OPTIONAL_INCLUDED -def test_resource(resource): # noqa: WPS218 +def test_resources(resources): # noqa: WPS218 """Test that all values from settings are present.""" - assert resource.APPS_MIDDLEWARE_INCLUDED - assert resource.BASE_INCLUDED - assert resource.DATABASE_INCLUDED - assert resource.LOCALE_INCLUDED - assert resource.LOGGING_INCLUDED - assert resource.STATIC_SETTINGS_INCLUDED - assert resource.TEMPLATES_INCLUDED - assert resource.OVERRIDE_WORKS + assert resources.APPS_MIDDLEWARE_INCLUDED + assert resources.BASE_INCLUDED + assert resources.DATABASE_INCLUDED + assert resources.LOCALE_INCLUDED + assert resources.LOGGING_INCLUDED + assert resources.STATIC_SETTINGS_INCLUDED + assert resources.TEMPLATES_INCLUDED + assert resources.OVERRIDE_WORKS + assert resources.UNFOUND_RESOURCE_IS_IOERROR + assert resources.UNFOUND_PACKAGE_IS_MODULE_ERROR def test_override(merged, monkeypatch): From 3d590866d9228a633b904b994c6c090629ec77fd Mon Sep 17 00:00:00 2001 From: Brian Kohan Date: Fri, 12 Jun 2020 16:14:48 -0700 Subject: [PATCH 7/9] - Allow resources to be included from multiple subdirectories - Handle temp file contexts appropriately - Update README.md - Update tests - Fix linting and typing issues --- README.md | 15 ++++ poetry.lock | 32 ++++++- pyproject.toml | 1 + split_settings/tools.py | 90 ++++++++++++------- tests/settings/resources/__init__.py | 5 ++ tests/settings/resources/base.conf | 14 --- tests/settings/resources/error.conf | 27 ++++++ .../resources/multiple/subdirs/file.conf | 3 + tests/test_import.py | 1 + tests/test_split_settings.py | 8 ++ 10 files changed, 146 insertions(+), 50 deletions(-) create mode 100644 tests/settings/resources/error.conf create mode 100644 tests/settings/resources/multiple/subdirs/file.conf diff --git a/README.md b/README.md index b365272..6938510 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,21 @@ previous files. We also made a in-depth [tutorial](https://sobolevn.me/2017/04/managing-djangos-settings). +## Package Resources + +You may also include package resources and use alternate extensions: +```python +from mypackge import settings +include(resource(settings, 'base.conf')) +include(optional(resource(settings, 'local.conf'))) +``` +Resources may be also be included by passing the module as a string: + +```python +include(resource('mypackage.settings', 'base.conf')) +``` +Note that resources included from archived packages (i.e. zip files), will have a temporary +file created, which will be deleted after the settings file has been compiled. ## Tips and tricks diff --git a/poetry.lock b/poetry.lock index f706447..4a0c48e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -90,7 +90,7 @@ version = "7.1.1" [[package]] category = "dev" description = "Cross-platform colored terminal text." -marker = "sys_platform == \"win32\" or platform_system == \"Windows\"" +marker = "platform_system == \"Windows\" or sys_platform == \"win32\"" name = "colorama" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -436,7 +436,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "1.2.0" [[package]] -category = "dev" +category = "main" description = "Read metadata from Python packages" marker = "python_version < \"3.8\"" name = "importlib-metadata" @@ -451,6 +451,26 @@ zipp = ">=0.5" docs = ["sphinx", "rst.linker"] testing = ["packaging", "importlib-resources"] +[[package]] +category = "main" +description = "Read resources from Python packages" +name = "importlib-resources" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +version = "2.0.0" + +[package.dependencies] +[package.dependencies.importlib-metadata] +python = "<3.8" +version = "*" + +[package.dependencies.zipp] +python = "<3.8" +version = ">=0.4" + +[package.extras] +docs = ["sphinx", "rst.linker", "jaraco.packaging"] + [[package]] category = "dev" description = "A Python utility / library to sort Python imports." @@ -1138,7 +1158,7 @@ python = "<3.8" version = "*" [[package]] -category = "dev" +category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" marker = "python_version < \"3.8\"" name = "zipp" @@ -1151,7 +1171,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "c6d306976f2e1114d6bc3d743d052f73b2ce76d4748d3462d2eb3b1867641d38" +content-hash = "87f3d446bcd2e9998483700a1ba347bffab8d3e6b4a5ca0d7ac3d40162d907f8" python-versions = "^3.6" [metadata.files] @@ -1339,6 +1359,10 @@ importlib-metadata = [ {file = "importlib_metadata-1.6.0-py2.py3-none-any.whl", hash = "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f"}, {file = "importlib_metadata-1.6.0.tar.gz", hash = "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"}, ] +importlib-resources = [ + {file = "importlib_resources-2.0.0-py2.py3-none-any.whl", hash = "sha256:a86462cf34a6d391d1d5d598a5e2f5aac9fc00b265d40542e1196328f015d1f6"}, + {file = "importlib_resources-2.0.0.tar.gz", hash = "sha256:7f6aae2ed252ba10f5d1af5676b0e35f3b3eb7d3cc103b8365cc92aec0c79258"}, +] isort = [ {file = "isort-4.3.21-py2.py3-none-any.whl", hash = "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"}, {file = "isort-4.3.21.tar.gz", hash = "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1"}, diff --git a/pyproject.toml b/pyproject.toml index dd97dc9..9d76ace 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.6" +importlib-resources = "^2.0.0" [tool.poetry.dev-dependencies] django = "^2.2" diff --git a/split_settings/tools.py b/split_settings/tools.py index 674d771..bdee0ea 100644 --- a/split_settings/tools.py +++ b/split_settings/tools.py @@ -7,27 +7,38 @@ settings files. """ +import contextlib import glob import inspect import os import sys import types -from importlib import import_module from importlib.machinery import SourceFileLoader from importlib.util import module_from_spec, spec_from_loader -from typing import Union +from typing import List, Union try: - from importlib import resources as pkg_resources # noqa: WPS433 + from importlib.resources import ( # type: ignore # noqa: WPS433 + files, + as_file, + ) except ImportError: - # Use backport to PY<3.7 `importlib_resources`. - import importlib_resources as pkg_resources # type: ignore # noqa: WPS433, WPS440, E501 + # Use backport to PY<3.9 `importlib_resources`. + # importlib_resources is included in python stdlib starting at 3.7 but + # the files function is not available until python 3.9 + from importlib_resources import files, as_file # noqa: WPS433, WPS440 __all__ = ('optional', 'include', 'resource') # noqa: WPS410 #: Special magic attribute that is sometimes set by `uwsgi` / `gunicord`. _INCLUDED_FILE = '__included_file__' +# If resources are located in archives, importlib will create temporary +# files to access them contained within contexts, we track the contexts +# here as opposed to the _Resource.__del__ method because invocation of +# that method is non-deterministic +__resource_file_contexts__: List[contextlib.ExitStack] = [] + def optional(filename: str) -> str: """ @@ -78,6 +89,8 @@ class _Resource(str): # noqa: WPS600 """ module_not_found = False + package: str + filename: str def __new__( cls, @@ -85,28 +98,33 @@ def __new__( filename: str, ) -> '_Resource': - # the type ignores workaround a known mypy bug + # the type ignores workaround a known mypy issue # https://github.com/python/mypy/issues/1021 - - if isinstance(package, str): - try: - package = import_module(package) - except ModuleNotFoundError: - rsrc = super().__new__(cls, package) # type: ignore - rsrc.module_not_found = True - return rsrc try: - with pkg_resources.path(package, filename) as pth: - return super().__new__(cls, str(pth)) # type: ignore - except FileNotFoundError: - # even if it doesnt exist - return what the path would be - return super().__new__( # type: ignore - cls, - os.path.join( - os.path.dirname(package.__file__), # noqa: WPS609 - filename, - ), - ) + ref = files(package) / filename + except ModuleNotFoundError: + rsrc = super().__new__(cls, '') # type: ignore + rsrc.module_not_found = True + return rsrc + + file_manager = contextlib.ExitStack() + __resource_file_contexts__.append(file_manager) + return super().__new__( # type: ignore + cls, + file_manager.enter_context(as_file(ref)), + ) + + def __init__( + self, + package: Union[str, types.ModuleType], + filename: str, + ) -> None: + super().__init__() + if isinstance(package, types.ModuleType): + self.package = package.__name__ + else: + self.package = package + self.filename = filename def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 @@ -153,15 +171,18 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 for conf_file in args: saved_included_file = scope.get(_INCLUDED_FILE) - pattern = os.path.join(conf_path, conf_file) + pattern = conf_file + # if a resource was not found the path will resolve to empty str here + if pattern: + pattern = os.path.join(conf_path, conf_file) # check if this include is a resource with an unfound module - # before we glob it - avoid the small chance there is a file - # with the same name as the package in cwd - if isinstance(conf_file, _Resource) and conf_file.module_not_found: - raise ModuleNotFoundError( - 'No module named {0}'.format(conf_file), - ) + # and issue a more specific exception + if isinstance(conf_file, _Resource): + if conf_file.module_not_found: + raise ModuleNotFoundError( + 'No module named {0}'.format(conf_file.package), + ) # find files per pattern, raise an error if not found # (unless file is optional) @@ -203,3 +224,8 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 scope[_INCLUDED_FILE] = saved_included_file elif _INCLUDED_FILE in scope: scope.pop(_INCLUDED_FILE) + + # close the contexts of any temporary files created to access + # resource contents thereby deleting them + for ctx in __resource_file_contexts__: + ctx.close() diff --git a/tests/settings/resources/__init__.py b/tests/settings/resources/__init__.py index a2bcb7b..b5dcbe3 100644 --- a/tests/settings/resources/__init__.py +++ b/tests/settings/resources/__init__.py @@ -10,12 +10,17 @@ resource('tests.settings.resources', 'apps_middleware'), resource(resources, 'static.settings'), resource(resources, 'templates.py'), + resource(resources, 'multiple/subdirs/file.conf'), optional(resource(resources, 'database.conf')), 'logging.py', # Missing file: optional(resource(resources, 'missing_file.py')), optional(resource('tests.settings.resources', 'missing_file.conf')), + resource('tests.settings.resources', 'error.conf'), + + # Missing module + optional(resource('module.does.not.exist', 'settings.conf')), # Scope: scope=globals(), # noqa: WPS421 diff --git a/tests/settings/resources/base.conf b/tests/settings/resources/base.conf index fd18fef..4ce8b13 100644 --- a/tests/settings/resources/base.conf +++ b/tests/settings/resources/base.conf @@ -1,18 +1,4 @@ # -*- coding: utf-8 -*- -from split_settings.tools import include, resource - BASE_INCLUDED = True OVERRIDE_WORKS = False - -UNFOUND_RESOURCE_IS_IOERROR = False -UNFOUND_PACKAGE_IS_MODULE_ERROR = False -try: - include(resource('tests.settings.resources', 'does_not_exist.conf')) -except IOError: - UNFOUND_RESOURCE_IS_IOERROR = True - -try: - include(resource('does.not.exist', 'database.conf')) -except ModuleNotFoundError: - UNFOUND_PACKAGE_IS_MODULE_ERROR = True diff --git a/tests/settings/resources/error.conf b/tests/settings/resources/error.conf new file mode 100644 index 0000000..ad5a7d1 --- /dev/null +++ b/tests/settings/resources/error.conf @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from split_settings.tools import include, resource + +UNFOUND_RESOURCE_IS_IOERROR = False +UNFOUND_RESOURCE_ERR_STR = '' +UNFOUND_PACKAGE_IS_MODULE_ERROR = False +UNFOUND_PACKAGE_ERR_STR = '' +try: + include(resource('tests.settings.resources', 'does_not_exist.conf')) +except IOError as ioe: + UNFOUND_RESOURCE_ERR_STR = str(ioe) + UNFOUND_RESOURCE_IS_IOERROR = True + +try: + include(resource('does.not.exist', 'database.conf')) +except ModuleNotFoundError as mnfe: + UNFOUND_PACKAGE_ERR_STR = str(mnfe) + UNFOUND_PACKAGE_IS_MODULE_ERROR = True + +try: + include(resource('tests.settings.resources', 'multiple/subdirs/does_not_exist.conf')) +except IOError: + UNFOUND_RESOURCE_IS_IOERROR &= True +except: + UNFOUND_RESOURCE_IS_IOERROR = False + diff --git a/tests/settings/resources/multiple/subdirs/file.conf b/tests/settings/resources/multiple/subdirs/file.conf new file mode 100644 index 0000000..76f00f1 --- /dev/null +++ b/tests/settings/resources/multiple/subdirs/file.conf @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +MULTIPLE_SUBDIR_FILE_INCLUDED = True diff --git a/tests/test_import.py b/tests/test_import.py index feded79..9229459 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -7,6 +7,7 @@ def test_wildcard_import(): """Imports all from all modules.""" assert 'optional' in __all__ assert 'include' in __all__ + assert 'resource' in __all__ def test_class_import(merged): diff --git a/tests/test_split_settings.py b/tests/test_split_settings.py index 561bcfa..fdbec15 100644 --- a/tests/test_split_settings.py +++ b/tests/test_split_settings.py @@ -25,8 +25,16 @@ def test_resources(resources): # noqa: WPS218 assert resources.STATIC_SETTINGS_INCLUDED assert resources.TEMPLATES_INCLUDED assert resources.OVERRIDE_WORKS + assert resources.MULTIPLE_SUBDIR_FILE_INCLUDED assert resources.UNFOUND_RESOURCE_IS_IOERROR assert resources.UNFOUND_PACKAGE_IS_MODULE_ERROR + assert resources.UNFOUND_RESOURCE_ERR_STR.startswith( + 'No such file', + ) + assert resources.UNFOUND_RESOURCE_ERR_STR.endswith( + 'does_not_exist.conf', + ) + assert resources.UNFOUND_PACKAGE_ERR_STR == 'No module named does.not.exist' def test_override(merged, monkeypatch): From abfa05312a4e4d443ceb916039066669be9dc786 Mon Sep 17 00:00:00 2001 From: Brian Kohan Date: Fri, 12 Jun 2020 16:25:41 -0700 Subject: [PATCH 8/9] add change log --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c84509..a9c7026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ We follow Semantic Version. +## 1.1.0 + +Improvements: + +- Packaged resources may now be included +- Alternate file extensions are allowed ## 1.0.1 From fe363d729ba8f7be150761e5534cda14ff715d28 Mon Sep 17 00:00:00 2001 From: Brian Kohan Date: Sat, 13 Jun 2020 16:48:38 -0700 Subject: [PATCH 9/9] fix relative import in doc string --- split_settings/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/split_settings/tools.py b/split_settings/tools.py index bdee0ea..7766ee4 100644 --- a/split_settings/tools.py +++ b/split_settings/tools.py @@ -136,7 +136,7 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 .. code:: python from split_settings.tools import optional, include, resource - from . import components + import components include( 'components/base.py',