diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c84509..a9c7026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ We follow Semantic Version. +## 1.1.0 + +Improvements: + +- Packaged resources may now be included +- Alternate file extensions are allowed ## 1.0.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0d9c33..5851629 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ We use `mypy` to run type checks on our code. To use it: ```bash -mypy django_split_settings +mypy split_settings ``` This step is mandatory during the CI. diff --git a/README.md b/README.md index b365272..6938510 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,21 @@ previous files. We also made a in-depth [tutorial](https://sobolevn.me/2017/04/managing-djangos-settings). +## Package Resources + +You may also include package resources and use alternate extensions: +```python +from mypackge import settings +include(resource(settings, 'base.conf')) +include(optional(resource(settings, 'local.conf'))) +``` +Resources may be also be included by passing the module as a string: + +```python +include(resource('mypackage.settings', 'base.conf')) +``` +Note that resources included from archived packages (i.e. zip files), will have a temporary +file created, which will be deleted after the settings file has been compiled. ## Tips and tricks diff --git a/poetry.lock b/poetry.lock index f706447..4a0c48e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -90,7 +90,7 @@ version = "7.1.1" [[package]] category = "dev" description = "Cross-platform colored terminal text." -marker = "sys_platform == \"win32\" or platform_system == \"Windows\"" +marker = "platform_system == \"Windows\" or sys_platform == \"win32\"" name = "colorama" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" @@ -436,7 +436,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "1.2.0" [[package]] -category = "dev" +category = "main" description = "Read metadata from Python packages" marker = "python_version < \"3.8\"" name = "importlib-metadata" @@ -451,6 +451,26 @@ zipp = ">=0.5" docs = ["sphinx", "rst.linker"] testing = ["packaging", "importlib-resources"] +[[package]] +category = "main" +description = "Read resources from Python packages" +name = "importlib-resources" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +version = "2.0.0" + +[package.dependencies] +[package.dependencies.importlib-metadata] +python = "<3.8" +version = "*" + +[package.dependencies.zipp] +python = "<3.8" +version = ">=0.4" + +[package.extras] +docs = ["sphinx", "rst.linker", "jaraco.packaging"] + [[package]] category = "dev" description = "A Python utility / library to sort Python imports." @@ -1138,7 +1158,7 @@ python = "<3.8" version = "*" [[package]] -category = "dev" +category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" marker = "python_version < \"3.8\"" name = "zipp" @@ -1151,7 +1171,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["jaraco.itertools", "func-timeout"] [metadata] -content-hash = "c6d306976f2e1114d6bc3d743d052f73b2ce76d4748d3462d2eb3b1867641d38" +content-hash = "87f3d446bcd2e9998483700a1ba347bffab8d3e6b4a5ca0d7ac3d40162d907f8" python-versions = "^3.6" [metadata.files] @@ -1339,6 +1359,10 @@ importlib-metadata = [ {file = "importlib_metadata-1.6.0-py2.py3-none-any.whl", hash = "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f"}, {file = "importlib_metadata-1.6.0.tar.gz", hash = "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e"}, ] +importlib-resources = [ + {file = "importlib_resources-2.0.0-py2.py3-none-any.whl", hash = "sha256:a86462cf34a6d391d1d5d598a5e2f5aac9fc00b265d40542e1196328f015d1f6"}, + {file = "importlib_resources-2.0.0.tar.gz", hash = "sha256:7f6aae2ed252ba10f5d1af5676b0e35f3b3eb7d3cc103b8365cc92aec0c79258"}, +] isort = [ {file = "isort-4.3.21-py2.py3-none-any.whl", hash = "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"}, {file = "isort-4.3.21.tar.gz", hash = "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1"}, diff --git a/pyproject.toml b/pyproject.toml index dd97dc9..9d76ace 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.6" +importlib-resources = "^2.0.0" [tool.poetry.dev-dependencies] django = "^2.2" diff --git a/setup.cfg b/setup.cfg index e3b6089..049932e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,9 +29,9 @@ per-file-ignores = # Our module is complex, there's nothing we can do: split_settings/tools.py: WPS232 # Tests contain examples with logic in init files: - tests/*/__init__.py: WPS412 + tests/*/__init__.py: WPS412, WPS226 # There are multiple fixtures, `assert`s, and subprocesses in tests: - tests/*.py: S101, S105, S404, S603, S607 + tests/*.py: S101, S105, S404, S603, S607, WPS202 [isort] diff --git a/split_settings/tools.py b/split_settings/tools.py index 307d347..7766ee4 100644 --- a/split_settings/tools.py +++ b/split_settings/tools.py @@ -7,17 +7,38 @@ settings files. """ +import contextlib import glob import inspect import os import sys -from importlib.util import module_from_spec, spec_from_file_location +import types +from importlib.machinery import SourceFileLoader +from importlib.util import module_from_spec, spec_from_loader +from typing import List, Union + +try: + from importlib.resources import ( # type: ignore # noqa: WPS433 + files, + as_file, + ) +except ImportError: + # Use backport to PY<3.9 `importlib_resources`. + # importlib_resources is included in python stdlib starting at 3.7 but + # the files function is not available until python 3.9 + from importlib_resources import files, as_file # noqa: WPS433, WPS440 -__all__ = ('optional', 'include') # noqa: WPS410 +__all__ = ('optional', 'include', 'resource') # noqa: WPS410 #: Special magic attribute that is sometimes set by `uwsgi` / `gunicord`. _INCLUDED_FILE = '__included_file__' +# If resources are located in archives, importlib will create temporary +# files to access them contained within contexts, we track the contexts +# here as opposed to the _Resource.__del__ method because invocation of +# that method is non-deterministic +__resource_file_contexts__: List[contextlib.ExitStack] = [] + def optional(filename: str) -> str: """ @@ -44,6 +65,68 @@ class _Optional(str): # noqa: WPS600 """ +def resource(package: Union[str, types.ModuleType], filename: str) -> str: + """ + Include a packaged resource as a settings file. + + Args: + package: the package as either an imported module, or a string + filename: the filename of the resource to include. + + Returns: + New instance of :class:`_Resource`. + + """ + return _Resource(package, filename) + + +class _Resource(str): # noqa: WPS600 + """ + Wrap an included package resource as a str. + + Resource includes may also be wrapped as Optional and record if the + package was found or not. + """ + + module_not_found = False + package: str + filename: str + + def __new__( + cls, + package: Union[str, types.ModuleType], + filename: str, + ) -> '_Resource': + + # the type ignores workaround a known mypy issue + # https://github.com/python/mypy/issues/1021 + try: + ref = files(package) / filename + except ModuleNotFoundError: + rsrc = super().__new__(cls, '') # type: ignore + rsrc.module_not_found = True + return rsrc + + file_manager = contextlib.ExitStack() + __resource_file_contexts__.append(file_manager) + return super().__new__( # type: ignore + cls, + file_manager.enter_context(as_file(ref)), + ) + + def __init__( + self, + package: Union[str, types.ModuleType], + filename: str, + ) -> None: + super().__init__() + if isinstance(package, types.ModuleType): + self.package = package.__name__ + else: + self.package = package + self.filename = filename + + def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 """ Used for including Django project settings from multiple files. @@ -52,11 +135,13 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 .. code:: python - from split_settings.tools import optional, include + from split_settings.tools import optional, include, resource + import components include( 'components/base.py', 'components/database.py', + resource(components, settings.conf), # package resource optional('local_settings.py'), scope=globals(), # optional scope @@ -68,6 +153,7 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 Raises: IOError: if a required settings file is not found. + ModuleNotFoundError: if a required resource package is not found. """ # we are getting globals() from previous frame @@ -85,7 +171,18 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 for conf_file in args: saved_included_file = scope.get(_INCLUDED_FILE) - pattern = os.path.join(conf_path, conf_file) + pattern = conf_file + # if a resource was not found the path will resolve to empty str here + if pattern: + pattern = os.path.join(conf_path, conf_file) + + # check if this include is a resource with an unfound module + # and issue a more specific exception + if isinstance(conf_file, _Resource): + if conf_file.module_not_found: + raise ModuleNotFoundError( + 'No module named {0}'.format(conf_file.package), + ) # find files per pattern, raise an error if not found # (unless file is optional) @@ -114,8 +211,12 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 rel_path[:rel_path.rfind('.')].replace('/', '.'), ) - spec = spec_from_file_location( - module_name, included_file, + spec = spec_from_loader( + module_name, + SourceFileLoader( + os.path.basename(included_file).split('.')[0], + included_file, + ), ) module = module_from_spec(spec) sys.modules[module_name] = module @@ -123,3 +224,8 @@ def include(*args: str, **kwargs) -> None: # noqa: WPS210, WPS231, C901 scope[_INCLUDED_FILE] = saved_included_file elif _INCLUDED_FILE in scope: scope.pop(_INCLUDED_FILE) + + # close the contexts of any temporary files created to access + # resource contents thereby deleting them + for ctx in __resource_file_contexts__: + ctx.close() diff --git a/tests/conftest.py b/tests/conftest.py index e14b3f7..74fae97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import pytest -class Scope(dict): # noqa: WPS600 +class Scope(dict): # noqa: WPS600, WPS202 """This class emulates `globals()`, but does not share state in tests.""" def __init__(self, *args, **kwargs): @@ -43,6 +43,20 @@ def merged(): return _merged +@pytest.fixture() +def alt_ext(): + """This fixture returns alt_ext settings example.""" + from tests.settings import alt_ext as _alt_ext # noqa: WPS433 + return _alt_ext + + +@pytest.fixture() +def resources(): + """This fixture returns resource settings example.""" + from tests.settings import resources as _resources # noqa: WPS433 + return _resources + + @pytest.fixture() def stacked(): """This fixture returns stacked settings example.""" diff --git a/tests/settings/alt_ext/__init__.py b/tests/settings/alt_ext/__init__.py new file mode 100644 index 0000000..3b43cfb --- /dev/null +++ b/tests/settings/alt_ext/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + +from split_settings.tools import include, optional + +# Includes files with non-standard extensions: +include( + 'include', + '*.conf', + optional('optional.ext'), +) diff --git a/tests/settings/alt_ext/include b/tests/settings/alt_ext/include new file mode 100644 index 0000000..a8feda4 --- /dev/null +++ b/tests/settings/alt_ext/include @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +NO_EXT_INCLUDED = True diff --git a/tests/settings/alt_ext/include.conf b/tests/settings/alt_ext/include.conf new file mode 100644 index 0000000..adae986 --- /dev/null +++ b/tests/settings/alt_ext/include.conf @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +DOT_CONF_INCLUDED = True diff --git a/tests/settings/alt_ext/include.double.conf b/tests/settings/alt_ext/include.double.conf new file mode 100644 index 0000000..8e31368 --- /dev/null +++ b/tests/settings/alt_ext/include.double.conf @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +DOUBLE_EXT_INCLUDED = True diff --git a/tests/settings/alt_ext/optional.ext b/tests/settings/alt_ext/optional.ext new file mode 100644 index 0000000..6cecc38 --- /dev/null +++ b/tests/settings/alt_ext/optional.ext @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +OPTIONAL_INCLUDED = True diff --git a/tests/settings/resources/__init__.py b/tests/settings/resources/__init__.py new file mode 100644 index 0000000..b5dcbe3 --- /dev/null +++ b/tests/settings/resources/__init__.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from split_settings.tools import include, optional, resource +from tests.settings import resources + +include( + # Components: + resource('tests.settings.resources', 'base.conf'), + resource('tests.settings.resources', 'locale.conf'), + resource('tests.settings.resources', 'apps_middleware'), + resource(resources, 'static.settings'), + resource(resources, 'templates.py'), + resource(resources, 'multiple/subdirs/file.conf'), + optional(resource(resources, 'database.conf')), + 'logging.py', + + # Missing file: + optional(resource(resources, 'missing_file.py')), + optional(resource('tests.settings.resources', 'missing_file.conf')), + resource('tests.settings.resources', 'error.conf'), + + # Missing module + optional(resource('module.does.not.exist', 'settings.conf')), + + # Scope: + scope=globals(), # noqa: WPS421 +) diff --git a/tests/settings/resources/apps_middleware b/tests/settings/resources/apps_middleware new file mode 100644 index 0000000..ea0c33b --- /dev/null +++ b/tests/settings/resources/apps_middleware @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +APPS_MIDDLEWARE_INCLUDED = True diff --git a/tests/settings/resources/base.conf b/tests/settings/resources/base.conf new file mode 100644 index 0000000..4ce8b13 --- /dev/null +++ b/tests/settings/resources/base.conf @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +BASE_INCLUDED = True +OVERRIDE_WORKS = False diff --git a/tests/settings/resources/database.conf b/tests/settings/resources/database.conf new file mode 100644 index 0000000..7683e1a --- /dev/null +++ b/tests/settings/resources/database.conf @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +DATABASE_INCLUDED = True diff --git a/tests/settings/resources/error.conf b/tests/settings/resources/error.conf new file mode 100644 index 0000000..ad5a7d1 --- /dev/null +++ b/tests/settings/resources/error.conf @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from split_settings.tools import include, resource + +UNFOUND_RESOURCE_IS_IOERROR = False +UNFOUND_RESOURCE_ERR_STR = '' +UNFOUND_PACKAGE_IS_MODULE_ERROR = False +UNFOUND_PACKAGE_ERR_STR = '' +try: + include(resource('tests.settings.resources', 'does_not_exist.conf')) +except IOError as ioe: + UNFOUND_RESOURCE_ERR_STR = str(ioe) + UNFOUND_RESOURCE_IS_IOERROR = True + +try: + include(resource('does.not.exist', 'database.conf')) +except ModuleNotFoundError as mnfe: + UNFOUND_PACKAGE_ERR_STR = str(mnfe) + UNFOUND_PACKAGE_IS_MODULE_ERROR = True + +try: + include(resource('tests.settings.resources', 'multiple/subdirs/does_not_exist.conf')) +except IOError: + UNFOUND_RESOURCE_IS_IOERROR &= True +except: + UNFOUND_RESOURCE_IS_IOERROR = False + diff --git a/tests/settings/resources/locale.conf b/tests/settings/resources/locale.conf new file mode 100644 index 0000000..3615b7b --- /dev/null +++ b/tests/settings/resources/locale.conf @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +LOCALE_INCLUDED = True +OVERRIDE_WORKS = True diff --git a/tests/settings/resources/logging.py b/tests/settings/resources/logging.py new file mode 100644 index 0000000..cbaf432 --- /dev/null +++ b/tests/settings/resources/logging.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +LOGGING_INCLUDED = True diff --git a/tests/settings/resources/multiple/subdirs/file.conf b/tests/settings/resources/multiple/subdirs/file.conf new file mode 100644 index 0000000..76f00f1 --- /dev/null +++ b/tests/settings/resources/multiple/subdirs/file.conf @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +MULTIPLE_SUBDIR_FILE_INCLUDED = True diff --git a/tests/settings/resources/static.settings b/tests/settings/resources/static.settings new file mode 100644 index 0000000..8728818 --- /dev/null +++ b/tests/settings/resources/static.settings @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +STATIC_SETTINGS_INCLUDED = True diff --git a/tests/settings/resources/templates.py b/tests/settings/resources/templates.py new file mode 100644 index 0000000..5d02f8d --- /dev/null +++ b/tests/settings/resources/templates.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +TEMPLATES_INCLUDED = True diff --git a/tests/test_import.py b/tests/test_import.py index feded79..9229459 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -7,6 +7,7 @@ def test_wildcard_import(): """Imports all from all modules.""" assert 'optional' in __all__ assert 'include' in __all__ + assert 'resource' in __all__ def test_class_import(merged): diff --git a/tests/test_split_settings.py b/tests/test_split_settings.py index 42ce703..fdbec15 100644 --- a/tests/test_split_settings.py +++ b/tests/test_split_settings.py @@ -7,6 +7,36 @@ def test_merge(merged): assert merged.STATIC_ROOT +def test_alt_ext(alt_ext): + """Test that all values from settings are present.""" + assert alt_ext.NO_EXT_INCLUDED + assert alt_ext.DOT_CONF_INCLUDED + assert alt_ext.DOUBLE_EXT_INCLUDED + assert alt_ext.OPTIONAL_INCLUDED + + +def test_resources(resources): # noqa: WPS218 + """Test that all values from settings are present.""" + assert resources.APPS_MIDDLEWARE_INCLUDED + assert resources.BASE_INCLUDED + assert resources.DATABASE_INCLUDED + assert resources.LOCALE_INCLUDED + assert resources.LOGGING_INCLUDED + assert resources.STATIC_SETTINGS_INCLUDED + assert resources.TEMPLATES_INCLUDED + assert resources.OVERRIDE_WORKS + assert resources.MULTIPLE_SUBDIR_FILE_INCLUDED + assert resources.UNFOUND_RESOURCE_IS_IOERROR + assert resources.UNFOUND_PACKAGE_IS_MODULE_ERROR + assert resources.UNFOUND_RESOURCE_ERR_STR.startswith( + 'No such file', + ) + assert resources.UNFOUND_RESOURCE_ERR_STR.endswith( + 'does_not_exist.conf', + ) + assert resources.UNFOUND_PACKAGE_ERR_STR == 'No module named does.not.exist' + + def test_override(merged, monkeypatch): """This setting must be overridden in the testing.py.""" monkeypatch.setenv('DJANGO_SETTINGS_MODULE', 'tests.settings.merged')