Reload config and read once and services (#83)

* @miguelgrubin Refactor config and memoize file 

* @miguelgrubin Reload config

* @miguelgrubin Fix error: removed memoize of service. This feature create an error on tests when create a instance of Microservice out of tests

* Reload services and app when the configuration was reloaded

Author: avara1986

Pipfile

@@ -15,6 +15,7 @@ flask-opentracing = "*"
 opentracing = ">=2.1"
 opentracing-instrumentation = "==3.2.1"
 prometheus_client = ">=0.7.1"
+
 [dev-packages]
 requests-mock = "*"
 coverage = "==4.5.4"

pyms/config/conf.py

@@ -2,9 +2,6 @@
 from pyms.exceptions import ServiceDoesNotExistException
 
 
-__service_configs = {}
-
-
 def get_conf(*args, **kwargs):
     """
     Returns an object with a set of attributes retrieved from the configuration file. Each subblock is a append of the
@@ -33,12 +30,12 @@ def get_conf(*args, **kwargs):
     block config
     :param args:
     :param kwargs:
+
     :return:
     """
     service = kwargs.pop('service', None)
-    memoize = kwargs.pop('memoize', True)
     if not service:
         raise ServiceDoesNotExistException("Service not defined")
-    if not memoize or service not in __service_configs:
-        __service_configs[service] = ConfFile(*args, **kwargs)
-    return getattr(__service_configs[service], service)
+
+    config = ConfFile(*args, **kwargs)
+    return getattr(config, service)

pyms/config/confile.py

@@ -1,7 +1,6 @@
 """Module to read yaml or json conf"""
 import logging
 import os
-from typing import Text
 
 import anyconfig
 
@@ -10,6 +9,8 @@
 
 logger = logging.getLogger(LOGGER_NAME)
 
+config_cache = {}
+
 
 class ConfFile(dict):
     """Recursive get configuration from dictionary, a config file in JSON or YAML format from a path or
@@ -18,8 +19,9 @@ class ConfFile(dict):
     * empty_init: Allow blank variables
     * default_file: search for config.yml file
     """
-    empty_init = False
-    default_file = "config.yml"
+    _empty_init = False
+    _default_file = "config.yml"
+    __path = None
 
     def __init__(self, *args, **kwargs):
         """
@@ -32,31 +34,38 @@ def __init__(self, *args, **kwargs):
             self[key] = getattr(obj, key)
         ```
         """
-        self.empty_init = kwargs.get("empty_init", False)
+        self._empty_init = kwargs.get("empty_init", False)
         config = kwargs.get("config")
-        uppercase = kwargs.get("uppercase", True)
         if config is None:
-            config = self._get_conf_from_file(kwargs.get("path")) or self._get_conf_from_env()
+            self.set_path(kwargs.get("path"))
+            config = self._get_conf_from_file() or self._get_conf_from_env()
 
         if not config:
-            if self.empty_init:
+            if self._empty_init:
                 config = {}
             else:
                 raise ConfigDoesNotFoundException("Configuration file not found")
 
+        config = self.set_config(config)
+
+        super(ConfFile, self).__init__(config)
+
+    def set_path(self, path):
+        self.__path = path
+
+    def to_flask(self):
+        return ConfFile(config={k.upper(): v for k, v in self.items()})
+
+    def set_config(self, config):
         config = dict(self.normalize_config(config))
         for k, v in config.items():
             setattr(self, k, v)
-            # Flask search for uppercase keys
-            if uppercase:
-                setattr(self, k.upper(), v)
-
-        super(ConfFile, self).__init__(config)
+        return config
 
     def normalize_config(self, config):
         for key, item in config.items():
             if isinstance(item, dict):
-                item = ConfFile(config=item, empty_init=self.empty_init)
+                item = ConfFile(config=item, empty_init=self._empty_init)
             yield self.normalize_keys(key), item
 
     @staticmethod
@@ -78,22 +87,32 @@ def __getattr__(self, name, *args, **kwargs):
                 aux_dict = aux_dict[k]
             return aux_dict
         except KeyError:
-            if self.empty_init:
-                return ConfFile(config={}, empty_init=self.empty_init)
+            if self._empty_init:
+                return ConfFile(config={}, empty_init=self._empty_init)
             raise AttrDoesNotExistException("Variable {} not exist in the config file".format(name))
 
     def _get_conf_from_env(self):
-        config_file = os.environ.get(CONFIGMAP_FILE_ENVIRONMENT, self.default_file)
+        config_file = os.environ.get(CONFIGMAP_FILE_ENVIRONMENT, self._default_file)
         logger.debug("[CONF] Searching file in ENV[{}]: {}...".format(CONFIGMAP_FILE_ENVIRONMENT, config_file))
-        return self._get_conf_from_file(config_file)
+        self.set_path(config_file)
+        return self._get_conf_from_file()
 
-    @staticmethod
-    def _get_conf_from_file(path: Text) -> dict:
-        if not path or not os.path.isfile(path):
+    def _get_conf_from_file(self) -> dict:
+        if not self.__path or not os.path.isfile(self.__path):
+            logger.debug("[CONF] Configmap {} NOT FOUND".format(self.__path))
             return {}
-        logger.debug("[CONF] Configmap {} found".format(path))
-        conf = anyconfig.load(path)
-        return conf
+        if self.__path not in config_cache:
+            logger.debug("[CONF] Configmap {} found".format(self.__path))
+            config_cache[self.__path] = anyconfig.load(self.__path)
+        return config_cache[self.__path]
+
+    def load(self):
+        config_src = self._get_conf_from_file() or self._get_conf_from_env()
+        self.set_config(config_src)
+
+    def reload(self):
+        config_cache.pop(self.__path, None)
+        self.load()
 
     def __setattr__(self, name, value, *args, **kwargs):
         super(ConfFile, self).__setattr__(name, value)

pyms/flask/app/create_app.py

@@ -90,6 +90,7 @@ def example():
     Current services are swagger, request, tracer, metrics
     """
     service = None
+    services = []
     application = None
     swagger = False
     request = False
@@ -98,7 +99,7 @@ def example():
 
     def __init__(self, *args, **kwargs):
         self.path = os.path.dirname(kwargs.get("path", __file__))
-        self.config = get_conf(service=CONFIG_BASE, memoize=self._singleton)
+        self.config = get_conf(service=CONFIG_BASE)
         self.init_services()
 
     def init_services(self) -> None:
@@ -107,9 +108,21 @@ def init_services(self) -> None:
         :return: None
         """
         service_manager = ServicesManager()
-        for service_name, service in service_manager.get_services(memoize=self._singleton):
+        for service_name, service in service_manager.get_services():
+            self.services.append(service_name)
             setattr(self, service_name, service)
 
+    def delete_services(self) -> None:
+        """
+        Set the Attributes of all service defined in config.yml and exists in `pyms.flask.service`
+        :return: None
+        """
+        for service_name in self.services:
+            try:
+                delattr(self, service_name)
+            except AttributeError:
+                pass
+
     def init_libs(self) -> Flask:
         """This function exists to override if you need to set more libs such as SQLAlchemy, CORs, and any else
         library needs to be init over flask, like the usual pattern [MYLIB].init_app(app)
@@ -153,7 +166,7 @@ def init_app(self) -> Flask:
         :return: None
         """
         if self._exists_service("swagger"):
-            application = self.swagger.init_app(config=self.config, path=self.path)
+            application = self.swagger.init_app(config=self.config.to_flask(), path=self.path)
         else:
             check_package_exists("flask")
             application = Flask(__name__, static_folder=os.path.join(self.path, 'static'),
@@ -175,6 +188,12 @@ def init_metrics(self):
             )
             self.metrics.monitor(self.application)
 
+    def reload_conf(self):
+        self.delete_services()
+        self.config.reload()
+        self.init_services()
+        self.create_app()
+
     def create_app(self):
         """Initialize the Flask app, register blueprints and initialize
         all libraries like Swagger, database,
@@ -183,7 +202,7 @@ def create_app(self):
         :return:
         """
         self.application = self.init_app()
-        self.application.config.from_object(self.config)
+        self.application.config.from_object(self.config.to_flask())
         self.application.tracer = None
         self.application.ms = self
 
@@ -199,6 +218,8 @@ def create_app(self):
 
         self.init_metrics()
 
+        logger.debug("Started app with PyMS and this services: {}".format(self.services))
+
         return self.application
 
     def _exists_service(self, service_name: Text) -> bool:

pyms/flask/services/driver.py

@@ -38,7 +38,7 @@ class DriverService:
 
     def __init__(self, *args, **kwargs):
         self.service = get_service_name(service=self.service)
-        self.config = get_conf(service=self.service, empty_init=True, memoize=kwargs.get("memoize", True))
+        self.config = get_conf(service=self.service, empty_init=True)
 
     def __getattr__(self, attr, *args, **kwargs):
         config_attribute = getattr(self.config, attr)
@@ -59,14 +59,14 @@ class ServicesManager:
     service = SERVICE_BASE
 
     def __init__(self):
-        self.config = get_conf(service=self.service, empty_init=True, memoize=False, uppercase=False)
+        self.config = get_conf(service=self.service, empty_init=True, uppercase=False)
 
-    def get_services(self, memoize: bool) -> Tuple[Text, DriverService]:
+    def get_services(self) -> Tuple[Text, DriverService]:
         for k in self.config.__dict__.keys():
-            if k.islower() and k not in ['empty_init', ]:
-                service = self.get_service(k, memoize=memoize)
+            if k.islower() and not k.startswith("_"):
+                service = self.get_service(k)
                 if service.is_enabled():
-                    yield (k, service)
+                    yield k, service
 
     @staticmethod
     def get_service(service: Text, *args, **kwargs) -> DriverService:

pyms/flask/services/tracer.py

@@ -68,7 +68,7 @@ def init_jaeger_tracer(self):
                     'reporting_host': self.host
                 }
             }
-        metrics_config = get_conf(service=get_service_name(service="metrics"), empty_init=True, memoize=False)
+        metrics_config = get_conf(service=get_service_name(service="metrics"), empty_init=True)
         metrics = ""
         if metrics_config:
             metrics = PrometheusMetricsFactory()

tests/config-tests-cache.yml

@@ -0,0 +1,12 @@
+pyms:
+  config:
+    debug: true
+    testing: true
+    my_cache: 1234
+    app_name: "Python Microservice"
+    application_root: /
+    test_var: general
+    subservice1:
+       test: input
+    subservice2:
+       test: output

tests/config-tests-cache2.yml

@@ -0,0 +1,12 @@
+pyms:
+  config:
+    debug: true
+    testing: true
+    my_cache: 12345678
+    app_name: "Python Microservice"
+    application_root: /
+    test_var: general
+    subservice1:
+       test: input
+    subservice2:
+       test: output

tests/config-tests-flask.yml

@@ -8,4 +8,6 @@ pyms:
     subservice1:
        test: input
     subservice2:
-       test: output
+       test: output
+    subservice_flask:
+       test: flask

tests/config-tests-requests-no-data.yml

@@ -1,6 +1,7 @@
 pyms:
   services:
-    requests: true
+    requests:
+      enabled: true
     swagger:
       path: ""
       file: "swagger.yaml"