License info from environment variables given higher preference.

Introducing module to house all environment variables used by app.

Author: prabhakk-mw

jupyter_matlab_proxy/app.py

@@ -409,14 +409,17 @@ def main():
     app = create_app()
 
     loop = asyncio.get_event_loop()
-    runner = web.AppRunner(app)
+
+    # Override default loggers
+    runner = web.AppRunner(app, logger=logger, access_log=logger)
+
     loop.run_until_complete(runner.setup())
     site = web.TCPSite(
         runner, host=app["settings"]["host_interface"], port=app["settings"]["app_port"]
     )
     loop.run_until_complete(site.start())
 
-    # Register handlers to trap termination signals 
+    # Register handlers to trap termination signals
     for signal in get_supported_termination_signals():
         logger.info(f"Installing handler for signal: {signal} ")
         loop.add_signal_handler(signal, lambda: loop.stop())

jupyter_matlab_proxy/app_state.py

@@ -12,16 +12,18 @@
 import socket
 import errno
 from collections import deque
-from .util import mw, mw_logger
+from .util import mw, mw_logger, validators
 from .util.exceptions import (
     LicensingError,
     InternalError,
     OnlineLicensingError,
     EntitlementError,
     MatlabInstallError,
+    NetworkLicensingError,
     log_error,
 )
 
+
 logger = mw_logger.get()
 
 
@@ -42,21 +44,59 @@ def __init__(self, settings):
             logger.error("'matlab' executable not found in PATH")
             return
 
+    def __get_cached_licensing_file(self):
+        """Get the cached licensing file
+
+        Returns:
+            Path : Path object to cached licensing file
+        """
+        return self.settings["matlab_config_file"]
+
+    def __delete_cached_licensing_file(self):
+        try:
+            logger.info(f"Deleting any cached licensing files!")
+            os.remove(self.__get_cached_licensing_file())
+        except FileNotFoundError:
+            # The file being absent is acceptable.
+            pass
+
+
+    def __reset_and_delete_cached_licensing(self):
+        logger.info(f"Resetting cached licensing information...")
+        self.licensing = None
+        self.__delete_cached_licensing_file()
+
     async def init_licensing(self):
-        """Initialise licensing from persisted details or environment variable."""
+        """Initialize licensing from environment variable or cached file.
+
+        Greater precedence is given to value specified in environment variable MLM_LICENSE_FILE
+            If specified, this function will delete previously cached licensing information.
+            This enforces a clear understanding of what was used to initialize licensing.
+            The contents of the environment variable are NEVER cached.
+        """
 
-        # Persisted licensing details present
-        if self.settings["matlab_config_file"].exists():
-            with open(self.settings["matlab_config_file"], "r") as f:
-                config = json.loads(f.read())
+        # Default value
+        self.licensing = None
 
-            if "licensing" in config:
-                # TODO Refactoring of config file reading/writing
-                licensing = config["licensing"]
+        # NLM Connection String set in environment
+        if self.settings["nlm_conn_str"] is not None:
+            nlm_licensing_str = self.settings["nlm_conn_str"]
+            logger.info(f"Found NLM:[{nlm_licensing_str}] set in environment")
+            logger.info(f"Using NLM string to connect ... ")
+            self.licensing = {
+                "type": "nlm",
+                "conn_str": nlm_licensing_str,
+            }
+            self.__delete_cached_licensing_file()
 
-                # If there is any problem loading config, remove it and persist
+        # If NLM connection string is not present, then look for persistent LNU info
+        elif self.__get_cached_licensing_file().exists():
+            with open(self.__get_cached_licensing_file(), "r") as f:
+                licensing = json.loads(f.read())
+                logger.info("Found cached licensing information...")
                 try:
                     if licensing["type"] == "nlm":
+                        # Note: Only NLM settings entered in browser were cached.
                         self.licensing = {
                             "type": "nlm",
                             "conn_str": licensing["conn_str"],
@@ -82,26 +122,19 @@ async def init_licensing(self):
                         ) - timedelta(hours=1)
 
                         if expiry_window > datetime.now(timezone.utc):
-                            await self.update_entitlements()
+                            successful_update = await self.update_entitlements()
+                            if successful_update:
+                                logger.info("Successful re-use of cached information.")
+                                self.persist_licensing()
+                            else:
+                                self.__reset_and_delete_cached_licensing()
                         else:
-                            # Reset licensing and persist
-                            self.licensing["identity_token"] = None
-                            self.licensing["source_id"] = None
-                            self.licensing["expiry"] = None
-                            self.licensing["entitlements"] = []
-                            self.persist_licensing()
+                            self.__reset_and_delete_cached_licensing()
+                    else:
+                        # Somethings wrong, licensing is neither NLM or MHLM
+                        self.__reset_and_delete_cached_licensing()
                 except Exception as e:
-                    logger.error("Error parsing config, resetting.")
-                    self.licensing = None
-                    self.persist_licensing()
-
-        # NLM Connection String set in environment
-        # TODO Validate connection string
-        elif self.settings["nlm_conn_str"] is not None:
-            self.licensing = {
-                "type": "nlm",
-                "conn_str": self.settings["nlm_conn_str"],
-            }
+                    self.__reset_and_delete_cached_licensing()
 
     def get_matlab_state(self):
         """Determine the state of MATLAB to be down/starting/up."""
@@ -208,6 +241,12 @@ def is_matlab_present(self):
         return self.settings["matlab_path"] is not None
 
     async def update_entitlements(self):
+        """Speaks to MW and updates MHLM entitlements
+
+        Returns: True if update was successful
+        Raises:
+            InternalError: OnlineLicensingError, EntitlementError
+        """
         if self.licensing is None or self.licensing["type"] != "mhlm":
             raise InternalError(
                 "MHLM licensing must be configured to update entitlements!"
@@ -230,7 +269,7 @@ async def update_entitlements(self):
         except OnlineLicensingError as e:
             self.error = e
             log_error(logger, e)
-            return
+            return False
         except EntitlementError as e:
             self.error = e
             log_error(logger, e)
@@ -244,48 +283,28 @@ async def update_entitlements(self):
             self.licensing["profile_id"] = None
             self.licensing["entitlements"] = []
             self.licensing["entitlement_id"] = None
-            return
+            return False
 
         self.licensing["entitlements"] = entitlements
 
         # If there is only one non-expired entitlement, set it as active
         # TODO Also, for now, set the first entitlement as active if there are multiple
         self.licensing["entitlement_id"] = entitlements[0]["id"]
 
-    def persist_licensing(self):
-        config_file = self.settings["matlab_config_file"]
-        if config_file.exists():
-            with open(config_file, "r") as f:
-                config = json.loads(f.read())
-        else:
-            config = {}
+        # Successful update
+        return True
 
+    def persist_licensing(self):
+        """Saves licensing information to file"""
         if self.licensing is None:
-            if "licensing" in config:
-                del config["licensing"]
-        elif self.licensing["type"] == "mhlm":
-            config["licensing"] = {
-                "type": "mhlm",
-                "identity_token": self.licensing["identity_token"],
-                "source_id": self.licensing["source_id"],
-                "expiry": self.licensing["expiry"],
-                "email_addr": self.licensing["email_addr"],
-                "first_name": self.licensing["first_name"],
-                "last_name": self.licensing["last_name"],
-                "display_name": self.licensing["display_name"],
-                "user_id": self.licensing["user_id"],
-                "profile_id": self.licensing["profile_id"],
-                "entitlement_id": self.licensing["entitlement_id"],
-            }
-        elif self.licensing["type"] == "nlm":
-            config["licensing"] = {
-                "type": "nlm",
-                "conn_str": self.licensing["conn_str"],
-            }
-
-        config_file.parent.mkdir(parents=True, exist_ok=True)
-        with open(config_file, "w") as f:
-            f.write(json.dumps(config))
+            self.__delete_cached_licensing_file()
+
+        elif self.licensing["type"] in ["mhlm", "nlm"]:
+            logger.info("Saving licensing information...")
+            cached_licensing_file = self.__get_cached_licensing_file()
+            cached_licensing_file.parent.mkdir(parents=True, exist_ok=True)
+            with open(cached_licensing_file, "w") as f:
+                f.write(json.dumps(self.licensing))
 
     def reserve_matlab_port(self):
         """Reserve a free port for MATLAB Embedded Connector in the allowed range."""

jupyter_matlab_proxy/mw_environment_variables.py

@@ -0,0 +1,55 @@
+# Copyright 2021 The MathWorks, Inc.
+"""This file lists and exposes the environment variables which are used by the integration."""
+
+
+def get_env_name_network_license_manager():
+    """Specifies the path to valid license file or address of a network license server"""
+    return "MLM_LICENSE_FILE"
+
+
+def get_env_name_logging_level():
+    """Specifies the logging level used by app's loggers"""
+    return "LOG_LEVEL"
+
+
+def get_env_name_log_file():
+    """Specifies a file into which logging content is directed"""
+    return "LOG_FILE"
+
+
+def get_env_name_base_url():
+    """Specifies the base url on which the website should run.
+    Eg: www.127.0.0.1:8888/base_url/index.html
+
+    Note: The website runs on a URL of the form:
+        www.<SERVER ADDRESS>:<PORT NUMBER>/<BASE_URL>/index.html
+    """
+    return "BASE_URL"
+
+
+def get_env_name_app_port():
+    """Specifies the port on which the website is running on the server.
+    Eg: www.127.0.0.1:PORT/index.html
+
+    Note: The website runs on a URL of the form:
+        www.<SERVER ADDRESS>:<PORT NUMBER>/<BASE_URL>/index.html
+    """
+    return "APP_PORT"
+
+
+def get_env_name_custom_http_headers():
+    """Specifies HTTP headers as JSON content, to be injected into responses sent to the browser"""
+    return "CUSTOM_HTTP_HEADERS"
+
+
+def get_env_name_app_host():
+    """Specifies the host on which the TCP site (aiohttp server) is being run."""
+    return "APP_HOST"
+
+
+def get_env_name_testing():
+    return "TEST"
+
+
+def get_env_name_development():
+    return "DEV"

jupyter_matlab_proxy/settings.py

@@ -4,9 +4,10 @@
 from pathlib import Path
 import tempfile
 import xml.etree.ElementTree as ET
-import uuid, socket
+import uuid
+import socket
 import shutil
-from .util import custom_http_headers
+from .util import custom_http_headers, validators
 
 
 def get_matlab_path():
@@ -131,7 +132,9 @@ def get(dev=False):
             "host_interface": os.environ.get("APP_HOST"),
             "mwapikey": str(uuid.uuid4()),
             "matlab_protocol": "https",
-            "nlm_conn_str": os.environ.get("MLM_LICENSE_FILE"),
+            "nlm_conn_str": validators.validate_mlm_license_file(
+                os.environ.get("MLM_LICENSE_FILE")
+            ),
             "matlab_config_file": Path.home() / ".matlab" / "proxy_app_config.json",
             "ws_env": ws_env,
             "mwa_api_endpoint": f"https://login{ws_env_suffix}.mathworks.com/authenticationws/service/v4",

jupyter_matlab_proxy/util/mw_logger.py

@@ -52,6 +52,9 @@ def __set_logging_configuration():
     # log_level is either set by environment or is the default value.
     logger.info(f"Initializing logger with log_level: {log_level}")
     logger.setLevel(log_level)
+
+    # Allow other libraries used by this integration to
+    # also print their logs at the specified level
     logging.basicConfig(level=log_level)
 
     return logger

jupyter_matlab_proxy/util/validators.py

@@ -0,0 +1,59 @@
+# Copyright 2020-2021 The MathWorks, Inc.
+"""This file contains validators for various runtime artefacts.
+A validator is defined as a function which verifies the input and 
+returns it unchanged if validation passes. 
+Returning inputs allows validators to be used inline with the input.
+
+Example: 
+Original code: if( input ):
+With validator: if (valid(input)):
+
+Exceptions are thrown to signal failure.
+"""
+
+
+def validate_mlm_license_file(nlm_conn_str):
+    """Validates and returns input if it passes validation.
+    Throws exception when validation fails.
+    The connection string should be in the form of port@hostname
+    OR path to valid license file
+    """
+    import re
+    import os
+    from . import mw_logger
+    from .exceptions import NetworkLicensingError
+
+    logger = mw_logger.get()
+
+    if nlm_conn_str is None:
+        return None
+
+    # TODO: The JS validation of this setting does not allow file path locations
+    # The JS validation occurs before reaching the set_licensing_info endpoint.
+
+    # Regular expression to match port@hostname,
+    # where port is any number and hostname is alphanumeric
+    # regex = Start of Line, Any number of 0-9 digits , @, any number of nonwhite space characters with "- _ ." allowed
+    regex = "^[0-9]+[@](\w|\_|\-|\.)+$"
+    if not re.search(regex, nlm_conn_str):
+        logger.debug("NLM info is not in the form of port@hostname")
+        if not os.path.isfile(nlm_conn_str):
+            logger.debug("NLM info is not a valid path to a license file")
+            error_message = (
+                f"MLM_LICENSE_FILE validation failed for {nlm_conn_str}. "
+                f"If set, the MLM_LICENSE_FILE environment variable must be a string which is either of the form port@hostname"
+                f" OR path to a valid license file."
+            )
+            logger.error(error_message)
+            raise NetworkLicensingError(error_message)
+        else:
+            logger.info(
+                f"MLM_LICENSE_FILE with value: {nlm_conn_str} is a path to a file. MATLAB will attempt to use it."
+            )
+    else:
+        logger.info(
+            f"MLM_LICENSE_FILE with value: {nlm_conn_str} is a license server, MATLAB will attempt to connect to it."
+        )
+
+    # Validation passed
+    return nlm_conn_str

tests/conftest.py

@@ -37,6 +37,9 @@ def delete_matlab_test_dir():
         # Delete matlab_config_file & its owning directory
         matlab_config_file = __get_matlab_config_file()
         matlab_config_dir = os.path.dirname(matlab_config_file)
-        shutil.rmtree(matlab_config_dir)
+        try:
+            shutil.rmtree(matlab_config_dir)
+        except FileNotFoundError:
+            pass
 
     request.addfinalizer(delete_matlab_test_dir)