fix: initialize separate logger for each module

solnlib/conf_manager.py

@@ -36,6 +36,11 @@
     InvalidHostnameError,
 )
 
+from solnlib.utils import get_solnlib_logger
+
+logger = get_solnlib_logger(__name__)
+
+
 __all__ = [
     "ConfFile",
     "ConfManager",
@@ -338,7 +343,7 @@ def delete(self, stanza_name: str):
         try:
             self._conf.delete(stanza_name)
         except KeyError:
-            logging.error(
+            logger().error(
                 "Delete stanza: %s error: %s.", stanza_name, traceback.format_exc()
             )
             raise ConfStanzaNotExistException(

solnlib/file_monitor.py

@@ -24,6 +24,11 @@
 import traceback
 from typing import Any, Callable, List
 
+from solnlib.utils import get_solnlib_logger
+
+logger = get_solnlib_logger(__name__)
+
+
 __all__ = ["FileChangesChecker", "FileMonitor"]
 
 
@@ -45,7 +50,10 @@ def __init__(self, callback: Callable[[List[str]], Any], files: List):
             try:
                 self.file_mtimes[k] = op.getmtime(k)
             except OSError:
-                logging.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}")
+                logging.debug(
+                    f"Getmtime for {k}, failed: {traceback.format_exc()}"
+                )  # deprecated
+                logger().debug(f"Getmtime for {k}, failed: {traceback.format_exc()}")
 
     def check_changes(self) -> bool:
         """Check files change.
@@ -56,7 +64,7 @@ def check_changes(self) -> bool:
         Returns:
             True if files changed else False
         """
-        logging.debug(f"Checking files={self._files}")
+        logger().debug(f"Checking files={self._files}")
         file_mtimes = self.file_mtimes
         changed_files = []
         for f, last_mtime in list(file_mtimes.items()):
@@ -65,7 +73,7 @@ def check_changes(self) -> bool:
                 if current_mtime != last_mtime:
                     file_mtimes[f] = current_mtime
                     changed_files.append(f)
-                    logging.info(f"Detect {f} has changed", f)
+                    logger().info(f"Detect {f} has changed", f)
             except OSError:
                 pass
         if changed_files:

solnlib/log.py

@@ -25,7 +25,14 @@
 from typing import Dict, Any
 
 from .pattern import Singleton
-from .splunkenv import make_splunkhome_path
+
+try:
+    from splunk.clilib.bundle_paths import make_splunkhome_path as msp
+except ImportError:
+
+    def msp(*args, **kwargs):
+        raise ImportError("This module requires Splunk to be installed.")
+
 
 __all__ = ["log_enter_exit", "LogException", "Logs"]
 
@@ -147,7 +154,7 @@ def _get_log_file(cls, name):
             directory = cls._default_directory
         else:
             try:
-                directory = make_splunkhome_path(["var", "log", "splunk"])
+                directory = msp(["var", "log", "splunk"])
             except KeyError:
                 raise LogException(
                     "Log directory is empty, please set log directory "

solnlib/modular_input/checkpointer.py

@@ -19,7 +19,6 @@
 
 import base64
 import json
-import logging
 import os
 import os.path as op
 import traceback
@@ -31,6 +30,10 @@
 
 from solnlib import _utils, utils
 
+from solnlib.utils import get_solnlib_logger
+
+logger = get_solnlib_logger(__name__)
+
 __all__ = ["CheckpointerException", "KVStoreCheckpointer", "FileCheckpointer"]
 
 
@@ -178,7 +181,7 @@ def get(self, key: str) -> Optional[Any]:
             record = self._collection_data.query_by_id(key)
         except binding.HTTPError as e:
             if e.status != 404:
-                logging.error(f"Get checkpoint failed: {traceback.format_exc()}.")
+                logger().error(f"Get checkpoint failed: {traceback.format_exc()}.")
                 raise
             return None
         return json.loads(record["state"])
@@ -199,7 +202,7 @@ def delete(self, key: str) -> None:
             self._collection_data.delete_by_id(key)
         except binding.HTTPError as e:
             if e.status != 404:
-                logging.error(f"Delete checkpoint failed: {traceback.format_exc()}.")
+                logger().error(f"Delete checkpoint failed: {traceback.format_exc()}.")
                 raise
 
 

solnlib/modular_input/modinput.py

@@ -19,7 +19,10 @@
 import traceback
 
 import solnlib.splunkenv as sp
-import logging
+
+from solnlib.utils import get_solnlib_logger
+
+logger = get_solnlib_logger(__name__)
 
 
 def _parse_modinput_configs(root, outer_block, inner_block):
@@ -62,7 +65,7 @@ def _parse_modinput_configs(root, outer_block, inner_block):
 
     confs = root.getElementsByTagName(outer_block)
     if not confs:
-        logging.error("Invalid config, missing %s section", outer_block)
+        logger().error("Invalid config, missing %s section", outer_block)
         raise Exception(f"Invalid config, missing {outer_block} section")
 
     configs = []
@@ -71,7 +74,7 @@ def _parse_modinput_configs(root, outer_block, inner_block):
         config = {}
         stanza_name = stanza.getAttribute("name")
         if not stanza_name:
-            logging.error("Invalid config, missing name")
+            logger().error("Invalid config, missing name")
             raise Exception("Invalid config, missing name")
 
         config["name"] = stanza_name
@@ -107,13 +110,13 @@ def parse_modinput_configs(config_str):
     for tag in meta_configs.keys():
         nodes = doc.getElementsByTagName(tag)
         if not nodes:
-            logging.error("Invalid config, missing %s section", tag)
+            logger().error("Invalid config, missing %s section", tag)
             raise Exception("Invalid config, missing %s section", tag)
 
         if nodes[0].firstChild and nodes[0].firstChild.nodeType == nodes[0].TEXT_NODE:
             meta_configs[tag] = nodes[0].firstChild.data
         else:
-            logging.error("Invalid config, expect text ndoe")
+            logger().error("Invalid config, expect text ndoe")
             raise Exception("Invalid config, expect text ndoe")
 
     if doc.nodeName == "input":
@@ -140,7 +143,7 @@ def get_modinput_configs_from_cli(modinput, modinput_stanza=None):
         cli, stdout=subprocess.PIPE, stderr=subprocess.PIPE
     ).communicate()
     if err:
-        logging.error("Failed to get modinput configs with error: %s", err)
+        logger().error("Failed to get modinput configs with error: %s", err)
         return None, None
     else:
         return parse_modinput_configs(out)
@@ -152,7 +155,7 @@ def get_modinput_config_str_from_stdin():
     try:
         return sys.stdin.read(5000)
     except Exception:
-        logging.error(traceback.format_exc())
+        logger().error(traceback.format_exc())
         raise
 
 

solnlib/modular_input/modular_input.py

@@ -16,7 +16,6 @@
 
 """This module provides a base class of Splunk modular input."""
 
-import logging
 import sys
 import traceback
 from abc import ABCMeta, abstractmethod
@@ -35,6 +34,10 @@
 from ..orphan_process_monitor import OrphanProcessMonitor
 from . import checkpointer, event_writer
 
+from solnlib.utils import get_solnlib_logger
+
+logger = get_solnlib_logger(__name__)
+
 __all__ = ["ModularInputException", "ModularInput"]
 
 
@@ -196,7 +199,7 @@ def _create_checkpointer(self):
                     port=self.server_port,
                 )
             except binding.HTTPError:
-                logging.error(
+                logger().error(
                     "Failed to init kvstore checkpointer: %s.", traceback.format_exc()
                 )
                 raise
@@ -234,7 +237,7 @@ def _create_event_writer(self):
                     global_settings_schema=self.hec_global_settings_schema,
                 )
             except binding.HTTPError:
-                logging.error(
+                logger().error(
                     "Failed to init HECEventWriter: %s.", traceback.format_exc()
                 )
                 raise
@@ -464,10 +467,10 @@ def execute(self):
                 else:
                     self.config_name = list(input_definition["inputs"].keys())[0]
                 self.do_run(input_definition["inputs"])
-                logging.info("Modular input: %s exit normally.", self.name)
+                logger().info("Modular input: %s exit normally.", self.name)
                 return 0
             except Exception:
-                logging.error(
+                logger().error(
                     "Modular input: %s exit with exception: %s.",
                     self.name,
                     traceback.format_exc(),
@@ -490,7 +493,7 @@ def execute(self):
                 self.do_validation(validation_definition["parameters"])
                 return 0
             except Exception as e:
-                logging.error(
+                logger().error(
                     "Modular input: %s validate arguments with exception: %s.",
                     self.name,
                     traceback.format_exc(),
@@ -501,7 +504,7 @@ def execute(self):
                 sys.stderr.flush()
                 return 1
         else:
-            logging.error(
+            logger().error(
                 'Modular input: %s run with invalid arguments: "%s".',
                 self.name,
                 " ".join(sys.argv[1:]),

solnlib/rest.py

@@ -23,6 +23,12 @@
 
 import logging
 
+from solnlib.log import log_exception
+
+from solnlib.utils import get_solnlib_logger
+
+logger = get_solnlib_logger(__name__)
+
 
 def splunkd_request(
     splunkd_uri,
@@ -34,7 +40,6 @@ def splunkd_request(
     retry=1,
     verify=False,
 ) -> Optional[requests.Response]:
-
     headers = headers if headers is not None else {}
     headers["Authorization"] = f"Splunk {session_key}"
     content_type = headers.get("Content-Type")
@@ -63,13 +68,22 @@ def splunkd_request(
                 timeout=timeout,
                 verify=verify,
             )
-        except Exception:
-            logging.error(msg_temp, splunkd_uri, "unknown", format_exc())
+        except Exception as e:
+            logging.error(msg_temp, splunkd_uri, "unknown", format_exc())  # deprecated
+            log_exception(
+                logger(),
+                e,
+                exc_label="unknown",
+                msg_before=f"Failed to send rest request={splunkd_uri}, errcode=unknown",
+            )
         else:
             if resp.status_code not in (200, 201):
                 if not (method == "GET" and resp.status_code == 404):
                     logging.debug(
                         msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp)
+                    )  # deprecated
+                    logger().debug(
+                        msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp)
                     )
             else:
                 return resp

solnlib/splunk_rest_client.py

@@ -21,7 +21,6 @@
 calling splunklib SDK directly in business logic code.
 """
 
-import logging
 import os
 import sys
 import traceback
@@ -35,6 +34,10 @@
 from .net_utils import validate_scheme_host_port
 from .splunkenv import get_splunkd_access_info
 
+from solnlib.utils import get_solnlib_logger
+
+logger = get_solnlib_logger(__name__)
+
 __all__ = ["SplunkRestClient"]
 MAX_REQUEST_RETRIES = 5
 
@@ -171,7 +174,7 @@ def request(url, message, **kwargs):
                 **kwargs,
             )
         except Exception:
-            logging.error(
+            logger().error(
                 "Failed to issue http request=%s to url=%s, error=%s",
                 method,
                 url,

solnlib/timer_queue.py

@@ -25,6 +25,11 @@
 
 import sortedcontainers as sc
 
+from solnlib.utils import get_solnlib_logger
+
+logger = get_solnlib_logger(__name__)
+
+
 __all__ = ["Timer", "TimerQueueStruct", "TimerQueue"]
 
 
@@ -119,10 +124,14 @@ def remove_timer(self, timer: Timer):
         """
 
         try:
+            raise ValueError
             self._timers.remove(timer)
         except ValueError:
             logging.info(
                 "Timer=%s is not in queue, move it to cancelling " "list", timer.ident
+            )  # deprecated
+            logger.info(
+                "Timer=%s is not in queue, move it to cancelling " "list", timer.ident
             )
         else:
             self._cancelling_timers[timer.ident] = timer
@@ -184,7 +193,8 @@ def check_and_execute(self) -> float:
             try:
                 timer()
             except Exception:
-                logging.error(traceback.format_exc())
+                logging.error(traceback.format_exc())  # deprecated
+                logger.error(traceback.format_exc())
 
         self.reset_timers(expired_timers)
         return _calc_sleep_time(next_expired_time)
@@ -231,7 +241,7 @@ def start(self):
         self._started = True
 
         self._thr.start()
-        logging.info("TimerQueue started.")
+        logger.info("TimerQueue started.")
 
     def stop(self):
         """Stop the timer queue."""
@@ -284,7 +294,8 @@ def _check_and_execute(self):
                     # Note, please make timer callback effective/short
                     timer()
                 except Exception:
-                    logging.error(traceback.format_exc())
+                    logging.error(traceback.format_exc())  # deprecated
+                    logger.error(traceback.format_exc())
 
             self._reset_timers(expired_timers)
 
@@ -295,7 +306,7 @@ def _check_and_execute(self):
                     break
             except Queue.Empty:
                 pass
-        logging.info("TimerQueue stopped.")
+        logger.info("TimerQueue stopped.")
 
     def _get_expired_timers(self):
         with self._lock: