From 0d5a4b7f55d7dc50e5721d93b830451228ef9d39 Mon Sep 17 00:00:00 2001 From: sgoral Date: Thu, 21 Aug 2025 10:14:40 +0200 Subject: [PATCH 1/8] fix: initialize separate logger for each module --- solnlib/conf_manager.py | 5 ++++- solnlib/file_monitor.py | 11 +++++++--- solnlib/modular_input/checkpointer.py | 8 ++++--- solnlib/modular_input/modinput.py | 16 ++++++++------ solnlib/modular_input/modular_input.py | 16 ++++++++------ solnlib/rest.py | 30 +++++++++++++++----------- solnlib/splunk_rest_client.py | 6 ++++-- solnlib/timer_queue.py | 17 +++++++++++---- solnlib/utils.py | 7 ++++-- 9 files changed, 75 insertions(+), 41 deletions(-) diff --git a/solnlib/conf_manager.py b/solnlib/conf_manager.py index 203164b4..6f87ff90 100644 --- a/solnlib/conf_manager.py +++ b/solnlib/conf_manager.py @@ -36,6 +36,9 @@ InvalidHostnameError, ) +from solnlib.log import Logs +logger = Logs().get_logger(__name__) + __all__ = [ "ConfFile", "ConfManager", @@ -338,7 +341,7 @@ def delete(self, stanza_name: str): try: self._conf.delete(stanza_name) except KeyError: - logging.error( + logger.error( "Delete stanza: %s error: %s.", stanza_name, traceback.format_exc() ) raise ConfStanzaNotExistException( diff --git a/solnlib/file_monitor.py b/solnlib/file_monitor.py index 971d1024..a8afe732 100644 --- a/solnlib/file_monitor.py +++ b/solnlib/file_monitor.py @@ -24,6 +24,10 @@ import traceback from typing import Any, Callable, List +from solnlib.log import Logs +logger = Logs().get_logger(__name__) + + __all__ = ["FileChangesChecker", "FileMonitor"] @@ -45,7 +49,8 @@ def __init__(self, callback: Callable[[List[str]], Any], files: List): try: self.file_mtimes[k] = op.getmtime(k) except OSError: - logging.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}") + logging.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}") # deprecated + logger.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}") def check_changes(self) -> bool: """Check files change. @@ -56,7 +61,7 @@ def check_changes(self) -> bool: Returns: True if files changed else False """ - logging.debug(f"Checking files={self._files}") + logger.debug(f"Checking files={self._files}") file_mtimes = self.file_mtimes changed_files = [] for f, last_mtime in list(file_mtimes.items()): @@ -65,7 +70,7 @@ def check_changes(self) -> bool: if current_mtime != last_mtime: file_mtimes[f] = current_mtime changed_files.append(f) - logging.info(f"Detect {f} has changed", f) + logger.info(f"Detect {f} has changed", f) except OSError: pass if changed_files: diff --git a/solnlib/modular_input/checkpointer.py b/solnlib/modular_input/checkpointer.py index 378b2f33..c43ec479 100644 --- a/solnlib/modular_input/checkpointer.py +++ b/solnlib/modular_input/checkpointer.py @@ -19,7 +19,6 @@ import base64 import json -import logging import os import os.path as op import traceback @@ -31,6 +30,9 @@ from solnlib import _utils, utils +from solnlib.log import Logs +logger = Logs().get_logger(__name__) + __all__ = ["CheckpointerException", "KVStoreCheckpointer", "FileCheckpointer"] @@ -178,7 +180,7 @@ def get(self, key: str) -> Optional[Any]: record = self._collection_data.query_by_id(key) except binding.HTTPError as e: if e.status != 404: - logging.error(f"Get checkpoint failed: {traceback.format_exc()}.") + logger.error(f"Get checkpoint failed: {traceback.format_exc()}.") raise return None return json.loads(record["state"]) @@ -199,7 +201,7 @@ def delete(self, key: str) -> None: self._collection_data.delete_by_id(key) except binding.HTTPError as e: if e.status != 404: - logging.error(f"Delete checkpoint failed: {traceback.format_exc()}.") + logger.error(f"Delete checkpoint failed: {traceback.format_exc()}.") raise diff --git a/solnlib/modular_input/modinput.py b/solnlib/modular_input/modinput.py index 0d591923..d808019b 100644 --- a/solnlib/modular_input/modinput.py +++ b/solnlib/modular_input/modinput.py @@ -19,7 +19,9 @@ import traceback import solnlib.splunkenv as sp -import logging + +from solnlib.log import Logs +logger = Logs().get_logger(__name__) def _parse_modinput_configs(root, outer_block, inner_block): @@ -62,7 +64,7 @@ def _parse_modinput_configs(root, outer_block, inner_block): confs = root.getElementsByTagName(outer_block) if not confs: - logging.error("Invalid config, missing %s section", outer_block) + logger.error("Invalid config, missing %s section", outer_block) raise Exception(f"Invalid config, missing {outer_block} section") configs = [] @@ -71,7 +73,7 @@ def _parse_modinput_configs(root, outer_block, inner_block): config = {} stanza_name = stanza.getAttribute("name") if not stanza_name: - logging.error("Invalid config, missing name") + logger.error("Invalid config, missing name") raise Exception("Invalid config, missing name") config["name"] = stanza_name @@ -107,13 +109,13 @@ def parse_modinput_configs(config_str): for tag in meta_configs.keys(): nodes = doc.getElementsByTagName(tag) if not nodes: - logging.error("Invalid config, missing %s section", tag) + logger.error("Invalid config, missing %s section", tag) raise Exception("Invalid config, missing %s section", tag) if nodes[0].firstChild and nodes[0].firstChild.nodeType == nodes[0].TEXT_NODE: meta_configs[tag] = nodes[0].firstChild.data else: - logging.error("Invalid config, expect text ndoe") + logger.error("Invalid config, expect text ndoe") raise Exception("Invalid config, expect text ndoe") if doc.nodeName == "input": @@ -140,7 +142,7 @@ def get_modinput_configs_from_cli(modinput, modinput_stanza=None): cli, stdout=subprocess.PIPE, stderr=subprocess.PIPE ).communicate() if err: - logging.error("Failed to get modinput configs with error: %s", err) + logger.error("Failed to get modinput configs with error: %s", err) return None, None else: return parse_modinput_configs(out) @@ -152,7 +154,7 @@ def get_modinput_config_str_from_stdin(): try: return sys.stdin.read(5000) except Exception: - logging.error(traceback.format_exc()) + logger.error(traceback.format_exc()) raise diff --git a/solnlib/modular_input/modular_input.py b/solnlib/modular_input/modular_input.py index 0889f7a9..2e04fe07 100644 --- a/solnlib/modular_input/modular_input.py +++ b/solnlib/modular_input/modular_input.py @@ -16,7 +16,6 @@ """This module provides a base class of Splunk modular input.""" -import logging import sys import traceback from abc import ABCMeta, abstractmethod @@ -35,6 +34,9 @@ from ..orphan_process_monitor import OrphanProcessMonitor from . import checkpointer, event_writer +from solnlib.log import Logs +logger = Logs().get_logger(__name__) + __all__ = ["ModularInputException", "ModularInput"] @@ -196,7 +198,7 @@ def _create_checkpointer(self): port=self.server_port, ) except binding.HTTPError: - logging.error( + logger.error( "Failed to init kvstore checkpointer: %s.", traceback.format_exc() ) raise @@ -234,7 +236,7 @@ def _create_event_writer(self): global_settings_schema=self.hec_global_settings_schema, ) except binding.HTTPError: - logging.error( + logger.error( "Failed to init HECEventWriter: %s.", traceback.format_exc() ) raise @@ -464,10 +466,10 @@ def execute(self): else: self.config_name = list(input_definition["inputs"].keys())[0] self.do_run(input_definition["inputs"]) - logging.info("Modular input: %s exit normally.", self.name) + logger.info("Modular input: %s exit normally.", self.name) return 0 except Exception: - logging.error( + logger.error( "Modular input: %s exit with exception: %s.", self.name, traceback.format_exc(), @@ -490,7 +492,7 @@ def execute(self): self.do_validation(validation_definition["parameters"]) return 0 except Exception as e: - logging.error( + logger.error( "Modular input: %s validate arguments with exception: %s.", self.name, traceback.format_exc(), @@ -501,7 +503,7 @@ def execute(self): sys.stderr.flush() return 1 else: - logging.error( + logger.error( 'Modular input: %s run with invalid arguments: "%s".', self.name, " ".join(sys.argv[1:]), diff --git a/solnlib/rest.py b/solnlib/rest.py index df13c303..362063f9 100644 --- a/solnlib/rest.py +++ b/solnlib/rest.py @@ -23,18 +23,21 @@ import logging +from solnlib.log import Logs, log_exception + +logger = Logs().get_logger(__name__) + def splunkd_request( - splunkd_uri, - session_key, - method="GET", - headers=None, - data=None, - timeout=300, - retry=1, - verify=False, + splunkd_uri, + session_key, + method="GET", + headers=None, + data=None, + timeout=300, + retry=1, + verify=False, ) -> Optional[requests.Response]: - headers = headers if headers is not None else {} headers["Authorization"] = f"Splunk {session_key}" content_type = headers.get("Content-Type") @@ -63,14 +66,17 @@ def splunkd_request( timeout=timeout, verify=verify, ) - except Exception: - logging.error(msg_temp, splunkd_uri, "unknown", format_exc()) + except Exception as e: + logging.error(msg_temp, splunkd_uri, "unknown", format_exc()) # deprecated + log_exception(logger, e, exc_label="unknown", + msg_before=f"Failed to send rest request={splunkd_uri}, errcode=unknown") else: if resp.status_code not in (200, 201): if not (method == "GET" and resp.status_code == 404): logging.debug( msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp) - ) + ) # deprecated + logger.debug(msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp)) else: return resp else: diff --git a/solnlib/splunk_rest_client.py b/solnlib/splunk_rest_client.py index a006d734..6b879ab5 100644 --- a/solnlib/splunk_rest_client.py +++ b/solnlib/splunk_rest_client.py @@ -21,7 +21,6 @@ calling splunklib SDK directly in business logic code. """ -import logging import os import sys import traceback @@ -35,6 +34,9 @@ from .net_utils import validate_scheme_host_port from .splunkenv import get_splunkd_access_info +from solnlib.log import Logs +logger = Logs().get_logger(__name__) + __all__ = ["SplunkRestClient"] MAX_REQUEST_RETRIES = 5 @@ -171,7 +173,7 @@ def request(url, message, **kwargs): **kwargs, ) except Exception: - logging.error( + logger.error( "Failed to issue http request=%s to url=%s, error=%s", method, url, diff --git a/solnlib/timer_queue.py b/solnlib/timer_queue.py index 53d7d439..0f0455c3 100644 --- a/solnlib/timer_queue.py +++ b/solnlib/timer_queue.py @@ -25,6 +25,10 @@ import sortedcontainers as sc +from solnlib.log import Logs +logger = Logs().get_logger(__name__) + + __all__ = ["Timer", "TimerQueueStruct", "TimerQueue"] @@ -123,6 +127,9 @@ def remove_timer(self, timer: Timer): except ValueError: logging.info( "Timer=%s is not in queue, move it to cancelling " "list", timer.ident + ) # deprecated + logger.info( + "Timer=%s is not in queue, move it to cancelling " "list", timer.ident ) else: self._cancelling_timers[timer.ident] = timer @@ -184,7 +191,8 @@ def check_and_execute(self) -> float: try: timer() except Exception: - logging.error(traceback.format_exc()) + logging.error(traceback.format_exc()) # deprecated + logger.error(traceback.format_exc()) self.reset_timers(expired_timers) return _calc_sleep_time(next_expired_time) @@ -231,7 +239,7 @@ def start(self): self._started = True self._thr.start() - logging.info("TimerQueue started.") + logger.info("TimerQueue started.") def stop(self): """Stop the timer queue.""" @@ -284,7 +292,8 @@ def _check_and_execute(self): # Note, please make timer callback effective/short timer() except Exception: - logging.error(traceback.format_exc()) + logging.error(traceback.format_exc()) # deprecated + logger.error(traceback.format_exc()) self._reset_timers(expired_timers) @@ -295,7 +304,7 @@ def _check_and_execute(self): break except Queue.Empty: pass - logging.info("TimerQueue stopped.") + logger.info("TimerQueue stopped.") def _get_expired_timers(self): with self._lock: diff --git a/solnlib/utils.py b/solnlib/utils.py index b7954208..8df877e6 100644 --- a/solnlib/utils.py +++ b/solnlib/utils.py @@ -17,7 +17,6 @@ """Common utilities.""" import datetime -import logging import os import signal import time @@ -26,6 +25,10 @@ from typing import Any, Callable, List, Tuple, Union from urllib import parse as urlparse + +from solnlib.log import Logs +logger = Logs().get_logger(__name__) + __all__ = [ "handle_teardown_signals", "datetime_to_seconds", @@ -152,7 +155,7 @@ def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Exception as e: - logging.warning( + logger.warning( "Run function: %s failed: %s.", func.__name__, traceback.format_exc(), From e9e7ae216953925f3a91994bdf3cc38421d0f7bf Mon Sep 17 00:00:00 2001 From: sgoral Date: Thu, 21 Aug 2025 13:19:13 +0200 Subject: [PATCH 2/8] chore: fix precommit --- solnlib/conf_manager.py | 1 + solnlib/file_monitor.py | 5 ++++- solnlib/log.py | 11 ++++++++-- solnlib/modular_input/checkpointer.py | 1 + solnlib/modular_input/modinput.py | 1 + solnlib/modular_input/modular_input.py | 1 + solnlib/rest.py | 28 ++++++++++++++++---------- solnlib/splunk_rest_client.py | 1 + solnlib/timer_queue.py | 2 ++ solnlib/utils.py | 1 + 10 files changed, 38 insertions(+), 14 deletions(-) diff --git a/solnlib/conf_manager.py b/solnlib/conf_manager.py index 6f87ff90..ed015e73 100644 --- a/solnlib/conf_manager.py +++ b/solnlib/conf_manager.py @@ -37,6 +37,7 @@ ) from solnlib.log import Logs + logger = Logs().get_logger(__name__) __all__ = [ diff --git a/solnlib/file_monitor.py b/solnlib/file_monitor.py index a8afe732..efc4e9aa 100644 --- a/solnlib/file_monitor.py +++ b/solnlib/file_monitor.py @@ -25,6 +25,7 @@ from typing import Any, Callable, List from solnlib.log import Logs + logger = Logs().get_logger(__name__) @@ -49,7 +50,9 @@ def __init__(self, callback: Callable[[List[str]], Any], files: List): try: self.file_mtimes[k] = op.getmtime(k) except OSError: - logging.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}") # deprecated + logging.debug( + f"Getmtime for {k}, failed: {traceback.format_exc()}" + ) # deprecated logger.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}") def check_changes(self) -> bool: diff --git a/solnlib/log.py b/solnlib/log.py index e1c0511c..5363cd2e 100644 --- a/solnlib/log.py +++ b/solnlib/log.py @@ -25,7 +25,14 @@ from typing import Dict, Any from .pattern import Singleton -from .splunkenv import make_splunkhome_path + +try: + from splunk.clilib.bundle_paths import make_splunkhome_path as msp +except ImportError: + + def msp(*args, **kwargs): + raise ImportError("This module requires Splunk to be installed.") + __all__ = ["log_enter_exit", "LogException", "Logs"] @@ -147,7 +154,7 @@ def _get_log_file(cls, name): directory = cls._default_directory else: try: - directory = make_splunkhome_path(["var", "log", "splunk"]) + directory = msp(["var", "log", "splunk"]) except KeyError: raise LogException( "Log directory is empty, please set log directory " diff --git a/solnlib/modular_input/checkpointer.py b/solnlib/modular_input/checkpointer.py index c43ec479..641928f5 100644 --- a/solnlib/modular_input/checkpointer.py +++ b/solnlib/modular_input/checkpointer.py @@ -31,6 +31,7 @@ from solnlib import _utils, utils from solnlib.log import Logs + logger = Logs().get_logger(__name__) __all__ = ["CheckpointerException", "KVStoreCheckpointer", "FileCheckpointer"] diff --git a/solnlib/modular_input/modinput.py b/solnlib/modular_input/modinput.py index d808019b..7635ddf6 100644 --- a/solnlib/modular_input/modinput.py +++ b/solnlib/modular_input/modinput.py @@ -21,6 +21,7 @@ import solnlib.splunkenv as sp from solnlib.log import Logs + logger = Logs().get_logger(__name__) diff --git a/solnlib/modular_input/modular_input.py b/solnlib/modular_input/modular_input.py index 2e04fe07..600f9253 100644 --- a/solnlib/modular_input/modular_input.py +++ b/solnlib/modular_input/modular_input.py @@ -35,6 +35,7 @@ from . import checkpointer, event_writer from solnlib.log import Logs + logger = Logs().get_logger(__name__) __all__ = ["ModularInputException", "ModularInput"] diff --git a/solnlib/rest.py b/solnlib/rest.py index 362063f9..3198c044 100644 --- a/solnlib/rest.py +++ b/solnlib/rest.py @@ -29,14 +29,14 @@ def splunkd_request( - splunkd_uri, - session_key, - method="GET", - headers=None, - data=None, - timeout=300, - retry=1, - verify=False, + splunkd_uri, + session_key, + method="GET", + headers=None, + data=None, + timeout=300, + retry=1, + verify=False, ) -> Optional[requests.Response]: headers = headers if headers is not None else {} headers["Authorization"] = f"Splunk {session_key}" @@ -68,15 +68,21 @@ def splunkd_request( ) except Exception as e: logging.error(msg_temp, splunkd_uri, "unknown", format_exc()) # deprecated - log_exception(logger, e, exc_label="unknown", - msg_before=f"Failed to send rest request={splunkd_uri}, errcode=unknown") + log_exception( + logger, + e, + exc_label="unknown", + msg_before=f"Failed to send rest request={splunkd_uri}, errcode=unknown", + ) else: if resp.status_code not in (200, 201): if not (method == "GET" and resp.status_code == 404): logging.debug( msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp) ) # deprecated - logger.debug(msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp)) + logger.debug( + msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp) + ) else: return resp else: diff --git a/solnlib/splunk_rest_client.py b/solnlib/splunk_rest_client.py index 6b879ab5..c0c42314 100644 --- a/solnlib/splunk_rest_client.py +++ b/solnlib/splunk_rest_client.py @@ -35,6 +35,7 @@ from .splunkenv import get_splunkd_access_info from solnlib.log import Logs + logger = Logs().get_logger(__name__) __all__ = ["SplunkRestClient"] diff --git a/solnlib/timer_queue.py b/solnlib/timer_queue.py index 0f0455c3..472574b4 100644 --- a/solnlib/timer_queue.py +++ b/solnlib/timer_queue.py @@ -26,6 +26,7 @@ import sortedcontainers as sc from solnlib.log import Logs + logger = Logs().get_logger(__name__) @@ -123,6 +124,7 @@ def remove_timer(self, timer: Timer): """ try: + raise ValueError self._timers.remove(timer) except ValueError: logging.info( diff --git a/solnlib/utils.py b/solnlib/utils.py index 8df877e6..468c1978 100644 --- a/solnlib/utils.py +++ b/solnlib/utils.py @@ -27,6 +27,7 @@ from solnlib.log import Logs + logger = Logs().get_logger(__name__) __all__ = [ From 2f5cfe4262d855e925bd4ddd84ee3ea7408c2d57 Mon Sep 17 00:00:00 2001 From: sgoral Date: Fri, 22 Aug 2025 09:47:41 +0200 Subject: [PATCH 3/8] test: fix current tests --- tests/unit/conftest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 8b3bf665..828cfbc6 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,12 +1,24 @@ import json import socket +import sys +import os.path as op from contextlib import closing from http.server import BaseHTTPRequestHandler, HTTPServer from threading import Thread +from unittest.mock import MagicMock import pytest +cur_dir = op.dirname(op.abspath(__file__)) + +mock_msp = MagicMock(return_value=op.sep.join([cur_dir, "data/mock_log"])) +mock_bundle_paths = MagicMock() +mock_bundle_paths.make_splunkhome_path = mock_msp + +sys.modules["splunk.clilib.bundle_paths"] = mock_bundle_paths + + @pytest.fixture(scope="session") def http_mock_server(): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: From debbc2120b30aa15fefe63dc605079e6a5191564 Mon Sep 17 00:00:00 2001 From: sgoral Date: Fri, 22 Aug 2025 09:55:12 +0200 Subject: [PATCH 4/8] test: add .gitkeep to empty dir --- tests/unit/data/mock_log/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/unit/data/mock_log/.gitkeep diff --git a/tests/unit/data/mock_log/.gitkeep b/tests/unit/data/mock_log/.gitkeep new file mode 100644 index 00000000..e69de29b From 3f734a7eea752c14852d64203168f26401293f1c Mon Sep 17 00:00:00 2001 From: sgoral Date: Fri, 22 Aug 2025 10:20:44 +0200 Subject: [PATCH 5/8] test: add .gitkeep to empty dir --- solnlib/utils.py | 6 ++++++ tests/integration/conftest.py | 10 ++++++++++ tests/integration/data/mock_log/.gitkeep | 0 3 files changed, 16 insertions(+) create mode 100644 tests/integration/data/mock_log/.gitkeep diff --git a/solnlib/utils.py b/solnlib/utils.py index 468c1978..1a441624 100644 --- a/solnlib/utils.py +++ b/solnlib/utils.py @@ -17,6 +17,7 @@ """Common utilities.""" import datetime +import logging import os import signal import time @@ -156,6 +157,11 @@ def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Exception as e: + logging.warning( + "Run function: %s failed: %s.", + func.__name__, + traceback.format_exc(), + ) # deprecated logger.warning( "Run function: %s failed: %s.", func.__name__, diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 9f9f6f0b..00233b7d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,10 +1,20 @@ import os import sys +import os.path as op +from unittest.mock import MagicMock import pytest import context +cur_dir = op.dirname(op.abspath(__file__)) + +mock_msp = MagicMock(return_value=op.sep.join([cur_dir, "data/mock_log"])) +mock_bundle_paths = MagicMock() +mock_bundle_paths.make_splunkhome_path = mock_msp + +sys.modules["splunk.clilib.bundle_paths"] = mock_bundle_paths + @pytest.fixture(autouse=True, scope="session") def setup_env(): diff --git a/tests/integration/data/mock_log/.gitkeep b/tests/integration/data/mock_log/.gitkeep new file mode 100644 index 00000000..e69de29b From 82d5d0a75bab1370eed7460d72b62122549de77e Mon Sep 17 00:00:00 2001 From: sgoral Date: Fri, 22 Aug 2025 12:21:14 +0200 Subject: [PATCH 6/8] test: add mock make_splunknhome_path to integration tests --- tests/integration/conftest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 00233b7d..51d4743d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,15 +7,27 @@ import context + cur_dir = op.dirname(op.abspath(__file__)) + +# Mock only once globally, before anything imports `log.py` mock_msp = MagicMock(return_value=op.sep.join([cur_dir, "data/mock_log"])) mock_bundle_paths = MagicMock() mock_bundle_paths.make_splunkhome_path = mock_msp +sys.modules["splunk"] = MagicMock() +sys.modules["splunk.clilib"] = MagicMock() sys.modules["splunk.clilib.bundle_paths"] = mock_bundle_paths +@pytest.fixture(autouse=True) +def patch_log_msp(monkeypatch): + """Ensure log.msp is patched in all tests after mocking sys.modules.""" + from solnlib import log # only import after sys.modules is patched + monkeypatch.setattr(log, "msp", mock_msp) + + @pytest.fixture(autouse=True, scope="session") def setup_env(): # path manipulation get the 'splunk' library for the imports while running on GH Actions From fe66d130c303bb8144045cc2498714ecb044f3c0 Mon Sep 17 00:00:00 2001 From: sgoral Date: Fri, 29 Aug 2025 11:03:32 +0200 Subject: [PATCH 7/8] test: adjust unit tests --- solnlib/conf_manager.py | 8 +++++--- solnlib/file_monitor.py | 12 +++++++----- solnlib/modular_input/checkpointer.py | 8 ++++---- solnlib/modular_input/modinput.py | 16 ++++++++-------- solnlib/modular_input/modular_input.py | 16 ++++++++-------- solnlib/rest.py | 10 ++++++---- solnlib/splunk_rest_client.py | 6 +++--- solnlib/timer_queue.py | 9 ++++++--- solnlib/utils.py | 18 +++++++++++------- tests/integration/conftest.py | 22 ---------------------- tests/integration/test_user_access.py | 7 ++++--- tests/unit/data/mock_log/.gitkeep | 0 tests/unit/test_log.py | 5 +++-- 13 files changed, 65 insertions(+), 72 deletions(-) delete mode 100644 tests/unit/data/mock_log/.gitkeep diff --git a/solnlib/conf_manager.py b/solnlib/conf_manager.py index ed015e73..386e54d0 100644 --- a/solnlib/conf_manager.py +++ b/solnlib/conf_manager.py @@ -36,9 +36,11 @@ InvalidHostnameError, ) -from solnlib.log import Logs +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) + -logger = Logs().get_logger(__name__) __all__ = [ "ConfFile", @@ -342,7 +344,7 @@ def delete(self, stanza_name: str): try: self._conf.delete(stanza_name) except KeyError: - logger.error( + logger().error( "Delete stanza: %s error: %s.", stanza_name, traceback.format_exc() ) raise ConfStanzaNotExistException( diff --git a/solnlib/file_monitor.py b/solnlib/file_monitor.py index efc4e9aa..d0f26477 100644 --- a/solnlib/file_monitor.py +++ b/solnlib/file_monitor.py @@ -24,9 +24,11 @@ import traceback from typing import Any, Callable, List -from solnlib.log import Logs +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) + -logger = Logs().get_logger(__name__) __all__ = ["FileChangesChecker", "FileMonitor"] @@ -53,7 +55,7 @@ def __init__(self, callback: Callable[[List[str]], Any], files: List): logging.debug( f"Getmtime for {k}, failed: {traceback.format_exc()}" ) # deprecated - logger.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}") + logger().debug(f"Getmtime for {k}, failed: {traceback.format_exc()}") def check_changes(self) -> bool: """Check files change. @@ -64,7 +66,7 @@ def check_changes(self) -> bool: Returns: True if files changed else False """ - logger.debug(f"Checking files={self._files}") + logger().debug(f"Checking files={self._files}") file_mtimes = self.file_mtimes changed_files = [] for f, last_mtime in list(file_mtimes.items()): @@ -73,7 +75,7 @@ def check_changes(self) -> bool: if current_mtime != last_mtime: file_mtimes[f] = current_mtime changed_files.append(f) - logger.info(f"Detect {f} has changed", f) + logger().info(f"Detect {f} has changed", f) except OSError: pass if changed_files: diff --git a/solnlib/modular_input/checkpointer.py b/solnlib/modular_input/checkpointer.py index 641928f5..340e10f4 100644 --- a/solnlib/modular_input/checkpointer.py +++ b/solnlib/modular_input/checkpointer.py @@ -30,9 +30,9 @@ from solnlib import _utils, utils -from solnlib.log import Logs +from solnlib.utils import get_solnlib_logger -logger = Logs().get_logger(__name__) +logger = get_solnlib_logger(__name__) __all__ = ["CheckpointerException", "KVStoreCheckpointer", "FileCheckpointer"] @@ -181,7 +181,7 @@ def get(self, key: str) -> Optional[Any]: record = self._collection_data.query_by_id(key) except binding.HTTPError as e: if e.status != 404: - logger.error(f"Get checkpoint failed: {traceback.format_exc()}.") + logger().error(f"Get checkpoint failed: {traceback.format_exc()}.") raise return None return json.loads(record["state"]) @@ -202,7 +202,7 @@ def delete(self, key: str) -> None: self._collection_data.delete_by_id(key) except binding.HTTPError as e: if e.status != 404: - logger.error(f"Delete checkpoint failed: {traceback.format_exc()}.") + logger().error(f"Delete checkpoint failed: {traceback.format_exc()}.") raise diff --git a/solnlib/modular_input/modinput.py b/solnlib/modular_input/modinput.py index 7635ddf6..600cef33 100644 --- a/solnlib/modular_input/modinput.py +++ b/solnlib/modular_input/modinput.py @@ -20,9 +20,9 @@ import solnlib.splunkenv as sp -from solnlib.log import Logs +from solnlib.utils import get_solnlib_logger -logger = Logs().get_logger(__name__) +logger = get_solnlib_logger(__name__) def _parse_modinput_configs(root, outer_block, inner_block): @@ -65,7 +65,7 @@ def _parse_modinput_configs(root, outer_block, inner_block): confs = root.getElementsByTagName(outer_block) if not confs: - logger.error("Invalid config, missing %s section", outer_block) + logger().error("Invalid config, missing %s section", outer_block) raise Exception(f"Invalid config, missing {outer_block} section") configs = [] @@ -74,7 +74,7 @@ def _parse_modinput_configs(root, outer_block, inner_block): config = {} stanza_name = stanza.getAttribute("name") if not stanza_name: - logger.error("Invalid config, missing name") + logger().error("Invalid config, missing name") raise Exception("Invalid config, missing name") config["name"] = stanza_name @@ -110,13 +110,13 @@ def parse_modinput_configs(config_str): for tag in meta_configs.keys(): nodes = doc.getElementsByTagName(tag) if not nodes: - logger.error("Invalid config, missing %s section", tag) + logger().error("Invalid config, missing %s section", tag) raise Exception("Invalid config, missing %s section", tag) if nodes[0].firstChild and nodes[0].firstChild.nodeType == nodes[0].TEXT_NODE: meta_configs[tag] = nodes[0].firstChild.data else: - logger.error("Invalid config, expect text ndoe") + logger().error("Invalid config, expect text ndoe") raise Exception("Invalid config, expect text ndoe") if doc.nodeName == "input": @@ -143,7 +143,7 @@ def get_modinput_configs_from_cli(modinput, modinput_stanza=None): cli, stdout=subprocess.PIPE, stderr=subprocess.PIPE ).communicate() if err: - logger.error("Failed to get modinput configs with error: %s", err) + logger().error("Failed to get modinput configs with error: %s", err) return None, None else: return parse_modinput_configs(out) @@ -155,7 +155,7 @@ def get_modinput_config_str_from_stdin(): try: return sys.stdin.read(5000) except Exception: - logger.error(traceback.format_exc()) + logger().error(traceback.format_exc()) raise diff --git a/solnlib/modular_input/modular_input.py b/solnlib/modular_input/modular_input.py index 600f9253..6ddd48f2 100644 --- a/solnlib/modular_input/modular_input.py +++ b/solnlib/modular_input/modular_input.py @@ -34,9 +34,9 @@ from ..orphan_process_monitor import OrphanProcessMonitor from . import checkpointer, event_writer -from solnlib.log import Logs +from solnlib.utils import get_solnlib_logger -logger = Logs().get_logger(__name__) +logger = get_solnlib_logger(__name__) __all__ = ["ModularInputException", "ModularInput"] @@ -199,7 +199,7 @@ def _create_checkpointer(self): port=self.server_port, ) except binding.HTTPError: - logger.error( + logger().error( "Failed to init kvstore checkpointer: %s.", traceback.format_exc() ) raise @@ -237,7 +237,7 @@ def _create_event_writer(self): global_settings_schema=self.hec_global_settings_schema, ) except binding.HTTPError: - logger.error( + logger().error( "Failed to init HECEventWriter: %s.", traceback.format_exc() ) raise @@ -467,10 +467,10 @@ def execute(self): else: self.config_name = list(input_definition["inputs"].keys())[0] self.do_run(input_definition["inputs"]) - logger.info("Modular input: %s exit normally.", self.name) + logger().info("Modular input: %s exit normally.", self.name) return 0 except Exception: - logger.error( + logger().error( "Modular input: %s exit with exception: %s.", self.name, traceback.format_exc(), @@ -493,7 +493,7 @@ def execute(self): self.do_validation(validation_definition["parameters"]) return 0 except Exception as e: - logger.error( + logger().error( "Modular input: %s validate arguments with exception: %s.", self.name, traceback.format_exc(), @@ -504,7 +504,7 @@ def execute(self): sys.stderr.flush() return 1 else: - logger.error( + logger().error( 'Modular input: %s run with invalid arguments: "%s".', self.name, " ".join(sys.argv[1:]), diff --git a/solnlib/rest.py b/solnlib/rest.py index 3198c044..70c30f0c 100644 --- a/solnlib/rest.py +++ b/solnlib/rest.py @@ -23,9 +23,11 @@ import logging -from solnlib.log import Logs, log_exception +from solnlib.log import log_exception -logger = Logs().get_logger(__name__) +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) def splunkd_request( @@ -69,7 +71,7 @@ def splunkd_request( except Exception as e: logging.error(msg_temp, splunkd_uri, "unknown", format_exc()) # deprecated log_exception( - logger, + logger(), e, exc_label="unknown", msg_before=f"Failed to send rest request={splunkd_uri}, errcode=unknown", @@ -80,7 +82,7 @@ def splunkd_request( logging.debug( msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp) ) # deprecated - logger.debug( + logger().debug( msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp) ) else: diff --git a/solnlib/splunk_rest_client.py b/solnlib/splunk_rest_client.py index c0c42314..990983c2 100644 --- a/solnlib/splunk_rest_client.py +++ b/solnlib/splunk_rest_client.py @@ -34,9 +34,9 @@ from .net_utils import validate_scheme_host_port from .splunkenv import get_splunkd_access_info -from solnlib.log import Logs +from solnlib.utils import get_solnlib_logger -logger = Logs().get_logger(__name__) +logger = get_solnlib_logger(__name__) __all__ = ["SplunkRestClient"] MAX_REQUEST_RETRIES = 5 @@ -174,7 +174,7 @@ def request(url, message, **kwargs): **kwargs, ) except Exception: - logger.error( + logger().error( "Failed to issue http request=%s to url=%s, error=%s", method, url, diff --git a/solnlib/timer_queue.py b/solnlib/timer_queue.py index 472574b4..924ab6d9 100644 --- a/solnlib/timer_queue.py +++ b/solnlib/timer_queue.py @@ -27,7 +27,7 @@ from solnlib.log import Logs -logger = Logs().get_logger(__name__) + __all__ = ["Timer", "TimerQueueStruct", "TimerQueue"] @@ -193,7 +193,8 @@ def check_and_execute(self) -> float: try: timer() except Exception: - logging.error(traceback.format_exc()) # deprecated + logging.error(traceback.format_exc()) + logger = Logs().get_logger(__name__)# deprecated logger.error(traceback.format_exc()) self.reset_timers(expired_timers) @@ -241,6 +242,7 @@ def start(self): self._started = True self._thr.start() + logger = Logs().get_logger(__name__) logger.info("TimerQueue started.") def stop(self): @@ -294,7 +296,8 @@ def _check_and_execute(self): # Note, please make timer callback effective/short timer() except Exception: - logging.error(traceback.format_exc()) # deprecated + logging.error(traceback.format_exc()) + logger = Logs().get_logger(__name__)# deprecated logger.error(traceback.format_exc()) self._reset_timers(expired_timers) diff --git a/solnlib/utils.py b/solnlib/utils.py index 1a441624..e77ec1a1 100644 --- a/solnlib/utils.py +++ b/solnlib/utils.py @@ -26,10 +26,8 @@ from typing import Any, Callable, List, Tuple, Union from urllib import parse as urlparse - from solnlib.log import Logs -logger = Logs().get_logger(__name__) __all__ = [ "handle_teardown_signals", @@ -162,11 +160,6 @@ def wrapper(*args, **kwargs): func.__name__, traceback.format_exc(), ) # deprecated - logger.warning( - "Run function: %s failed: %s.", - func.__name__, - traceback.format_exc(), - ) if not exceptions or any( isinstance(e, exception) for exception in exceptions ): @@ -229,3 +222,14 @@ def get_appname_from_path(absolute_path): pass continue return "-" + + +def get_solnlib_logger(name): + _logger = None + + def logger(): + nonlocal _logger + if _logger is None: + _logger = Logs().get_logger(name) + return _logger + return logger \ No newline at end of file diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 51d4743d..9f9f6f0b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,33 +1,11 @@ import os import sys -import os.path as op -from unittest.mock import MagicMock import pytest import context -cur_dir = op.dirname(op.abspath(__file__)) - - -# Mock only once globally, before anything imports `log.py` -mock_msp = MagicMock(return_value=op.sep.join([cur_dir, "data/mock_log"])) -mock_bundle_paths = MagicMock() -mock_bundle_paths.make_splunkhome_path = mock_msp - -sys.modules["splunk"] = MagicMock() -sys.modules["splunk.clilib"] = MagicMock() -sys.modules["splunk.clilib.bundle_paths"] = mock_bundle_paths - - -@pytest.fixture(autouse=True) -def patch_log_msp(monkeypatch): - """Ensure log.msp is patched in all tests after mocking sys.modules.""" - from solnlib import log # only import after sys.modules is patched - monkeypatch.setattr(log, "msp", mock_msp) - - @pytest.fixture(autouse=True, scope="session") def setup_env(): # path manipulation get the 'splunk' library for the imports while running on GH Actions diff --git a/tests/integration/test_user_access.py b/tests/integration/test_user_access.py index 09ba9857..b16497bb 100644 --- a/tests/integration/test_user_access.py +++ b/tests/integration/test_user_access.py @@ -18,7 +18,7 @@ import conftest import os.path as op import sys - +from unittest import mock import pytest from solnlib import user_access @@ -26,7 +26,8 @@ sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -def test_object_acl_manager(): +def test_object_acl_manager(monkeypatch): + conftest.mock_splunk(monkeypatch) session_key = context.get_session_key() oaclm = user_access.ObjectACLManager( "object_acls_collection", @@ -183,7 +184,7 @@ def test_get_user_roles(): def test_user_access(monkeypatch): - test_object_acl_manager() + test_object_acl_manager(monkeypatch) test_app_capability_manager() test_check_user_access(monkeypatch) test_get_current_username() diff --git a/tests/unit/data/mock_log/.gitkeep b/tests/unit/data/mock_log/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/test_log.py b/tests/unit/test_log.py index 2e565829..006bf118 100644 --- a/tests/unit/test_log.py +++ b/tests/unit/test_log.py @@ -30,7 +30,9 @@ from solnlib import log -reset_root_log_path = os.path.join(".", ".root_log") +cur_dir = os.path.dirname(os.path.abspath(__file__)) + +reset_root_log_path = os.path.sep.join([cur_dir, "data/mock_log"]) def setup_module(module): @@ -45,7 +47,6 @@ def setup_module(module): def teardown_module(module): shutil.rmtree("./.log") - shutil.rmtree(reset_root_log_path) def multiprocess_worker(logger_ref): From c8fa0f3361b848a04d65091d0648d4ea32e0afd5 Mon Sep 17 00:00:00 2001 From: sgoral Date: Fri, 29 Aug 2025 11:08:07 +0200 Subject: [PATCH 8/8] test: adjust unit tests --- solnlib/conf_manager.py | 1 - solnlib/file_monitor.py | 2 -- solnlib/timer_queue.py | 11 ++++------- solnlib/utils.py | 4 +++- tests/integration/test_user_access.py | 1 - tests/unit/data/mock_log/.gitkeep | 0 6 files changed, 7 insertions(+), 12 deletions(-) create mode 100644 tests/unit/data/mock_log/.gitkeep diff --git a/solnlib/conf_manager.py b/solnlib/conf_manager.py index 386e54d0..af72cb17 100644 --- a/solnlib/conf_manager.py +++ b/solnlib/conf_manager.py @@ -41,7 +41,6 @@ logger = get_solnlib_logger(__name__) - __all__ = [ "ConfFile", "ConfManager", diff --git a/solnlib/file_monitor.py b/solnlib/file_monitor.py index d0f26477..6ca81bef 100644 --- a/solnlib/file_monitor.py +++ b/solnlib/file_monitor.py @@ -29,8 +29,6 @@ logger = get_solnlib_logger(__name__) - - __all__ = ["FileChangesChecker", "FileMonitor"] diff --git a/solnlib/timer_queue.py b/solnlib/timer_queue.py index 924ab6d9..684eee74 100644 --- a/solnlib/timer_queue.py +++ b/solnlib/timer_queue.py @@ -25,9 +25,9 @@ import sortedcontainers as sc -from solnlib.log import Logs - +from solnlib.utils import get_solnlib_logger +logger = get_solnlib_logger(__name__) __all__ = ["Timer", "TimerQueueStruct", "TimerQueue"] @@ -193,8 +193,7 @@ def check_and_execute(self) -> float: try: timer() except Exception: - logging.error(traceback.format_exc()) - logger = Logs().get_logger(__name__)# deprecated + logging.error(traceback.format_exc()) # deprecated logger.error(traceback.format_exc()) self.reset_timers(expired_timers) @@ -242,7 +241,6 @@ def start(self): self._started = True self._thr.start() - logger = Logs().get_logger(__name__) logger.info("TimerQueue started.") def stop(self): @@ -296,8 +294,7 @@ def _check_and_execute(self): # Note, please make timer callback effective/short timer() except Exception: - logging.error(traceback.format_exc()) - logger = Logs().get_logger(__name__)# deprecated + logging.error(traceback.format_exc()) # deprecated logger.error(traceback.format_exc()) self._reset_timers(expired_timers) diff --git a/solnlib/utils.py b/solnlib/utils.py index e77ec1a1..a5c4f02c 100644 --- a/solnlib/utils.py +++ b/solnlib/utils.py @@ -30,6 +30,7 @@ __all__ = [ + "get_solnlib_logger", "handle_teardown_signals", "datetime_to_seconds", "is_true", @@ -232,4 +233,5 @@ def logger(): if _logger is None: _logger = Logs().get_logger(name) return _logger - return logger \ No newline at end of file + + return logger diff --git a/tests/integration/test_user_access.py b/tests/integration/test_user_access.py index b16497bb..bf8fdeaa 100644 --- a/tests/integration/test_user_access.py +++ b/tests/integration/test_user_access.py @@ -18,7 +18,6 @@ import conftest import os.path as op import sys -from unittest import mock import pytest from solnlib import user_access diff --git a/tests/unit/data/mock_log/.gitkeep b/tests/unit/data/mock_log/.gitkeep new file mode 100644 index 00000000..e69de29b