diff --git a/solnlib/conf_manager.py b/solnlib/conf_manager.py index 203164b4..af72cb17 100644 --- a/solnlib/conf_manager.py +++ b/solnlib/conf_manager.py @@ -36,6 +36,11 @@ InvalidHostnameError, ) +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) + + __all__ = [ "ConfFile", "ConfManager", @@ -338,7 +343,7 @@ def delete(self, stanza_name: str): try: self._conf.delete(stanza_name) except KeyError: - logging.error( + logger().error( "Delete stanza: %s error: %s.", stanza_name, traceback.format_exc() ) raise ConfStanzaNotExistException( diff --git a/solnlib/file_monitor.py b/solnlib/file_monitor.py index 971d1024..6ca81bef 100644 --- a/solnlib/file_monitor.py +++ b/solnlib/file_monitor.py @@ -24,6 +24,11 @@ import traceback from typing import Any, Callable, List +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) + + __all__ = ["FileChangesChecker", "FileMonitor"] @@ -45,7 +50,10 @@ def __init__(self, callback: Callable[[List[str]], Any], files: List): try: self.file_mtimes[k] = op.getmtime(k) except OSError: - logging.debug(f"Getmtime for {k}, failed: {traceback.format_exc()}") + logging.debug( + f"Getmtime for {k}, failed: {traceback.format_exc()}" + ) # deprecated + logger().debug(f"Getmtime for {k}, failed: {traceback.format_exc()}") def check_changes(self) -> bool: """Check files change. @@ -56,7 +64,7 @@ def check_changes(self) -> bool: Returns: True if files changed else False """ - logging.debug(f"Checking files={self._files}") + logger().debug(f"Checking files={self._files}") file_mtimes = self.file_mtimes changed_files = [] for f, last_mtime in list(file_mtimes.items()): @@ -65,7 +73,7 @@ def check_changes(self) -> bool: if current_mtime != last_mtime: file_mtimes[f] = current_mtime changed_files.append(f) - logging.info(f"Detect {f} has changed", f) + logger().info(f"Detect {f} has changed", f) except OSError: pass if changed_files: diff --git a/solnlib/log.py b/solnlib/log.py index e1c0511c..5363cd2e 100644 --- a/solnlib/log.py +++ b/solnlib/log.py @@ -25,7 +25,14 @@ from typing import Dict, Any from .pattern import Singleton -from .splunkenv import make_splunkhome_path + +try: + from splunk.clilib.bundle_paths import make_splunkhome_path as msp +except ImportError: + + def msp(*args, **kwargs): + raise ImportError("This module requires Splunk to be installed.") + __all__ = ["log_enter_exit", "LogException", "Logs"] @@ -147,7 +154,7 @@ def _get_log_file(cls, name): directory = cls._default_directory else: try: - directory = make_splunkhome_path(["var", "log", "splunk"]) + directory = msp(["var", "log", "splunk"]) except KeyError: raise LogException( "Log directory is empty, please set log directory " diff --git a/solnlib/modular_input/checkpointer.py b/solnlib/modular_input/checkpointer.py index 378b2f33..340e10f4 100644 --- a/solnlib/modular_input/checkpointer.py +++ b/solnlib/modular_input/checkpointer.py @@ -19,7 +19,6 @@ import base64 import json -import logging import os import os.path as op import traceback @@ -31,6 +30,10 @@ from solnlib import _utils, utils +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) + __all__ = ["CheckpointerException", "KVStoreCheckpointer", "FileCheckpointer"] @@ -178,7 +181,7 @@ def get(self, key: str) -> Optional[Any]: record = self._collection_data.query_by_id(key) except binding.HTTPError as e: if e.status != 404: - logging.error(f"Get checkpoint failed: {traceback.format_exc()}.") + logger().error(f"Get checkpoint failed: {traceback.format_exc()}.") raise return None return json.loads(record["state"]) @@ -199,7 +202,7 @@ def delete(self, key: str) -> None: self._collection_data.delete_by_id(key) except binding.HTTPError as e: if e.status != 404: - logging.error(f"Delete checkpoint failed: {traceback.format_exc()}.") + logger().error(f"Delete checkpoint failed: {traceback.format_exc()}.") raise diff --git a/solnlib/modular_input/modinput.py b/solnlib/modular_input/modinput.py index 0d591923..600cef33 100644 --- a/solnlib/modular_input/modinput.py +++ b/solnlib/modular_input/modinput.py @@ -19,7 +19,10 @@ import traceback import solnlib.splunkenv as sp -import logging + +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) def _parse_modinput_configs(root, outer_block, inner_block): @@ -62,7 +65,7 @@ def _parse_modinput_configs(root, outer_block, inner_block): confs = root.getElementsByTagName(outer_block) if not confs: - logging.error("Invalid config, missing %s section", outer_block) + logger().error("Invalid config, missing %s section", outer_block) raise Exception(f"Invalid config, missing {outer_block} section") configs = [] @@ -71,7 +74,7 @@ def _parse_modinput_configs(root, outer_block, inner_block): config = {} stanza_name = stanza.getAttribute("name") if not stanza_name: - logging.error("Invalid config, missing name") + logger().error("Invalid config, missing name") raise Exception("Invalid config, missing name") config["name"] = stanza_name @@ -107,13 +110,13 @@ def parse_modinput_configs(config_str): for tag in meta_configs.keys(): nodes = doc.getElementsByTagName(tag) if not nodes: - logging.error("Invalid config, missing %s section", tag) + logger().error("Invalid config, missing %s section", tag) raise Exception("Invalid config, missing %s section", tag) if nodes[0].firstChild and nodes[0].firstChild.nodeType == nodes[0].TEXT_NODE: meta_configs[tag] = nodes[0].firstChild.data else: - logging.error("Invalid config, expect text ndoe") + logger().error("Invalid config, expect text ndoe") raise Exception("Invalid config, expect text ndoe") if doc.nodeName == "input": @@ -140,7 +143,7 @@ def get_modinput_configs_from_cli(modinput, modinput_stanza=None): cli, stdout=subprocess.PIPE, stderr=subprocess.PIPE ).communicate() if err: - logging.error("Failed to get modinput configs with error: %s", err) + logger().error("Failed to get modinput configs with error: %s", err) return None, None else: return parse_modinput_configs(out) @@ -152,7 +155,7 @@ def get_modinput_config_str_from_stdin(): try: return sys.stdin.read(5000) except Exception: - logging.error(traceback.format_exc()) + logger().error(traceback.format_exc()) raise diff --git a/solnlib/modular_input/modular_input.py b/solnlib/modular_input/modular_input.py index 0889f7a9..6ddd48f2 100644 --- a/solnlib/modular_input/modular_input.py +++ b/solnlib/modular_input/modular_input.py @@ -16,7 +16,6 @@ """This module provides a base class of Splunk modular input.""" -import logging import sys import traceback from abc import ABCMeta, abstractmethod @@ -35,6 +34,10 @@ from ..orphan_process_monitor import OrphanProcessMonitor from . import checkpointer, event_writer +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) + __all__ = ["ModularInputException", "ModularInput"] @@ -196,7 +199,7 @@ def _create_checkpointer(self): port=self.server_port, ) except binding.HTTPError: - logging.error( + logger().error( "Failed to init kvstore checkpointer: %s.", traceback.format_exc() ) raise @@ -234,7 +237,7 @@ def _create_event_writer(self): global_settings_schema=self.hec_global_settings_schema, ) except binding.HTTPError: - logging.error( + logger().error( "Failed to init HECEventWriter: %s.", traceback.format_exc() ) raise @@ -464,10 +467,10 @@ def execute(self): else: self.config_name = list(input_definition["inputs"].keys())[0] self.do_run(input_definition["inputs"]) - logging.info("Modular input: %s exit normally.", self.name) + logger().info("Modular input: %s exit normally.", self.name) return 0 except Exception: - logging.error( + logger().error( "Modular input: %s exit with exception: %s.", self.name, traceback.format_exc(), @@ -490,7 +493,7 @@ def execute(self): self.do_validation(validation_definition["parameters"]) return 0 except Exception as e: - logging.error( + logger().error( "Modular input: %s validate arguments with exception: %s.", self.name, traceback.format_exc(), @@ -501,7 +504,7 @@ def execute(self): sys.stderr.flush() return 1 else: - logging.error( + logger().error( 'Modular input: %s run with invalid arguments: "%s".', self.name, " ".join(sys.argv[1:]), diff --git a/solnlib/rest.py b/solnlib/rest.py index df13c303..70c30f0c 100644 --- a/solnlib/rest.py +++ b/solnlib/rest.py @@ -23,6 +23,12 @@ import logging +from solnlib.log import log_exception + +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) + def splunkd_request( splunkd_uri, @@ -34,7 +40,6 @@ def splunkd_request( retry=1, verify=False, ) -> Optional[requests.Response]: - headers = headers if headers is not None else {} headers["Authorization"] = f"Splunk {session_key}" content_type = headers.get("Content-Type") @@ -63,13 +68,22 @@ def splunkd_request( timeout=timeout, verify=verify, ) - except Exception: - logging.error(msg_temp, splunkd_uri, "unknown", format_exc()) + except Exception as e: + logging.error(msg_temp, splunkd_uri, "unknown", format_exc()) # deprecated + log_exception( + logger(), + e, + exc_label="unknown", + msg_before=f"Failed to send rest request={splunkd_uri}, errcode=unknown", + ) else: if resp.status_code not in (200, 201): if not (method == "GET" and resp.status_code == 404): logging.debug( msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp) + ) # deprecated + logger().debug( + msg_temp, splunkd_uri, resp.status_code, code_to_msg(resp) ) else: return resp diff --git a/solnlib/splunk_rest_client.py b/solnlib/splunk_rest_client.py index a006d734..990983c2 100644 --- a/solnlib/splunk_rest_client.py +++ b/solnlib/splunk_rest_client.py @@ -21,7 +21,6 @@ calling splunklib SDK directly in business logic code. """ -import logging import os import sys import traceback @@ -35,6 +34,10 @@ from .net_utils import validate_scheme_host_port from .splunkenv import get_splunkd_access_info +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) + __all__ = ["SplunkRestClient"] MAX_REQUEST_RETRIES = 5 @@ -171,7 +174,7 @@ def request(url, message, **kwargs): **kwargs, ) except Exception: - logging.error( + logger().error( "Failed to issue http request=%s to url=%s, error=%s", method, url, diff --git a/solnlib/timer_queue.py b/solnlib/timer_queue.py index 53d7d439..684eee74 100644 --- a/solnlib/timer_queue.py +++ b/solnlib/timer_queue.py @@ -25,6 +25,11 @@ import sortedcontainers as sc +from solnlib.utils import get_solnlib_logger + +logger = get_solnlib_logger(__name__) + + __all__ = ["Timer", "TimerQueueStruct", "TimerQueue"] @@ -119,10 +124,14 @@ def remove_timer(self, timer: Timer): """ try: + raise ValueError self._timers.remove(timer) except ValueError: logging.info( "Timer=%s is not in queue, move it to cancelling " "list", timer.ident + ) # deprecated + logger.info( + "Timer=%s is not in queue, move it to cancelling " "list", timer.ident ) else: self._cancelling_timers[timer.ident] = timer @@ -184,7 +193,8 @@ def check_and_execute(self) -> float: try: timer() except Exception: - logging.error(traceback.format_exc()) + logging.error(traceback.format_exc()) # deprecated + logger.error(traceback.format_exc()) self.reset_timers(expired_timers) return _calc_sleep_time(next_expired_time) @@ -231,7 +241,7 @@ def start(self): self._started = True self._thr.start() - logging.info("TimerQueue started.") + logger.info("TimerQueue started.") def stop(self): """Stop the timer queue.""" @@ -284,7 +294,8 @@ def _check_and_execute(self): # Note, please make timer callback effective/short timer() except Exception: - logging.error(traceback.format_exc()) + logging.error(traceback.format_exc()) # deprecated + logger.error(traceback.format_exc()) self._reset_timers(expired_timers) @@ -295,7 +306,7 @@ def _check_and_execute(self): break except Queue.Empty: pass - logging.info("TimerQueue stopped.") + logger.info("TimerQueue stopped.") def _get_expired_timers(self): with self._lock: diff --git a/solnlib/utils.py b/solnlib/utils.py index b7954208..a5c4f02c 100644 --- a/solnlib/utils.py +++ b/solnlib/utils.py @@ -26,7 +26,11 @@ from typing import Any, Callable, List, Tuple, Union from urllib import parse as urlparse +from solnlib.log import Logs + + __all__ = [ + "get_solnlib_logger", "handle_teardown_signals", "datetime_to_seconds", "is_true", @@ -156,7 +160,7 @@ def wrapper(*args, **kwargs): "Run function: %s failed: %s.", func.__name__, traceback.format_exc(), - ) + ) # deprecated if not exceptions or any( isinstance(e, exception) for exception in exceptions ): @@ -219,3 +223,15 @@ def get_appname_from_path(absolute_path): pass continue return "-" + + +def get_solnlib_logger(name): + _logger = None + + def logger(): + nonlocal _logger + if _logger is None: + _logger = Logs().get_logger(name) + return _logger + + return logger diff --git a/tests/integration/data/mock_log/.gitkeep b/tests/integration/data/mock_log/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/test_user_access.py b/tests/integration/test_user_access.py index 09ba9857..bf8fdeaa 100644 --- a/tests/integration/test_user_access.py +++ b/tests/integration/test_user_access.py @@ -18,7 +18,6 @@ import conftest import os.path as op import sys - import pytest from solnlib import user_access @@ -26,7 +25,8 @@ sys.path.insert(0, op.dirname(op.dirname(op.abspath(__file__)))) -def test_object_acl_manager(): +def test_object_acl_manager(monkeypatch): + conftest.mock_splunk(monkeypatch) session_key = context.get_session_key() oaclm = user_access.ObjectACLManager( "object_acls_collection", @@ -183,7 +183,7 @@ def test_get_user_roles(): def test_user_access(monkeypatch): - test_object_acl_manager() + test_object_acl_manager(monkeypatch) test_app_capability_manager() test_check_user_access(monkeypatch) test_get_current_username() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 8b3bf665..828cfbc6 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,12 +1,24 @@ import json import socket +import sys +import os.path as op from contextlib import closing from http.server import BaseHTTPRequestHandler, HTTPServer from threading import Thread +from unittest.mock import MagicMock import pytest +cur_dir = op.dirname(op.abspath(__file__)) + +mock_msp = MagicMock(return_value=op.sep.join([cur_dir, "data/mock_log"])) +mock_bundle_paths = MagicMock() +mock_bundle_paths.make_splunkhome_path = mock_msp + +sys.modules["splunk.clilib.bundle_paths"] = mock_bundle_paths + + @pytest.fixture(scope="session") def http_mock_server(): with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: diff --git a/tests/unit/data/mock_log/.gitkeep b/tests/unit/data/mock_log/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/test_log.py b/tests/unit/test_log.py index 2e565829..006bf118 100644 --- a/tests/unit/test_log.py +++ b/tests/unit/test_log.py @@ -30,7 +30,9 @@ from solnlib import log -reset_root_log_path = os.path.join(".", ".root_log") +cur_dir = os.path.dirname(os.path.abspath(__file__)) + +reset_root_log_path = os.path.sep.join([cur_dir, "data/mock_log"]) def setup_module(module): @@ -45,7 +47,6 @@ def setup_module(module): def teardown_module(module): shutil.rmtree("./.log") - shutil.rmtree(reset_root_log_path) def multiprocess_worker(logger_ref):