Merge branch 'develop' into py3-code-migration

Author: ashah-splunk

.github/workflows/release.yml

@@ -9,17 +9,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source
-        uses: actions/checkout@v2.3.2
+        uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.7
       - name: Install dependencies
         run: pip install twine
       - name: Build package
         run: python setup.py sdist
       - name: Publish package to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.3.1
+        uses: pypa/gh-action-pypi-publish@v1.8.8
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.github/workflows/test.yml

@@ -20,13 +20,13 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Run docker-compose
         run: SPLUNK_VERSION=${{matrix.splunk-version}} docker-compose up -d
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
 

CHANGELOG.md

@@ -13,6 +13,14 @@
 * Updated CI test matrix to run with Python versions - 3.7, 3.9 and 3.10.7
 * Refactored Code throwing `deprecation` warnings
 * Refactored Code violating Pylint rules
+## Version 1.7.4
+
+### Bug fixes
+* [#532](https://github.com/splunk/splunk-sdk-python/pull/532) update encoding errors mode to 'replace' [[issue#505](https://github.com/splunk/splunk-sdk-python/issues/505)]
+* [#507](https://github.com/splunk/splunk-sdk-python/pull/507) masked sensitive data in logs [[issue#506](https://github.com/splunk/splunk-sdk-python/issues/506)]
+
+### Minor changes
+* [#530](https://github.com/splunk/splunk-sdk-python/pull/530) Update GitHub CI build status in README and removed RTD(Read The Docs) reference
 
 ## Version 1.7.3
 

README.md

@@ -1,9 +1,10 @@
-[![Build Status](https://travis-ci.org/splunk/splunk-sdk-python.svg?branch=master)](https://travis-ci.org/splunk/splunk-sdk-python)
-[![Documentation Status](https://readthedocs.org/projects/splunk-python-sdk/badge/?version=latest)](https://splunk-python-sdk.readthedocs.io/en/latest/?badge=latest)
+[![Build Status](https://github.com/splunk/splunk-sdk-python/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/splunk/splunk-sdk-python/actions/workflows/test.yml)
+
+[Reference Docs](https://dev.splunk.com/enterprise/reference)
 
 # The Splunk Enterprise Software Development Kit for Python
 
-#### Version 1.7.3
+#### Version 1.7.4
 
 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code designed to enable developers to build applications using the Splunk platform.
 
@@ -127,7 +128,7 @@ The Splunk Enterprise SDK for Python contains a collection of unit tests. To run
 
 You can also run individual test files, which are located in **/splunk-sdk-python/tests**. To run a specific test, enter:
 
-    make specific_test_name
+    make test_specific
 
 The test suite uses Python's standard library, the built-in `unittest` library, `pytest`, and `tox`.
 

scripts/test_specific.sh

@@ -1,2 +1,4 @@
 echo "To run a specific test:"
-echo "  tox -e py27,py37 [test_file_path]::[test_name]"
+echo "  tox -e py27,py37 [test_file_path]::[TestClassName]::[test_method]"
+echo "For Example, To run 'test_autologin' testcase from 'test_service.py' file run"
+echo "  tox -e py37 -- tests/test_service.py::ServiceTestCase::test_autologin"

splunklib/binding.py

@@ -25,6 +25,7 @@
 """
 
 import io
+import json
 import logging
 import socket
 import ssl
@@ -57,6 +58,10 @@
     "namespace"
 ]
 
+SENSITIVE_KEYS = ['Authorization', 'Cookie', 'action.email.auth_password', 'auth', 'auth_password', 'clear_password', 'clientId',
+                  'crc-salt', 'encr_password', 'oldpassword', 'passAuth', 'password', 'session', 'suppressionKey',
+                  'token']
+
 # If you change these, update the docstring
 # on _authority as well.
 DEFAULT_HOST = "localhost"
@@ -76,6 +81,28 @@ def new_f(*args, **kwargs):
     return new_f
 
 
+def mask_sensitive_data(data):
+    '''
+    Masked sensitive fields data for logging purpose
+    '''
+    if not isinstance(data, dict):
+        try:
+            data = json.loads(data)
+        except Exception as ex:
+            return data
+
+    # json.loads will return "123"(str) as 123(int), so return the data if it's not 'dict' type
+    if not isinstance(data, dict):
+        return data
+    mdata = {}
+    for k, v in data.items():
+        if k in SENSITIVE_KEYS:
+            mdata[k] = "******"
+        else:
+            mdata[k] = mask_sensitive_data(v)
+    return mdata
+
+
 def _parse_cookies(cookie_str, dictionary):
     """Tries to parse any key-value pairs of cookies in a string,
     then updates the the dictionary with any key-value pairs found.
@@ -635,7 +662,7 @@ def delete(self, path_segment, owner=None, app=None, sharing=None, **query):
         """
         path = self.authority + self._abspath(path_segment, owner=owner,
                                               app=app, sharing=sharing)
-        logger.debug("DELETE request to %s (body: %s)", path, repr(query))
+        logger.debug("DELETE request to %s (body: %s)", path, mask_sensitive_data(query))
         response = self.http.delete(path, self._auth_headers, **query)
         return response
 
@@ -698,7 +725,7 @@ def get(self, path_segment, owner=None, app=None, headers=None, sharing=None, **
 
         path = self.authority + self._abspath(path_segment, owner=owner,
                                               app=app, sharing=sharing)
-        logger.debug("GET request to %s (body: %s)", path, repr(query))
+        logger.debug("GET request to %s (body: %s)", path, mask_sensitive_data(query))
         all_headers = headers + self.additional_headers + self._auth_headers
         response = self.http.get(path, all_headers, **query)
         return response
@@ -777,12 +804,7 @@ def post(self, path_segment, owner=None, app=None, sharing=None, headers=None, *
 
         path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing)
 
-        # To avoid writing sensitive data in debug logs
-        endpoint_having_sensitive_data = ["/storage/passwords"]
-        if any(endpoint in path for endpoint in endpoint_having_sensitive_data):
-            logger.debug("POST request to %s ", path)
-        else:
-            logger.debug("POST request to %s (body: %s)", path, repr(query))
+        logger.debug("POST request to %s (body: %s)", path, mask_sensitive_data(query))
         all_headers = headers + self.additional_headers + self._auth_headers
         response = self.http.post(path, all_headers, **query)
         return response
@@ -849,8 +871,7 @@ def request(self, path_segment, method="GET", headers=None, body={},
 
         all_headers = headers + self.additional_headers + self._auth_headers
         logger.debug("%s request to %s (headers: %s, body: %s)",
-                     method, path, str(all_headers), repr(body))
-
+                     method, path, str(mask_sensitive_data(dict(all_headers))), mask_sensitive_data(body))
         if body:
             body = _encode(**body)
 

splunklib/modularinput/event_writer.py

@@ -72,7 +72,7 @@ def write_xml_document(self, document):
 
         :param document: An ``ElementTree`` object.
         """
-        self._out.write(ensure_str(ET.tostring(document)))
+        self._out.write(ensure_str(ET.tostring(document), errors="replace"))
         self._out.flush()
 
     def close(self):

splunklib/searchcommands/search_command.py

@@ -919,7 +919,7 @@ def _read_chunk(istream):
         except Exception as error:
             raise RuntimeError(f'Failed to read body of length {body_length}: {error}')
 
-        return metadata, splunklib.ensure_str(body)
+        return metadata, splunklib.ensure_str(body,errors="replace")
 
     _header = re.compile(r'chunked\s+1.0\s*,\s*(\d+)\s*,\s*(\d+)\s*\n')
 

tests/searchcommands/test_csc_apps.py

@@ -15,10 +15,12 @@
 # under the License.
 
 import unittest
+import pytest
+
 from tests import testlib
 from splunklib import results
 
-
+@pytest.mark.smoke
 class TestCSC(testlib.SDKTestCase):
 
     def test_eventing_app(self):

tests/test_binding.py

@@ -633,6 +633,7 @@ def test_got_updated_cookie_with_get(self):
                 self.assertEqual(list(new_cookies.values())[0], list(old_cookies.values())[0])
         self.assertTrue(found)
 
+    @pytest.mark.smoke
     def test_login_fails_with_bad_cookie(self):
         # We should get an error if using a bad cookie
         try:
@@ -641,6 +642,7 @@ def test_login_fails_with_bad_cookie(self):
         except AuthenticationError as ae:
             self.assertEqual(str(ae), "Login failed.")
 
+    @pytest.mark.smoke
     def test_login_with_multiple_cookies(self):
         # We should get an error if using a bad cookie
         new_context = binding.Context()