Merge commit '4a1d31a9c5bd70759c19bf6628951810eab82ce2' into fix-searchcommands_app-example

# Conflicts:
#	examples/searchcommands_app/setup.py

Author: fantavlik

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Splunk Enterprise SDK for Python Changelog
 
+## Version 1.6.17
+
+### Bug fixes
+
+* [#383](https://github.com/splunk/splunk-sdk-python/pull/383) Implemented the possibility to provide a SSLContext object to the connect method
+* [#396](https://github.com/splunk/splunk-sdk-python/pull/396) Updated KVStore Methods to support dictionaries
+* [#397](https://github.com/splunk/splunk-sdk-python/pull/397) Added code changes for encoding '/' in _key parameter in kvstore.data APIs.
+* [#398](https://github.com/splunk/splunk-sdk-python/pull/398) Added dictionary support for KVStore "query" methods.
+* [#402](https://github.com/splunk/splunk-sdk-python/pull/402) Fixed regression introduced in 1.6.15 to once again allow processing of empty input records in custom search commands (fix [#376](https://github.com/splunk/splunk-sdk-python/issues/376))
+* [#404](https://github.com/splunk/splunk-sdk-python/pull/404) Fixed test case failure for 8.0 and latest(8.2.x) splunk version
+
+### Minor changes
+
+* [#381](https://github.com/splunk/splunk-sdk-python/pull/381) Updated current year in conf.py
+* [#389](https://github.com/splunk/splunk-sdk-python/pull/389) Fixed few typos
+* [#391](https://github.com/splunk/splunk-sdk-python/pull/391) Fixed spelling error in client.py
+* [#393](https://github.com/splunk/splunk-sdk-python/pull/393) Updated development status past 3
+* [#394](https://github.com/splunk/splunk-sdk-python/pull/394) Updated Readme steps to run examples
+* [#395](https://github.com/splunk/splunk-sdk-python/pull/395) Updated random_number.py
+* [#399](https://github.com/splunk/splunk-sdk-python/pull/399) Moved CI tests to GitHub Actions
+* [#403](https://github.com/splunk/splunk-sdk-python/pull/403) Removed usage of Easy_install to install SDK
+
 ## Version 1.6.16
 
 ### Bug fixes

README.md

@@ -3,7 +3,7 @@
 
 # The Splunk Enterprise Software Development Kit for Python
 
-#### Version 1.6.16
+#### Version 1.6.17
 
 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code and examples designed to enable developers to build applications using the Splunk platform.
 
@@ -112,8 +112,18 @@ Save the file as **.splunkrc** in the current user's home directory.
 
 Examples are located in the **/splunk-sdk-python/examples** directory. To run the examples at the command line, use the Python interpreter and include any arguments that are required by the example. In the commands below, replace "examplename" with the name of the specific example in the directory that you want to run:
 
+Using username and Password
+    
     python examplename.py --username="admin" --password="changeme"
 
+Using Bearer token
+    
+    python examplename.py --bearerToken=<value>
+
+Using Session key
+    
+    python examplename.py --sessionKey="<value>"
+
 If you saved your login credentials in the **.splunkrc** file, you can omit those arguments:
 
     python examplename.py
@@ -150,6 +160,53 @@ The test suite uses Python's standard library, the built-in `unittest` library,
 |/tests     | Source for unit tests                                      |
 |/utils     | Source for utilities shared by the examples and unit tests |
 
+### Customization
+* When working with custom search commands such as Custom Streaming Commands or Custom Generating Commands, We may need to add new fields to the records based on certain conditions.
+* Structural changes like this may not be preserved.
+* Make sure to use ``add_field(record, fieldname, value)`` method from SearchCommand to add a new field and value to the record.
+* ___Note:__ Usage of ``add_field`` method is completely optional, if you are not facing any issues with field retention._
+
+Do
+```python
+class CustomStreamingCommand(StreamingCommand):
+    def stream(self, records):
+        for index, record in enumerate(records):
+            if index % 1 == 0:
+                self.add_field(record, "odd_record", "true")
+            yield record
+```
+
+Don't
+```python
+class CustomStreamingCommand(StreamingCommand):
+    def stream(self, records):
+        for index, record in enumerate(records):
+            if index % 1 == 0:
+                record["odd_record"] = "true"
+            yield record
+```
+### Customization for Generating Custom Search Command
+* Generating Custom Search Command is used to generate events using SDK code.
+* Make sure to use ``gen_record()`` method from SearchCommand to add a new record and pass event data as a key=value pair separated by , (mentioned in below example).
+
+Do
+```python
+@Configuration()
+    class GeneratorTest(GeneratingCommand):
+        def generate(self):
+            yield self.gen_record(_time=time.time(), one=1)
+            yield self.gen_record(_time=time.time(), two=2)
+```
+
+Don't
+```python
+@Configuration()
+    class GeneratorTest(GeneratingCommand):
+        def generate(self):
+            yield {'_time': time.time(), 'one': 1}
+            yield {'_time': time.time(), 'two': 2}
+```
+
 ### Changelog
 
 The [CHANGELOG](CHANGELOG.md) contains a description of changes for each version of the SDK. For the latest version, see the [CHANGELOG.md](https://github.com/splunk/splunk-sdk-python/blob/master/CHANGELOG.md) on GitHub.

docs/searchcommands.rst

@@ -3,7 +3,7 @@ splunklib.searchcommands
 
 .. automodule:: splunklib.searchcommands
 
-.. autofunction:: dispatch(command_class[, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None])
+.. autofunction:: dispatch(command_class[, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_input=True])
 
 .. autoclass:: EventingCommand
     :members:
@@ -31,7 +31,7 @@ splunklib.searchcommands
 
     .. automethod:: splunklib.searchcommands::GeneratingCommand.generate
 
-    .. automethod:: splunklib.searchcommands::GeneratingCommand.process(args=sys.argv[, input_file=sys.stdin, output_file=sys.stdout])
+    .. automethod:: splunklib.searchcommands::GeneratingCommand.process(args=sys.argv[, input_file=sys.stdin, output_file=sys.stdout, allow_empty_input=True])
 
 .. autoclass:: ReportingCommand
     :members:
@@ -59,7 +59,7 @@ splunklib.searchcommands
         :inherited-members:
         :exclude-members: configuration_settings, fix_up, items, keys
 
-    .. automethod:: splunklib.searchcommands::StreamingCommand.process(args=sys.argv[, input_file=sys.stdin, output_file=sys.stdout])
+    .. automethod:: splunklib.searchcommands::StreamingCommand.process(args=sys.argv[, input_file=sys.stdin, output_file=sys.stdout, allow_empty_input=True])
 
     .. automethod:: splunklib.searchcommands::StreamingCommand.stream
 

scripts/templates/splunkrc.template

@@ -10,3 +10,7 @@ password=$password
 scheme=$scheme
 # Your version of Splunk (default: 6.2)
 version=$version
+# Bearer token for authentication
+#bearerToken=<Bearer-token>
+# Session key for authentication
+#sessionKey=<Session-Key>

splunklib/__init__.py

@@ -16,5 +16,5 @@
 
 from __future__ import absolute_import
 from splunklib.six.moves import map
-__version_info__ = (1, 6, 16)
+__version_info__ = (1, 6, 17)
 __version__ = ".".join(map(str, __version_info__))

splunklib/binding.py

@@ -30,7 +30,6 @@
 import logging
 import socket
 import ssl
-import sys
 from base64 import b64encode
 from contextlib import contextmanager
 from datetime import datetime
@@ -39,7 +38,6 @@
 from xml.etree.ElementTree import XML
 
 from splunklib import six
-from splunklib.six import StringIO
 from splunklib.six.moves import urllib
 
 from .data import record
@@ -1391,7 +1389,7 @@ def request(url, message, **kwargs):
         head = {
             "Content-Length": str(len(body)),
             "Host": host,
-            "User-Agent": "splunk-sdk-python/1.6.16",
+            "User-Agent": "splunk-sdk-python/1.6.17",
             "Accept": "*/*",
             "Connection": "Close",
         } # defaults

splunklib/client.py

@@ -403,6 +403,7 @@ class Service(_BaseService):
     def __init__(self, **kwargs):
         super(Service, self).__init__(**kwargs)
         self._splunk_version = None
+        self._kvstore_owner = None
 
     @property
     def apps(self):
@@ -675,12 +676,34 @@ def splunk_version(self):
             self._splunk_version = tuple([int(p) for p in self.info['version'].split('.')])
         return self._splunk_version
 
+    @property
+    def kvstore_owner(self):
+        """Returns the KVStore owner for this instance of Splunk.
+
+        By default is the kvstore owner is not set, it will return "nobody"
+        :return: A string with the KVStore owner.
+        """
+        if self._kvstore_owner is None:
+            self._kvstore_owner = "nobody"
+        return self._kvstore_owner
+
+    @kvstore_owner.setter
+    def kvstore_owner(self, value):
+        """
+        kvstore is refreshed, when the owner value is changed
+        """
+        self._kvstore_owner = value
+        self.kvstore
+
     @property
     def kvstore(self):
         """Returns the collection of KV Store collections.
 
+        sets the owner for the namespace, before retrieving the KVStore Collection
+
         :return: A :class:`KVStoreCollections` collection of :class:`KVStoreCollection` entities.
         """
+        self.namespace['owner'] = self.kvstore_owner
         return KVStoreCollections(self)
 
     @property

splunklib/modularinput/event_writer.py

@@ -15,7 +15,6 @@
 from __future__ import absolute_import
 import sys
 
-from io import TextIOWrapper, TextIOBase
 from splunklib.six import ensure_str
 from .event import ET
 

splunklib/searchcommands/generating_command.py

@@ -15,6 +15,7 @@
 # under the License.
 
 from __future__ import absolute_import, division, print_function, unicode_literals
+import sys
 
 from .decorators import ConfigurationSetting
 from .search_command import SearchCommand
@@ -212,13 +213,49 @@ def _execute(self, ifile, process):
 
     def _execute_chunk_v2(self, process, chunk):
         count = 0
+        records = []
         for row in process:
-            self._record_writer.write_record(row)
+            records.append(row)
             count += 1
             if count == self._record_writer._maxresultrows:
-                self._finished = False
-                return
-        self._finished = True
+                break
+
+        for row in records:
+            self._record_writer.write_record(row)
+
+        if count == self._record_writer._maxresultrows:
+            self._finished = False
+        else:
+            self._finished = True
+
+    def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_input=True):
+        """ Process data.
+
+        :param argv: Command line arguments.
+        :type argv: list or tuple
+
+        :param ifile: Input data file.
+        :type ifile: file
+
+        :param ofile: Output data file.
+        :type ofile: file
+
+        :param allow_empty_input: For generating commands, it must be true. Doing otherwise will cause an error.
+        :type allow_empty_input: bool
+
+        :return: :const:`None`
+        :rtype: NoneType
+
+        """
+
+        # Generating commands are expected to run on an empty set of inputs as the first command being run in a search,
+        # also this class implements its own separate _execute_chunk_v2 method which does not respect allow_empty_input
+        # so ensure that allow_empty_input is always True
+
+        if not allow_empty_input:
+            raise ValueError("allow_empty_input cannot be False for Generating Commands")
+        else:
+            return super(GeneratingCommand, self).process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=True)
 
     # endregion
 

splunklib/searchcommands/internals.py

@@ -508,6 +508,7 @@ def __init__(self, ofile, maxresultrows=None):
         self._chunk_count = 0
         self._pending_record_count = 0
         self._committed_record_count = 0
+        self.custom_fields = set()
 
     @property
     def is_flushed(self):
@@ -572,6 +573,7 @@ def write_record(self, record):
 
     def write_records(self, records):
         self._ensure_validity()
+        records = list(records)
         write_record = self._write_record
         for record in records:
             write_record(record)
@@ -593,6 +595,7 @@ def _write_record(self, record):
 
         if fieldnames is None:
             self._fieldnames = fieldnames = list(record.keys())
+            self._fieldnames.extend([i for i in self.custom_fields if i not in self._fieldnames])
             value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames)
             self._writerow(list(chain.from_iterable(value_list)))