Merge branch 'develop' into tests_and_docstring_for_post_body

Author: zenmoto

.travis.yml

@@ -12,14 +12,12 @@ before_install:
     - echo password=changed! >> $HOME/.splunkrc
     # Set SPLUNK_HOME
     - export SPLUNK_HOME="/opt/splunk"
-    # Pull docker image
-    - docker pull splunk/splunk-sdk-travis-ci:$SPLUNK_VERSION
     # Add DOCKER to iptables, 1/10 times this is needed, force 0 exit status
     - sudo iptables -N DOCKER || true
-    # Start Docker container
-    - docker run -p 127.0.0.1:8089:8089 -d splunk/splunk-sdk-travis-ci:$SPLUNK_VERSION
-    # curl Splunk until it returns valid data indicating it has been setup, try 20 times maximum
-    - for i in `seq 0 20`; do if curl --fail -k https://localhost:8089/services/server/info &> /dev/null; then break; fi; echo $i; sleep 1; done
+    # Start docker-compose in detached mode
+    - docker-compose up -d
+    # Health Check (3 minutes)
+    - for i in `seq 0 180`; do if docker exec -it splunk /sbin/checkstate.sh &> /dev/null; then break; fi; echo $i; sleep 1; done
     # The upload test needs to refer to a file that Splunk has in the docker
     # container
     - export INPUT_EXAMPLE_UPLOAD=$SPLUNK_HOME/var/log/splunk/splunkd_ui_access.log
@@ -32,8 +30,8 @@ before_install:
 language: python
 
 env:
-    - SPLUNK_VERSION=7.3-sdk
-    - SPLUNK_VERSION=8.0-sdk
+    - SPLUNK_VERSION=7.3
+    - SPLUNK_VERSION=8.0
 
 python:
   - "2.7"

README.md

@@ -13,7 +13,7 @@ The Splunk platform is popular with system administrators for aggregation and mo
 
 The Splunk developer platform enables developers to take advantage of the same technology used by the Splunk platform to build exciting new applications.
 
-For more information, see [Splunk Enterprise SDK for Python](https://dev.splunk.com/enterprise/docs/devtools/python/sdk-python/) on the Splunk Developer Portal.
+## Getting started with the Splunk SDK for Python
 
 
 ## Get started with the Splunk Enterprise SDK for Python
@@ -37,7 +37,6 @@ Here's what you need to get going with the Splunk Enterprise SDK for Python.
 
   Get the Splunk Enterprise SDK for Python from [PyPI](https://pypi.org/project/splunk-sdk/). If you want to contribute to the SDK, clone the repository from [GitHub](https://github.com/splunk/splunk-sdk-python).
 
-
 ### Install the SDK
 
 Use the following commands to install the Splunk Enterprise SDK for Python libraries in different ways. However, it's not necessary to install the libraries to run the examples and unit tests from the SDK.
@@ -58,8 +57,17 @@ Install the sources you cloned from GitHub:
 
     [sudo] python setup.py install
 
+## Testing Quickstart
+
+You'll need `docker` and `docker-compose` to get up and running using this method.
 
-### Run the examples and unit tests
+```
+make up SPLUNK_VERSION=8.0
+make wait_up
+make splunkrc_default
+make test
+make down
+```
 
 To run the examples and unit tests, you must put the root of the SDK on your PYTHONPATH. For example, if you downloaded the SDK to your home folder and are running OS X or Linux, add the following line to your **.bash_profile** file:
 

docker-compose.yml

@@ -0,0 +1,20 @@
+version: '3.6'
+
+services:
+    splunk:
+        image: "splunk/splunk:${SPLUNK_VERSION}"
+        container_name: splunk
+        environment:
+            - SPLUNK_START_ARGS=--accept-license
+            - SPLUNK_HEC_TOKEN=11111111-1111-1111-1111-1111111111113
+            - SPLUNK_PASSWORD=changed!
+            - SPLUNK_APPS_URL=https://github.com/splunk/sdk-app-collection/releases/download/v1.0.0/sdk-app-collection.tgz
+        ports:
+            - 8000:8000
+            - 8088:8088
+            - 8089:8089
+        healthcheck:
+            test: ['CMD', 'curl', '-f', 'http://localhost:8000']
+            interval: 5s
+            timeout: 5s
+            retries: 20

docs/conf.py

@@ -43,7 +43,7 @@
 
 # General information about the project.
 project = u'Splunk SDK for Python'
-copyright = u'2014, Splunk Inc'
+copyright = u'2020, Splunk Inc'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -95,7 +95,7 @@
 # a list of builtin themes.
 
 # agogo, default, epub, haiku, nature, pyramid, scrolls, sphinxdoc, traditional
-html_theme = 'default' 
+html_theme = 'default'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

scripts/build-splunkrc.py

@@ -25,9 +25,9 @@
     'host': 'localhost',
     'port': '8089',
     'username': 'admin',
-    'password': 'changeme',
+    'password': 'changed!',
     'scheme': 'https',
-    'version': '6.3'
+    'version': '8.0'
 }
 
 DEFAULT_SPLUNKRC_PATH = os.path.join(str(Path.home()), '.splunkrc')

splunklib/client.py

@@ -1872,7 +1872,7 @@ def delete(self, username, realm=None):
             name = UrlEncoded(realm, encode_slash=True) + ":" + UrlEncoded(username, encode_slash=True)
 
         # Append the : expected at the end of the name
-        if name[-1] is not ":":
+        if name[-1] != ":":
             name = name + ":"
         return Collection.delete(self, name)
 

splunklib/searchcommands/generating_command.py

@@ -204,19 +204,21 @@ def _execute(self, ifile, process):
 
         """
         if self._protocol_version == 2:
-            result = self._read_chunk(self._as_binary_stream(ifile))
+            self._execute_v2(ifile, self.generate())
+        else:
+            assert self._protocol_version == 1
+            self._record_writer.write_records(self.generate())
+        self.finish()
 
-            if not result:
+    def _execute_chunk_v2(self, process, chunk):
+        count = 0
+        for row in process:
+            self._record_writer.write_record(row)
+            count += 1
+            if count == self._record_writer._maxresultrows:
+                self._finished = False
                 return
-
-            metadata, body = result
-            action = getattr(metadata, 'action', None)
-
-            if action != 'execute':
-                raise RuntimeError('Expected execute action, not {}'.format(action))
-
-        self._record_writer.write_records(self.generate())
-        self.finish()
+        self._finished = True
 
     # endregion
 

splunklib/searchcommands/internals.py

@@ -35,6 +35,7 @@
 import os
 import re
 import sys
+import warnings
 
 from . import environment
 
@@ -74,7 +75,7 @@ def set_binary_mode(fh):
 
 
 class CommandLineParser(object):
-    """ Parses the arguments to a search command.
+    r""" Parses the arguments to a search command.
 
     A search command line is described by the following syntax.
 
@@ -505,8 +506,8 @@ def __init__(self, ofile, maxresultrows=None):
 
         self._inspector = OrderedDict()
         self._chunk_count = 0
-        self._record_count = 0
-        self._total_record_count = 0
+        self._pending_record_count = 0
+        self._committed_record_count = 0
 
     @property
     def is_flushed(self):
@@ -524,6 +525,30 @@ def ofile(self):
     def ofile(self, value):
         self._ofile = set_binary_mode(value)
 
+    @property
+    def pending_record_count(self):
+        return self._pending_record_count
+
+    @property
+    def _record_count(self):
+        warnings.warn(
+            "_record_count will be deprecated soon. Use pending_record_count instead.",
+             PendingDeprecationWarning
+        )
+        return self.pending_record_count
+
+    @property
+    def committed_record_count(self):
+        return self._committed_record_count
+
+    @property
+    def _total_record_count(self):
+        warnings.warn(
+            "_total_record_count will be deprecated soon. Use committed_record_count instead.",
+             PendingDeprecationWarning
+        )
+        return self.committed_record_count
+
     def write(self, data):
         bytes_type = bytes if sys.version_info >= (3, 0) else str
         if not isinstance(data, bytes_type):
@@ -555,8 +580,7 @@ def _clear(self):
         self._buffer.seek(0)
         self._buffer.truncate()
         self._inspector.clear()
-        self._record_count = 0
-        self._flushed = False
+        self._pending_record_count = 0
 
     def _ensure_validity(self):
         if self._finished is True:
@@ -651,9 +675,9 @@ def _write_record(self, record):
             values += (repr(value), None)
 
         self._writerow(values)
-        self._record_count += 1
+        self._pending_record_count += 1
 
-        if self._record_count >= self._maxresultrows:
+        if self.pending_record_count >= self._maxresultrows:
             self.flush(partial=True)
 
     try:
@@ -690,7 +714,7 @@ def flush(self, finished=None, partial=None):
 
         RecordWriter.flush(self, finished, partial)  # validates arguments and the state of this instance
 
-        if self._record_count > 0 or (self._chunk_count == 0 and 'messages' in self._inspector):
+        if self.pending_record_count > 0 or (self._chunk_count == 0 and 'messages' in self._inspector):
 
             messages = self._inspector.get('messages')
 
@@ -728,9 +752,9 @@ def flush(self, finished=None, partial=None):
                     print(level, text, file=stderr)
 
             self.write(self._buffer.getvalue())
-            self._clear()
             self._chunk_count += 1
-            self._total_record_count += self._record_count
+            self._committed_record_count += self.pending_record_count
+            self._clear()
 
         self._finished = finished is True
 
@@ -748,37 +772,36 @@ class RecordWriterV2(RecordWriter):
     def flush(self, finished=None, partial=None):
 
         RecordWriter.flush(self, finished, partial)  # validates arguments and the state of this instance
-        inspector = self._inspector
-
-        if self._flushed is False:
-
-            self._total_record_count += self._record_count
-            self._chunk_count += 1
-
-            # TODO: DVPL-6448: splunklib.searchcommands | Add support for partial: true when it is implemented in
-            # ChunkedExternProcessor (See SPL-103525)
-            #
-            # We will need to replace the following block of code with this block:
-            #
-            # metadata = [
-            #     ('inspector', self._inspector if len(self._inspector) else None),
-            #     ('finished', finished),
-            #     ('partial', partial)]
-
-            if len(inspector) == 0:
-                inspector = None
-
-            if partial is True:
-                finished = False
 
-            metadata = [item for item in (('inspector', inspector), ('finished', finished))]
-            self._write_chunk(metadata, self._buffer.getvalue())
-            self._clear()
+        if partial or not finished:
+            # Don't flush partial chunks, since the SCP v2 protocol does not
+            # provide a way to send partial chunks yet.
+            return
 
-        elif finished is True:
-            self._write_chunk((('finished', True),), '')
+        if not self.is_flushed:
+            self.write_chunk(finished=True)
 
-        self._finished = finished is True
+    def write_chunk(self, finished=None):
+        inspector = self._inspector
+        self._committed_record_count += self.pending_record_count
+        self._chunk_count += 1
+
+        # TODO: DVPL-6448: splunklib.searchcommands | Add support for partial: true when it is implemented in
+        # ChunkedExternProcessor (See SPL-103525)
+        #
+        # We will need to replace the following block of code with this block:
+        #
+        # metadata = [item for item in (('inspector', inspector), ('finished', finished), ('partial', partial))]
+        #
+        # if partial is True:
+        #     finished = False
+
+        if len(inspector) == 0:
+            inspector = None
+
+        metadata = [item for item in (('inspector', inspector), ('finished', finished))]
+        self._write_chunk(metadata, self._buffer.getvalue())
+        self._clear()
 
     def write_metadata(self, configuration):
         self._ensure_validity()
@@ -793,7 +816,7 @@ def write_metric(self, name, value):
         self._inspector['metric.' + name] = value
 
     def _clear(self):
-        RecordWriter._clear(self)
+        super(RecordWriterV2, self)._clear()
         self._fieldnames = None
 
     def _write_chunk(self, metadata, body):
@@ -818,4 +841,4 @@ def _write_chunk(self, metadata, body):
         self.write(metadata)
         self.write(body)
         self._ofile.flush()
-        self._flushed = False
+        self._flushed = True