Fix up searchcommands and add example searches and results

Author: fantavlik

examples/searchcommands_app/README.md

@@ -7,7 +7,6 @@ This app provides several examples of custom search commands that illustrate eac
 :---------------- |:-----------|:-------------------------------------------------------------------------------------------
  countmatches     | Streaming  | Counts the number of non-overlapping matches to a regular expression in a set of fields.
  generatetext     | Generating | Generates a specified number of events containing a specified text string.
- pypygeneratetext | Generating | Runs generatetext with the string 'PyPy'. 
  simulate         | Generating | Generates a sequence of events drawn from a csv file using repeated random sampling with replacement.
  generatehello    | Generating | Generates a specified number of events containing the text string 'hello'.
  sum              | Reporting  | Adds all of the numbers in a set of fields.
@@ -19,12 +18,10 @@ The app is tested on Splunk 5 and 6. Here is its manifest:
 ├── bin
 │   ├── countmatches.py .......... CountMatchesCommand implementation
 │   ├── generatetext.py .......... GenerateTextCommand implementation
-│   ├── pypygeneratetext.py ...... Runs generatetext.py with PyPy
 │   ├── simulate.py .............. SimulateCommand implementation
 │   └── sum.py ................... SumCommand implementation
 ├── lib
-|   └── splunklib
-│      └── searchcommands ....... splunklib.searchcommands module
+|   └── splunklib ................ splunklib module
 ├── default
 │   ├── data
 │   │   └── ui
@@ -35,41 +32,83 @@ The app is tested on Splunk 5 and 6. Here is its manifest:
 │   ├── logging.conf ............. Python logging[3] configuration in ConfigParser[4] format
 │   └── searchbnf.conf ........... Search assistant configuration [5]
 └── metadata
-    └── local.meta ............... Permits the search assistant to use searchbnf.conf[6]
+    └── default.meta ............. Permits the search assistant to use searchbnf.conf[6]
 ```
 **References**  
-[1] [app.conf](http://docs.splunk.com/Documentation/Splunk/latest/Admin/Appconf app.conf)  
-[2] [commands.conf](http://docs.splunk.com/Documentation/Splunk/latest/Admin/Commandsconf)  
-[3] [Python Logging HOWTO](http://docs.python.org/2/howto/logging.html)  
-[4] [ConfigParser—Configuration file parser](http://docs.python.org/2/library/configparser.html)
-[5] [searchbnf.conf](http://docs.splunk.com/Documentation/Splunk/latest/admin/Searchbnfconf)
-[6] [Set permissions in the file system](http://docs.splunk.com/Documentation/Splunk/latest/AdvancedDev/SetPermissions#Set_permissions_in_the_filesystem)
+[1] [app.conf](https://docs.splunk.com/Documentation/Splunk/latest/Admin/Appconf app.conf)  
+[2] [commands.conf](https://docs.splunk.com/Documentation/Splunk/latest/Admin/Commandsconf)  
+[3] [Python Logging HOWTO](https://docs.python.org/2/howto/logging.html)  
+[4] [ConfigParser—Configuration file parser](https://docs.python.org/2/library/configparser.html)
+[5] [searchbnf.conf](https://docs.splunk.com/Documentation/Splunk/latest/admin/Searchbnfconf)
+[6] [Set permissions in the file system](https://docs.splunk.com/Documentation/Splunk/latest/AdvancedDev/SetPermissions#Set_permissions_in_the_filesystem)
 
 ## Installation
 
-+ Link the app to $SPLUNK_HOME/etc/apps/searchcommands_app by running this command:
++ Bring up Dockerized Splunk with the app installed from the root of this repository via:
 
   ```
-  ./setup.py link --scp-version {1|2}
+  SPLUNK_VERSION=latest docker compose up -d
   ```
 
-+ Or build a tarball to install on any Splunk instance by running this command:
++ When the `splunk` service is healthy (`health: starting` -> `healthy`) login and run test searches within the app via http://localhost:8000/en-US/app/searchcommands_app/search
 
-  ```
-  ./setup.py build --scp-version {1|2}
-  ```
+### Example searches
 
-  The tarball is build as build/searchcommands_app-1.5.0-private.tar.gz.
-  
-+ Then (re)start Splunk so that the app is recognized.
+#### countmatches
+```
+| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text
+```
+Results:
+text | word_count
+:----|:---|
+excellent review my friend loved it yours always guppyman @GGreeny62... http://t.co/fcvq7NDHxl | 14
+Tú novia te ama mucho | 5
+... |
 
-## Dashboards and Searches
+#### filter
+```
+| generatetext text="Hello world! How the heck are you?" count=6 \
+| filter predicate="(int(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')"
+```
+Results:
+Event |
+:-----|
+2. Hello Splunk! How the heck are you? |
+4. Hello Splunk! How the heck are you? |
+6. Hello Splunk! How the heck are you? |
 
-+ TODO: Add saved search(es) for each example.
+#### generatetext
+```
+| generatetext count=3 text="Hello there"
+```
+Results:
+Event |
+:-----|
+1. Hello there | 
+2. Hello there |
+3. Hello there |
 
-### Searches
+#### simulate
+```
+| simulate csv="/opt/splunk/etc/apps/searchcommands_app/data/population.csv" rate=10 interval=00:00:01 duration=00:00:02 seed=9
+```
+Results:
+Event |
+:-----|
+text = Margarita (8) |
+text = RT @Habibies: When you were born, you cried and the world rejoiced. Live your life so that when you die, the world will cry and you will re... |
+text = @dudaribeiro_13 q engraçado em. |
 
-+ TODO: Describe saved searches.
+#### sum
+```
+| inputlookup tweets 
+| countmatches fieldname=word_count pattern="\\w+" text
+| sum total=word_counts word_count
+```
+Results:
+word_counts |
+:-----|
+4497.0 |
 
 ## License
 

examples/searchcommands_app/package/bin/filter.py

@@ -49,7 +49,7 @@ class FilterCommand(EventingCommand):
 
     .. code-block::
         | generatetext text="Hello world! How the heck are you?" count=6
-        | filter predicate="(long(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')"
+        | filter predicate="(int(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')"
 
     """
     predicate = Option(doc='''

examples/searchcommands_app/package/bin/pypygeneratetext.py

@@ -47,9 +47,7 @@ class SimulateCommand(GeneratingCommand):
     ##Example
 
     .. code-block::
-        | simulate csv=population.csv rate=50 interval=00:00:01
-            duration=00:00:05 | countmatches fieldname=word_count
-            pattern="\\w+" text | stats mean(word_count) stdev(word_count)
+        | simulate csv="/opt/splunk/etc/apps/searchcommands_app/data/population.csv" rate=10 interval=00:00:01 duration=00:00:02 seed=1
 
     This example generates events drawn from repeated random sampling of events from :code:`tweets.csv`. Events are
     drawn at an average rate of 50 per second for a duration of 5 seconds. Events are piped to the example
@@ -85,28 +83,20 @@ class SimulateCommand(GeneratingCommand):
         **Description:** Value for initializing the random number generator ''')
 
     def generate(self):
-
-        if not self.records:
-            if self.seed is not None:
-                random.seed(self.seed)
-            self.records = [record for record in csv.DictReader(self.csv_file)]
-            self.lambda_value = 1.0 / (self.rate / float(self.interval))
+        if self.seed is not None:
+            random.seed(self.seed)
+        records = [record for record in csv.DictReader(self.csv_file)]
+        lambda_value = 1.0 / (self.rate / float(self.interval))
 
         duration = self.duration
-
         while duration > 0:
-            count = int(round(random.expovariate(self.lambda_value)))
+            count = int(round(random.expovariate(lambda_value)))
             start_time = time.clock()
-            for record in random.sample(self.records, count):
+            for record in random.sample(records, count):
                 yield record
             interval = time.clock() - start_time
             if interval < self.interval:
                 time.sleep(self.interval - interval)
             duration -= max(interval, self.interval)
 
-    def __init__(self):
-        super(SimulateCommand, self).__init__()
-        self.lambda_value = None
-        self.records = None
-
 dispatch(SimulateCommand, sys.argv, sys.stdin, sys.stdout, __name__)

examples/searchcommands_app/package/bin/simulate.py

@@ -16,11 +16,6 @@ filename = generatetext.py
 chunked = true
 python.version = python3
 
-[pypygeneratetext]
-filename = pypygeneratetext.py
-chunked = true
-python.version = python3
-
 [simulate]
 filename = simulate.py
 chunked = true

examples/searchcommands_app/package/default/commands.conf

@@ -35,7 +35,7 @@ comment1 = \
     of the records produced by the generatetext command.
 example1 = \
     | generatetext text="Hello world! How the heck are you?" count=6 \
-        | filter predicate="(long(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')"
+        | filter predicate="(int(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')"
 category = events
 appears-in = 1.5
 maintainer = dnoble
@@ -58,23 +58,6 @@ maintainer = dnoble
 usage = public
 tags = searchcommands_app
 
-[pypygeneratetext-command]
-syntax = PYPYGENERATETEXT COUNT=<event_count> TEXT=<string>
-alias =
-shortdesc = Generates a sequence of occurrences of a text string on the streams pipeline under control of PyPy.
-description = \
-    This command generates COUNT occurrences of a TEXT string under control of PyPy. Each occurrence is prefixed \
-    by its _SERIAL number and stored in the _RAW field of each record. This command assumes that PyPy is on Splunkd's \
-    PATH.
-comment1 = \
-    This example generates 10 occurrences of the string "Hello world!".
-example1 = | pypygeneratetext count=10 text="Hello world!"
-category = external generating
-appears-in = 1.5
-maintainer = dnoble
-usage = public
-tags = searchcommands_app
-
 [simulate-command]
 syntax = SIMULATE CSV=<path> RATE=<expected_event_count> INTERVAL=<sampling_period> DURATION=<execution_period> \
     [SEED=<string>]?
@@ -90,9 +73,7 @@ comment1 = \
     countmatches command which adds a word_count field containing the number of words in the text field of each event. \
     The mean and standard deviation of the word_count are then computed by the builtin stats command.
 example1 = \
-    | simulate csv=population.csv rate=50 interval=00:00:01 duration=00:00:05 \
-        | countmatches fieldname=word_count pattern="\\w+" text \
-        | stats mean(word_count) stdev(word_count)
+    | simulate csv="/opt/splunk/etc/apps/searchcommands_app/data/population.csv" rate=10 interval=00:00:01 duration=00:00:02 seed=1
 category = generating
 appears-in = 1.2
 maintainer = dnoble

examples/searchcommands_app/package/default/searchbnf.conf

@@ -468,7 +468,6 @@ def run(self):
                 os.path.join('package', 'bin', 'filter.py'),
                 os.path.join('package', 'bin', 'generatehello.py'),
                 os.path.join('package', 'bin', 'generatetext.py'),
-                os.path.join('package', 'bin', 'pypygeneratetext.py'),
                 os.path.join('package', 'bin', 'simulate.py'),
                 os.path.join('package', 'bin', 'sum.py')
             ]

examples/searchcommands_app/setup.py

@@ -199,26 +199,6 @@ def test_generatehello_as_unit(self):
 
         return
 
-    @skipUnless(pypy(), 'Skipping TestSearchCommandsApp.test_pypygeneratetext_as_unit because pypy is not on PATH.')
-    def test_pypygeneratetext_as_unit(self):
-
-        expected, output, errors, exit_status = self._run_command('pypygeneratetext', action='getinfo', protocol=1)
-        self.assertEqual(0, exit_status, msg=six.text_type(errors))
-        self.assertEqual('', errors, msg=six.text_type(errors))
-        self._compare_csv_files_time_sensitive(expected, output)
-
-        expected, output, errors, exit_status = self._run_command('pypygeneratetext', action='execute', protocol=1)
-        self.assertEqual(0, exit_status, msg=six.text_type(errors))
-        self.assertEqual('', errors, msg=six.text_type(errors))
-        self._compare_csv_files_time_insensitive(expected, output)
-
-        expected, output, errors, exit_status = self._run_command('pypygeneratetext')
-        self.assertEqual(0, exit_status, msg=six.text_type(errors))
-        self.assertEqual('', errors, msg=six.text_type(errors))
-        self._compare_chunks(expected, output, time_sensitive=False)
-
-        return
-
     def test_sum_as_unit(self):
 
         expected, output, errors, exit_status = self._run_command('sum', action='getinfo', phase='reduce', protocol=1)