Changes

* Gzipped recordings are working well enough
* Smarter chunked- and csv-file differencing is in place (including corrections to two recordings)

Verified:
100% test pass on OS X

Author: David Noble

tests/searchcommands/recordings/scpv1/Splunk-6.3/generatehello.getinfo.output

@@ -1,3 +1,3 @@
 
-generates_timeorder,__mv_generates_timeorder,generating,__mv_generating,retainsevents,__mv_retainsevents,streaming,__mv_streaming
-0,,1,,0,,1,
+generates_timeorder,__mv_generates_timeorder,generating,__mv_generating,local,__mv_local,retainsevents,__mv_retainsevents,streaming,__mv_streaming
+0,,1,,0,,0,,1,

tests/searchcommands/recordings/scpv1/Splunk-6.3/pypygeneratetext.execute.output

@@ -1,5 +1,5 @@
 
-_time,__mv__time,event_no,__mv_event_no,_raw,__mv__raw
+_time,__mv__time,_serial,__mv__serial,_raw,__mv__raw
 1434910895.01,,1,,1. Hello World!,
 1434910895.01,,2,,2. Hello World!,
 1434910895.01,,3,,3. Hello World!,

tests/searchcommands/recordings/scpv2/Splunk-6.3/TestInternals.test_record_writer_with_recordings.1443140714.71.input.gz

@@ -1,7 +1,7 @@
 chunked 1.0,38,0
 {"generating":true,"type":"streaming"}
-chunked 1.0,17,37842
-{"finished":true}_time,__mv__time,event_no,__mv_event_no,_raw,__mv__raw
+chunked 1.0,17,39840
+{"finished":true}_time,__mv__time,_serial,__mv__serial,_raw,__mv__raw
 1433261371.25,,1,,1. Hello World!,
 1433261371.25,,2,,2. Hello World!,
 1433261371.25,,3,,3. Hello World!,

tests/searchcommands/recordings/scpv2/Splunk-6.3/TestInternals.test_record_writer_with_recordings.1443140714.71.output

@@ -19,7 +19,7 @@
 
 from splunklib.searchcommands.internals import MetadataDecoder, MetadataEncoder, Recorder, RecordWriterV2
 from splunklib.searchcommands import SearchMetric
-from collections import deque, OrderedDict
+from collections import deque, namedtuple, OrderedDict
 from cStringIO import StringIO
 from functools import wraps
 from glob import iglob
@@ -63,7 +63,7 @@ def random_dict():
     # contain utf-8 encoded byte strings or--better still--unicode strings. This is because the json package
     # converts all bytes strings to unicode strings before serializing them.
 
-    return {'a': random_float(), 'b': random_unicode(), '福 酒吧': {'fu': random_float(), 'bar': random_float()}}
+    return OrderedDict((('a', random_float()), ('b', random_unicode()), ('福 酒吧', OrderedDict((('fu', random_float()), ('bar', random_float()))))))
 
 
 def random_float():
@@ -260,8 +260,8 @@ def test_record_writer_with_recordings(self):
 
         for input_file in iglob(base_path + '*.input.gz'):
 
-            with gzip.open(input_file, 'rb') as f:
-                test_data = pickle.load(f)
+            with gzip.open(input_file, 'rb') as ifile:
+                test_data = pickle.load(ifile)
 
             writer = RecordWriterV2(StringIO(), maxresultrows=10)  # small for the purposes of this unit test
             write_record = writer.write_record
@@ -282,13 +282,76 @@ def test_record_writer_with_recordings(self):
 
             writer.flush(finished=True)
 
-            with io.open(os.path.splitext(os.path.splitext(input_file)[0])[0] + '.output', 'rb') as f:
-                expected = f.read()
+            # Read expected data
 
-            self.assertMultiLineEqual(writer._ofile.getvalue(), expected)
+            expected_path = os.path.splitext(os.path.splitext(input_file)[0])[0] + '.output'
+
+            with io.open(expected_path, 'rb') as ifile:
+                expected = ifile.read()
+
+            expected = self._load_chunks(StringIO(expected))
+
+            # Read observed data
+
+            ifile = writer._ofile
+            ifile.seek(0)
+
+            observed = self._load_chunks(ifile)
+
+            # Write observed data (as an aid to diagnostics)
+
+            observed_path = expected_path + '.observed'
+            observed_value = ifile.getvalue()
+
+            with io.open(observed_path, 'wb') as ifile:
+                ifile.write(observed_value)
+
+            self._compare_chunks(observed, expected)
 
         return
 
+    def _compare_chunks(self, chunks_1, chunks_2):
+        self.assertEqual(len(chunks_1), len(chunks_2))
+        n = 0
+        for chunk_1, chunk_2 in izip(chunks_1, chunks_2):
+            self.assertDictEqual(
+                chunk_1.metadata, chunk_2.metadata,
+                'Chunk {0}: metadata error: "{1}" != "{2}"'.format(n, chunk_1.metadata, chunk_2.metadata))
+            self.assertMultiLineEqual(chunk_1.body, chunk_2.body, 'Chunk {0}: data error'.format(n))
+            n += 1
+        return
+
+    def _load_chunks(self, ifile):
+        import re
+
+        pattern = re.compile(r'chunked 1.0,(?P<metadata_length>\d+),(?P<body_length>\d+)\n')
+        decoder = json.JSONDecoder()
+
+        chunks = []
+
+        while True:
+
+            line = ifile.readline()
+
+            if len(line) == 0:
+                break
+
+            match = pattern.match(line)
+            self.assertIsNotNone(match)
+
+            metadata_length = int(match.group('metadata_length'))
+            metadata = ifile.read(metadata_length)
+            metadata = decoder.decode(metadata)
+
+            body_length = int(match.group('body_length'))
+            body = ifile.read(body_length) if body_length > 0 else ''
+
+            chunks.append(TestInternals._Chunk(metadata, body))
+
+        return chunks
+
+    _Chunk = namedtuple('Chunk', (b'metadata', b'body'))
+
     _dictionary = {
         'a': 1,
         'b': 2,