FT.CREATE - support MAXTEXTFIELDS, TEMPORARY, NOHL, NOFREQS, SKIPINITIALSCAN (#144)

* add tags to client creat_index function

* test expire

* test skip initial scan and summarization disabled

* add testMaxTextFields

Author: AvitalFineRedis

redisearch/client.py

@@ -107,7 +107,8 @@ class IndexDefinition(object):
     IndexDefinition is used to define a index definition for automatic indexing on Hash or Json update.
     """
 
-    def __init__(self, prefix=[], filter=None, language_field=None, language=None, score_field=None, score=1.0, payload_field=None, index_type=None):
+    def __init__(self, prefix=[], filter=None, language_field=None, language=None, score_field=None,
+                 score=1.0, payload_field=None, index_type=None):
         args = []
 
         if index_type is IndexType.HASH:
@@ -182,7 +183,12 @@ class Client(object):
 
     NOOFFSETS = 'NOOFFSETS'
     NOFIELDS = 'NOFIELDS'
+    NOHL = 'NOHL'
+    NOFREQS = 'NOFREQS'
+    MAXTEXTFIELDS = 'MAXTEXTFIELDS'
+    TEMPORARY = 'TEMPORARY'
     STOPWORDS = 'STOPWORDS'
+    SKIPINITIALSCAN = 'SKIPINITIALSCAN'
 
     class BatchIndexer(object):
         """
@@ -256,7 +262,9 @@ def batch_indexer(self, chunk_size=100):
         return Client.BatchIndexer(self, chunk_size=chunk_size)
 
     def create_index(self, fields, no_term_offsets=False,
-                     no_field_flags=False, stopwords=None, definition=None):
+                     no_field_flags=False, stopwords=None, definition=None,
+                     max_text_fields=False, temporary=None, no_highlight=False,
+                     no_term_frequencies=False, skip_initial_scan=False):
         """
         Create the search index. The index must not already exist.
 
@@ -266,15 +274,33 @@ def create_index(self, fields, no_term_offsets=False,
         - **no_term_offsets**: If true, we will not save term offsets in the index
         - **no_field_flags**: If true, we will not save field flags that allow searching in specific fields
         - **stopwords**: If not None, we create the index with this custom stopword list. The list can be empty
+        - **max_text_fields**: If true, we will encode indexes as if there were more than 32 text fields,
+        which allows you to add additional fields (beyond 32).
+        - **temporary**: Create a lightweight temporary index which will expire after the specified period of
+        inactivity (in seconds). The internal idle timer is reset whenever the index is searched or added to.
+        - **no_highlight**: If true, disabling highlighting support. Also implied by no_term_offsets.
+        - **no_term_frequencies**: If true, we avoid saving the term frequencies in the index.
+        - **skip_initial_scan**: If true, we do not scan and index.
         """
 
         args = [self.CREATE_CMD, self.index_name]
         if definition is not None:
-            args += definition.args            
+            args += definition.args
+        if max_text_fields:
+            args.append(self.MAXTEXTFIELDS)
+        if temporary is not None and isinstance(temporary, int):
+            args.append(self.TEMPORARY)
+            args.append(temporary)
         if no_term_offsets:
             args.append(self.NOOFFSETS)
+        if no_highlight:
+            args.append(self.NOHL)
         if no_field_flags:
             args.append(self.NOFIELDS)
+        if no_term_frequencies:
+            args.append(self.NOFREQS)
+        if skip_initial_scan:
+            args.append(self.SKIPINITIALSCAN)
         if stopwords is not None and isinstance(stopwords, (list, tuple, set)):
             args += [self.STOPWORDS, len(stopwords)]
             if len(stopwords) > 0:

test/test.py

@@ -300,6 +300,28 @@ def testReplace(self):
             self.assertEqual(1, res.total)
             self.assertEqual('doc1', res.docs[0].id)
 
+    def testExpire(self):
+        client = self.getCleanClient('idx')
+        client.create_index((TextField('txt', sortable=True),), temporary=4)
+
+        redis_client = redis.client.Redis()
+        ttl = redis_client.execute_command('ft.debug', 'TTL', 'idx')
+        self.assertTrue(ttl > 2)
+        while ttl > 2:
+            ttl = redis_client.execute_command('ft.debug', 'TTL', 'idx')
+            time.sleep(0.01)
+
+        # add document - should reset the ttl
+        client.add_document('doc', txt='foo bar', text='this is a simple test')
+        ttl = redis_client.execute_command('ft.debug', 'TTL', 'idx')
+        self.assertTrue(ttl > 2)
+        try:
+            while True:
+                ttl = redis_client.execute_command('ft.debug', 'TTL', 'idx')
+                time.sleep(0.5)
+        except redis.exceptions.ResponseError:
+            self.assertEqual(ttl, 0)
+
     def testStopwords(self):
         # Creating a client with a given index name
         client = self.getCleanClient('idx')
@@ -314,6 +336,18 @@ def testStopwords(self):
         self.assertEqual(0, res1.total)
         self.assertEqual(1, res2.total)
 
+    def testSkipInitialScan(self):
+        client = self.getCleanClient('idx')
+        client.redis.hset("doc1", "foo", "bar")
+        q = Query('@foo:bar')
+
+        client1 = self.getCleanClient('idx1')
+        client1.create_index((TextField('foo'),))
+        self.assertEqual(1, client1.search(q).total)
+        client2 = self.getCleanClient('idx2')
+        client2.create_index((TextField('foo'),), skip_initial_scan=True)
+        self.assertEqual(0, client2.search(q).total)
+
     def testFilters(self):
         conn = self.redis()
 
@@ -602,6 +636,25 @@ def testSummarize(self):
             self.assertEqual('ACT I SCENE I. London. The palace. Enter <b>KING</b> <b>HENRY</b>, LORD JOHN OF LANCASTER, the EARL of WESTMORELAND, SIR... ',
                             doc.txt)
 
+    def testSummarizeDisabled(self):
+        # test NOOFFSETS
+        client = self.getCleanClient('idx')
+        client.create_index((TextField('txt'),), no_term_offsets=True)
+        client.add_document('doc1', txt='foo bar')
+        with self.assertRaises(Exception) as context:
+            client.search(Query('foo').summarize(fields=['txt']))
+        self.assertEqual('Cannot use highlight/summarize because NOOFSETS was specified at index level',
+                         str(context.exception))
+
+        # test NOHL
+        client = self.getCleanClient('idx')
+        client.create_index((TextField('txt'),), no_highlight=True)
+        client.add_document('doc1', txt='foo bar')
+        with self.assertRaises(Exception) as context:
+            client.search(Query('foo').summarize(fields=['txt']))
+        self.assertEqual('Cannot use highlight/summarize because NOOFSETS was specified at index level',
+                         str(context.exception))
+
     def testAlias(self):
         conn = self.redis()
         with conn as r:
@@ -735,6 +788,35 @@ def testTextFieldSortableNostem(self):
             self.assertIn('SORTABLE', response['attributes'][0])
             self.assertIn('NOSTEM', response['attributes'][0])
 
+    def testMaxTextFields(self):
+        conn = self.redis()
+
+        with conn as r:
+            # Creating a client
+            client = Client('idx1', port=conn.port)
+            client.redis.flushdb()
+
+            # Creating the index definition
+            client.create_index((TextField('f0'),))
+            # Fill the index with fields
+            for x in range(1, 32):
+                client.alter_schema_add((TextField('f{}'.format(x)),))
+            # OK for now.
+
+            # Should be too many indexes
+            with self.assertRaises(redis.ResponseError):
+                client.alter_schema_add((TextField('f{}'.format(x)),))
+
+            # Creating new client
+            client = Client('idx2', port=conn.port)
+            client.redis.flushdb()
+
+            # Creating the index definition
+            client.create_index((TextField('f0'),), max_text_fields=True)
+            # Fill the index with fields
+            for x in range(1, 50):
+                    client.alter_schema_add((TextField('f{}'.format(x)),))
+
     def testAlterSchemaAdd(self):
         conn = self.redis()