Change default buffer size to 64MB (#68)

Author: jeffreylovitz

README.md

@@ -53,8 +53,8 @@ python3 redisgraph_bulk_loader/bulk_insert.py GRAPHNAME [OPTIONS]
 |  -e   | --skip-invalid-edges       |            Skip edges that use invalid IDs for endpoints instead of exiting with an error            |
 |  -q   | --quote INT                | The quoting format used in the CSV file. QUOTE_MINIMAL=0,QUOTE_ALL=1,QUOTE_NONNUMERIC=2,QUOTE_NONE=3 |
 |  -t   | --max-token-count INT      |            (Debug argument) Max number of tokens sent in each Redis query (default 1024)             |
-|  -b   | --max-buffer-size INT      |               (Debug argument) Max batch size (MBs) of each Redis query (default 4096)               |
-|  -c   | --max-token-size INT       |              (Debug argument) Max size (MBs) of each token sent to Redis (default 500)               |
+|  -b   | --max-buffer-size INT      |                (Debug argument) Max batch size (MBs) of each Redis query (default 64)                |
+|  -c   | --max-token-size INT       |               (Debug argument) Max size (MBs) of each token sent to Redis (default 64)               |
 |  -i   | --index Label:Property     |              After bulk import, create an Index on provided Label:Property pair (optional)           |
 |  -f   | --full-text-index Label:Property     |              After bulk import, create an full text index on provided Label:Property pair (optional)           |
 
@@ -70,7 +70,7 @@ The label (for nodes) or relationship type (for relationships) is derived from t
 RedisGraph does not impose a schema on properties, so the same property key can have values of differing types, such as strings and integers. As such, the bulk loader's default behaviour is to infer the type for each field independently for each value. This can cause unexpected behaviors when, for example, a property expected to always have string values has a field that can be cast to an integer or double. To avoid this, use the `--enforce-schema` flag and update your CSV headers as described in [Input Schemas](#input-schemas).
 
 ### Extended parameter descriptions
-The flags for `max-token-count`, `max-buffer-size`, and `max-token-size` are typically not required. They should only be specified if the memory overhead of graph creation is too high. The bulk loader builds large graphs by sending binary tokens (each of which holds multiple nodes or relations) to Redis in batches. By lowering these limits from their defaults, the size of each transmission to Redis is lowered and fewer entities are held in memory, at the expense of a longer overall runtime.
+The flags for `max-token-count`, `max-buffer-size`, and `max-token-size` are typically not required. They should only be specified if the memory overhead of graph creation is too high, or raised if the volume of Redis calls is too high. The bulk loader builds large graphs by sending binary tokens (each of which holds multiple nodes or relations) to Redis in batches.
 
 `--quote` is maintained for backwards compatibility, and allows some control over Python's type inference in the default mode. `--enforce-schema-type` is preferred.
 

redisgraph_bulk_loader/bulk_insert.py

@@ -66,8 +66,8 @@ def process_entities(entities):
 @click.option('--escapechar', '-x', default='\\', help='the escape char used for the CSV reader (default \\). Use "none" for None.')
 # Buffer size restrictions
 @click.option('--max-token-count', '-c', default=1024, help='max number of processed CSVs to send per query (default 1024)')
-@click.option('--max-buffer-size', '-b', default=128, help='max buffer size in megabytes (default 128, max 1024)')
-@click.option('--max-token-size', '-t', default=128, help='max size of each token in megabytes (default 128, max 512)')
+@click.option('--max-buffer-size', '-b', default=64, help='max buffer size in megabytes (default 64, max 1024)')
+@click.option('--max-token-size', '-t', default=64, help='max size of each token in megabytes (default 64, max 512)')
 @click.option('--index', '-i', multiple=True, help='Label:Propery on which to create an index')
 @click.option('--full-text-index', '-f', multiple=True, help='Label:Propery on which to create an full text search index')
 def bulk_insert(graph, host, port, password, user, unix_socket_path, nodes, nodes_with_label, relations, relations_with_type, separator, enforce_schema, skip_invalid_nodes, skip_invalid_edges, escapechar, quote, max_token_count, max_buffer_size, max_token_size, index, full_text_index):

redisgraph_bulk_loader/config.py

@@ -1,5 +1,5 @@
 class Config:
-    def __init__(self, max_token_count=1024 * 1023, max_buffer_size=128, max_token_size=128, enforce_schema=False, skip_invalid_nodes=False, skip_invalid_edges=False, separator=',', quoting=3, store_node_identifiers=False, escapechar='\\'):
+    def __init__(self, max_token_count=1024 * 1023, max_buffer_size=64, max_token_size=64, enforce_schema=False, skip_invalid_nodes=False, skip_invalid_edges=False, separator=',', quoting=3, store_node_identifiers=False, escapechar='\\'):
         """Settings for this run of the bulk loader"""
         # Maximum number of tokens per query
         # 1024 * 1024 is the hard-coded Redis maximum. We'll set a slightly lower limit so

test/test_config.py

@@ -7,8 +7,8 @@ def test01_default_values(self):
         """Verify the default values in the Config class."""
         config = Config()
         self.assertEqual(config.max_token_count, 1024 * 1023)
-        self.assertEqual(config.max_buffer_size, 128_000_000)
-        self.assertEqual(config.max_token_size, 128_000_000)
+        self.assertEqual(config.max_buffer_size, 64_000_000)
+        self.assertEqual(config.max_token_size, 64_000_000)
         self.assertEqual(config.enforce_schema, False)
         self.assertEqual(config.skip_invalid_nodes, False)
         self.assertEqual(config.skip_invalid_edges, False)