Added --Streaming option for very large csv files

Author: datsom1

db_diff/__init__.py

@@ -123,6 +123,76 @@ def compare(previous, current, show_unchanged=False, fields=None, ignorefields=N
     return result
 
 
+def streaming_compare_csv(prev_path, curr_path, key, compare_columns=None, ignorefields=None, encoding='utf-8', dialect='excel'):
+    """
+    Compare two sorted CSV files by streaming, returning a diff dict.
+    """
+    import csv
+    result = {
+        "added": [],
+        "removed": [],
+        "changed": [],
+        "columns_added": [],
+        "columns_removed": [],
+    }
+
+    with open(prev_path, newline='', encoding=encoding) as f1, open(curr_path, newline='', encoding=encoding) as f2:
+        reader1 = csv.DictReader(f1, dialect=dialect)
+        reader2 = csv.DictReader(f2, dialect=dialect)
+        prev_row = next(reader1, None)
+        curr_row = next(reader2, None)
+
+        prev_columns = set(reader1.fieldnames)
+        curr_columns = set(reader2.fieldnames)
+
+        # Determine columns to compare
+        if compare_columns:
+            compare_columns = set(compare_columns)
+        elif ignorefields:
+            compare_columns = (prev_columns | curr_columns) - set(ignorefields)
+        else:
+            compare_columns = prev_columns | curr_columns
+
+        result["columns_added"] = [c for c in curr_columns if c not in prev_columns and c in compare_columns]
+        result["columns_removed"] = [c for c in prev_columns if c not in curr_columns and c in compare_columns]
+
+        while prev_row or curr_row:
+            if prev_row and curr_row:
+                if key not in prev_row or key not in curr_row:
+                    raise KeyError(f"Key column '{key}' missing in one of the rows.")
+                prev_key = str(prev_row[key])
+                curr_key = str(curr_row[key])
+                if prev_key == curr_key:
+                    # Check for changes
+                    changed_fields = {
+                        col: [prev_row.get(col), curr_row.get(col)]
+                        for col in compare_columns
+                        if prev_row.get(col) != curr_row.get(col)
+                    }
+                    if changed_fields:
+                        result["changed"].append({
+                            "key": prev_key,
+                            "changes": changed_fields
+                        })
+                    prev_row = next(reader1, None)
+                    curr_row = next(reader2, None)
+                elif prev_key < curr_key:
+                    # Row removed
+                    result["removed"].append(prev_row)
+                    prev_row = next(reader1, None)
+                else:
+                    # Row added
+                    result["added"].append(curr_row)
+                    curr_row = next(reader2, None)
+            elif prev_row:
+                result["removed"].append(prev_row)
+                prev_row = next(reader1, None)
+            elif curr_row:
+                result["added"].append(curr_row)
+                curr_row = next(reader2, None)
+    return result
+
+
 def human_text(result, key=None, current=None, extras=None):
     title = []
     summary = []

db_diff/cli.py

@@ -3,7 +3,7 @@
 import time as time_module
 import os
 import datetime
-from . import load_csv, load_json, compare, human_text
+from . import load_csv, load_json, compare, human_text, streaming_compare_csv
 
 @click.command(
     context_settings={"help_option_names": ["-h", "--help"]},
@@ -81,11 +81,16 @@
     show_default=True,
     help="Input File Encoding. Available: (utf-8|utf-16|utf-16le|utf-16be|latin1|cp1252|ascii|...).",
 )
+@click.option(
+    "--streaming",
+    is_flag=True,
+    help="Use streaming mode for very large CSV/TSV files (requires files to be sorted by key).",
+)
 @click.version_option()
 def cli(
     previous, current, key, input_format,
     show_unchanged, encoding, show_time, output, output_file, output_path,
-    fields, ignorefields
+    fields, ignorefields, streaming
 ):
     """Compare the differences between two CSV or JSON files."""
     dialect = {
@@ -148,16 +153,34 @@ def load(filename, key):
     fields_set = set(f.strip() for f in fields.split(",")) if fields else None
     ignorefields_set = set(f.strip() for f in ignorefields.split(",")) if ignorefields else None
 
-    previous_data = load(previous, key)
-    current_data = load(current, key)
-    diff = compare(
-        previous_data,
-        current_data,
-        show_unchanged,
-        fields=fields_set,
-        ignorefields=ignorefields_set,
-    )
- # test edit
+    # --- Streaming logic ---
+    if streaming and (input_format in ("csv", "tsv", None)):
+        # Default to csv if not specified
+        fmt = input_format or previous.split(".")[-1].lower()
+        if fmt not in ("csv", "tsv"):
+            raise click.ClickException("--streaming only works with CSV/TSV files.")
+        diff = streaming_compare_csv(
+            previous,
+            current,
+            key=key,
+            compare_columns=fields_set,
+            ignorefields=ignorefields_set,
+            encoding=encoding,
+            dialect=dialect.get(fmt, "excel"),
+        )
+        # For human_text, we need current_data for extras (if used)
+        current_data = None
+    else:
+        previous_data = load(previous, key)
+        current_data = load(current, key)
+        diff = compare(
+            previous_data,
+            current_data,
+            show_unchanged,
+            fields=fields_set,
+            ignorefields=ignorefields_set,
+        )
+
     if output == "json":
         print(std_json.dumps(diff, indent=4))
     elif output == "jsonfile":