generate component specific grammars (#181)

rough first cut at a reader supported by typed grammar generation. typed parser/transformer working for a few test pkgs. period data is just defined in terms of nondescript records, everything else is properly defined. maybe eventually we can properly type record fields. I included generated grammars here for now so we can easily see what they look like. but maybe at some point it will be cleaner not to version them.

Author: wpbonelli

.gitignore

@@ -9,3 +9,4 @@ venv/
 *.egg-info
 temp/
 .coverage
+flopy4/mf6/codec/reader/grammar/generated/*.lark

flopy4/mf6/codec/filters.py

@@ -0,0 +1,31 @@
+"""Filters shared by both reader and writer."""
+
+from typing import Any
+
+import xarray as xr
+from modflow_devtools.dfn.schema.field import Field
+from modflow_devtools.dfn.schema.v2 import FieldType
+
+
+def field_type(value: Any) -> FieldType:
+    """Get a value's type according to the MF6 specification."""
+
+    if isinstance(value, Field):
+        return value.type
+    if isinstance(value, bool):
+        return "keyword"
+    if isinstance(value, int):
+        return "integer"
+    if isinstance(value, float):
+        return "double"
+    if isinstance(value, str):
+        return "string"
+    if isinstance(value, tuple):
+        return "record"
+    if isinstance(value, xr.DataArray):
+        if value.dtype == "object":
+            return "list"
+        return "array"
+    if isinstance(value, (list, dict, xr.Dataset)):
+        return "list"
+    raise ValueError(f"Unsupported field type: {type(value)}")

flopy4/mf6/codec/reader/__init__.py

@@ -1,9 +1,9 @@
 from typing import IO, Any
 
-from flopy4.mf6.codec.reader.parser import make_basic_parser
+from flopy4.mf6.codec.reader.parser import get_basic_parser
 from flopy4.mf6.codec.reader.transformer import BasicTransformer
 
-BASIC_PARSER = make_basic_parser()
+BASIC_PARSER = get_basic_parser()
 BASIC_TRANSFORMER = BasicTransformer()
 
 

flopy4/mf6/codec/reader/dfn2lark.py

@@ -0,0 +1,44 @@
+"""Convert (TOML/v2) DFNs to Lark grammars."""
+
+import argparse
+from os import PathLike
+from pathlib import Path
+
+from modflow_devtools.dfn import load_flat, map
+
+from flopy4.mf6.codec.reader.grammar import make_all_grammars
+
+_GRAMMAR_MODULE = Path(__file__).parent / "grammar"
+_GRAMMAR_GEN_DIR = _GRAMMAR_MODULE / "generated"
+
+
+def generate(dfndir: PathLike, outdir: PathLike):
+    """Generate lark grammars from DFNs."""
+    dfndir = Path(dfndir).expanduser().absolute()
+    outdir = Path(outdir).expanduser().absolute()
+    outdir.mkdir(exist_ok=True, parents=True)
+    dfns_v1 = load_flat(dfndir)
+    dfns_v2 = {name: map(dfn, schema_version=2) for name, dfn in dfns_v1.items()}
+    make_all_grammars(dfns_v2, outdir)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate lark grammars from DFNs.")
+    parser.add_argument(
+        "--dfndir",
+        "-d",
+        type=str,
+        help="Directory containing DFN files.",
+    )
+    parser.add_argument(
+        "--outdir",
+        "-o",
+        help="Output directory.",
+        default=_GRAMMAR_GEN_DIR,
+    )
+    args = parser.parse_args()
+    generate(args.dfndir, args.outdir)
+
+
+if __name__ == "__main__":
+    main()

flopy4/mf6/codec/reader/grammar/__init__.py

@@ -0,0 +1,72 @@
+from os import PathLike
+from pathlib import Path
+
+import jinja2
+from modflow_devtools.dfn import Dfn
+
+from flopy4.mf6.codec.reader.grammar import filters
+
+
+def _get_template_env():
+    loader = jinja2.PackageLoader("flopy4", "mf6/codec/reader/grammar/templates/")
+    env = jinja2.Environment(
+        loader=loader,
+        trim_blocks=True,
+        lstrip_blocks=True,
+        keep_trailing_newline=True,
+    )
+    env.filters["field_type"] = filters.field_type
+    env.filters["record_child_type"] = filters.record_child_type
+    return env
+
+
+def _get_template_data(blocks) -> tuple[list[dict], dict[str, object]]:
+    all_blocks = []
+    all_fields = {}
+
+    for block_name, block_fields in blocks.items():
+        period_groups = filters.group_period_fields(block_fields)
+        has_index = block_name == "period"
+
+        recarrays = []
+        grouped_field_names = set()
+        if period_groups:
+            for field_names in period_groups.values():
+                recarray_name = filters.get_recarray_name(block_name)
+                recarrays.append({"name": recarray_name, "fields": field_names})
+                grouped_field_names.update(field_names)
+
+        all_field_names = list(block_fields.keys())
+        standalone_fields = [f for f in all_field_names if f not in grouped_field_names]
+
+        all_fields.update(block_fields)
+        all_blocks.append(
+            {
+                "name": block_name,
+                "has_index": has_index,
+                "standalone_fields": standalone_fields,
+                "recarrays": recarrays,
+            }
+        )
+
+    return all_blocks, all_fields
+
+
+def make_grammar(dfn: Dfn, outdir: PathLike):
+    """Generate a Lark grammar file for a single component."""
+    outdir = Path(outdir).expanduser().resolve().absolute()
+    env = _get_template_env()
+    template = env.get_template("component.lark.jinja")
+    target_path = outdir / f"{dfn.name}.lark"
+    blocks, fields = _get_template_data(dfn.blocks)
+    with open(target_path, "w") as f:
+        name = dfn.name
+        f.write(template.render(name=name, blocks=blocks, fields=fields))
+
+
+def make_all_grammars(dfns: dict[str, Dfn], outdir: PathLike):
+    """Generate grammars for all components."""
+    outdir = Path(outdir).expanduser().resolve().absolute()
+    outdir.mkdir(parents=True, exist_ok=True)
+    for dfn in dfns.values():
+        make_grammar(dfn, outdir)

flopy4/mf6/codec/reader/grammar/filters.py

@@ -0,0 +1,72 @@
+from collections.abc import Mapping
+
+from modflow_devtools.dfn.schema.v2 import FieldV2
+
+
+def field_type(field: FieldV2) -> str:
+    match field.type:
+        case t if t in ["string", "integer", "double"] and field.shape:
+            if "period" in field.block:
+                return "list"
+            return "array"
+        case "keyword":
+            return ""
+        case "union":
+            return ""  # keystrings generate their own union rules
+        case _:
+            return field.type
+
+
+def record_child_type(field: FieldV2) -> str:
+    """Get the grammar type for a field within a record context."""
+    match field.type:
+        case t if t in ["string", "double", "integer"]:
+            return t
+        case "keyword":
+            return ""
+        case "union":
+            return ""  # keystrings generate their own union rules
+        case _:
+            return field.type
+
+
+def is_period_list_field(field: FieldV2) -> bool:
+    """Check if a field is part of a period block list/recarray."""
+    if not field.shape or not field.block:
+        return False
+    return (
+        "period" in field.block
+        and field.type in ["string", "integer", "double"]
+        and field.shape is not None
+    )
+
+
+def group_period_fields(block_fields: Mapping[str, FieldV2]) -> dict[str, list[str]]:
+    """
+    Group period block fields that should be combined into a single list.
+
+    Returns a dict mapping the first field name to a list of all field names
+    in the group. Fields are grouped if they share similar shapes (same base
+    dimensions like nper, nnodes).
+    """
+    period_fields = {
+        name: field for name, field in block_fields.items() if is_period_list_field(field)
+    }
+
+    if not period_fields:
+        return {}
+
+    # All period fields in the same block should be combined into one recarray
+    # Return a single group with all field names
+    field_names = list(period_fields.keys())
+    if field_names:
+        return {field_names[0]: field_names}
+    return {}
+
+
+def get_recarray_name(block_name: str) -> str:
+    """Get the name for a recarray representing period data in a block."""
+    # Use similar naming to V1: stress_period_data, perioddata, etc.
+    if block_name == "period":
+        return "stress_period_data"
+    return f"{block_name}data"

flopy4/mf6/codec/reader/grammar/generated/chf-cdb.lark

@@ -0,0 +1,32 @@
+// Auto-generated grammar for MF6 CHF-CDB
+
+%import typed.integer -> integer
+%import typed.double -> double
+%import typed.number -> number
+%import typed.string -> string
+%import typed.array -> array
+%import typed.record -> record
+%import typed.NEWLINE -> NEWLINE
+%import common.WS
+%import common.SH_COMMENT
+
+%ignore WS
+%ignore SH_COMMENT
+
+start: block*
+block: options_block | dimensions_block | period_block
+options_block: "begin"i "options"i options_fields "end"i "options"i
+dimensions_block: "begin"i "dimensions"i dimensions_fields "end"i "dimensions"i
+period_block: "begin"i "period"i block_index period_fields "end"i "period"i block_index
+block_index: integer
+options_fields: (auxiliary | boundnames | print_input | print_flows | save_flows | obs_filerecord)*
+dimensions_fields: (maxbound)*
+period_fields: (stress_period_data)*
+auxiliary: "auxiliary"i array
+boundnames: "boundnames"i 
+print_input: "print_input"i 
+print_flows: "print_flows"i 
+save_flows: "save_flows"i 
+obs_filerecord: "filein"i "obs6"i string
+maxbound: "maxbound"i integer
+stress_period_data: record+

flopy4/mf6/codec/reader/grammar/generated/chf-chd.lark

@@ -0,0 +1,34 @@
+// Auto-generated grammar for MF6 CHF-CHD
+
+%import typed.integer -> integer
+%import typed.double -> double
+%import typed.number -> number
+%import typed.string -> string
+%import typed.array -> array
+%import typed.record -> record
+%import typed.NEWLINE -> NEWLINE
+%import common.WS
+%import common.SH_COMMENT
+
+%ignore WS
+%ignore SH_COMMENT
+
+start: block*
+block: options_block | dimensions_block | period_block
+options_block: "begin"i "options"i options_fields "end"i "options"i
+dimensions_block: "begin"i "dimensions"i dimensions_fields "end"i "dimensions"i
+period_block: "begin"i "period"i block_index period_fields "end"i "period"i block_index
+block_index: integer
+options_fields: (auxiliary | auxmultname | boundnames | print_input | print_flows | save_flows | ts_filerecord | obs_filerecord)*
+dimensions_fields: (maxbound)*
+period_fields: (stress_period_data)*
+auxiliary: "auxiliary"i array
+auxmultname: "auxmultname"i string
+boundnames: "boundnames"i 
+print_input: "print_input"i 
+print_flows: "print_flows"i 
+save_flows: "save_flows"i 
+ts_filerecord: "ts6"i "filein"i string
+obs_filerecord: "filein"i "obs6"i string
+maxbound: "maxbound"i integer
+stress_period_data: record+

flopy4/mf6/codec/reader/grammar/generated/chf-cxs.lark

@@ -0,0 +1,30 @@
+// Auto-generated grammar for MF6 CHF-CXS
+
+%import typed.integer -> integer
+%import typed.double -> double
+%import typed.number -> number
+%import typed.string -> string
+%import typed.array -> array
+%import typed.record -> record
+%import typed.NEWLINE -> NEWLINE
+%import common.WS
+%import common.SH_COMMENT
+
+%ignore WS
+%ignore SH_COMMENT
+
+start: block*
+block: options_block | dimensions_block | packagedata_block | crosssectiondata_block
+options_block: "begin"i "options"i options_fields "end"i "options"i
+dimensions_block: "begin"i "dimensions"i dimensions_fields "end"i "dimensions"i
+packagedata_block: "begin"i "packagedata"i packagedata_fields "end"i "packagedata"i
+crosssectiondata_block: "begin"i "crosssectiondata"i crosssectiondata_fields "end"i "crosssectiondata"i
+options_fields: (print_input)*
+dimensions_fields: (nsections | npoints)*
+packagedata_fields: (packagedata)*
+crosssectiondata_fields: (crosssectiondata)*
+print_input: "print_input"i 
+nsections: "nsections"i integer
+npoints: "npoints"i integer
+packagedata: "packagedata"i recarray
+crosssectiondata: "crosssectiondata"i recarray

flopy4/mf6/codec/reader/grammar/generated/chf-dfw.lark

@@ -0,0 +1,32 @@
+// Auto-generated grammar for MF6 CHF-DFW
+
+%import typed.integer -> integer
+%import typed.double -> double
+%import typed.number -> number
+%import typed.string -> string
+%import typed.array -> array
+%import typed.record -> record
+%import typed.NEWLINE -> NEWLINE
+%import common.WS
+%import common.SH_COMMENT
+
+%ignore WS
+%ignore SH_COMMENT
+
+start: block*
+block: options_block | griddata_block
+options_block: "begin"i "options"i options_fields "end"i "options"i
+griddata_block: "begin"i "griddata"i griddata_fields "end"i "griddata"i
+options_fields: (central_in_space | length_conversion | time_conversion | save_flows | print_flows | save_velocity | obs_filerecord | export_array_ascii | dev_swr_conductance)*
+griddata_fields: (manningsn | idcxs)*
+central_in_space: "central_in_space"i 
+length_conversion: "length_conversion"i double
+time_conversion: "time_conversion"i double
+save_flows: "save_flows"i 
+print_flows: "print_flows"i 
+save_velocity: "save_velocity"i 
+obs_filerecord: "obs6"i "filein"i string
+export_array_ascii: "export_array_ascii"i 
+dev_swr_conductance: "dev_swr_conductance"i 
+manningsn: "manningsn"i array
+idcxs: "idcxs"i array