Use a default fill_value of NaN for floats in Zarr v3 (#10757)

* Use a default fill_value of NaN for floats in Zarr v3

Zarr stores written with Xarray now consistently use a default Zarr fill value
of ``NaN`` for float variables, for both Zarr v2 and v3. All
other dtypes still use the Zarr default ``fill_value`` of zero. To customize,
explicitly set encoding in :py:meth:`~Dataset.to_zarr`, e.g.,
``encoding=dict.fromkey(ds.data_vars, {'fill_value': 0})``.

Fixes #10646

* Update note on fill_value in to_zarr()

* Fix for zarr v2

* improve test_default_zarr_fill_value

* tweak test for zarr v2

* Fix error regex

* revert inadvertent test change

* Fix test failure

* Use generic np.nan

Author: shoyer

doc/whats-new.rst

@@ -43,6 +43,13 @@ Breaking changes
   sub-groups (:pull:`10785`).
   By `Stephan Hoyer <https://github.com/shoyer>`_.
 
+- Zarr stores written with Xarray now consistently use a default Zarr fill value
+  of ``NaN`` for float variables, for both Zarr v2 and v3 (:issue:`10646``). All
+  other dtypes still use the Zarr default ``fill_value`` of zero. To customize,
+  explicitly set encoding in :py:meth:`~Dataset.to_zarr`, e.g.,
+  ``encoding=dict.fromkey(ds.data_vars, {'fill_value': 0})``.
+  By `Stephan Hoyer <https://github.com/shoyer>`_.
+
 Deprecations
 ~~~~~~~~~~~~
 

xarray/backends/zarr.py

@@ -1178,6 +1178,11 @@ def set_variables(
                 fill_value = attrs.pop("_FillValue", None)
             else:
                 fill_value = v.encoding.pop("fill_value", None)
+                if fill_value is None and v.dtype.kind == "f":
+                    # For floating point data, Xarray defaults to a fill_value
+                    # of NaN (unlike Zarr, which uses zero):
+                    # https://github.com/pydata/xarray/issues/10646
+                    fill_value = np.nan
                 if "_FillValue" in attrs:
                     # replace with encoded fill value
                     fv = attrs.pop("_FillValue")

xarray/core/dataset.py

@@ -2350,10 +2350,15 @@ def to_zarr(
             used. Override any existing encodings by providing the ``encoding`` kwarg.
 
         ``fill_value`` handling:
-            There exists a subtlety in interpreting zarr's ``fill_value`` property. For zarr v2 format
-            arrays, ``fill_value`` is *always* interpreted as an invalid value similar to the ``_FillValue`` attribute
-            in CF/netCDF. For Zarr v3 format arrays, only an explicit ``_FillValue`` attribute will be used
-            to mask the data if requested using ``mask_and_scale=True``. See this `Github issue <https://github.com/pydata/xarray/issues/5475>`_
+            There exists a subtlety in interpreting zarr's ``fill_value`` property.
+            For Zarr v2 format arrays, ``fill_value`` is *always* interpreted as an
+            invalid value similar to the ``_FillValue`` attribute in CF/netCDF.
+            For Zarr v3 format arrays, only an explicit ``_FillValue`` attribute
+            will be used to mask the data if requested using ``mask_and_scale=True``.
+            To customize the fill value Zarr uses as a default for unwritten
+            chunks on disk, set ``_FillValue`` in encoding for Zarr v2 or
+            ``fill_value`` for Zarr v3.
+            See this `Github issue <https://github.com/pydata/xarray/issues/5475>`_
             for more.
 
         See Also

xarray/tests/test_backends.py

@@ -4380,6 +4380,23 @@ def roundtrip_dir(
         ) as ds:
             yield ds
 
+    @requires_dask
+    def test_default_zarr_fill_value(self):
+        inputs = xr.Dataset({"floats": ("x", [1.0]), "ints": ("x", [1])}).chunk()
+        expected = xr.Dataset({"floats": ("x", [np.nan]), "ints": ("x", [0])})
+        with self.temp_dir() as (d, store):
+            inputs.to_zarr(store, compute=False)
+            with open_dataset(store) as on_disk:
+                assert np.isnan(on_disk.variables["floats"].encoding["_FillValue"])
+                assert (
+                    "_FillValue" not in on_disk.variables["ints"].encoding
+                )  # use default
+                if not has_zarr_v3:
+                    # zarr-python v2 interprets fill_value=None inconsistently
+                    del on_disk["ints"]
+                    del expected["ints"]
+                assert_identical(expected, on_disk)
+
     @pytest.mark.parametrize("consolidated", [True, False, None])
     @pytest.mark.parametrize("write_empty", [True, False, None])
     def test_write_empty(
@@ -4418,14 +4435,13 @@ def assert_expected_files(expected: list[str], store: str) -> None:
                 "0.1.1",
             ]
 
+        # use nan for default fill_value behaviour
+        data = np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2))
+
         if zarr_format_3:
-            data = np.array([0.0, 0, 1.0, 0]).reshape((1, 2, 2))
             # transform to the path style of zarr 3
             # e.g. 0/0/1
             expected = [e.replace(".", "/") for e in expected]
-        else:
-            # use nan for default fill_value behaviour
-            data = np.array([np.nan, np.nan, 1.0, np.nan]).reshape((1, 2, 2))
 
         ds = xr.Dataset(data_vars={"test": (("Z", "Y", "X"), data)})