Add rf_local_data and rf_local_no_data #244

Signed-off-by: Jason T. Brown <jason@astraea.earth>

Author: vpipkt

core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala

@@ -371,6 +371,12 @@ trait RasterFunctions {
   /** Cellwise inequality comparison between a tile and a scalar. */
   def rf_local_unequal[T: Numeric](tileCol: Column, value: T): Column = Unequal(tileCol, value)
 
+  /** Return a tile with ones where the input is NoData, otherwise zero */
+  def rf_local_no_data(tileCol: Column): Column = Undefined(tileCol)
+
+  /** Return a tile with zeros where the input is NoData, otherwise one*/
+  def rf_local_data(tileCol: Column): Column = Defined(tileCol)
+
   /** Round cell values to nearest integer without chaning cell type. */
   def rf_round(tileCol: Column): Column = Round(tileCol)
 

core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala

@@ -0,0 +1,48 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2019 Astraea, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *     [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.locationtech.rasterframes.expressions.localops
+
+import geotrellis.raster.Tile
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
+import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp}
+
+@ExpressionDescription(
+  usage = "_FUNC_(tile) - Return a tile with zeros where the input is NoData, otherwise one.",
+  arguments = """
+  Arguments:
+    * tile - tile column to inspect""",
+  examples = """
+  Examples:
+    > SELECT  _FUNC_(tile);
+       ..."""
+)
+case class Defined(child: Expression) extends UnaryLocalRasterOp
+  with NullToValue with CodegenFallback {
+  override def nodeName: String = "rf_local_data"
+  override def na: Any = null
+  override protected def op(child: Tile): Tile = child.localDefined()
+}
+object Defined{
+  def apply(tile: Column): Column = new Column(Defined(tile.expr))
+}

core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala

@@ -0,0 +1,48 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2019 Astraea, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *     [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.locationtech.rasterframes.expressions.localops
+
+import geotrellis.raster.Tile
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
+import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp}
+
+@ExpressionDescription(
+  usage = "_FUNC_(tile) - Return a tile with ones where the input is NoData, otherwise zero.",
+  arguments = """
+  Arguments:
+    * tile - tile column to inspect""",
+  examples = """
+  Examples:
+    > SELECT  _FUNC_(tile);
+       ..."""
+)
+case class Undefined(child: Expression) extends UnaryLocalRasterOp
+  with NullToValue with CodegenFallback {
+  override def nodeName: String = "rf_local_no_data"
+  override def na: Any = null
+  override protected def op(child: Tile): Tile = child.localUndefined()
+}
+object Undefined{
+  def apply(tile: Column): Column = new Column(Undefined(tile.expr))
+}

core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala

@@ -87,6 +87,8 @@ package object expressions {
     registry.registerExpression[GreaterEqual]("rf_local_greater_equal")
     registry.registerExpression[Equal]("rf_local_equal")
     registry.registerExpression[Unequal]("rf_local_unequal")
+    registry.registerExpression[Undefined]("rf_local_no_data")
+    registry.registerExpression[Defined]("rf_local_data")
     registry.registerExpression[Sum]("rf_tile_sum")
     registry.registerExpression[Round]("rf_round")
     registry.registerExpression[Abs]("rf_abs")

core/src/test/scala/org/locationtech/rasterframes/RasterFunctionsSpec.scala

@@ -912,4 +912,19 @@ class RasterFunctionsSpec extends TestEnvironment with RasterMatchers {
     countNewNd should be (0L)
 
   }
+
+  it("should return local data and nodata"){
+    checkDocs("rf_local_data")
+    checkDocs("rf_local_no_data")
+
+    val df = Seq(randNDPRT).toDF("t")
+      .withColumn("ld", rf_local_data($"t"))
+      .withColumn("lnd", rf_local_no_data($"t"))
+
+    val ndResult = df.select($"lnd").as[Tile].first()
+    ndResult should be (randNDPRT.localUndefined())
+
+    val dResult = df.select($"ld").as[Tile].first()
+    dResult should be (randNDPRT.localDefined())
+  }
 }

docs/src/main/paradox/release-notes.md

@@ -4,7 +4,7 @@
 
 ### 0.8.1
 
-* Added `rf_interpret_cell_type_as` raster function.
+* Added `rf_local_no_data`, `rf_local_data` and `rf_interpret_cell_type_as` raster functions.
 * Added `toMarkdown()` and `toHTML()` extension methods for `DataFrame`, and registered them with the IPython formatter system when `rf_ipython` is imported.  
 * Fixed: Removed false return type garauntee in cases where an `Expression` accepts either `Tile` or `ProjectedRasterTile` [(#295)](https://github.com/locationtech/rasterframes/issues/295)
 

pyrasterframes/src/main/python/docs/reference.pymd

@@ -223,6 +223,20 @@ Generate a `tile` with the values from `data_tile`, with NoData in cells where t
 
 Returns true if `tile` contains only NoData. By definition returns false if cell type does not support NoData. To count NoData cells or data cells, see @ref:[`rf_no_data_cells`](reference.md#rf-no-data-cells), @ref:[`rf_data_cells`](reference.md#rf-data-cells), @ref:[`rf_agg_no_data_cells`](reference.md#rf-agg-no-data-cells), @ref:[`rf_agg_data_cells`](reference.md#rf-agg-data-cells), @ref:[`rf_agg_local_no_data_cells`](reference.md#rf-agg-local-no-data-cells), and @ref:[`rf_agg_local_data_cells`](reference.md#rf-agg-local-data-cells). This function is distinguished from @ref:[`rf_for_all`](reference.md#rf-for-all), which tests that values are not NoData and nonzero.
 
+### rf_local_no_data
+
+    Tile rf_local_no_data(Tile tile)
+
+Returns a tile with values of 1 in each cell where the input tile contains NoData. Otherwise values are 0.
+
+### rf_local_data
+
+    Tile rf_local_no_data(Tile tile)
+
+Returns a tile with values of 0 in each cell where the input tile contains NoData. Otherwise values are 1.
+
+### rf_local_data
+
 ### rf_with_no_data
 
     Tile rf_with_no_data(Tile tile, Double no_data_value)

pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py

@@ -246,6 +246,13 @@ def rf_local_unequal_int(tile_col, scalar):
     """Return a Tile with values equal 1 if the cell is not equal to a scalar, otherwise 0"""
     return _apply_scalar_to_tile('rf_local_unequal_int', tile_col, scalar)
 
+def rf_local_no_data(tile_col):
+    """Return a tile with ones where the input is NoData, otherwise zero."""
+    return _apply_column_function('rf_local_no_data', tile_col)
+
+def rf_local_data(tile_col):
+    """Return a tile with zeros where the input is NoData, otherwise one."""
+    return _apply_column_function('rf_local_data', tile_col)
 
 def _apply_column_function(name, *args):
     jfcn = RFContext.active().lookup(name)

pyrasterframes/src/main/python/tests/RasterFunctionsTests.py

@@ -288,9 +288,28 @@ def test_rf_interpret_cell_type_as(self):
         df = self.spark.createDataFrame([
             Row(t=Tile(np.array([[1, 3, 4], [5, 0, 3]]), CellType.uint8().with_no_data_value(5)))
         ])
-        df = df.withColumn('tile', rf_interpret_cell_type_as('t', 'uint8ud3')) # threes become ND
+        df = df.withColumn('tile', rf_interpret_cell_type_as('t', 'uint8ud3'))  # threes become ND
         result = df.select(rf_tile_sum(rf_local_equal('t', lit(3))).alias('threes')).first()['threes']
         self.assertEqual(result, 2)
 
         result_5 = df.select(rf_tile_sum(rf_local_equal('t', lit(5))).alias('fives')).first()['fives']
         self.assertEqual(result_5, 0)
+
+    def test_rf_local_data_and_no_data(self):
+        from pyspark.sql import Row
+        from pyrasterframes.rf_types import Tile
+        import numpy as np
+        from numpy.testing import assert_equal
+
+        t = Tile(np.array([[1, 3, 4], [5, 0, 3]]), CellType.uint8().with_no_data_value(5))
+        #note the convert is due to issue #188
+        df = self.spark.createDataFrame([Row(t=t)])\
+            .withColumn('lnd', rf_convert_cell_type(rf_local_no_data('t'), 'uint8')) \
+            .withColumn('ld',  rf_convert_cell_type(rf_local_data('t'),    'uint8'))
+
+        result = df.first()
+        result_nd = result['lnd']
+        assert_equal(result_nd.cells, t.cells.mask)
+
+        result_d = result['ld']
+        assert_equal(result_d.cells, np.invert(t.cells.mask))