Skip to content

Commit 20cf328

Browse files
metasimmetasim
committed
Added tile sample rendering to DataFrame renderers.
1 parent 9dc6a63 commit 20cf328

File tree

15 files changed

+208
-185
lines changed

15 files changed

+208
-185
lines changed

‎core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
package org.locationtech.rasterframes
2323
import geotrellis.proj4.CRS
2424
import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp
25+
import geotrellis.raster.render.ColorRamp
2526
import geotrellis.raster.{CellType, Tile}
2627
import geotrellis.vector.Extent
2728
import org.apache.spark.annotation.Experimental
@@ -34,7 +35,7 @@ import org.locationtech.rasterframes.expressions.aggregates._
3435
import org.locationtech.rasterframes.expressions.generators._
3536
import org.locationtech.rasterframes.expressions.localops._
3637
import org.locationtech.rasterframes.expressions.tilestats._
37-
import org.locationtech.rasterframes.expressions.transformers.RenderPNG.RenderCompositePNG
38+
import org.locationtech.rasterframes.expressions.transformers.RenderPNG.{RenderCompositePNG, RenderColorRampPNG}
3839
import org.locationtech.rasterframes.expressions.transformers._
3940
import org.locationtech.rasterframes.model.TileDimensions
4041
import org.locationtech.rasterframes.stats._
@@ -322,12 +323,16 @@ trait RasterFunctions {
322323
ReprojectGeometry(sourceGeom, srcCRSCol, dstCRSCol)
323324

324325
/** Render Tile as ASCII string, for debugging purposes. */
325-
def rf_render_ascii(col: Column): TypedColumn[Any, String] =
326-
DebugRender.RenderAscii(col)
326+
def rf_render_ascii(tile: Column): TypedColumn[Any, String] =
327+
DebugRender.RenderAscii(tile)
327328

328329
/** Render Tile cell values as numeric values, for debugging purposes. */
329-
def rf_render_matrix(col: Column): TypedColumn[Any, String] =
330-
DebugRender.RenderMatrix(col)
330+
def rf_render_matrix(tile: Column): TypedColumn[Any, String] =
331+
DebugRender.RenderMatrix(tile)
332+
333+
/** Converts tiles in a column into PNG encoded byte array, using given ColorRamp to assign values to colors. */
334+
def rf_render_png(tile: Column, colors: ColorRamp): TypedColumn[Any, Array[Byte]] =
335+
RenderColorRampPNG(tile, colors)
331336

332337
/** Converts columns of tiles representing RGB channels into a PNG encoded byte array. */
333338
def rf_render_png(red: Column, green: Column, blue: Column): TypedColumn[Any, Array[Byte]] =

‎core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,28 +21,29 @@
2121

2222
package org.locationtech.rasterframes.expressions.transformers
2323

24-
import org.locationtech.rasterframes.expressions.UnaryRasterOp
25-
import org.locationtech.rasterframes.util.TileAsMatrix
26-
import geotrellis.raster.Tile
2724
import geotrellis.raster.render.ascii.AsciiArtEncoder
25+
import geotrellis.raster.{Tile, isNoData}
2826
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
2927
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
3028
import org.apache.spark.sql.types.{DataType, StringType}
3129
import org.apache.spark.sql.{Column, TypedColumn}
3230
import org.apache.spark.unsafe.types.UTF8String
31+
import org.locationtech.rasterframes.expressions.UnaryRasterOp
3332
import org.locationtech.rasterframes.model.TileContext
33+
import spire.syntax.cfor.cfor
3434

3535
abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterOp
36-
with CodegenFallback with Serializable {
37-
override def dataType: DataType = StringType
36+
with CodegenFallback with Serializable {
37+
import org.locationtech.rasterframes.expressions.transformers.DebugRender.TileAsMatrix
38+
override def dataType: DataType = StringType
3839

39-
override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = {
40-
UTF8String.fromString(if (asciiArt)
41-
s"\n${tile.renderAscii(AsciiArtEncoder.Palette.NARROW)}\n"
42-
else
43-
s"\n${tile.renderMatrix(6)}\n"
44-
)
45-
}
40+
override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = {
41+
UTF8String.fromString(if (asciiArt)
42+
s"\n${tile.renderAscii(AsciiArtEncoder.Palette.NARROW)}\n"
43+
else
44+
s"\n${tile.renderMatrix(6)}\n"
45+
)
46+
}
4647
}
4748

4849
object DebugRender {
@@ -75,4 +76,29 @@ object DebugRender {
7576
def apply(tile: Column): TypedColumn[Any, String] =
7677
new Column(RenderMatrix(tile.expr)).as[String]
7778
}
79+
80+
implicit class TileAsMatrix(val tile: Tile) extends AnyVal {
81+
def renderMatrix(significantDigits: Int): String = {
82+
val ND = s"%${significantDigits+5}s".format(Double.NaN)
83+
val fmt = s"% ${significantDigits+5}.${significantDigits}g"
84+
val buf = new StringBuilder("[")
85+
cfor(0)(_ < tile.rows, _ + 1) { row =>
86+
if(row > 0) buf.append(' ')
87+
buf.append('[')
88+
cfor(0)(_ < tile.cols, _ + 1) { col =>
89+
val v = tile.getDouble(col, row)
90+
if (isNoData(v)) buf.append(ND)
91+
else buf.append(fmt.format(v))
92+
93+
if (col < tile.cols - 1)
94+
buf.append(',')
95+
}
96+
buf.append(']')
97+
if (row < tile.rows - 1)
98+
buf.append(",\n")
99+
}
100+
buf.append("]")
101+
buf.toString()
102+
}
103+
}
78104
}

‎core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,19 @@ object RenderPNG {
6161
def apply(red: Column, green: Column, blue: Column): TypedColumn[Any, Array[Byte]] =
6262
new Column(RenderCompositePNG(RGBComposite(red.expr, green.expr, blue.expr))).as[Array[Byte]]
6363
}
64+
65+
@ExpressionDescription(
66+
usage = "_FUNC_(tile) - Encode the given tile as a PNG using a color ramp with assignemnts from quantile computation",
67+
arguments = """
68+
Arguments:
69+
* tile - tile to render"""
70+
)
71+
case class RenderColorRampPNG(child: Expression, colors: ColorRamp) extends RenderPNG(child, Some(colors)) {
72+
override def nodeName: String = "rf_render_png"
73+
}
74+
75+
object RenderColorRampPNG {
76+
def apply(tile: Column, colors: ColorRamp): TypedColumn[Any, Array[Byte]] =
77+
new Column(RenderColorRampPNG(tile.expr, colors)).as[Array[Byte]]
78+
}
6479
}

‎core/src/main/scala/org/locationtech/rasterframes/util/DataFrameRenderers.scala‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
* This software is licensed under the Apache 2 license, quoted below.
3+
*
4+
* Copyright 2019 Astraea, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
7+
* use this file except in compliance with the License. You may obtain a copy of
8+
* the License at
9+
*
10+
* [http://www.apache.org/licenses/LICENSE-2.0]
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
* License for the specific language governing permissions and limitations under
16+
* the License.
17+
*
18+
* SPDX-License-Identifier: Apache-2.0
19+
*
20+
*/
21+
22+
package org.locationtech.rasterframes.util
23+
24+
import geotrellis.raster.render.ColorRamps
25+
import org.apache.spark.sql.Dataset
26+
import org.apache.spark.sql.functions.{base64, concat, concat_ws, length, lit, substring, when}
27+
import org.apache.spark.sql.types.{StringType, StructField}
28+
import org.locationtech.rasterframes.expressions.DynamicExtractors
29+
import org.locationtech.rasterframes.{rfConfig, rf_render_png, rf_resample}
30+
31+
/**
32+
* DataFrame extensiosn for rendering sample content in a number of ways
33+
*/
34+
trait DataFrameRenderers {
35+
private val truncateWidth = rfConfig.getInt("max-truncate-row-element-length")
36+
37+
implicit class DFWithPrettyPrint(val df: Dataset[_]) {
38+
39+
private def stringifyRowElements(cols: Seq[StructField], truncate: Boolean, renderTiles: Boolean) = {
40+
cols
41+
.map(c => {
42+
val resolved = df.col(s"`${c.name}`")
43+
if (renderTiles && DynamicExtractors.tileExtractor.isDefinedAt(c.dataType))
44+
concat(
45+
lit("<img src=\"data:image/png;base64,"),
46+
base64(rf_render_png(rf_resample(resolved, 0.5), ColorRamps.Viridis)), // TODO: how to expose?
47+
lit("\"></img>")
48+
)
49+
else {
50+
val str = resolved.cast(StringType)
51+
if (truncate)
52+
when(length(str) > lit(truncateWidth),
53+
concat(substring(str, 1, truncateWidth), lit("..."))
54+
)
55+
.otherwise(str)
56+
else str
57+
}
58+
})
59+
}
60+
61+
def toMarkdown(numRows: Int = 5, truncate: Boolean = false, renderTiles: Boolean = true): String = {
62+
import df.sqlContext.implicits._
63+
val cols = df.schema.fields
64+
val header = cols.map(_.name).mkString("| ", " | ", " |") + "\n" + ("|---" * cols.length) + "|\n"
65+
val stringifiers = stringifyRowElements(cols, truncate, renderTiles)
66+
val cat = concat_ws(" | ", stringifiers: _*)
67+
val rows = df
68+
.select(cat)
69+
.limit(numRows)
70+
.as[String]
71+
.collect()
72+
.map(_.replaceAll("\\[", "\\\\["))
73+
.map(_.replace('\n', '↩'))
74+
75+
val body = rows
76+
.mkString("| ", " |\n| ", " |")
77+
78+
val caption = if (rows.length >= numRows) s"\n_Showing only top $numRows rows_.\n\n" else ""
79+
caption + header + body
80+
}
81+
82+
def toHTML(numRows: Int = 5, truncate: Boolean = false, renderTiles: Boolean = true): String = {
83+
import df.sqlContext.implicits._
84+
val cols = df.schema.fields
85+
val header = "<thead>\n" + cols.map(_.name).mkString("<tr><th>", "</th><th>", "</th></tr>\n") + "</thead>\n"
86+
val stringifiers = stringifyRowElements(cols, truncate, renderTiles)
87+
val cat = concat_ws("</td><td>", stringifiers: _*)
88+
val rows = df
89+
.select(cat).limit(numRows)
90+
.as[String]
91+
.collect()
92+
93+
val body = rows
94+
.mkString("<tr><td>", "</td></tr>\n<tr><td>", "</td></tr>\n")
95+
96+
val caption = if (rows.length >= numRows) s"<caption>Showing only top $numRows rows</caption>\n" else ""
97+
98+
"<table>\n" + caption + header + "<tbody>\n" + body + "</tbody>\n" + "</table>"
99+
}
100+
}
101+
}

‎core/src/main/scala/org/locationtech/rasterframes/util/package.scala‎

Lines changed: 4 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -22,26 +22,24 @@
2222
package org.locationtech.rasterframes
2323

2424
import com.typesafe.scalalogging.Logger
25+
import geotrellis.raster.CellGrid
2526
import geotrellis.raster.crop.TileCropMethods
2627
import geotrellis.raster.io.geotiff.reader.GeoTiffReader
2728
import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp
2829
import geotrellis.raster.mask.TileMaskMethods
2930
import geotrellis.raster.merge.TileMergeMethods
3031
import geotrellis.raster.prototype.TilePrototypeMethods
31-
import geotrellis.raster.{CellGrid, Tile, isNoData}
3232
import geotrellis.spark.Bounds
3333
import geotrellis.spark.tiling.TilerKeyMethods
3434
import geotrellis.util.{ByteReader, GetComponent}
35+
import org.apache.spark.sql._
3536
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
3637
import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, NamedExpression}
3738
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
3839
import org.apache.spark.sql.catalyst.rules.Rule
39-
import org.apache.spark.sql.functions._
4040
import org.apache.spark.sql.rf._
41-
import org.apache.spark.sql.types.{StringType, StructField}
42-
import org.apache.spark.sql._
41+
import org.apache.spark.sql.types.StringType
4342
import org.slf4j.LoggerFactory
44-
import spire.syntax.cfor._
4543

4644
import scala.Boolean.box
4745

@@ -50,7 +48,7 @@ import scala.Boolean.box
5048
*
5149
* @since 12/18/17
5250
*/
53-
package object util {
51+
package object util extends DataFrameRenderers {
5452
@transient
5553
protected lazy val logger: Logger =
5654
Logger(LoggerFactory.getLogger("org.locationtech.rasterframes"))
@@ -159,86 +157,6 @@ package object util {
159157
analyzer(sqlContext).extendedResolutionRules
160158
}
161159

162-
implicit class TileAsMatrix(val tile: Tile) extends AnyVal {
163-
def renderMatrix(significantDigits: Int): String = {
164-
val ND = s"%${significantDigits+5}s".format(Double.NaN)
165-
val fmt = s"% ${significantDigits+5}.${significantDigits}g"
166-
val buf = new StringBuilder("[")
167-
cfor(0)(_ < tile.rows, _ + 1) { row =>
168-
if(row > 0) buf.append(' ')
169-
buf.append('[')
170-
cfor(0)(_ < tile.cols, _ + 1) { col =>
171-
val v = tile.getDouble(col, row)
172-
if (isNoData(v)) buf.append(ND)
173-
else buf.append(fmt.format(v))
174-
175-
if (col < tile.cols - 1)
176-
buf.append(',')
177-
}
178-
buf.append(']')
179-
if (row < tile.rows - 1)
180-
buf.append(",\n")
181-
}
182-
buf.append("]")
183-
buf.toString()
184-
}
185-
}
186-
187-
private val truncateWidth = rfConfig.getInt("max-truncate-row-element-length")
188-
189-
implicit class DFWithPrettyPrint(val df: Dataset[_]) extends AnyVal {
190-
191-
def stringifyRowElements(cols: Seq[StructField], truncate: Boolean) = {
192-
cols
193-
.map(c => s"`${c.name}`")
194-
.map(c => df.col(c).cast(StringType))
195-
.map(c => if (truncate) {
196-
when(length(c) > lit(truncateWidth), concat(substring(c, 1, truncateWidth), lit("...")))
197-
.otherwise(c)
198-
} else c)
199-
}
200-
201-
def toMarkdown(numRows: Int = 5, truncate: Boolean = false): String = {
202-
import df.sqlContext.implicits._
203-
val cols = df.schema.fields
204-
val header = cols.map(_.name).mkString("| ", " | ", " |") + "\n" + ("|---" * cols.length) + "|\n"
205-
val stringifiers = stringifyRowElements(cols, truncate)
206-
val cat = concat_ws(" | ", stringifiers: _*)
207-
val rows = df
208-
.select(cat)
209-
.limit(numRows)
210-
.as[String]
211-
.collect()
212-
.map(_.replaceAll("\\[", "\\\\["))
213-
.map(_.replace('\n', '↩'))
214-
215-
val body = rows
216-
.mkString("| ", " |\n| ", " |")
217-
218-
val caption = if (rows.length >= numRows) s"\n_Showing only top $numRows rows_.\n\n" else ""
219-
caption + header + body
220-
}
221-
222-
def toHTML(numRows: Int = 5, truncate: Boolean = false): String = {
223-
import df.sqlContext.implicits._
224-
val cols = df.schema.fields
225-
val header = "<thead>\n" + cols.map(_.name).mkString("<tr><th>", "</th><th>", "</th></tr>\n") + "</thead>\n"
226-
val stringifiers = stringifyRowElements(cols, truncate)
227-
val cat = concat_ws("</td><td>", stringifiers: _*)
228-
val rows = df
229-
.select(cat).limit(numRows)
230-
.as[String]
231-
.collect()
232-
233-
val body = rows
234-
.mkString("<tr><td>", "</td></tr>\n<tr><td>", "</td></tr>\n")
235-
236-
val caption = if (rows.length >= numRows) s"<caption>Showing only top $numRows rows</caption>\n" else ""
237-
238-
"<table>\n" + caption + header + "<tbody>\n" + body + "</tbody>\n" + "</table>"
239-
}
240-
}
241-
242160
object Shims {
243161
// GT 1.2.1 to 2.0.0
244162
def toArrayTile[T <: CellGrid](tile: T): T =
@@ -281,5 +199,4 @@ package object util {
281199
result.asInstanceOf[GeoTiffReader.GeoTiffInfo]
282200
}
283201
}
284-
285202
}

‎core/src/test/scala/org/locationtech/rasterframes/ExtensionMethodSpec.scala‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,16 +115,18 @@ class ExtensionMethodSpec extends TestEnvironment with TestData with SubdivideSu
115115
it("should render Markdown") {
116116
val md = rf.toMarkdown()
117117
md.count(_ == '|') shouldBe >=(3 * 5)
118-
md.count(_ == '\n') should be >=(6)
118+
md.count(_ == '\n') should be >= 6
119119

120120
val md2 = rf.toMarkdown(truncate=true)
121121
md2 should include ("...")
122122
}
123123

124124
it("should render HTML") {
125+
val html = rf.toHTML()
125126
noException shouldBe thrownBy {
126-
XhtmlParser(scala.io.Source.fromString(rf.toHTML()))
127+
XhtmlParser(scala.io.Source.fromString(html))
127128
}
129+
println(html)
128130
}
129131
}
130132
}

0 commit comments

Comments
 (0)