Support for On-The-Fly GeoJson Layers and a new Mutator Transform (#2095)

* first stab at external layers and mutator transform

* change name to landmarks

* rename test as pr number

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* lint my changes let others unlinted

* move landmarks to official layer and rely on other tile queue changes to load inline layers

* dont need special fixture env for landmarks testing

* rename layer_path, update mutator error text and change test to expect non string values

* origin coords for landmark pois can be in list form as we dont send them downstream anyway

* transforms must be re-entrant, since this one destructively modifies, take a copy

* match proper kinds vernacular

* more nomenclature changes

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Kevin Kreiser <kkreiser@snapchat.com>

Author: 3 people

integration-test/2095_inline_layers.py

@@ -0,0 +1,80 @@
+# -*- encoding: utf-8 -*-
+import json
+import math
+import os
+import shutil
+
+import dsl
+from shapely.wkt import loads as wkt_loads
+
+from . import FixtureTest
+
+
+class InlineLayers(FixtureTest):
+    def test_no_landmarks_buildings_merged(self):
+        """
+        First make sure that even if there is no external layer everything still loads fine and the buildings do merge
+        """
+        # just in case a failed test didnt cleanup
+        shutil.rmtree('metatile-layers', ignore_errors=True)
+
+        # get two adjacent buildings there so they will merge
+        self.generate_fixtures(dsl.way(777, wkt_loads('POLYGON ((0 0, 0 .001, .001 .001, .001 0, 0 0))'),
+                                       {u'building': u'yes', u'layer': u'2', u'name': u'foo'}),
+                               dsl.way(778, wkt_loads('POLYGON ((.001 0, .001 .001, .002 .001, .002 0, .001 0))'),
+                                       {u'building': u'yes', u'layer': u'2', u'name': u'bar'}),
+                               )
+
+        # prove they merged, there should be 1 building there
+        self.assert_n_matching_features(15, 16384, 16383, 'buildings', {'kind': 'building'}, 1)
+
+        # prove there are no auxiliary buildings pois
+        self.assert_no_matching_feature(15, 16384, 16383, 'landmarks', {})
+
+    def test_landmarks_buildings_unmerged(self):
+        # we can temporarily add the external layer, here we build a square structure over the center of the map
+        # note that the origin of the feature is in the upper right quadrant but should appear in all quadrants
+        null_island = {
+            'type': 'FeatureCollection',
+            'name': 'foo',
+            'crs': {'type': 'name', 'properties': {'name': 'urn:ogc:def:crs:EPSG::3857'}},
+            'features': [
+                {'type': 'Feature',
+                 'properties': {'name': 'null island hut', 'supersede': True, 'height': math.pi, 'origin': [1, 1], 'id': 42},
+                 'geometry': {'type': 'Polygon', 'coordinates':
+                              [[[111, 111], [-111, 111],  [-111, -111], [111, -111], [111, 111]]]}}
+            ]
+        }
+
+        # write it to disk
+        shutil.rmtree('metatile-layers', ignore_errors=True)
+        os.makedirs('metatile-layers')
+        with open('metatile-layers/landmarks.geojson', 'w') as f:
+            json.dump(null_island, f)
+
+        # get a building or two in there right next to each other so they will merge
+        self.generate_fixtures(dsl.way(777, wkt_loads('POLYGON ((0 0, 0 .001, .001 .001, .001 0, 0 0))'),
+                                       {u'building': u'yes', u'layer': u'2', u'name': u'foo'}),
+                               dsl.way(778, wkt_loads('POLYGON ((.001 0, .001 .001, .002 .001, .002 0, .001 0))'),
+                                       {u'building': u'yes', u'layer': u'2', u'name': u'bar'}),
+                               )
+
+        # prove they didnt merge, there should still be 2 buildings there
+        self.assert_n_matching_features(15, 16384, 16383, 'buildings', {'kind': 'building'}, 2)
+
+        # check that the property was copied to the right building
+        self.assert_has_feature(17, 65536, 65535, 'buildings', {'superseded_by': True, 'name': 'foo'})
+
+        # check that the pois made it in properly mutated from the original feature
+        props = {'name': 'null island hut', 'height': math.pi, 'id': 42}
+        self.assert_has_feature(15, 16383, 16383, 'landmarks', props)
+        self.assert_feature_geom_type(15, 16383, 16383, 'landmarks', 42, 'Point')
+        self.assert_has_feature(15, 16384, 16383, 'landmarks', props)
+        self.assert_feature_geom_type(15, 16384, 16383, 'landmarks', 42, 'Point')
+        self.assert_has_feature(15, 16383, 16384, 'landmarks', props)
+        self.assert_feature_geom_type(15, 16383, 16384, 'landmarks', 42, 'Point')
+        self.assert_has_feature(15, 16384, 16384, 'landmarks', props)
+        self.assert_feature_geom_type(15, 16384, 16384, 'landmarks', 42, 'Point')
+
+        # clean up external layer
+        shutil.rmtree('metatile-layers')

integration-test/__init__.py

@@ -39,14 +39,14 @@
 from tilequeue.tile import coord_to_mercator_bounds
 from tilequeue.tile import reproject_lnglat_to_mercator
 from tilequeue.tile import reproject_mercator_to_lnglat
-from yaml import load as load_yaml
 
 from vectordatasource.meta import find_yaml_path
 from vectordatasource.meta.python import make_function_name_min_zoom
 from vectordatasource.meta.python import make_function_name_props
 from vectordatasource.meta.python import output_kind
 from vectordatasource.meta.python import output_min_zoom
 from vectordatasource.meta.python import parse_layers
+from yaml import load as load_yaml
 
 
 # the Overpass server is used to download data about OSM elements. the

queries.yaml

@@ -9,6 +9,7 @@ all:
   - boundaries
   - transit
   - admin_areas
+  - landmarks
 
 sources:
 
@@ -338,6 +339,15 @@ layers:
       - vectordatasource.transform.add_id_to_properties
       - vectordatasource.transform.remove_feature_id
     area-inclusion-threshold: 1
+  # optional inline layer, if the file is found on disk it will be used
+  landmarks:
+    geometry_types: [Point, Polygon, MultiPolygon]
+    clip: false
+    area-inclusion-threshold: 1
+    simplify_before_intersect: false
+    simplify_start: 0,
+    pre_processed_layer_path: ./metatile-layers/landmarks.geojson
+
 post_process:
   - fn: vectordatasource.transform.build_fence
     params:
@@ -542,6 +552,23 @@ post_process:
       source_layer: roads
       properties: [layer]
 
+  # supersede existing osm buildings with data from landmark geojson layer
+  - fn: vectordatasource.transform.overlap
+    params:
+      base_layer: buildings
+      cutting_layer: landmarks
+      attribute: supersede
+      target_attribute: superseded_by
+      linear: false
+
+  # turn the landmark geojson polygons into point features
+  - fn: vectordatasource.transform.mutate
+    params:
+      layer: landmarks
+      start_zoom: 13
+      geometry_expression: Point({properties}['origin'])
+      properties_expression: "dict(filter(lambda p: p[0] not in ['supersede', 'origin'], {properties}.items()))"
+
   # cut with admin_areas to put country_code attributes on roads
   # which are mostly within a particular country.
   - fn: vectordatasource.transform.overlap
@@ -1865,6 +1892,7 @@ post_process:
         - addr_housenumber
         - addr_street
         - osm_relation
+      exclude: ['superseded_by']
       # NOTE: max_merged_features is set to keep the time taken for geometry
       # merging down, as it seems to go up with the square of the number of
       # features.

requirements.txt

@@ -7,13 +7,15 @@ future==0.16.0
 gunicorn==19.9.0
 hanzidentifier==1.0.2
 hiredis==0.2.0
+kdtree
 lxml==4.6.2
 mapbox-vector-tile==1.2.0
 ModestMaps==1.4.7
 protobuf==3.4.0
 psycopg2==2.7.3.2
 pyclipper==1.0.6
 pycountry==17.9.23
+pyshp==2.3.0
 python-dateutil==2.6.1
 PyYAML==4.2b4
 git+https://github.com/tilezen/raw_tiles@master#egg=raw_tiles

vectordatasource/transform.py

@@ -1,5 +1,6 @@
 # -*- encoding: utf-8 -*-
 # transformation functions to apply to features
+import copy
 import csv
 import re
 from collections import defaultdict
@@ -9758,3 +9759,44 @@ def override_with_ne_names(shape, props, fid, zoom):
             props['name:' + language] = ne_name
 
     return shape, props, fid
+
+
+def mutate(ctx):
+    """
+    We take a layer and we modify the geometry using the python expression in the query. The expressions have access to
+    both the existing shape and existing properties via python string format replacements {shape} and {properties}
+    respectively. Each expression is then eval'd to replace the existing feature in the layer with the result of the
+    expressions. By default the expressions are a no-op
+    """
+
+    layer = ctx.params.get('layer')
+    assert layer, 'regenerate_geometry: missing layer'
+    geometry_expression = ctx.params.get('geometry_expression', '{shape}')
+    properties_expression = ctx.params.get('properties_expression', '{properties}')
+    assert geometry_expression or properties_expression, \
+        'mutate: requires at least one geometry or properties expression'
+    geometry_expression = geometry_expression.format(shape='shape', properties='props')
+    properties_expression = properties_expression.format(shape='shape', properties='props')
+
+    zoom = ctx.nominal_zoom
+    start_zoom = ctx.params.get('start_zoom', 0)
+    end_zoom = ctx.params.get('end_zoom')
+    if zoom < start_zoom:
+        return None
+    if end_zoom is not None and zoom >= end_zoom:
+        return None
+
+    # for the max zoom a transform needs to be re-entrant so we take a copy here
+    layer = copy.deepcopy(_find_layer(ctx.feature_layers, layer))
+    if layer is None:
+        return None
+
+    new_features = []
+    for feature in layer['features']:
+        shape, props, fid = feature
+        shape = eval(geometry_expression)
+        props = eval(properties_expression)
+        new_features.append((shape, props, fid))
+
+    layer['features'] = new_features
+    return layer