3751661366

Build Type

push

github

Committed by GitHub

Commit Message

Improvements (#122)

Run Details

88 of 99 new or added lines in 7 files covered. (88.89%)

6 existing lines in 3 files now uncovered.

543 of 709 relevant lines covered (76.59%)

3.83 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.69

/mapbox_vector_tile/encoder.py

from numbers import Number

from shapely.geometry.base import BaseGeometry
from shapely.geometry.multipolygon import MultiPolygon
from shapely.geometry.polygon import orient, Polygon
from shapely.ops import transform
from shapely.wkb import loads as load_wkb
from shapely.wkt import loads as load_wkt

from mapbox_vector_tile.Mapbox import vector_tile_pb2 as vector_tile
from mapbox_vector_tile.geom_encoder import GeometryEncoder
from mapbox_vector_tile.polygon import make_it_valid
from mapbox_vector_tile.simple_shape import SimpleShape


def on_invalid_geometry_raise(shape):
    raise ValueError(f"Invalid geometry: {shape.wkt}")


def on_invalid_geometry_ignore(shape):
    return None


def on_invalid_geometry_make_valid(shape):
    return make_it_valid(shape)


class VectorTile:
    def __init__(self, extents, on_invalid_geometry=None, max_geometry_validate_tries=5, check_winding_order=True):
        self.tile = vector_tile.tile()
        self.extents = extents
        self.on_invalid_geometry = on_invalid_geometry
        self.check_winding_order = check_winding_order
        self.max_geometry_validate_tries = max_geometry_validate_tries

        self.layer = None
        self.key_idx = 0
        self.val_idx = 0
        self.seen_keys_idx = {}
        self.seen_values_idx = {}
        self.seen_values_bool_idx = {}

    def add_features(self, features, layer_name="", quantize_bounds=None, y_coord_down=False):
        self.layer = self.tile.layers.add()
        self.layer.name = layer_name
        self.layer.version = 1
        self.layer.extent = self.extents

        self.key_idx = 0
        self.val_idx = 0
        self.seen_keys_idx = {}
        self.seen_values_idx = {}
        self.seen_values_bool_idx = {}

        for feature in features:
            # skip missing or empty geometries
            geometry_spec = feature.get("geometry")
            if geometry_spec is None:
                continue
            shape = self._load_geometry(geometry_spec)

            if shape is None:
                raise NotImplementedError("Can't do geometries that are not wkt, wkb, or shapely geometries")

            if shape.is_empty:
                continue

            if quantize_bounds:
                shape = self.quantize(shape, quantize_bounds)
            if self.check_winding_order:
                shape = self.enforce_winding_order(shape, y_coord_down)

            if shape is not None and not shape.is_empty:
                self.add_feature(feature, shape, y_coord_down)

    def enforce_winding_order(self, shape, y_coord_down, n_try=1):
        if shape.type == "MultiPolygon":
            # If we are a multipolygon, we need to ensure that the winding orders of the constituent polygons are
            # correct. In particular, the winding order of the interior rings need to be the opposite of the exterior
            # ones, and all interior rings need to follow the exterior one. This is how the end of one polygon and
            # the beginning of another are signaled.
            shape = self.enforce_multipolygon_winding_order(shape, y_coord_down, n_try)

        elif shape.type == "Polygon":
            # Ensure that polygons are also oriented with the appropriate winding order. Their exterior rings must
            # have a clockwise order, which is translated into a clockwise order in MVT's tile-local coordinates with
            # the Y axis in "screen" (i.e: +ve down) configuration. Note that while the Y axis flips, we also invert
            # the Y coordinate to get the tile-local value, which means the clockwise orientation is unchanged.
            shape = self.enforce_polygon_winding_order(shape, y_coord_down, n_try)

        # other shapes just get passed through
        return shape

    def quantize(self, shape, bounds):
        minx, miny, maxx, maxy = bounds

        def fn(x, y, z=None):
            xfac = self.extents / (maxx - minx)
            yfac = self.extents / (maxy - miny)
            x = xfac * (x - minx)
            y = yfac * (y - miny)
            return round(x), round(y)

        return transform(fn, shape)

    def handle_shape_validity(self, shape, y_coord_down, n_try):
        if shape.is_valid:
            return shape

        if n_try >= self.max_geometry_validate_tries:
            # ensure that we don't recurse indefinitely with an invalid geometry handler that doesn't validate
            # geometries
            return None

        if self.on_invalid_geometry:
            shape = self.on_invalid_geometry(shape)
            if shape is not None and not shape.is_empty:
                # This means that we have a handler that might have altered the geometry. We'll run through the process
                # again, but keep track of which attempt we are on to terminate the recursion.
                shape = self.enforce_winding_order(shape, y_coord_down, n_try + 1)

        return shape

    def enforce_multipolygon_winding_order(self, shape, y_coord_down, n_try):
        assert shape.type == "MultiPolygon"

        parts = []
        for part in shape.geoms:
            part = self.enforce_polygon_winding_order(part, y_coord_down, n_try)
            if part is not None and not part.is_empty:
                if part.geom_type == "MultiPolygon":
                    parts.extend(part.geoms)
                else:
                    parts.append(part)

        if not parts:
            return None

        if len(parts) == 1:
            oriented_shape = parts[0]
        else:
            oriented_shape = MultiPolygon(parts)

        oriented_shape = self.handle_shape_validity(oriented_shape, y_coord_down, n_try)
        return oriented_shape

    def enforce_polygon_winding_order(self, shape, y_coord_down, n_try):
        assert shape.type == "Polygon"

        def fn(point):
            x, y = point
            return round(x), round(y)

        exterior = self.apply_map(fn, shape.exterior.coords)
        rings = None

        if len(shape.interiors) > 0:
            rings = [self.apply_map(fn, ring.coords) for ring in shape.interiors]

        sign = 1.0 if y_coord_down else -1.0
        oriented_shape = orient(Polygon(exterior, rings), sign=sign)
        oriented_shape = self.handle_shape_validity(oriented_shape, y_coord_down, n_try)
        return oriented_shape

    @staticmethod
    def apply_map(fn, x):
        return list(map(fn, x))

    @staticmethod
    def _load_geometry(geometry_spec):
        if isinstance(geometry_spec, BaseGeometry):
            return geometry_spec

        if isinstance(geometry_spec, dict):
            return SimpleShape(geometry_spec["coordinates"], geometry_spec["type"])

        try:
            return load_wkb(geometry_spec)
        except Exception:
            try:
                return load_wkt(geometry_spec)
            except Exception:
                return None

    def add_feature(self, feature, shape, y_coord_down):
        geom_encoder = GeometryEncoder(y_coord_down, self.extents)
        geometry = geom_encoder.encode(shape)

        feature_type = self._get_feature_type(shape)
        if len(geometry) == 0:
            # Don't add geometry if it's too small
            return
        f = self.layer.features.add()

        fid = feature.get("id")
        if fid is not None:
            if isinstance(fid, Number) and fid >= 0:
                f.id = fid

        # properties
        properties = feature.get("properties")
        if properties is not None:
            self._handle_attr(self.layer, f, properties)

        f.type = feature_type
        f.geometry.extend(geometry)

    def _get_feature_type(self, shape):
        if shape.type == "Point" or shape.type == "MultiPoint":
            return self.tile.Point
        elif shape.type == "LineString" or shape.type == "MultiLineString":
            return self.tile.LineString
        elif shape.type == "Polygon" or shape.type == "MultiPolygon":
            return self.tile.Polygon
        elif shape.type == "GeometryCollection":
            raise ValueError("Encoding geometry collections not supported")
        else:
            raise ValueError(f"Cannot encode unknown geometry type: {shape.type}")

    @staticmethod
    def _chunker(seq, size):
        return [seq[pos : pos + size] for pos in range(0, len(seq), size)]

    @staticmethod
    def _can_handle_key(k):
        return isinstance(k, str)

    @staticmethod
    def _can_handle_val(v):
        return isinstance(v, (str, bool, int, float))

    @classmethod
    def _can_handle_attr(cls, k, v):
        return cls._can_handle_key(k) and cls._can_handle_val(v)

    def _handle_attr(self, layer, feature, props):
        for k, v in props.items():
            if self._can_handle_attr(k, v):
                if k not in self.seen_keys_idx:
                    layer.keys.append(k)
                    self.seen_keys_idx[k] = self.key_idx
                    self.key_idx += 1

                feature.tags.append(self.seen_keys_idx[k])

                if isinstance(v, bool):
                    values_idx = self.seen_values_bool_idx
                else:
                    values_idx = self.seen_values_idx

                if v not in values_idx:
                    values_idx[v] = self.val_idx
                    self.val_idx += 1

                    val = layer.values.add()
                    if isinstance(v, bool):
                        val.bool_value = v
                    elif isinstance(v, str):
                        val.string_value = v
                    elif isinstance(v, int):
                        val.int_value = v
                    elif isinstance(v, float):
                        val.double_value = v

                feature.tags.append(values_idx[v])

1	from numbers import Number	5✔
2
3	from shapely.geometry.base import BaseGeometry	5✔
4	from shapely.geometry.multipolygon import MultiPolygon	5✔
5	from shapely.geometry.polygon import orient, Polygon	5✔
6	from shapely.ops import transform	5✔
7	from shapely.wkb import loads as load_wkb	5✔
8	from shapely.wkt import loads as load_wkt	5✔
9
10	from mapbox_vector_tile.Mapbox import vector_tile_pb2 as vector_tile	5✔
11	from mapbox_vector_tile.geom_encoder import GeometryEncoder	5✔
12	from mapbox_vector_tile.polygon import make_it_valid	5✔
13	from mapbox_vector_tile.simple_shape import SimpleShape	5✔
14
15
16	def on_invalid_geometry_raise(shape):	5✔
17	raise ValueError(f"Invalid geometry: {shape.wkt}")	5✔
18
19
20	def on_invalid_geometry_ignore(shape):	5✔
21	return None	5✔
22
23
24	def on_invalid_geometry_make_valid(shape):	5✔
25	return make_it_valid(shape)	5✔
26
27
28	class VectorTile:	5✔
29	def __init__(self, extents, on_invalid_geometry=None, max_geometry_validate_tries=5, check_winding_order=True):	5✔
30	self.tile = vector_tile.tile()	5✔
31	self.extents = extents	5✔
32	self.on_invalid_geometry = on_invalid_geometry	5✔
33	self.check_winding_order = check_winding_order	5✔
34	self.max_geometry_validate_tries = max_geometry_validate_tries	5✔
35
36	self.layer = None	5✔
37	self.key_idx = 0	5✔
38	self.val_idx = 0	5✔
39	self.seen_keys_idx = {}	5✔
40	self.seen_values_idx = {}	5✔
41	self.seen_values_bool_idx = {}	5✔
42
43	def add_features(self, features, layer_name="", quantize_bounds=None, y_coord_down=False):	5✔
44	self.layer = self.tile.layers.add()	5✔
45	self.layer.name = layer_name	5✔
46	self.layer.version = 1	5✔
47	self.layer.extent = self.extents	5✔
48
49	self.key_idx = 0	5✔
50	self.val_idx = 0	5✔
51	self.seen_keys_idx = {}	5✔
52	self.seen_values_idx = {}	5✔
53	self.seen_values_bool_idx = {}	5✔
54
55	for feature in features:	5✔
56	# skip missing or empty geometries
57	geometry_spec = feature.get("geometry")	5✔
58	if geometry_spec is None:	5✔
59	continue	×
60	shape = self._load_geometry(geometry_spec)	5✔
61
62	if shape is None:	5✔
63	raise NotImplementedError("Can't do geometries that are not wkt, wkb, or shapely geometries")	5✔
64
65	if shape.is_empty:	5✔
66	continue	×
67
68	if quantize_bounds:	5✔
69	shape = self.quantize(shape, quantize_bounds)	5✔
70	if self.check_winding_order:	5✔
71	shape = self.enforce_winding_order(shape, y_coord_down)	5✔
72
73	if shape is not None and not shape.is_empty:	5✔
74	self.add_feature(feature, shape, y_coord_down)	5✔
75
76	def enforce_winding_order(self, shape, y_coord_down, n_try=1):	5✔
77	if shape.type == "MultiPolygon":	5✔
78	# If we are a multipolygon, we need to ensure that the winding orders of the constituent polygons are
79	# correct. In particular, the winding order of the interior rings need to be the opposite of the exterior
80	# ones, and all interior rings need to follow the exterior one. This is how the end of one polygon and
81	# the beginning of another are signaled.
82	shape = self.enforce_multipolygon_winding_order(shape, y_coord_down, n_try)	5✔
83
84	elif shape.type == "Polygon":	5✔
85	# Ensure that polygons are also oriented with the appropriate winding order. Their exterior rings must
86	# have a clockwise order, which is translated into a clockwise order in MVT's tile-local coordinates with
87	# the Y axis in "screen" (i.e: +ve down) configuration. Note that while the Y axis flips, we also invert
88	# the Y coordinate to get the tile-local value, which means the clockwise orientation is unchanged.
89	shape = self.enforce_polygon_winding_order(shape, y_coord_down, n_try)	5✔
90
91	# other shapes just get passed through
92	return shape	5✔
93
94	def quantize(self, shape, bounds):	5✔
95	minx, miny, maxx, maxy = bounds	5✔
96
97	def fn(x, y, z=None):	5✔
98	xfac = self.extents / (maxx - minx)	5✔
99	yfac = self.extents / (maxy - miny)	5✔
100	x = xfac * (x - minx)	5✔
101	y = yfac * (y - miny)	5✔
102	return round(x), round(y)	5✔
103
104	return transform(fn, shape)	5✔
105
106	def handle_shape_validity(self, shape, y_coord_down, n_try):	5✔
107	if shape.is_valid:	5✔
108	return shape	5✔
109
110	if n_try >= self.max_geometry_validate_tries:	5✔
111	# ensure that we don't recurse indefinitely with an invalid geometry handler that doesn't validate
112	# geometries
113	return None	×
114
115	if self.on_invalid_geometry:	5✔
116	shape = self.on_invalid_geometry(shape)	5✔
117	if shape is not None and not shape.is_empty:	5✔
118	# This means that we have a handler that might have altered the geometry. We'll run through the process
119	# again, but keep track of which attempt we are on to terminate the recursion.
120	shape = self.enforce_winding_order(shape, y_coord_down, n_try + 1)	5✔
121
122	return shape	5✔
123
124	def enforce_multipolygon_winding_order(self, shape, y_coord_down, n_try):	5✔
125	assert shape.type == "MultiPolygon"	5✔
126
127	parts = []	5✔
128	for part in shape.geoms:	5✔
129	part = self.enforce_polygon_winding_order(part, y_coord_down, n_try)	5✔
130	if part is not None and not part.is_empty:	5✔
131	if part.geom_type == "MultiPolygon":	5✔
132	parts.extend(part.geoms)	5✔
133	else:
134	parts.append(part)	5✔
135
136	if not parts:	5✔
137	return None	×
138
139	if len(parts) == 1:	5✔
140	oriented_shape = parts[0]	5✔
141	else:
142	oriented_shape = MultiPolygon(parts)	5✔
143
144	oriented_shape = self.handle_shape_validity(oriented_shape, y_coord_down, n_try)	5✔
145	return oriented_shape	5✔
146
147	def enforce_polygon_winding_order(self, shape, y_coord_down, n_try):	5✔
148	assert shape.type == "Polygon"	5✔
149
150	def fn(point):	5✔
151	x, y = point	5✔
152	return round(x), round(y)	5✔
153
154	exterior = self.apply_map(fn, shape.exterior.coords)	5✔
155	rings = None	5✔
156
157	if len(shape.interiors) > 0:	5✔
158	rings = [self.apply_map(fn, ring.coords) for ring in shape.interiors]	5✔
159
160	sign = 1.0 if y_coord_down else -1.0	5✔
161	oriented_shape = orient(Polygon(exterior, rings), sign=sign)	5✔
162	oriented_shape = self.handle_shape_validity(oriented_shape, y_coord_down, n_try)	5✔
163	return oriented_shape	5✔
164
165	@staticmethod	5✔
166	def apply_map(fn, x):	4✔
167	return list(map(fn, x))	5✔
168
169	@staticmethod	5✔
170	def _load_geometry(geometry_spec):	4✔
171	if isinstance(geometry_spec, BaseGeometry):	5✔
172	return geometry_spec	5✔
173
174	if isinstance(geometry_spec, dict):	5✔
175	return SimpleShape(geometry_spec["coordinates"], geometry_spec["type"])	5✔
176
177	try:	5✔
178	return load_wkb(geometry_spec)	5✔
179	except Exception:	5✔
180	try:	5✔
181	return load_wkt(geometry_spec)	5✔
182	except Exception:	5✔
183	return None	5✔
184
185	def add_feature(self, feature, shape, y_coord_down):	5✔
186	geom_encoder = GeometryEncoder(y_coord_down, self.extents)	5✔
187	geometry = geom_encoder.encode(shape)	5✔
188
189	feature_type = self._get_feature_type(shape)	5✔
190	if len(geometry) == 0:	5✔
191	# Don't add geometry if it's too small
192	return	5✔
193	f = self.layer.features.add()	5✔
194
195	fid = feature.get("id")	5✔
196	if fid is not None:	5✔
197	if isinstance(fid, Number) and fid >= 0:	5✔
198	f.id = fid	5✔
199
200	# properties
201	properties = feature.get("properties")	5✔
202	if properties is not None:	5✔
203	self._handle_attr(self.layer, f, properties)	5✔
204
205	f.type = feature_type	5✔
206	f.geometry.extend(geometry)	5✔
207
208	def _get_feature_type(self, shape):	5✔
209	if shape.type == "Point" or shape.type == "MultiPoint":	5✔
210	return self.tile.Point	5✔
211	elif shape.type == "LineString" or shape.type == "MultiLineString":	5✔
212	return self.tile.LineString	5✔
213	elif shape.type == "Polygon" or shape.type == "MultiPolygon":	5✔
214	return self.tile.Polygon	5✔
215	elif shape.type == "GeometryCollection":	5✔
216	raise ValueError("Encoding geometry collections not supported")	5✔
217	else:
218	raise ValueError(f"Cannot encode unknown geometry type: {shape.type}")	×
219
220	@staticmethod	5✔
221	def _chunker(seq, size):	4✔
UNCOV 222	return [seq[pos : pos + size] for pos in range(0, len(seq), size)]	×
223
224	@staticmethod	5✔
225	def _can_handle_key(k):	4✔
226	return isinstance(k, str)	5✔
227
228	@staticmethod	5✔
229	def _can_handle_val(v):	4✔
230	return isinstance(v, (str, bool, int, float))	5✔
231
232	@classmethod	5✔
233	def _can_handle_attr(cls, k, v):	4✔
234	return cls._can_handle_key(k) and cls._can_handle_val(v)	5✔
235
236	def _handle_attr(self, layer, feature, props):	5✔
237	for k, v in props.items():	5✔
238	if self._can_handle_attr(k, v):	5✔
239	if k not in self.seen_keys_idx:	5✔
240	layer.keys.append(k)	5✔
241	self.seen_keys_idx[k] = self.key_idx	5✔
242	self.key_idx += 1	5✔
243
244	feature.tags.append(self.seen_keys_idx[k])	5✔
245
246	if isinstance(v, bool):	5✔
247	values_idx = self.seen_values_bool_idx	5✔
248	else:
249	values_idx = self.seen_values_idx	5✔
250
251	if v not in values_idx:	5✔
252	values_idx[v] = self.val_idx	5✔
253	self.val_idx += 1	5✔
254
255	val = layer.values.add()	5✔
256	if isinstance(v, bool):	5✔
257	val.bool_value = v	5✔
258	elif isinstance(v, str):	5✔
259	val.string_value = v	5✔
260	elif isinstance(v, int):	5✔
261	val.int_value = v	5✔
262	elif isinstance(v, float):	5✔
263	val.double_value = v	5✔
264
265	feature.tags.append(values_idx[v])	5✔

tilezen / mapbox-vector-tile / 3751661366

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous