Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions deployment/docker/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,13 @@ redis==4.3.4

# Library to handle geospatial functions
GeoAlchemy2==0.15.1
# GeoAlchemy2 0.15.1 only declares "SQLAlchemy>=1.4", so an unpinned install
# drifts to whatever is newest - SQLAlchemy 2.1 made ColumnCollection an
# abstract base class GeoAlchemy2 0.15.1 still instantiates directly
# (TypeError on every geopanda_to_postgis/create_id_field call), and its
# default driver for a bare "postgresql://" URL became psycopg (v3), not
# installed here (only psycopg2-binary is). Capped at the last 2.0.x release.
SQLAlchemy==2.0.55
geopandas==0.14.4
drf-yasg==1.21.7

Expand Down
12 changes: 12 additions & 0 deletions processing/app/routers/pmtiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ class PMTilesRequest(BaseModel):
# Each layer is converted to its own PMTiles file.
layers: Optional[List[str]] = None
thumbnail: bool = False
# The zoom tippecanoe's maxzoom guess (-zg) is floored to, so a layer of
# many small, densely-packed features (e.g. neighbourhood polygons) isn't
# left looking coarse at the zoom it's actually viewed at - see
# shapefile_to_pmtiles.DEFAULT_MIN_MEANINGFUL_ZOOM for the default and the
# trade-off (a higher floor means bigger files and longer tiling, floored
# only when the data's own guess is lower).
min_meaningful_zoom: Optional[int] = None
# Upload each layer's results here rather than keeping them to collect.
uploads: Optional[List[LayerUploads]] = None

Expand All @@ -38,6 +45,10 @@ def create_pmtiles(body: PMTilesRequest):
GET /api/v1/jobs/{job_id} for status. With `uploads`, each layer's
results go straight to those presigned URLs and the finished job
reports them as `outputs`.

`min_meaningful_zoom` (optional): see PMTilesRequest - raise it if a
layer of small, densely-packed polygons (e.g. admin boundaries) still
looks too simplified at the zoom it's actually viewed at.
"""

def work(job_id: str) -> dict:
Expand All @@ -46,6 +57,7 @@ def work(job_id: str) -> dict:
layers=body.layers,
job_id=job_id,
with_thumbnails=body.thumbnail,
min_meaningful_zoom=body.min_meaningful_zoom,
)
return deliver(files, errors, workdir, body.uploads, job_id)

Expand Down
107 changes: 74 additions & 33 deletions processing/app/services/shapefile_to_pmtiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,22 @@
TIPPECANOE_TIMEOUT_PER_GB = 20 * 60
TIPPECANOE_MAX_TIMEOUT = 3 * 60 * 60

# -zg's own maxzoom guess, floored to this zoom (--smallest-maximum-zoom-guess)
# rather than used as-is. -zg guesses from the *typical* spacing between
# features, which a layer with both dense and sparse areas (a country's
# neighbourhood polygons: small and packed in cities, large and sparse
# elsewhere) can get dragged down by - e.g. the whole dataset guessing z5 only
# because of its sparser areas, which then makes every zoom above z5 a client-
# side blow-up of that same z5 tile: coarse and blocky exactly at the zoom
# (e.g. "the whole country at once") someone actually wants to look at. Each
# zoom level floored up roughly doubles that layer's tippecanoe output, so
# this is deliberately modest; a request can ask for more
# (min_meaningful_zoom).
# A floor forced well above a genuinely low natural guess (a single huge,
# simple polygon) costs real tiling time - the per-GB tippecanoe_timeout below
# is what bounds that, same as it already bounds -zg's own worst case.
DEFAULT_MIN_MEANINGFUL_ZOOM = 9

# A step's report of its progress: (what it's doing, fraction of it done).
Report = Callable[[str, float], None]

Expand Down Expand Up @@ -140,39 +156,46 @@ def _tile(
layer_name: str,
geojson_path: str,
report: Report = lambda *_: None,
min_meaningful_zoom: Optional[int] = None,
) -> None:
"""Tiles geojson_path with an adaptively-guessed max zoom (-zg).

geojson_path is line-delimited GeoJSON (GeoJSONSeq), which tippecanoe
reads in parallel (-P). Falls back to a low fixed zoom only for the
one case -zg can't handle at all — too few distinct feature locations
to guess from (e.g. a single point). A blanket fixed zoom is
deliberately avoided otherwise: forcing a high zoom on a huge, simple
feature (e.g. a near-world-sized polygon) makes tippecanoe try to tile
it down to street level, which can run for hours — -zg picks a zoom
that actually matches the data.
"""Tile geojson_path at an adaptively-guessed max zoom.

Floored at `min_meaningful_zoom` (default
DEFAULT_MIN_MEANINGFUL_ZOOM; see there). geojson_path is
line-delimited GeoJSON (GeoJSONSeq), which tippecanoe reads in
parallel (-P). --smallest-maximum-zoom-guess is -zg's own
density-based guess with a floor: unlike plain -zg it never
refuses to guess (even a single feature just gets the floor), so
there is no separate fallback to handle here. A blanket *fixed*
zoom is still avoided: a feature denser than the floor keeps
whatever higher zoom its own guess calls for.
"""
# --force: -zg can fail partway through after already creating
# pmtiles_path, so a same-path retry (the -z0 fallback below) would
# otherwise hit "tileset already exists" instead of actually retrying.
base_cmd = [
"tippecanoe",
"--force",
"-P",
"--projection=EPSG:4326",
"-o",
pmtiles_path,
"-l",
layer_name,
geojson_path,
]
timeout = tippecanoe_timeout(geojson_path)
try:
_run_tippecanoe(base_cmd[:1] + ["-zg"] + base_cmd[1:], timeout, report)
except ConversionError as e:
if "Can't guess maxzoom" not in e.message:
raise
_run_tippecanoe(base_cmd[:1] + ["-z0"] + base_cmd[1:], timeout, report)
floor = (
DEFAULT_MIN_MEANINGFUL_ZOOM
if min_meaningful_zoom is None
else min_meaningful_zoom
)
_run_tippecanoe(
[
"tippecanoe",
"--force",
"-P",
"--projection=EPSG:4326",
f"--smallest-maximum-zoom-guess={floor}",
# Simplify a shared edge between adjacent polygons (e.g.
# neighbouring admin boundaries) the same way on both sides,
# so low zoom doesn't show a gap or sliver along it.
"--no-simplification-of-shared-nodes",
"-o",
pmtiles_path,
"-l",
layer_name,
geojson_path,
],
timeout,
report,
)


def _source_srs(source: str, layer_name: Optional[str]) -> list:
Expand Down Expand Up @@ -288,6 +311,7 @@ def _convert_layer(
workdir: str,
report: Report,
with_thumbnail: bool = False,
min_meaningful_zoom: Optional[int] = None,
) -> tuple:
"""Convert one layer: (pmtiles path, parquet path, thumbnail or None).

Expand Down Expand Up @@ -339,7 +363,13 @@ def tiling(step: str, fraction: float) -> None:
f"{step} (PMTiles): {percent}% · {note()}", best["fraction"]
)

_tile(pmtiles_path, tile_layer, geojson_path, tiling)
_tile(
pmtiles_path,
tile_layer,
geojson_path,
tiling,
min_meaningful_zoom,
)
os.remove(geojson_path)
waiting = [name for name, task in side.items() if not task.done()]
if waiting:
Expand Down Expand Up @@ -434,6 +464,7 @@ def _convert_geopackage(
layers: Optional[List[str]],
job_id: Optional[str],
with_thumbnails: bool = False,
min_meaningful_zoom: Optional[int] = None,
) -> tuple:
"""Convert each requested layer to its own PMTiles + GeoParquet pair.

Expand Down Expand Up @@ -478,6 +509,7 @@ def _convert_geopackage(
workdir,
report,
with_thumbnails,
min_meaningful_zoom,
)
except ConversionError as e:
logger.warning(
Expand Down Expand Up @@ -520,6 +552,7 @@ def convert(
layers: Optional[List[str]] = None,
job_id: Optional[str] = None,
with_thumbnails: bool = False,
min_meaningful_zoom: Optional[int] = None,
) -> tuple:
"""Convert source: a URL or local path.

Expand All @@ -532,7 +565,9 @@ def convert(
conversion.
`job_id`, if given, receives live per-layer progress via
app.jobs.update_detail. `with_thumbnails` also renders each layer's
"{stem}_thumbnail.png" from its default style.
"{stem}_thumbnail.png" from its default style. `min_meaningful_zoom`
floors the maxzoom tiling guesses - see
_tile / DEFAULT_MIN_MEANINGFUL_ZOOM.

Returns a tuple of ([{'name', 'path', 'media_type', 'info'}, ...],
[{'name', 'error'}, ...], workdir) — the second list is any GeoPackage
Expand All @@ -549,7 +584,12 @@ def convert(
input_path = resolve_source(source, workdir)
if input_path.lower().endswith(".gpkg"):
files, errors = _convert_geopackage(
input_path, workdir, layers, job_id, with_thumbnails
input_path,
workdir,
layers,
job_id,
with_thumbnails,
min_meaningful_zoom,
)
return files, errors, workdir

Expand All @@ -567,6 +607,7 @@ def convert(
workdir,
report,
with_thumbnails,
min_meaningful_zoom,
)
files = _layer_outputs(
"output", pmtiles_path, parquet_path, thumbnail_path
Expand Down
Loading