From 43006ff5f500feb240d15b6fb6ca6f9d082bfd35 Mon Sep 17 00:00:00 2001 From: Oliver Tonnhofer Date: Thu, 20 Jun 2024 08:36:53 +0200 Subject: [PATCH] diff: use adaptive zoom level for expire tiles to avoid huge lists for large geometries --- docs/tutorial.rst | 3 +- expire/tilelist.go | 79 +++++++++++++++++------------ expire/tilelist_test.go | 109 +++++++++++++++++++++++++++++----------- 3 files changed, 128 insertions(+), 63 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 7b1e6a48..33e5967d 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -235,5 +235,6 @@ Remember that you have to make the initial import with the ``-diff`` option. See Expire tiles ------------ -Imposm can log where the OSM data was changed when it imports diff files. You can use the ``-expiretiles-dir`` option to specify a location where Imposm should log this information. Imposm creates files in the format `YYYYmmdd/HHMMSS.sss.tiles`` (e.g. ``20161129/212345.123.tiles``) inside this directory. The timestamp is the current time of the diff import, not the creation time of the diff. Each file contains a list with webmercator tiles in the format ``z/x/y`` (e.g. ``14/7321/1339``). All tiles are based on zoom level 14. You can change this with the ``-expiretiles-zoom`` option. +Imposm can log where the OSM data was changed when it imports diff files. You can use the ``-expiretiles-dir`` option to specify a location where Imposm should log this information. Imposm creates files in the format `YYYYmmdd/HHMMSS.sss.tiles`` (e.g. ``20240629/212345.123.tiles``) inside this directory. The timestamp is the current time of the diff import, not the creation time of the diff. Each file contains a list with webmercator tiles in the format ``z/x/y`` (e.g. ``14/7321/1339``). All tiles are based on zoom level 14. You can change this with the ``-expiretiles-zoom`` option. +Imposm tries to keep the number of change tiles reasonable for large changes by "zooming out", e.g. a continent wide change would result in a few handful of tiles in zoom level 6, and not millions of tiles in level 14. Both expire options can be set as ``expiretiles_dir`` and ``expiretiles_zoom`` in the JSON configuration. diff --git a/expire/tilelist.go b/expire/tilelist.go index 66b4a31f..9bd1ba25 100644 --- a/expire/tilelist.go +++ b/expire/tilelist.go @@ -43,10 +43,10 @@ func tileCoord(long, lat float64, zoom int) (float64, float64) { type TileList struct { mu sync.Mutex - tiles map[tileKey]struct{} + tiles []map[tileKey]struct{} - zoom int - out string + maxZoom int + out string } type tileKey struct { @@ -55,12 +55,16 @@ type tileKey struct { } func NewTileList(zoom int, out string) *TileList { - return &TileList{ - tiles: make(map[tileKey]struct{}), - zoom: zoom, - mu: sync.Mutex{}, - out: out, + tl := TileList{ + maxZoom: zoom, + mu: sync.Mutex{}, + out: out, } + for i := 0; i <= tl.maxZoom; i++ { + tl.tiles = append(tl.tiles, make(map[tileKey]struct{})) + } + + return &tl } func (tl *TileList) Expire(long, lat float64) { @@ -71,16 +75,21 @@ func (tl *TileList) ExpireNodes(nodes []osm.Node, closed bool) { if len(nodes) == 0 { return } - if closed { - box := nodesBbox(nodes) - tiles := numBboxTiles(box, tl.zoom) - if tiles > 500 { - tl.expireLine(nodes) - } else if !box.isEmpty() { - tl.expireBox(box) + box := nodesBbox(nodes) + + for zoom := tl.maxZoom; zoom > 0; zoom-- { + numTiles := numBboxTiles(box, zoom) + if closed { + if numTiles < 64 { + tl.expireBox(box, zoom) + return + } + } else { + if numTiles < 500 { + tl.expireLine(nodes, zoom) + return + } } - } else { - tl.expireLine(nodes) } } @@ -90,10 +99,10 @@ func (tl *TileList) addCoord(long, lat float64) { // fraction of a tile that is added as a padding around a single node const tilePadding = 0.2 tl.mu.Lock() - tileX, tileY := tileCoord(long, lat, tl.zoom) + tileX, tileY := tileCoord(long, lat, tl.maxZoom) for x := uint32(tileX - tilePadding); x <= uint32(tileX+tilePadding); x++ { for y := uint32(tileY - tilePadding); y <= uint32(tileY+tilePadding); y++ { - tl.tiles[tileKey{x, y}] = struct{}{} + tl.tiles[tl.maxZoom][tileKey{x, y}] = struct{}{} } } tl.mu.Unlock() @@ -101,7 +110,7 @@ func (tl *TileList) addCoord(long, lat float64) { // expireLine expires all tiles that are intersected by the line segments // between the nodes -func (tl *TileList) expireLine(nodes []osm.Node) { +func (tl *TileList) expireLine(nodes []osm.Node, zoom int) { if len(nodes) == 1 { tl.addCoord(nodes[0].Long, nodes[0].Lat) return @@ -113,36 +122,38 @@ func (tl *TileList) expireLine(nodes []osm.Node) { if (nodes[i].Long == 0 && nodes[i].Lat == 0) || (nodes[i+1].Long == 0 && nodes[i+1].Lat == 0) { continue } - x1, y1 := tileCoord(nodes[i].Long, nodes[i].Lat, tl.zoom) - x2, y2 := tileCoord(nodes[i+1].Long, nodes[i+1].Lat, tl.zoom) + x1, y1 := tileCoord(nodes[i].Long, nodes[i].Lat, zoom) + x2, y2 := tileCoord(nodes[i+1].Long, nodes[i+1].Lat, zoom) if int(x1) == int(x2) && int(y1) == int(y2) { - tl.tiles[tileKey{X: uint32(x1), Y: uint32(y1)}] = struct{}{} + tl.tiles[zoom][tileKey{X: uint32(x1), Y: uint32(y1)}] = struct{}{} } else { for _, tk := range bresenham(x1, y1, x2, y2) { - tl.tiles[tk] = struct{}{} + tl.tiles[zoom][tk] = struct{}{} } } } } // expireBox expires all tiles inside the bbox -func (tl *TileList) expireBox(b bbox) { +func (tl *TileList) expireBox(b bbox, zoom int) { tl.mu.Lock() defer tl.mu.Unlock() - x1, y1 := tileCoord(b.minx, b.maxy, tl.zoom) - x2, y2 := tileCoord(b.maxx, b.miny, tl.zoom) + x1, y1 := tileCoord(b.minx, b.maxy, zoom) + x2, y2 := tileCoord(b.maxx, b.miny, zoom) for x := uint32(x1); x <= uint32(x2); x++ { for y := uint32(y1); y <= uint32(y2); y++ { - tl.tiles[tileKey{x, y}] = struct{}{} + tl.tiles[zoom][tileKey{x, y}] = struct{}{} } } } func (tl *TileList) writeTiles(w io.Writer) error { - for tileKey := range tl.tiles { - _, err := fmt.Fprintf(w, "%d/%d/%d\n", tl.zoom, tileKey.X, tileKey.Y) - if err != nil { - return err + for zoom, tiles := range tl.tiles { + for tileKey := range tiles { + _, err := fmt.Fprintf(w, "%d/%d/%d\n", zoom, tileKey.X, tileKey.Y) + if err != nil { + return err + } } } return nil @@ -171,7 +182,9 @@ func (tl *TileList) Flush() error { if err != nil { return err } - tl.tiles = make(map[tileKey]struct{}) + for i := 0; i <= tl.maxZoom; i++ { + tl.tiles = append(tl.tiles, make(map[tileKey]struct{})) + } // wrote to .tiles~ and now atomically move file to .tiles return os.Rename(fileName, fileName[0:len(fileName)-1]) } diff --git a/expire/tilelist_test.go b/expire/tilelist_test.go index 4bcb9319..0d09b474 100644 --- a/expire/tilelist_test.go +++ b/expire/tilelist_test.go @@ -1,83 +1,134 @@ package expire import ( + "bufio" + "bytes" + "regexp" "testing" osm "github.com/omniscale/go-osm" ) -func TestTileList_ExpireNodes(t *testing.T) { - tests := []struct { - nodes []osm.Node - expected int - polygon bool +func TestTileList_ExpireNodesAdaptive(t *testing.T) { + for _, test := range []struct { + nodes []osm.Node + expectedNum int + expectedLevel int + closed bool }{ // point - {[]osm.Node{{Long: 8.30, Lat: 53.26}}, 1, false}, + {[]osm.Node{{Long: 8.30, Lat: 53.26}}, 1, 14, false}, // point + paddings - {[]osm.Node{{Long: 0, Lat: 0}}, 4, false}, - {[]osm.Node{{Long: 0.01, Lat: 0}}, 2, false}, - {[]osm.Node{{Long: 0, Lat: 0.01}}, 2, false}, - {[]osm.Node{{Long: 0.01, Lat: 0.01}}, 1, false}, + {[]osm.Node{{Long: 0, Lat: 0}}, 4, 14, false}, + {[]osm.Node{{Long: 0.01, Lat: 0}}, 2, 14, false}, + {[]osm.Node{{Long: 0, Lat: 0.01}}, 2, 14, false}, + {[]osm.Node{{Long: 0.01, Lat: 0.01}}, 1, 14, false}, // line {[]osm.Node{ {Long: 8.30, Lat: 53.25}, {Long: 8.30, Lat: 53.30}, - }, 5, false}, + }, 5, 14, false}, // same line, but split into multiple segments {[]osm.Node{ {Long: 8.30, Lat: 53.25}, {Long: 8.30, Lat: 53.27}, {Long: 8.30, Lat: 53.29}, {Long: 8.30, Lat: 53.30}, - }, 5, false}, + }, 5, 14, false}, // L-shape {[]osm.Node{ {Long: 8.30, Lat: 53.25}, {Long: 8.30, Lat: 53.30}, {Long: 8.35, Lat: 53.30}, - }, 8, false}, + }, 8, 14, false}, - // closed line (triangle) + // line (triangle) {[]osm.Node{ {Long: 8.30, Lat: 53.25}, {Long: 8.30, Lat: 53.30}, {Long: 8.35, Lat: 53.30}, {Long: 8.30, Lat: 53.25}, - }, 11, false}, - // same closed line but polygon (triangle), whole bbox (4x5 tiles) is expired + }, 11, 14, false}, + // same line but closed/polygon (triangle), whole bbox (4x5 tiles) is expired {[]osm.Node{ {Long: 8.30, Lat: 53.25}, {Long: 8.30, Lat: 53.30}, {Long: 8.35, Lat: 53.30}, {Long: 8.30, Lat: 53.25}, - }, 20, true}, + }, 20, 14, true}, - // large triangle, only outline expired for polygons and lines + // large triangle, moved zoom level up {[]osm.Node{ {Long: 8.30, Lat: 53.25}, {Long: 8.30, Lat: 53.90}, {Long: 8.85, Lat: 53.90}, {Long: 8.30, Lat: 53.25}, - }, 124, true}, + }, 28, 11, true}, + // same large triangle but as line, moved just one zoom level up to be + // able to follow the outline more precise {[]osm.Node{ {Long: 8.30, Lat: 53.25}, {Long: 8.30, Lat: 53.90}, {Long: 8.85, Lat: 53.90}, {Long: 8.30, Lat: 53.25}, - }, 124, false}, - } - for _, test := range tests { - tl := NewTileList(14, "") - tl.ExpireNodes(test.nodes, test.polygon) - if len(tl.tiles) != test.expected { - t.Errorf("expected %d tiles, got %d", test.expected, len(tl.tiles)) - for tk := range tl.tiles { - t.Errorf("\t%v", tk) + }, 63, 13, false}, + // long line, accross world + {[]osm.Node{ + {Long: -170, Lat: -80}, + {Long: 170, Lat: 80}, + }, 17, 4, false}, + // large polygon, accross world + {[]osm.Node{ + {Long: -160, Lat: -70}, + {Long: 160, Lat: -70}, + {Long: 160, Lat: 70}, + {Long: -160, Lat: 70}, + }, 48, 3, true}, + } { + t.Run("", func(t *testing.T) { + tl := NewTileList(14, "") + tl.ExpireNodes(test.nodes, test.closed) + for z := 0; z <= tl.maxZoom; z++ { + expected := 0 + if z == test.expectedLevel { + expected = test.expectedNum + } + if len(tl.tiles[z]) != expected { + t.Errorf("expected %d tiles, got %d in z=%d", expected, len(tl.tiles[z]), z) + for tk := range tl.tiles[z] { + t.Errorf("\t%v", tk) + } + } + } + buf := bytes.Buffer{} + if err := tl.writeTiles(&buf); err != nil { + t.Errorf("error writing tiles list: %s", err) + } + + tileRe := regexp.MustCompile(`^\d+/\d+/\d+$`) + scanner := bufio.NewScanner(&buf) + + lines := 0 + + for scanner.Scan() { + line := scanner.Text() + lines++ + + if !tileRe.MatchString(line) { + t.Errorf("line %d does is not a tile coordinate: %s", lines, line) + } + } + + if err := scanner.Err(); err != nil { + t.Fatalf("rrror reading buffer: %v", err) + } + + if lines != test.expectedNum { + t.Errorf("expected %d lines, but got %d", test.expectedNum, lines) } - } + }) } }