From dc7cfd1708da6fceb268d2efaaf92ef513a20234 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 11 Mar 2024 14:48:24 +0100 Subject: [PATCH 1/4] look for postcode areas when finding something in the postcode table --- lib-sql/indices.sql | 5 +++++ nominatim/api/search/db_searches.py | 16 +++++++++++++++- nominatim/tools/migration.py | 10 ++++++++++ nominatim/version.py | 2 +- test/bdd/db/query/postcodes.feature | 16 ++++++++++++++++ 5 files changed, 47 insertions(+), 2 deletions(-) diff --git a/lib-sql/indices.sql b/lib-sql/indices.sql index b802a660e7..9c31f55699 100644 --- a/lib-sql/indices.sql +++ b/lib-sql/indices.sql @@ -21,6 +21,11 @@ CREATE INDEX IF NOT EXISTS idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) {{db.tablespace.search_index}} WHERE parent_place_id IS NOT NULL; --- +-- Used to find postcode areas after a search in location_postcode. +CREATE INDEX IF NOT EXISTS idx_placex_postcode_areas + ON placex USING BTREE (country_code, postcode) {{db.tablespace.search_index}} + WHERE osm_type = 'R' AND class = 'boundary' AND type = 'postal_code'; +--- CREATE INDEX IF NOT EXISTS idx_placex_geometry ON placex USING GIST (geometry) {{db.tablespace.search_index}}; -- Index is needed during import but can be dropped as soon as a full diff --git a/nominatim/api/search/db_searches.py b/nominatim/api/search/db_searches.py index be88395327..6631c7cb52 100644 --- a/nominatim/api/search/db_searches.py +++ b/nominatim/api/search/db_searches.py @@ -602,7 +602,21 @@ async def lookup(self, conn: SearchConnection, results = nres.SearchResults() for row in await conn.execute(sql, _details_to_bind_params(details)): - result = nres.create_from_postcode_row(row, nres.SearchResult) + p = conn.t.placex + placex_sql = _select_placex(p).add_columns(p.c.importance)\ + .where(sa.text("""class = 'boundary' + AND type = 'postal_code' + AND osm_type = 'R'"""))\ + .where(p.c.country_code == row.country_code)\ + .where(p.c.postcode == row.postcode)\ + .where(_exclude_places(p))\ + .limit(1) + for prow in await conn.execute(placex_sql, _details_to_bind_params(details)): + result = nres.create_from_placex_row(prow, nres.SearchResult) + break + else: + result = nres.create_from_postcode_row(row, nres.SearchResult) + assert result result.accuracy = row.accuracy results.append(result) diff --git a/nominatim/tools/migration.py b/nominatim/tools/migration.py index ffeb4958f4..e864ce5254 100644 --- a/nominatim/tools/migration.py +++ b/nominatim/tools/migration.py @@ -382,3 +382,13 @@ def add_improved_geometry_reverse_placenode_index(conn: Connection, **_: Any) -> WHERE rank_address between 4 and 25 AND type != 'postcode' AND name is not null AND linked_place_id is null AND osm_type = 'N' """) + +@_migration(4, 4, 99, 0) +def create_postcode_ara_lookup_index(conn: Connection, **_: Any) -> None: + """ Create index needed for looking up postcode areas from postocde points. + """ + with conn.cursor() as cur: + cur.execute("""CREATE INDEX IF NOT EXISTS idx_placex_postcode_areas + ON placex USING BTREE (country_code, postcode) + WHERE osm_type = 'R' AND class = 'boundary' AND type = 'postal_code' + """) diff --git a/nominatim/version.py b/nominatim/version.py index 88112cf6c6..76da1dbd97 100644 --- a/nominatim/version.py +++ b/nominatim/version.py @@ -34,7 +34,7 @@ def __str__(self) -> str: return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}" -NOMINATIM_VERSION = NominatimVersion(4, 4, 0, 0) +NOMINATIM_VERSION = NominatimVersion(4, 4, 99, 0) POSTGRESQL_REQUIRED_VERSION = (9, 6) POSTGIS_REQUIRED_VERSION = (2, 2) diff --git a/test/bdd/db/query/postcodes.feature b/test/bdd/db/query/postcodes.feature index 78a26a90f5..e0a622d1c0 100644 --- a/test/bdd/db/query/postcodes.feature +++ b/test/bdd/db/query/postcodes.feature @@ -95,3 +95,19 @@ Feature: Querying fo postcode variants | type | display_name | | postcode | E4 7EA, United Kingdom | + + Scenario: Postcode areas are preferred over postcode points + Given the grid with origin DE + | 1 | 2 | + | 4 | 3 | + Given the places + | osm | class | type | postcode | geometry | + | R23 | boundary | postal_code | 12345 | (1,2,3,4,1) | + When importing + Then location_postcode contains exactly + | country | postcode | + | de | 12345 | + When sending search query "12345, de" + Then results contain + | osm | + | R23 | From 6e688a01138dd18c643cf3a5e3541506d9f63b20 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 11 Mar 2024 15:18:57 +0100 Subject: [PATCH 2/4] postcodes: exclude seen places later The seen list will only have the postcode area when available but we want the postcode point excluded as well if the area has been seen. --- nominatim/api/search/db_searches.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nominatim/api/search/db_searches.py b/nominatim/api/search/db_searches.py index 6631c7cb52..5a13061e09 100644 --- a/nominatim/api/search/db_searches.py +++ b/nominatim/api/search/db_searches.py @@ -609,7 +609,6 @@ async def lookup(self, conn: SearchConnection, AND osm_type = 'R'"""))\ .where(p.c.country_code == row.country_code)\ .where(p.c.postcode == row.postcode)\ - .where(_exclude_places(p))\ .limit(1) for prow in await conn.execute(placex_sql, _details_to_bind_params(details)): result = nres.create_from_placex_row(prow, nres.SearchResult) @@ -618,8 +617,9 @@ async def lookup(self, conn: SearchConnection, result = nres.create_from_postcode_row(row, nres.SearchResult) assert result - result.accuracy = row.accuracy - results.append(result) + if result.place_id not in details.excluded: + result.accuracy = row.accuracy + results.append(result) return results From 9de6bdd7dd155d7e0138aefd2d4544b2a3826e2b Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 11 Mar 2024 15:28:57 +0100 Subject: [PATCH 3/4] exclude postcode test from legacy and php --- test/bdd/db/query/postcodes.feature | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/bdd/db/query/postcodes.feature b/test/bdd/db/query/postcodes.feature index e0a622d1c0..9f0249594a 100644 --- a/test/bdd/db/query/postcodes.feature +++ b/test/bdd/db/query/postcodes.feature @@ -96,6 +96,8 @@ Feature: Querying fo postcode variants | postcode | E4 7EA, United Kingdom | + @fail-legacy + @v1-api-python-only Scenario: Postcode areas are preferred over postcode points Given the grid with origin DE | 1 | 2 | From 50beac83055c0d7915d59ae51c15031983b998e8 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 11 Mar 2024 21:07:14 +0100 Subject: [PATCH 4/4] remove BDD test for place_id details It is not reliable. --- test/bdd/api/details/simple.feature | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/bdd/api/details/simple.feature b/test/bdd/api/details/simple.feature index 99d3422334..0e456aa5bb 100644 --- a/test/bdd/api/details/simple.feature +++ b/test/bdd/api/details/simple.feature @@ -3,14 +3,6 @@ Feature: Object details Check details page for correctness - Scenario: Details by place ID - When sending details query for 107077 - Then the result is valid json - And results contain - | place_id | - | 107077 | - - Scenario Outline: Details via OSM id When sending details query for Then the result is valid json