Support for importing non-area relations as MultiLineString

Author: talaj

database/postgis/postgis.go

@@ -72,8 +72,8 @@ func addGeometryColumn(tx *sql.Tx, tableName string, spec TableSpec) error {
 	}
 
 	geomType := strings.ToUpper(spec.GeometryType)
-	if geomType == "POLYGON" {
-		geomType = "GEOMETRY" // for multipolygon support
+	if geomType == "POLYGON" || geomType == "LINESTRING" {
+		geomType = "GEOMETRY" // for multigeometry support
 	}
 	sql := fmt.Sprintf("SELECT AddGeometryColumn('%s', '%s', '%s', '%d', '%s', 2);",
 		spec.Schema, tableName, colName, spec.Srid, geomType)

geom/geom.go

@@ -3,6 +3,7 @@ package geom
 import (
 	"errors"
 	"math"
+	"runtime"
 
 	osm "github.com/omniscale/go-osm"
 	"github.com/omniscale/imposm3/geom/geos"
@@ -136,6 +137,41 @@ func Polygon(g *geos.Geos, nodes []osm.Node) (*geos.Geom, error) {
 	return geom, nil
 }
 
+func MultiLinestring(rel *osm.Relation, srid int) (*geos.Geom, error) {
+	g := geos.NewGeos()
+	g.SetHandleSrid(srid)
+	defer g.Finish()
+
+	var lines []*geos.Geom
+
+	for _, member := range rel.Members {
+		if member.Way == nil {
+			continue
+		}
+
+		line, err := LineString(g, member.Way.Nodes)
+
+		// Clear the finalizer created in LineString()
+		// as we want to make the object a part of MultiLineString.
+		runtime.SetFinalizer(line, nil)
+
+		if err != nil {
+			return nil, err
+		}
+
+		lines = append(lines, line)
+	}
+
+	result := g.MultiLineString(lines)
+	if result == nil {
+		return nil, errors.New("Error while building multi-linestring.")
+	}
+
+	g.DestroyLater(result)
+
+	return result, nil
+}
+
 func AsGeomElement(g *geos.Geos, geom *geos.Geom) (Geometry, error) {
 	wkb := g.AsEwkbHex(geom)
 	if wkb == nil {

geom/multipolygon_test.go

@@ -659,3 +659,38 @@ func TestClosedAndOpenRing(t *testing.T) {
 		t.Fatal("geometry not valid", g.AsWkt(geom.Geom))
 	}
 }
+
+func TestSimpleMultiLineString(t *testing.T) {
+	w1 := makeWay(1, osm.Tags{}, []coord{
+		{1, 1, 0},
+		{2, 2, 0},
+	})
+	w2 := makeWay(2, osm.Tags{}, []coord{
+		{3, 2, 0},
+		{4, 3, 0},
+	})
+
+	rel := osm.Relation{
+		Element: osm.Element{ID: 1, Tags: osm.Tags{}}}
+	rel.Members = []osm.Member{
+		{ID: 1, Type: osm.WayMember, Role: "", Way: &w1},
+		{ID: 2, Type: osm.WayMember, Role: "", Way: &w2},
+	}
+
+	geom, err := MultiLinestring(&rel, 3857)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	g := geos.NewGeos()
+	defer g.Finish()
+
+	if !g.IsValid(geom) {
+		t.Fatal("geometry not valid", g.AsWkt(geom))
+	}
+
+	if length := geom.Length(); length != 2 {
+		t.Fatal("length invalid", length)
+	}
+}

import_/import.go

@@ -188,6 +188,7 @@ func Import(importOpts config.Import) {
 			tagmapping.Conf.SingleIDSpace,
 			relations,
 			db, progress,
+			tagmapping.LineStringMatcher,
 			tagmapping.PolygonMatcher,
 			tagmapping.RelationMatcher,
 			tagmapping.RelationMemberMatcher,

mapping/mapping.go

@@ -84,7 +84,7 @@ const (
 type Mapping struct {
 	Conf                  config.Mapping
 	PointMatcher          NodeMatcher
-	LineStringMatcher     WayMatcher
+	LineStringMatcher     RelWayMatcher
 	PolygonMatcher        RelWayMatcher
 	RelationMatcher       RelationMatcher
 	RelationMemberMatcher RelationMatcher
@@ -356,6 +356,11 @@ func (m *Mapping) addRelationFilters(tableType TableType, filters tableElementFi
 					return false
 				}
 				filters[name] = append(filters[name], f)
+			} else if TableType(t.Type) == LineStringTable {
+				f := func(tags osm.Tags, key Key, closed bool) bool {
+					return false
+				}
+				filters[name] = append(filters[name], f)
 			}
 		}
 	}

mapping/matcher.go

@@ -20,17 +20,20 @@ func (m *Mapping) pointMatcher() (NodeMatcher, error) {
 	}, err
 }
 
-func (m *Mapping) lineStringMatcher() (WayMatcher, error) {
+func (m *Mapping) lineStringMatcher() (RelWayMatcher, error) {
 	mappings := make(TagTableMapping)
 	m.mappings(LineStringTable, mappings)
 	filters := make(tableElementFilters)
 	m.addFilters(filters)
 	m.addTypedFilters(LineStringTable, filters)
+	relFilters := make(tableElementFilters)
+	m.addRelationFilters(LineStringTable, relFilters)
 	tables, err := m.tables(LineStringTable)
 	return &tagMatcher{
 		mappings:   mappings,
 		filters:    filters,
 		tables:     tables,
+		relFilters: relFilters,
 		matchAreas: false,
 	}, err
 }

test/multilinestring.osc

@@ -0,0 +1,37 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<osmChange version="1">
+  <create>
+    <node id="10009" version="1" timestamp="2016-01-01T00:00:00Z" lat="51.5512379" lon="-0.0405630"/>
+    <node id="10010" version="1" timestamp="2016-01-01T00:00:00Z" lat="51.5515113" lon="-0.0406333"/>
+    <way id="1006" version="1" timestamp="2016-01-01T00:00:00Z">
+      <nd ref="10009"/>
+      <nd ref="10010"/>
+    </way>
+
+    <node id="10008" version="1" timestamp="2016-01-01T00:00:00Z" lat="51.5515683" lon="-0.0406486"/>
+    <node id="10007" version="1" timestamp="2016-01-01T00:00:00Z" lat="51.5515883" lon="-0.0406757"/>
+    <way id="1007" version="1" timestamp="2016-01-01T00:00:00Z">
+      <nd ref="10008"/>
+      <nd ref="10007"/>
+    </way>
+
+    <relation id="102" version="1" timestamp="2016-01-01T00:00:00Z">
+      <member type="way" ref="1006" role=""/>
+      <member type="way" ref="1007" role=""/>
+      <tag k="network" v="lcn"/>
+      <tag k="route" v="bicycle"/>
+      <tag k="type" v="route"/>
+    </relation>
+  </create>
+
+  <modify>
+    <way id="1004" version="2" timestamp="2016-01-01T00:00:00Z">
+      <nd ref="10004"/>
+      <nd ref="10005"/>
+      <nd ref="10006"/>
+      <nd ref="10004"/>
+      <tag k="building" v="residential"/>
+      <!-- tag with area=yes removed -->
+    </way>
+  </modify>
+</osmChange>

test/multilinestring.osm

@@ -0,0 +1,52 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<osm version="1">
+  <node id="10002" version="1" timestamp="2015-12-31T23:59:99Z" lat="51.5107973" lon="-0.0930300" />
+  <node id="10003" version="1" timestamp="2015-12-31T23:59:99Z" lat="51.5108542" lon="-0.0930091" />
+  <way id="1000" version="1" timestamp="2015-12-31T23:59:99Z">
+    <nd ref="10002"/>
+    <nd ref="10003"/>
+    <tag k="highway" v="trunk"/>
+  </way>
+
+  <node id="10000" version="1" timestamp="2015-12-31T23:59:99Z" lat="51.5107776" lon="-0.0930375"/>
+  <way id="1005" version="1" timestamp="2015-12-31T23:59:99Z">
+    <nd ref="10000"/>
+    <nd ref="10002"/>
+  </way>
+
+  <node id="10001" version="1" timestamp="2015-12-31T23:59:99Z" lat="51.5106434" lon="-0.0930826" />
+  <way id="1001" version="1" timestamp="2015-12-31T23:59:99Z">
+    <nd ref="10001"/>
+    <nd ref="10000"/>
+  </way>
+  <relation id="100" version="1" timestamp="2015-12-31T23:59:99Z">
+    <member type="way" ref="1000" role=""/>
+    <member type="way" ref="1005" role=""/>
+    <member type="way" ref="1001" role=""/>
+    <tag k="route" v="bicycle"/>
+    <tag k="type" v="route"/>
+  </relation>
+
+  <node id="10004" version="1" timestamp="2015-12-31T23:59:99Z" lat="51.4860918" lon="-0.0873508"/>
+  <node id="10006" version="1" timestamp="2015-12-31T23:59:99Z" lat="51.4860020" lon="-0.0875372"/>
+  <node id="10005" version="1" timestamp="2015-12-31T23:59:99Z" lat="51.4853370" lon="-0.0864133"/>
+  <way id="1004" version="1" timestamp="2015-12-31T23:59:99Z">
+    <nd ref="10004"/>
+    <nd ref="10005"/>
+    <nd ref="10006"/>
+    <nd ref="10004"/>
+    <tag k="building" v="residential"/>
+    <tag k="area" v="yes"/>
+  </way>
+
+  <node id="10011" version="1" timestamp="2015-12-31T23:59:99Z" lat="51.5004084" lon="-0.0824810"/>
+  <node id="10012" version="1" timestamp="2015-12-31T23:59:99Z" lat="51.5003998" lon="-0.0833872"/>
+  <node id="10013" version="1" timestamp="2015-12-31T23:59:99Z" lat="51.5003917" lon="-0.0836970"/>
+  <way id="1008" version="1" timestamp="2015-12-31T23:59:99Z">
+    <nd ref="10011"/>
+    <nd ref="10012"/>
+    <nd ref="10013"/>
+    <nd ref="10011"/>
+    <tag k="leisure" v="park"/>
+  </way>
+</osm>

test/multilinestring_mapping.yml

@@ -0,0 +1,42 @@
+areas:
+    area_tags:
+        - leisure
+tables:
+    multilinestring:
+        type: linestring
+        columns:
+            - name: osm_id
+              type: id
+            - name: geometry
+              type: geometry
+            - name: name
+              type: string
+              key: name
+            - name: type
+              type: mapping_value
+        relation_types:
+            - route
+        mapping:
+            type:
+                - route
+            highway:
+                - trunk
+            building:
+                - residential
+            leisure:
+                - park
+    multilinestring_no_relations:
+        type: linestring
+        columns:
+            - name: osm_id
+              type: id
+            - name: geometry
+              type: geometry
+            - name: name
+              type: string
+              key: name
+            - name: type
+              type: mapping_value
+        mapping:
+            type:
+                - route

test/multilinestring_test.go

@@ -0,0 +1,110 @@
+package test
+
+import (
+	"database/sql"
+	"io/ioutil"
+
+	"testing"
+
+	"github.com/omniscale/imposm3/geom/geos"
+)
+
+func TestMultiLineString(t *testing.T) {
+	if testing.Short() {
+		t.Skip("system test skipped with -test.short")
+	}
+	t.Parallel()
+
+	ts := importTestSuite{
+		name: "multilinestring",
+	}
+
+	t.Run("Prepare", func(t *testing.T) {
+		var err error
+
+		ts.dir, err = ioutil.TempDir("", "imposm_test")
+		if err != nil {
+			t.Fatal(err)
+		}
+		ts.config = importConfig{
+			connection:      "postgis://",
+			cacheDir:        ts.dir,
+			osmFileName:     "build/multilinestring.pbf",
+			mappingFileName: "multilinestring_mapping.yml",
+		}
+		ts.g = geos.NewGeos()
+
+		ts.db, err = sql.Open("postgres", "sslmode=disable")
+		if err != nil {
+			t.Fatal(err)
+		}
+		ts.dropSchemas()
+	})
+
+	const mlsTable = "osm_multilinestring"
+
+	t.Run("Import", func(t *testing.T) {
+		if ts.tableExists(t, ts.dbschemaImport(), mlsTable) != false {
+			t.Fatalf("table %s exists in schema %s", mlsTable, ts.dbschemaImport())
+		}
+		ts.importOsm(t)
+		if ts.tableExists(t, ts.dbschemaImport(), mlsTable) != true {
+			t.Fatalf("table %s does not exists in schema %s", mlsTable, ts.dbschemaImport())
+		}
+	})
+
+	t.Run("Deploy", func(t *testing.T) {
+		ts.deployOsm(t)
+		if ts.tableExists(t, ts.dbschemaImport(), mlsTable) != false {
+			t.Fatalf("table %s exists in schema %s", mlsTable, ts.dbschemaImport())
+		}
+		if ts.tableExists(t, ts.dbschemaProduction(), mlsTable) != true {
+			t.Fatalf("table %s does not exists in schema %s", mlsTable, ts.dbschemaProduction())
+		}
+	})
+
+	t.Run("CheckMultiLineStringGeometry", func(t *testing.T) {
+		element := checkElem{mlsTable, -100, "*", nil}
+		ts.assertGeomType(t, element, "MultiLineString")
+		ts.assertGeomValid(t, element)
+		ts.assertGeomLength(t, element, 38)
+	})
+
+	t.Run("CheckLineStringGeometry", func(t *testing.T) {
+		element := checkElem{mlsTable, 1000, "*", nil}
+		ts.assertGeomType(t, element, "LineString")
+		ts.assertGeomValid(t, element)
+		ts.assertGeomLength(t, element, 10)
+	})
+
+	t.Run("CheckFilters", func(t *testing.T) {
+		if records := ts.queryRows(t, mlsTable, 1008); len(records) > 0 {
+			t.Fatalf("The way 1008 should be filtered out by typed filter")
+		}
+		if records := ts.queryRows(t, mlsTable, 1004); len(records) > 0 {
+			t.Fatalf("The way 1004 should be filtered out as it is closed path with area=yes")
+		}
+	})
+
+	t.Run("RelationTypesFilter", func(t *testing.T) {
+		if records := ts.queryRows(t, "osm_multilinestring_no_relations", -100); len(records) > 0 {
+			t.Fatalf("The relation -100 should not be imported due to empty relation_types")
+		}
+	})
+
+	t.Run("Update", func(t *testing.T) {
+		ts.updateOsm(t, "build/multilinestring.osc.gz")
+	})
+
+	t.Run("CheckFilters2", func(t *testing.T) {
+		if records := ts.queryRows(t, mlsTable, 1004); len(records) == 0 {
+			t.Fatalf("The way 1004 should now be there as we removed area=yes in the update")
+		}
+	})
+
+	t.Run("CheckNewRelation", func(t *testing.T) {
+		if records := ts.queryRows(t, mlsTable, -102); len(records) == 0 {
+			t.Fatalf("The relation -102 should be created")
+		}
+	})
+}