Skip to content

Commit 344dc33

Browse files
bchapuisbchapuis
authored
Importing natural earth generates an error (#668)
* Add integration tests * Address concurrency issue in the postgres store
1 parent 90680de commit 344dc33

File tree

10 files changed

+171
-46
lines changed

10 files changed

+171
-46
lines changed

‎baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresNodeRepository.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,7 @@ public void copy(List<Node> values) throws RepositoryException {
280280
writer.writeJsonb(PostgresJsonbMapper.toJson(value.getTags()));
281281
writer.writeDouble(value.getLon());
282282
writer.writeDouble(value.getLat());
283-
writer.writePostgisGeometry(value.getGeometry());
283+
writer.writeGeometry(value.getGeometry());
284284
}
285285
}
286286
} catch (IOException | SQLException e) {

‎baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresRelationRepository.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ public void copy(List<Relation> values) throws RepositoryException {
290290
.map(MemberType::ordinal).collect(Collectors.toList()));
291291
writer
292292
.write(value.getMembers().stream().map(Member::getRole).collect(Collectors.toList()));
293-
writer.writePostgisGeometry(value.getGeometry());
293+
writer.writeGeometry(value.getGeometry());
294294
}
295295
}
296296
} catch (IOException | SQLException ex) {

‎baremaps-core/src/main/java/org/apache/baremaps/openstreetmap/postgres/PostgresWayRepository.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ public void copy(List<Way> values) throws RepositoryException {
281281
writer.writeLong(value.getInfo().getChangeset());
282282
writer.writeJsonb(PostgresJsonbMapper.toJson(value.getTags()));
283283
writer.writeLongList(value.getNodes());
284-
writer.writePostgisGeometry(value.getGeometry());
284+
writer.writeGeometry(value.getGeometry());
285285
}
286286
}
287287
} catch (IOException | SQLException e) {

‎baremaps-core/src/main/java/org/apache/baremaps/postgres/copy/CopyWriter.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -332,8 +332,8 @@ public void writeJsonb(String value) throws IOException {
332332
* @param value
333333
* @throws IOException
334334
*/
335-
public void writePostgisGeometry(Geometry value) throws IOException {
336-
new PostgisGeometryValueHandler().handle(data, value);
335+
public void writeGeometry(Geometry value) throws IOException {
336+
new GeometryValueHandler().handle(data, value);
337337
}
338338

339339
/** Close the writer. */

‎baremaps-core/src/main/java/org/apache/baremaps/postgres/copy/PostgisGeometryValueHandler.java renamed to ‎baremaps-core/src/main/java/org/apache/baremaps/postgres/copy/GeometryValueHandler.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
import org.locationtech.jts.geom.Geometry;
2121
import org.locationtech.jts.io.WKBWriter;
2222

23-
public class PostgisGeometryValueHandler extends BaseValueHandler<Geometry> {
23+
public class GeometryValueHandler extends BaseValueHandler<Geometry> {
2424

2525
private final WKBWriter writer = new WKBWriter(2, wkbNDR, true);
2626

‎baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresStore.java

Lines changed: 74 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
import java.util.stream.Collectors;
2828
import javax.sql.DataSource;
2929
import org.apache.baremaps.postgres.copy.CopyWriter;
30-
import org.apache.baremaps.postgres.copy.PostgisGeometryValueHandler;
30+
import org.apache.baremaps.postgres.copy.GeometryValueHandler;
3131
import org.apache.baremaps.postgres.metadata.DatabaseMetadata;
3232
import org.apache.baremaps.postgres.metadata.TableMetadata;
3333
import org.apache.baremaps.storage.*;
@@ -81,28 +81,6 @@ public class PostgresStore implements Store {
8181
Map.entry("time", LocalTime.class),
8282
Map.entry("timestamp", LocalDateTime.class));
8383

84-
public static final Map<Class, BaseValueHandler> typeToHandler = Map.ofEntries(
85-
Map.entry(String.class, new StringValueHandler()),
86-
Map.entry(Short.class, new ShortValueHandler()),
87-
Map.entry(Integer.class, new IntegerValueHandler()),
88-
Map.entry(Long.class, new LongValueHandler()),
89-
Map.entry(Float.class, new FloatValueHandler()),
90-
Map.entry(Double.class, new DoubleValueHandler()),
91-
Map.entry(Geometry.class, new PostgisGeometryValueHandler()),
92-
Map.entry(MultiPoint.class, new PostgisGeometryValueHandler()),
93-
Map.entry(Point.class, new PostgisGeometryValueHandler()),
94-
Map.entry(LineString.class, new PostgisGeometryValueHandler()),
95-
Map.entry(MultiLineString.class, new PostgisGeometryValueHandler()),
96-
Map.entry(Polygon.class, new PostgisGeometryValueHandler()),
97-
Map.entry(MultiPolygon.class, new PostgisGeometryValueHandler()),
98-
Map.entry(LinearRing.class, new PostgisGeometryValueHandler()),
99-
Map.entry(GeometryCollection.class, new PostgisGeometryValueHandler()),
100-
Map.entry(Inet4Address.class, new Inet4AddressValueHandler()),
101-
Map.entry(Inet6Address.class, new Inet6AddressValueHandler()),
102-
Map.entry(LocalDate.class, new LocalDateValueHandler()),
103-
Map.entry(LocalTime.class, new LocalTimeValueHandler()),
104-
Map.entry(LocalDateTime.class, new LocalDateTimeValueHandler()));
105-
10684
private final DataSource dataSource;
10785

10886
/**
@@ -163,22 +141,23 @@ public void add(Table table) {
163141
}
164142

165143
// Copy the data
166-
PGConnection pgConnection = connection.unwrap(PGConnection.class);
144+
var pgConnection = connection.unwrap(PGConnection.class);
167145
var copyQuery = copy(schema);
168146
logger.debug(copyQuery);
169147
try (var writer = new CopyWriter(new PGCopyOutputStream(pgConnection, copyQuery))) {
170148
writer.writeHeader();
171-
var rowIterator = table.iterator();
172-
while (rowIterator.hasNext()) {
173-
var row = rowIterator.next();
174-
var columns = getColumns(schema);
149+
var columns = getColumns(schema);
150+
var handlers = getHandlers(schema);
151+
for (Row row : table) {
175152
writer.startRow(columns.size());
176-
for (Column column : columns) {
153+
for (int i = 0; i < columns.size(); i++) {
154+
var column = columns.get(i);
155+
var handler = handlers.get(i);
177156
var value = row.get(column.name());
178157
if (value == null) {
179158
writer.writeNull();
180159
} else {
181-
writer.write(typeToHandler.get(value.getClass()), value);
160+
writer.write(handler, value);
182161
}
183162
}
184163
}
@@ -290,6 +269,71 @@ protected List<Column> getColumns(Schema schema) {
290269
.collect(Collectors.toList());
291270
}
292271

272+
/**
273+
* Get the handlers for the columns of the schema.
274+
*
275+
* @param schema the schema
276+
* @return the handlers
277+
*/
278+
protected List<BaseValueHandler> getHandlers(Schema schema) {
279+
return getColumns(schema).stream()
280+
.map(column -> getHandler(column.type()))
281+
.collect(Collectors.toList());
282+
}
283+
284+
/**
285+
* Get the handler for a type. Handlers are used to write values to the copy stream. They are not
286+
* thread safe and should not be reused or shared between threads.
287+
*
288+
* @param type the type
289+
* @return the handler
290+
*/
291+
protected BaseValueHandler getHandler(Class type) {
292+
if (type == String.class) {
293+
return new StringValueHandler();
294+
} else if (type == Short.class) {
295+
return new ShortValueHandler<Short>();
296+
} else if (type == Integer.class) {
297+
return new IntegerValueHandler<Integer>();
298+
} else if (type == Long.class) {
299+
return new LongValueHandler<Long>();
300+
} else if (type == Float.class) {
301+
return new FloatValueHandler<Float>();
302+
} else if (type == Double.class) {
303+
return new DoubleValueHandler<Double>();
304+
} else if (type == Geometry.class) {
305+
return new GeometryValueHandler();
306+
} else if (type == MultiPoint.class) {
307+
return new GeometryValueHandler();
308+
} else if (type == Point.class) {
309+
return new GeometryValueHandler();
310+
} else if (type == LineString.class) {
311+
return new GeometryValueHandler();
312+
} else if (type == MultiLineString.class) {
313+
return new GeometryValueHandler();
314+
} else if (type == Polygon.class) {
315+
return new GeometryValueHandler();
316+
} else if (type == MultiPolygon.class) {
317+
return new GeometryValueHandler();
318+
} else if (type == LinearRing.class) {
319+
return new GeometryValueHandler();
320+
} else if (type == GeometryCollection.class) {
321+
return new GeometryValueHandler();
322+
} else if (type == Inet4Address.class) {
323+
return new Inet4AddressValueHandler();
324+
} else if (type == Inet6Address.class) {
325+
return new Inet6AddressValueHandler();
326+
} else if (type == LocalDate.class) {
327+
return new LocalDateValueHandler();
328+
} else if (type == LocalTime.class) {
329+
return new LocalTimeValueHandler();
330+
} else if (type == LocalDateTime.class) {
331+
return new LocalDateTimeValueHandler();
332+
} else {
333+
throw new IllegalArgumentException("Unsupported type " + type);
334+
}
335+
}
336+
293337
/**
294338
* Check if the column type is supported by postgres.
295339
*

‎baremaps-core/src/main/java/org/apache/baremaps/storage/postgres/PostgresTable.java

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -96,12 +96,12 @@ public boolean add(Row row) {
9696
var query = insert(schema);
9797
try (var connection = dataSource.getConnection();
9898
var statement = connection.prepareStatement(query)) {
99-
for (int i = 0; i < schema.columns().size(); i++) {
100-
var value = row.get(schema.columns().get(i).name());
101-
if (value instanceof Geometry) {
102-
statement.setBytes(i + 1, GeometryUtils.serialize((Geometry) value));
99+
for (int i = 1; i <= schema.columns().size(); i++) {
100+
var value = row.get(schema.columns().get(i - 1).name());
101+
if (value instanceof Geometry geometry) {
102+
statement.setBytes(i, GeometryUtils.serialize(geometry));
103103
} else {
104-
statement.setObject(i + 1, value);
104+
statement.setObject(i, value);
105105
}
106106
}
107107
return statement.executeUpdate() > 0;
@@ -118,12 +118,12 @@ public boolean addAll(Collection<? extends Row> rows) {
118118
try (var connection = dataSource.getConnection();
119119
var statement = connection.prepareStatement(insert(schema))) {
120120
for (var row : rows) {
121-
for (int i = 0; i < schema.columns().size(); i++) {
122-
var value = row.get(schema.columns().get(i).name());
123-
if (value instanceof Geometry) {
124-
statement.setBytes(i + 1, GeometryUtils.serialize((Geometry) value));
121+
for (int i = 1; i <= schema.columns().size(); i++) {
122+
var value = row.get(schema.columns().get(i - 1).name());
123+
if (value instanceof Geometry geometry) {
124+
statement.setBytes(i, GeometryUtils.serialize(geometry));
125125
} else {
126-
statement.setObject(i + 1, value);
126+
statement.setObject(i, value);
127127
}
128128
}
129129
statement.addBatch();

‎baremaps-core/src/test/java/org/apache/baremaps/storage/geopackage/GeoPackageStoreTest.java

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
3+
* in compliance with the License. You may obtain a copy of the License at
4+
*
5+
* http://www.apache.org/licenses/LICENSE-2.0
6+
*
7+
* Unless required by applicable law or agreed to in writing, software distributed under the License
8+
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
9+
* or implied. See the License for the specific language governing permissions and limitations under
10+
* the License.
11+
*/
12+
13+
package org.apache.baremaps.storage.geopackage;
14+
15+
import static org.junit.jupiter.api.Assertions.assertEquals;
16+
17+
import org.apache.baremaps.testing.TestFiles;
18+
import org.junit.jupiter.api.Test;
19+
20+
class GeoPackageStoreTest {
21+
22+
@Test
23+
void schema() {
24+
var geoPackageStore = new GeoPackageStore(TestFiles.resolve("countries.gpkg"));
25+
var table = geoPackageStore.get("countries");
26+
var schema = table.schema();
27+
assertEquals(schema.name(), "countries");
28+
assertEquals(schema.columns().size(), 4);
29+
}
30+
31+
@Test
32+
void read() {
33+
var geoPackageStore = new GeoPackageStore(TestFiles.resolve("countries.gpkg"));
34+
var table = geoPackageStore.get("countries");
35+
assertEquals(179, table.sizeAsLong());
36+
assertEquals(179, table.stream().count());
37+
}
38+
}

‎baremaps-core/src/test/java/org/apache/baremaps/storage/geopackage/GeoPackageToPostgresTest.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
3+
* in compliance with the License. You may obtain a copy of the License at
4+
*
5+
* http://www.apache.org/licenses/LICENSE-2.0
6+
*
7+
* Unless required by applicable law or agreed to in writing, software distributed under the License
8+
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
9+
* or implied. See the License for the specific language governing permissions and limitations under
10+
* the License.
11+
*/
12+
13+
package org.apache.baremaps.storage.geopackage;
14+
15+
import static org.junit.jupiter.api.Assertions.*;
16+
17+
import org.apache.baremaps.storage.postgres.PostgresStore;
18+
import org.apache.baremaps.testing.PostgresContainerTest;
19+
import org.apache.baremaps.testing.TestFiles;
20+
import org.junit.jupiter.api.Tag;
21+
import org.junit.jupiter.api.Test;
22+
23+
public class GeoPackageToPostgresTest extends PostgresContainerTest {
24+
25+
@Test
26+
@Tag("integration")
27+
void schema() {
28+
// Open the GeoPackage
29+
var geoPackageStore = new GeoPackageStore(TestFiles.resolve("countries.gpkg"));
30+
var geoPackageTable = geoPackageStore.get("countries");
31+
32+
// Copy the table to Postgres
33+
var postgresStore = new PostgresStore(dataSource());
34+
postgresStore.add(geoPackageTable);
35+
36+
// Check the table in Postgres
37+
var postgresTable = postgresStore.get("countries");
38+
assertEquals("countries", postgresTable.schema().name());
39+
assertEquals(4, postgresTable.schema().columns().size());
40+
assertEquals(179l, postgresTable.sizeAsLong());
41+
assertEquals(179l, postgresTable.stream().count());
42+
}
43+
}

‎baremaps-core/src/test/resources/countries.gpkg

324 KB
Binary file not shown.

0 commit comments

Comments
 (0)