Skip to content

Commit

Permalink
web ui postgis layer detail page with vector features geojson downloa…
Browse files Browse the repository at this point in the history 
…d, vector feature properties csv download; postgis layer geojson streaming writer, skip null fields; postgis layer csv writer for feature properties; separate web request logging; update java dependencies
  • Loading branch information
@swoellauer
swoellauer committed Feb 2, 2024
1 parent 4def7d5 commit 55dad5e
Show file tree
Hide file tree
Showing 19 changed files with 748 additions and 188 deletions.
7 changes: 6 additions & 1 deletion app/src/components/AddLayer.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,12 @@
<q-select
v-model="selectedLayer"
:options="filteredOptionsLayers"
:label="'Layer (' + filteredOptionsLayers.length + ')'"
:label="
'Layer' +
(filteredOptionsLayers.length > 0
? ' (' + filteredOptionsLayers.length + ')'
: '')
"
option-label="name"
clearable
use-input
Expand Down
18 changes: 9 additions & 9 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,10 @@ dependencies {

implementation group: 'org.json', name: 'json', version: '20231013'
implementation group: 'org.yaml', name: 'snakeyaml', version: '1.33' // TODO: check compatibility with newer version 2.x
implementation group: 'com.opencsv', name: 'opencsv', version: '5.8'
implementation group: 'com.opencsv', name: 'opencsv', version: '5.9'
implementation group: 'com.github.mreutegg', name: 'laszip4j', version: '0.17'
implementation group: 'ar.com.hjg', name: 'pngj', version: '2.1.0'
implementation group: 'com.samskivert', name: 'jmustache', version: '1.15'
implementation group: 'com.samskivert', name: 'jmustache', version: '1.16'

implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
implementation group: 'com.mkobos', name: 'pca_transform', version: '1.0.2'
Expand All @@ -76,15 +76,15 @@ dependencies {
implementation group: 'org.xerial.snappy', name: 'snappy-java', version: '1.1.10.5' // caution: newer versions may break storage layout (last tested: 1.1.7.7)
implementation group: 'me.lemire.integercompression', name: 'JavaFastPFOR', version: '0.1.11' // data format changed from 0.1.11 to 0.1.12
implementation group: 'me.lemire.integercompression', name: 'JavaFastPFOR', version: '0.1.12' // caution: newer versions may break storage layout
implementation group: 'com.github.luben', name: 'zstd-jni', version: '1.5.5-10' // caution: newer versions may break storage layout (last tested: 1.4.5-6)
implementation group: 'com.github.luben', name: 'zstd-jni', version: '1.5.5-11' // caution: newer versions may break storage layout (last tested: 1.4.5-6)

implementation group: 'org.antlr', name: 'antlr4-runtime', version: '4.9.3' // caution: newer versions may break with generated DSL Java source files

implementation group: 'org.eclipse.jetty', name: 'jetty-server', version: '11.0.18' // keep v11.x, newer Java version is needed for v12.x
implementation group: 'org.eclipse.jetty', name: 'jetty-security', version: '11.0.18' // keep v11.x, newer Java version is needed for v12.x
implementation group: 'org.eclipse.jetty.http2', name: 'http2-server', version: '11.0.18' // keep v11.x, newer Java version is needed for v12.x
implementation group: 'org.eclipse.jetty', name: 'jetty-alpn-java-server', version: '11.0.18' // keep v11.x, newer Java version is needed for v12.x
implementation group: 'org.eclipse.jetty', name: 'jetty-client', version: '11.0.18' // keep v11.x, newer Java version is needed for v12.x
implementation group: 'org.eclipse.jetty', name: 'jetty-server', version: '11.0.19' // keep v11.x, newer Java version is needed for v12.x
implementation group: 'org.eclipse.jetty', name: 'jetty-security', version: '11.0.19' // keep v11.x, newer Java version is needed for v12.x
implementation group: 'org.eclipse.jetty.http2', name: 'http2-server', version: '11.0.19' // keep v11.x, newer Java version is needed for v12.x
implementation group: 'org.eclipse.jetty', name: 'jetty-alpn-java-server', version: '11.0.19' // keep v11.x, newer Java version is needed for v12.x
implementation group: 'org.eclipse.jetty', name: 'jetty-client', version: '11.0.19' // keep v11.x, newer Java version is needed for v12.x

implementation 'io.jsonwebtoken:jjwt-api:0.11.5' // TODO: check for breaking changes in v0.12.x
runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.11.5' // TODO: check for breaking changes in v0.12.x
Expand All @@ -99,7 +99,7 @@ dependencies {

//implementation group: 'com.github.aelstad', name: 'keccakj', version: '1.1.0' // server not always reachable

implementation group: 'net.postgis', name: 'postgis-jdbc', version: '2021.1.0'
implementation group: 'net.postgis', name: 'postgis-jdbc', version: '2023.1.0'
}

sourceSets.main.java.srcDir 'src'
Expand Down
23 changes: 14 additions & 9 deletions src/postgis/FeatureConsumer.java
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
package postgis;

import postgis.PostgisLayer.PostgisColumn;
import util.collections.array.ReadonlyArray;

public interface FeatureConsumer {
void acceptFeatureStart(boolean isFirstFeature);
void acceptFeatureGeometry(String geometry);
void acceptFeatureFieldsStart();
void acceptFeatureField(PostgisColumn field, String fieldValue, boolean isFirstFeatureField);
void acceptFeatureFieldNull(PostgisColumn field, boolean isFirstFeatureField);
void acceptFeatureFieldInt32(PostgisColumn field, int fieldValue, boolean isFirstFeatureField);
void acceptFeatureFieldsEnd();
void acceptFeatureEnd();
public interface FeatureConsumer extends FieldsConsumer {

// FieldsConsumer methods

void acceptFields(ReadonlyArray<PostgisColumn> fields);

void acceptFeatureStart(int i);

void acceptGeometry(String geometry);

// FieldsConsumer methods

void acceptFeatureEnd(int i);
}
23 changes: 23 additions & 0 deletions src/postgis/FieldsConsumer.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package postgis;

import postgis.PostgisLayer.PostgisColumn;
import util.collections.array.ReadonlyArray;

public interface FieldsConsumer {

void acceptStart();

void acceptFields(ReadonlyArray<PostgisColumn> fields);

void acceptCellsStart(int i);

void acceptCell(int i, String fieldValue);
void acceptCellNull(int i);
void acceptCellInt16(int i, short fieldValue);
void acceptCellInt32(int i, int fieldValue);
void acceptCellFloat64(int i, double fieldValue);

void acceptCellsEnd(int i);

void acceptEnd();
}
131 changes: 119 additions & 12 deletions src/postgis/PostgisLayer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -452,7 +452,7 @@ public ReadonlyArray<String> getGeometryTypes() {
}
return e;
}

private synchronized ReadonlyArray<String> calcUsedGeometryTypes() {
if(geometryTypes != null) {
return geometryTypes;
Expand Down Expand Up @@ -812,39 +812,146 @@ public long forEachGeoJSONWithFields(Rect2d rect2d, boolean crop, FeatureConsume
Logger.info(sql);
PreparedStatement stmt = conn.prepareStatement(sql);
ResultSet rs = stmt.executeQuery();
long featureCount = 0;
consumer.acceptFields(fields);
consumer.acceptStart();
int featureCount = 0;
while(rs.next()) {
consumer.acceptFeatureStart(featureCount == 0);
consumer.acceptFeatureStart(featureCount);
String geometry = rs.getString(1);
consumer.acceptFeatureGeometry(geometry);
consumer.acceptFeatureFieldsStart();
consumer.acceptGeometry(geometry);
consumer.acceptCellsStart(featureCount);
for (int i = 0; i < flds.length; i++) {
PostgisColumn field = flds[i];

switch(field.type) {
case "int2": {
short fieldValue = rs.getShort(i + 2);
if(rs.wasNull()) {
consumer.acceptCellNull(i);
} else {
consumer.acceptCellInt16(i, fieldValue);
}
break;
}
case "serial":
case "int4": {
int fieldValue = rs.getInt(i + 2);
if(rs.wasNull()) {
consumer.acceptFeatureFieldNull(field, i == 0);
consumer.acceptCellNull(i);
} else {
consumer.acceptFeatureFieldInt32(field, fieldValue, i == 0);
consumer.acceptCellInt32(i, fieldValue);
}
break;
}
}
case "float8": {
double fieldValue = rs.getDouble(i + 2);
if(rs.wasNull()) {
consumer.acceptCellNull(i);
} else {
consumer.acceptCellFloat64(i, fieldValue);
}
break;
}
case "text":
case "timestamp": // included as text for simplicity (data type: java.sql.Timestamp)
case "varchar":
default: {
//Logger.info("field type: " + field.type);
String fieldValue = rs.getString(i + 2);
if(rs.wasNull()) {
consumer.acceptFeatureFieldNull(field, i == 0);
consumer.acceptCellNull(i);
} else {
consumer.acceptCell(i, fieldValue);
}
}
}
}
consumer.acceptCellsEnd(featureCount);
consumer.acceptFeatureEnd(featureCount);
featureCount++;
}
consumer.acceptEnd();
Logger.info("features " + featureCount);
Logger.info(Timer.stop("query"));
return featureCount;
} catch (SQLException e) {
e.printStackTrace();
return -1;
}
}

/**
*
* @param rect2d nullable
* @param consumer
* @return
*/
public long forEachWithFields(Rect2d rect2d, FieldsConsumer consumer) {
try (Connection conn = postgisConnector.getConnection()) {
Timer.start("query");
String sql;
if(rect2d == null) {
sql = String.format("SELECT %s FROM %s", selectorFields, name);
} else {
int epsg = getEPSG();
String sql_ST_Intersects = SQL_ST_Intersects(primaryGeometryColumn, rect2d, epsg);
sql = String.format("SELECT %s FROM %s WHERE %s", selectorFields, name, sql_ST_Intersects);
}
Logger.info(sql);
PreparedStatement stmt = conn.prepareStatement(sql);
ResultSet rs = stmt.executeQuery();
consumer.acceptFields(fields);
int featureCount = 0;
while(rs.next()) {
consumer.acceptCellsStart(featureCount);
for (int i = 0; i < flds.length; i++) {
PostgisColumn field = flds[i];
switch(field.type) {
case "int2": {
short fieldValue = rs.getShort(i + 1);
if(rs.wasNull()) {
consumer.acceptCellNull(i);
} else {
consumer.acceptCellInt16(i, fieldValue);
}
break;
}
case "serial":
case "int4": {
int fieldValue = rs.getInt(i + 1);
if(rs.wasNull()) {
consumer.acceptCellNull(i);
} else {
consumer.acceptCellInt32(i, fieldValue);
}
break;
}
case "float8": {
double fieldValue = rs.getDouble(i + 1);
if(rs.wasNull()) {
consumer.acceptCellNull(i);
} else {
consumer.acceptCellFloat64(i, fieldValue);
}
break;
}
case "text":
case "timestamp": // included as text for simplicity (data type: java.sql.Timestamp)
case "varchar":
default: {
/*if(!(field.type.equals("varchar") || field.type.equals("timestamp") || field.type.equals("text"))) {
Logger.info("field type: " + field.type);
}*/
String fieldValue = rs.getString(i + 1);
if(rs.wasNull()) {
consumer.acceptCellNull(i);
} else {
consumer.acceptFeatureField(field, fieldValue, i == 0);
consumer.acceptCell(i, fieldValue);
}
}
}
}
consumer.acceptFeatureFieldsEnd();
consumer.acceptFeatureEnd();
consumer.acceptCellsEnd(featureCount);
featureCount++;
}
Logger.info("features " + featureCount);
Expand Down
37 changes: 35 additions & 2 deletions src/server/InjectHandler.java
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,24 @@
package server;

import java.io.IOException;
import java.time.LocalDateTime;

import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;


import org.tinylog.Logger;
import org.tinylog.TaggedLogger;
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.UserIdentity;
import org.eclipse.jetty.server.handler.AbstractHandler;

import util.Web;

public class InjectHandler extends AbstractHandler {


private static final TaggedLogger log = Logger.tag("REQ");

@Override
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
Expand All @@ -41,7 +45,36 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques
String h = hs.nextElement();
Logger.info("header " + h + ": " + request.getHeader(h));
}*/
Logger.tag("REQ").info(Web.getRequestLogString("REQ", baseRequest.getMethod() + " " + baseRequest.getRequestURL().toString(), baseRequest));
Logger.info(Web.getRequestLogString("REQ", baseRequest.getMethod() + " " + baseRequest.getRequestURL().toString(), baseRequest));
log.trace(getRequestLogString("REQ", baseRequest.getMethod() + " " + baseRequest.getRequestURL().toString(), baseRequest));
}
}

private static StringBuilder getRequestLogString(String handlerText, String target, Request request) {
String user = "?";
UserIdentity userIdentity = Web.getUserIdentity(request);
if(userIdentity!=null) {
//Logger.info("userIdentity "+userIdentity);
user = userIdentity.getUserPrincipal().getName();
}
StringBuilder s = new StringBuilder();
s.append(Web.timestampText(LocalDateTime.now()));
s.append(" ");
s.append(user);
s.append(" ");
s.append(request.getRemoteAddr());
s.append(" ");
s.append(target);
String qs = request.getQueryString();
if(qs != null) {
s.append("?");
s.append(request.getQueryString());
}
String referer = request.getHeader("Referer");
if(referer!=null) {
s.append("\t\tref ");
s.append(referer);
}
return s;
}
}
Loading

0 comments on commit 55dad5e

Please sign in to comment.