Feature processing (#14)

* new schema version for feature provider data

* license headers

* Add additional provider schema information

* Fix constraints field

* upgrade immutables

* Fixes to provider schema

* migrate provider data from v1 to v2

* Implement migration to the new format

* fix wrong path in test case

* add special case with table name in square brackets to test case

* Fix paths and property names

* refactor provider migration

* provider migration for deeply nested properties

* add oneo test case

* schemas, en/decoder, transactions

* license headers

* fix unit tests

* version bump

* reverse migration

* nested schema integration

* version bump

Co-authored-by: ulugbek <[email protected]>

Author: azahnen

build.gradle

@@ -8,12 +8,12 @@ allprojects {
     group = 'de.interactive_instruments'
 }
 
-version = '3.3.0'
+version = '3.3.1'
 
 dependencies {
-    feature group: 'de.interactive_instruments', name: 'xtraplatform-base', version: '2.2.+'
-    feature group: 'de.interactive_instruments', name: 'xtraplatform-store', version: '2.2.+'
-    feature group: 'de.interactive_instruments', name: 'xtraplatform-web', version: '2.2.+'
+    feature group: 'de.interactive_instruments', name: 'xtraplatform-base', version: '2.2.3'
+    feature group: 'de.interactive_instruments', name: 'xtraplatform-store', version: '2.2.9'
+    feature group: 'de.interactive_instruments', name: 'xtraplatform-web', version: '2.2.3'
 
     bundle subprojects
 }

license-headers.gradle

@@ -21,6 +21,7 @@ subprojects {
                 "**/*.ico",
                 "**/*.xcf",
                 "**/*.txt",
+                "**/*.yml",
                 "**/*.json"])
         ext.year = Calendar.getInstance().get(Calendar.YEAR)
         ext.name = "interactive instruments GmbH"

unit-tests.gradle

@@ -11,6 +11,7 @@ subprojects {
             transitive = false // this avoids affecting your version of Groovy/Spock
         }
         testCompile group: 'org.mockito', name: 'mockito-core', version: '1.9.5'
+        testCompile "net.bytebuddy:byte-buddy:1.10.9"
     }
 
     test {

xtraplatform-feature-provider-api/src/main/java/de/ii/xtraplatform/feature/provider/api/ConnectorFactory.java

@@ -9,13 +9,14 @@
 
 import de.ii.xtraplatform.features.domain.FeatureProviderConnector;
 import de.ii.xtraplatform.features.domain.FeatureProviderDataV1;
+import de.ii.xtraplatform.features.domain.FeatureProviderDataV2;
 
 /**
  * @author zahnen
  */
 public interface ConnectorFactory {
 
-    FeatureProviderConnector<?, ?, ?> createConnector(FeatureProviderDataV1 featureProviderData);
+    FeatureProviderConnector<?, ?, ?> createConnector(FeatureProviderDataV2 featureProviderData);
 
     void disposeConnector(FeatureProviderConnector<?, ?, ?> connector);
 }

xtraplatform-feature-provider-pgis/src/main/java/de/ii/xtraplatform/feature/provider/pgis/SqlFeatureCreator.java

@@ -121,8 +121,8 @@ public CompletionStage<String> runQueries() {
             return previousId != null ? previousId : id;
         };
 
-        return SlickSql.source(session, queryFunctions, mapper, valueContainer, materializer.system())
-                .runWith(Sink.fold("", (id1, id2) -> id1.isEmpty() ? id2 : id1), materializer);
+        return null;//SlickSql.source(session, materializer.system().dispatcher(), queryFunctions, mapper, valueContainer)
+                //.runWith(Sink.fold("", (id1, id2) -> id1.isEmpty() ? id2 : id1), materializer);
 
         /*return Source.from(queries)
                      .flatMapConcat(queryFunction -> {
@@ -223,8 +223,8 @@ public CompletionStage<String> runQueries(String id) {
             return previousId != null ? previousId : id2;
         };
 
-        return SlickSql.source(session, queryFunctions, mapper, valueContainer, materializer.system())
-                       .runWith(Sink.fold("", (id1, id2) -> id1.isEmpty() ? id2 : id1), materializer);
+        return null;//SlickSql.source(session, materializer.system().dispatcher(), queryFunctions, mapper, valueContainer)
+                      // .runWith(Sink.fold("", (id1, id2) -> id1.isEmpty() ? id2 : id1), materializer);
 
         /*return Source.from(queries)
                      .flatMapConcat(queryFunction -> {

xtraplatform-feature-provider-pgis/src/test/java/de/ii/xtraplatform/feature/provider/pgis/SqlFeatureInsertsTest.java

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2020 interactive instruments GmbH
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+package de.ii.xtraplatform.feature.provider.sql;
+
+import com.google.common.collect.ImmutableList;
+import de.ii.xtraplatform.features.domain.FeatureSchema;
+import de.ii.xtraplatform.features.domain.FeatureTypeV2;
+
+import java.util.List;
+
+public class SqlFeatureTypeParser2 {
+
+    private final SqlPathSyntax syntax;
+
+    public SqlFeatureTypeParser2(SqlPathSyntax syntax) {
+        this.syntax = syntax;
+    }
+
+
+    public List<String> parse(FeatureTypeV2 featureType) {
+        return featureType.getProperties()
+                          .values()
+                                  .stream()
+                                  .map(this::toPathWithFlags)
+                                  .collect(ImmutableList.toImmutableList());
+    }
+
+    private String toPathWithFlags(FeatureSchema property) {
+        String path = property.getSourcePath().orElse("");
+
+        path = syntax.setQueryableFlag(path, property.getName());
+
+        boolean isOid = property.isId();
+
+        /*Optional<Integer> sortPriority = generalMapping.map(TargetMapping::getSortPriority);
+        boolean isOid = generalMapping.flatMap(targetMapping -> Optional.ofNullable(targetMapping.getType()))
+                                      .map(enumValue -> enumValue.toString()
+                                                                 .equals("ID"))
+                                      .orElse(false);
+        Optional<String> queryableName = generalMapping.flatMap(targetMapping -> Optional.ofNullable(targetMapping.getName()));*/
+        boolean isSpatial = property.getType() == FeatureSchema.Type.GEOMETRY;
+
+        if (isOid) {
+            path = syntax.setOidFlag(path);
+        }
+        /*if (sortPriority.isPresent()) {
+            path = syntax.setPriorityFlag(path, sortPriority.get());
+        }
+        if (queryableName.isPresent()) {
+            path = syntax.setQueryableFlag(path, queryableName.get());
+        }*/
+        if (isSpatial) {
+            path = syntax.setSpatialFlag(path);
+        }
+
+        return path;
+    }
+}

xtraplatform-feature-provider-sql/src/main/java/de/ii/xtraplatform/feature/provider/sql/SqlFeatureTypeParser2.java

@@ -246,11 +246,25 @@ default String getJoinConditionPattern() {
                 Pattern.quote(getJoinConditionEnd());
     }
 
+    @Value.Derived
+    default String getJoinConditionPlainPattern() {
+        return Pattern.quote(getJoinConditionStart()) +
+                "(?:" + getIdentifierPattern() + ")" +
+                Pattern.quote(getJoinConditionSeparator()) +
+                "(?:" + getIdentifierPattern() + ")" +
+                Pattern.quote(getJoinConditionEnd());
+    }
+
     @Value.Derived
     default String getTablePatternString() {
         return "(?:" + getJoinConditionPattern() + ")?" + "(?<" + MatcherGroups.TABLE + ">" + getIdentifierPattern() + ")" + "(?<" + MatcherGroups.TABLE_FLAGS + ">" + getFlagsPattern() + ")?";
     }
 
+    @Value.Derived
+    default String getTablePatternPlainString() {
+        return "(?:" + getJoinConditionPlainPattern() + ")?" + "(?:" + getIdentifierPattern() + ")" + "(?:" + getFlagsPattern() + ")?";
+    }
+
     @Value.Derived
     default Pattern getTablePattern() {
         return Pattern.compile(getTablePatternString());
@@ -266,6 +280,21 @@ default Pattern getColumnPathPattern() {
         return Pattern.compile("^(?<" + MatcherGroups.PATH + ">" + "(?:" + getPathSeparator() + getTablePatternString() + ")+)" + getPathSeparator() + "(?<" + MatcherGroups.COLUMNS + ">" + getColumnPattern() + ")" + "(?<" + MatcherGroups.PATH_FLAGS + ">" + getFlagsPattern() + ")?$");
     }
 
+    @Value.Derived
+    default Pattern getPartialColumnPathPattern() {
+        return Pattern.compile(getPathPatternString() + "(?:" + getPathSeparator() + "(?<" + MatcherGroups.COLUMNS + ">" + getColumnPattern() + "))" + "(?<" + MatcherGroups.PATH_FLAGS + ">" + getFlagsPattern() + ")?$");
+    }
+
+    @Value.Derived
+    default Pattern getPathPattern() {
+        return Pattern.compile(getPathPatternString());
+    }
+
+    @Value.Derived
+    default String getPathPatternString() {
+        return "(?<" + MatcherGroups.PATH + ">" + getPathSeparator() + "?" + getTablePatternPlainString() + "(?:" + getPathSeparator() + getTablePatternPlainString() + ")*)";
+    }
+
     @Value.Default
     default Options getOptions() {
         return new ImmutableOptions.Builder()

xtraplatform-feature-provider-sql/src/main/java/de/ii/xtraplatform/feature/provider/sql/SqlPathSyntax.java

@@ -0,0 +1,120 @@
+/**
+ * Copyright 2020 interactive instruments GmbH
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+package de.ii.xtraplatform.feature.provider.sql.app;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ListMultimap;
+import de.ii.xtraplatform.features.domain.FeatureStoreRelation;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * @author zahnen
+ */
+class FeatureCreatorValues {
+    final List<FeatureStoreRelation> path;
+    final ListMultimap<List<FeatureStoreRelation>, FeatureCreatorValues> relatedValues;
+    final Map<List<FeatureStoreRelation>, List<Integer>> rowCounts;
+    final Map<String, String> values;
+    final Map<String, String> ids;
+
+    FeatureCreatorValues(List<FeatureStoreRelation> path) {
+        this.path = path;
+        this.relatedValues = ArrayListMultimap.create();
+        this.rowCounts = new LinkedHashMap<>();
+        this.values = new LinkedHashMap<>();
+        this.ids = new LinkedHashMap<>();
+    }
+
+    FeatureCreatorValues getRelatedValues(List<FeatureStoreRelation> path, List<Integer> parentRows) {
+        int row = parentRows.size() < path.size() ? 0 : parentRows.get(path.size() - 1);
+
+        return getRelatedValues(path, row);
+    }
+
+    FeatureCreatorValues getCurrentRelatedValues(List<FeatureStoreRelation> path) {
+        if (path.equals(this.path)) {
+            return this;
+        }
+
+        int row = Optional.ofNullable(relatedValues.get(path))
+                          .map(List::size)
+                          .orElse(0);
+
+        return getRelatedValues(path, row);
+    }
+
+    Map<List<FeatureStoreRelation>, List<Integer>> getRowCounts() {
+        return rowCounts;
+    }
+
+    private FeatureCreatorValues getRelatedValues(List<FeatureStoreRelation> path, int row) {
+        if (!relatedValues.containsKey(path)) {
+            throw new IllegalStateException(String.format("No values found for path %s", path));
+        }
+
+        List<FeatureCreatorValues> values = relatedValues.get(path);
+
+        if (row > values.size() - 1) {
+            throw new IllegalStateException(String.format("No values found for row %s of %s", row, path));
+        }
+
+        return values.get(row);
+    }
+
+    //TODO: value escaping to SqlSyntax
+    void addValue(List<FeatureStoreRelation> path, String attribute, String value) {
+        FeatureCreatorValues related = getCurrentRelatedValues(path);
+        related.values.put(attribute, value != null ? "'" + value.replaceAll("'", "''") + "'" : null);
+    }
+
+    void addRow(List<FeatureStoreRelation> path) {
+        if (!relatedValues.containsKey(path)) {
+            throw new IllegalStateException(String.format("No values found for path %s", path));
+        }
+
+        relatedValues.put(path, new FeatureCreatorValues(path));
+
+        rowCounts.put(path, getIncrementedRowCounts(path));
+    }
+
+    private List<Integer> getIncrementedRowCounts(List<FeatureStoreRelation> path) {
+        List<Integer> currentRowCounts = rowCounts.getOrDefault(path, getDefaultRowCounts(path));
+        int currentRowCount = currentRowCounts.get(currentRowCounts.size() - 1);
+
+        currentRowCounts.add(currentRowCounts.size() - 1, currentRowCount + 1);
+
+        return currentRowCounts;
+    }
+
+    private List<Integer> getDefaultRowCounts(List<FeatureStoreRelation> path) {
+        int currentParentRowCount = getCurrentParentRowCount(path);
+
+        return IntStream.range(0, currentParentRowCount)
+                        .mapToObj(i -> 0)
+                        .collect(Collectors.toList());
+    }
+
+    private int getCurrentParentRowCount(List<FeatureStoreRelation> path) {
+        for (int i = path.size() - 1; i > 0; i--) {
+            if (rowCounts.containsKey(path.subList(0, i))) {
+                List<Integer> parentRowCounts = rowCounts.get(path.subList(0, i));
+
+                return parentRowCounts.get(parentRowCounts.size() - 1);
+            }
+        }
+
+        return 1;
+    }
+
+}