the create index command now supports creating partial or full indexe…

…s for the property value columns
3dcitydb · Dec 18, 2024 · cd33295 · cd33295
1 parent cb8f32f
commit cd33295
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 41 deletions.
diff --git a/citydb-cli/src/main/java/org/citydb/cli/index/CreateIndexCommand.java b/citydb-cli/src/main/java/org/citydb/cli/index/CreateIndexCommand.java
@@ -31,12 +31,18 @@
 import picocli.CommandLine;
 
 import java.sql.SQLException;
-import java.util.List;
 
 @CommandLine.Command(
         name = "create",
         description = "Create indexes on the database tables.")
 public class CreateIndexCommand extends IndexController {
+    enum Mode {partial, full}
+
+    @CommandLine.Option(names = {"-m", "--index-mode"}, paramLabel = "<mode>", defaultValue = "partial",
+            description = "Index mode for property value columns: ${COMPLETION-CANDIDATES} " +
+                    "(default: ${DEFAULT-VALUE}). Null values are not indexed in partial mode.")
+    private Mode mode;
+
     private final Logger logger = LoggerManager.getInstance().getLogger(CreateIndexCommand.class);
 
     @Override
@@ -47,12 +53,11 @@ public Integer call() throws ExecutionException {
         logger.info("Creating database indexes.");
         logger.info("Depending on the database size, this operation may take some time.");
 
-        List<Index> indexes = IndexHelper.DEFAULT_INDEXES;
-        for (int i = 0; i < indexes.size(); i++) {
+        int i = 1, size = IndexHelper.DEFAULT_INDEXES.size();
+        for (Index index : IndexHelper.DEFAULT_INDEXES) {
             try {
-                Index index = indexes.get(i);
-                logger.info("[{}|{}] Creating database index on {}.", i + 1, indexes.size(), index);
-                indexHelper.create(index);
+                logger.info("[{}|{}] Creating database index on {}.", i++, size, index);
+                indexHelper.create(index, mode == Mode.partial && IndexHelper.DEFAULT_PARTIAL_INDEXES.contains(index));
             } catch (SQLException e) {
                 throw new ExecutionException("Failed to create database indexes.", e);
             }

diff --git a/citydb-cli/src/main/java/org/citydb/cli/index/DropIndexCommand.java b/citydb-cli/src/main/java/org/citydb/cli/index/DropIndexCommand.java
@@ -31,7 +31,6 @@
 import picocli.CommandLine;
 
 import java.sql.SQLException;
-import java.util.List;
 
 @CommandLine.Command(
         name = "drop",
@@ -46,11 +45,10 @@ public Integer call() throws ExecutionException {
 
         logger.info("Dropping database indexes.");
 
-        List<Index> indexes = IndexHelper.DEFAULT_INDEXES;
-        for (int i = 0; i < indexes.size(); i++) {
+        int i = 1, size = IndexHelper.DEFAULT_INDEXES.size();
+        for (Index index : IndexHelper.DEFAULT_INDEXES) {
             try {
-                Index index = indexes.get(i);
-                logger.info("[{}|{}] Dropping database index on {}.", i + 1, indexes.size(), index);
+                logger.info("[{}|{}] Dropping database index on {}.", i++, size, index);
                 indexHelper.drop(index);
             } catch (SQLException e) {
                 throw new ExecutionException("Failed to drop database indexes.", e);

diff --git a/citydb-cli/src/main/java/org/citydb/cli/index/IndexStatusCommand.java b/citydb-cli/src/main/java/org/citydb/cli/index/IndexStatusCommand.java
@@ -31,7 +31,6 @@
 import picocli.CommandLine;
 
 import java.sql.SQLException;
-import java.util.List;
 
 @CommandLine.Command(
         name = "status",
@@ -47,11 +46,10 @@ public Integer call() throws ExecutionException {
         helper.logIndexStatus(Level.INFO, databaseManager.getAdapter());
         logger.info("Indexes list:");
 
-        List<Index> indexes = IndexHelper.DEFAULT_INDEXES;
-        for (int i = 0; i < indexes.size(); i++) {
+        int i = 1, size = IndexHelper.DEFAULT_INDEXES.size();
+        for (Index index : IndexHelper.DEFAULT_INDEXES) {
             try {
-                Index index = indexes.get(i);
-                logger.info("[{}|{}] Database index on {}: {}", i + 1, indexes.size(), index,
+                logger.info("[{}|{}] Database index on {}: {}", i++, size, index,
                         indexHelper.exists(index) ? "on" : "off");
             } catch (SQLException e) {
                 throw new ExecutionException("Failed to query status of database indexes.", e);

diff --git a/citydb-cli/src/main/java/org/citydb/cli/util/CommandHelper.java b/citydb-cli/src/main/java/org/citydb/cli/util/CommandHelper.java
@@ -49,7 +49,6 @@
 import java.nio.file.Path;
 import java.sql.SQLException;
 import java.util.Collection;
-import java.util.List;
 import java.util.function.Consumer;
 
 public class CommandHelper {
@@ -142,11 +141,10 @@ public String getFormattedSql(SqlObject object, DatabaseAdapter adapter) {
     public void createIndexes(DatabaseAdapter adapter) throws ExecutionException {
         try {
             IndexHelper indexHelper = adapter.getSchemaAdapter().getIndexHelper();
-            List<Index> indexes = IndexHelper.DEFAULT_INDEXES;
-            for (int i = 0; i < indexes.size(); i++) {
-                Index index = indexes.get(i);
-                logger.debug("Creating database index {} of {} on {}.", i + 1, indexes.size(), index);
-                indexHelper.create(index);
+            int i = 1, size = IndexHelper.DEFAULT_INDEXES.size();
+            for (Index index : IndexHelper.DEFAULT_INDEXES) {
+                logger.debug("Creating database index {} of {} on {}.", i++, size, index);
+                indexHelper.create(index, IndexHelper.DEFAULT_PARTIAL_INDEXES.contains(index));
             }
         } catch (SQLException e) {
             throw new ExecutionException("Failed to create database indexes.", e);
@@ -156,10 +154,9 @@ public void createIndexes(DatabaseAdapter adapter) throws ExecutionException {
     public void dropIndexes(DatabaseAdapter adapter) throws ExecutionException {
         try {
             IndexHelper indexHelper = adapter.getSchemaAdapter().getIndexHelper();
-            List<Index> indexes = IndexHelper.DEFAULT_INDEXES;
-            for (int i = 0; i < indexes.size(); i++) {
-                Index index = indexes.get(i);
-                logger.debug("Dropping database index {} of {} on {}.", i + 1, indexes.size(), index);
+            int i = 1, size = IndexHelper.DEFAULT_INDEXES.size();
+            for (Index index : IndexHelper.DEFAULT_INDEXES) {
+                logger.debug("Dropping database index {} of {} on {}.", i++, size, index);
                 indexHelper.drop(index);
             }
         } catch (SQLException e) {

diff --git a/citydb-database-postgres/src/main/java/org/citydb/database/postgres/SchemaAdapter.java b/citydb-database-postgres/src/main/java/org/citydb/database/postgres/SchemaAdapter.java
@@ -183,11 +183,19 @@ public Select getRecursiveLodQuery(Set<String> lods, boolean requireAll, int sea
     }
 
     @Override
-    public String getCreateIndex(Index index) {
-        return "create index if not exists " + index.getName() +
+    public String getCreateIndex(Index index, boolean ignoreNulls) {
+        String stmt = "create index if not exists " + index.getName() +
                 " on " + adapter.getConnectionDetails().getSchema() + "." + index.getTable().getName() +
                 (index.getType() == Index.Type.SPATIAL ? " using gist " : " ") +
                 "(" + String.join(", ", index.getColumns()) + ")";
+
+        if (ignoreNulls) {
+            stmt += " where " + index.getColumns().stream()
+                    .map(column -> column + " is not null")
+                    .collect(Collectors.joining(" and "));
+        }
+
+        return stmt;
     }
 
     @Override

diff --git a/citydb-database/src/main/java/org/citydb/database/adapter/SchemaAdapter.java b/citydb-database/src/main/java/org/citydb/database/adapter/SchemaAdapter.java
@@ -63,7 +63,7 @@ protected SchemaAdapter(DatabaseAdapter adapter) {
 
     public abstract Select getRecursiveLodQuery(Set<String> lods, boolean requireAll, int searchDepth, Table table);
 
-    public abstract String getCreateIndex(Index index);
+    public abstract String getCreateIndex(Index index, boolean ignoreNulls);
 
     public abstract String getDropIndex(Index index);
 
@@ -94,4 +94,8 @@ public SqlHelper getSqlHelper() {
     public IndexHelper getIndexHelper() {
         return indexHelper;
     }
+
+    public String getCreateIndex(Index index) {
+        return getCreateIndex(index, false);
+    }
 }
diff --git a/citydb-database/src/main/java/org/citydb/database/util/IndexHelper.java b/citydb-database/src/main/java/org/citydb/database/util/IndexHelper.java
@@ -28,12 +28,11 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
+import java.util.*;
+import java.util.function.Function;
 
 public class IndexHelper {
-    public static final List<Index> DEFAULT_INDEXES = List.of(
+    public static final Set<Index> DEFAULT_INDEXES = new LinkedHashSet<>(List.of(
             Index.FEATURE_OBJECTID,
             Index.FEATURE_IDENTIFIER,
             Index.FEATURE_ENVELOPE,
@@ -48,14 +47,23 @@ public class IndexHelper {
             Index.PROPERTY_VAL_LOD,
             Index.PROPERTY_VAL_STRING,
             Index.PROPERTY_VAL_UOM,
-            Index.PROPERTY_VAL_URI);
+            Index.PROPERTY_VAL_URI));
 
-    public static final List<Index> DEFAULT_NORMAL_INDEXES = DEFAULT_INDEXES.stream()
+    public static final Set<Index> DEFAULT_PARTIAL_INDEXES = new LinkedHashSet<>(List.of(
+            Index.PROPERTY_VAL_TIMESTAMP,
+            Index.PROPERTY_VAL_DOUBLE,
+            Index.PROPERTY_VAL_INT,
+            Index.PROPERTY_VAL_LOD,
+            Index.PROPERTY_VAL_STRING,
+            Index.PROPERTY_VAL_UOM,
+            Index.PROPERTY_VAL_URI));
+
+    public static final Set<Index> DEFAULT_NORMAL_INDEXES = new LinkedHashSet<>(DEFAULT_INDEXES.stream()
             .filter(index -> index.getType() == Index.Type.NORMAL)
-            .toList();
-    public static final List<Index> DEFAULT_SPATIAL_INDEXES = DEFAULT_INDEXES.stream()
+            .toList());
+    public static final Set<Index> DEFAULT_SPATIAL_INDEXES = new LinkedHashSet<>(DEFAULT_INDEXES.stream()
             .filter(index -> index.getType() == Index.Type.SPATIAL)
-            .toList();
+            .toList());
 
     public enum Status {
         ON,
@@ -75,22 +83,35 @@ public static IndexHelper newInstance(DatabaseAdapter adapter) {
     }
 
     public void create(Index index) throws SQLException {
+        create(index, false);
+    }
+
+    public void create(Index index, boolean ignoreNulls) throws SQLException {
         try (Connection connection = adapter.getPool().getConnection(true)) {
             if (!exists(index, connection)) {
                 try (Statement stmt = createStatement(connection)) {
-                    stmt.executeUpdate(adapter.getSchemaAdapter().getCreateIndex(index));
+                    stmt.executeUpdate(adapter.getSchemaAdapter().getCreateIndex(index, ignoreNulls));
                 }
             }
         }
     }
 
     public void createAll(Index... indexes) throws SQLException {
-        createAll(Arrays.asList(indexes));
+        createAll(Arrays.asList(indexes), index -> false);
+    }
+
+    public void createAll(Function<Index, Boolean> ignoreNulls, Index... indexes) throws SQLException {
+        createAll(Arrays.asList(indexes), ignoreNulls);
     }
 
     public void createAll(Collection<Index> indexes) throws SQLException {
+        createAll(indexes, index -> false);
+    }
+
+    public void createAll(Collection<Index> indexes, Function<Index, Boolean> ignoreNulls) throws SQLException {
         for (Index index : indexes) {
-            create(index);
+            Boolean result = ignoreNulls != null ? ignoreNulls.apply(index) : null;
+            create(index, result != null ? result : false);
         }
     }