From cd33295f4f2770f8e90cb3465e59546b8d91097a Mon Sep 17 00:00:00 2001 From: Claus Nagel Date: Wed, 18 Dec 2024 14:12:58 +0100 Subject: [PATCH] the create index command now supports creating partial or full indexes for the property value columns --- .../citydb/cli/index/CreateIndexCommand.java | 17 ++++--- .../citydb/cli/index/DropIndexCommand.java | 8 ++-- .../citydb/cli/index/IndexStatusCommand.java | 8 ++-- .../org/citydb/cli/util/CommandHelper.java | 17 +++---- .../database/postgres/SchemaAdapter.java | 12 ++++- .../database/adapter/SchemaAdapter.java | 6 ++- .../org/citydb/database/util/IndexHelper.java | 45 ++++++++++++++----- 7 files changed, 72 insertions(+), 41 deletions(-) diff --git a/citydb-cli/src/main/java/org/citydb/cli/index/CreateIndexCommand.java b/citydb-cli/src/main/java/org/citydb/cli/index/CreateIndexCommand.java index 6419b93f..ce000a5b 100644 --- a/citydb-cli/src/main/java/org/citydb/cli/index/CreateIndexCommand.java +++ b/citydb-cli/src/main/java/org/citydb/cli/index/CreateIndexCommand.java @@ -31,12 +31,18 @@ import picocli.CommandLine; import java.sql.SQLException; -import java.util.List; @CommandLine.Command( name = "create", description = "Create indexes on the database tables.") public class CreateIndexCommand extends IndexController { + enum Mode {partial, full} + + @CommandLine.Option(names = {"-m", "--index-mode"}, paramLabel = "", defaultValue = "partial", + description = "Index mode for property value columns: ${COMPLETION-CANDIDATES} " + + "(default: ${DEFAULT-VALUE}). Null values are not indexed in partial mode.") + private Mode mode; + private final Logger logger = LoggerManager.getInstance().getLogger(CreateIndexCommand.class); @Override @@ -47,12 +53,11 @@ public Integer call() throws ExecutionException { logger.info("Creating database indexes."); logger.info("Depending on the database size, this operation may take some time."); - List indexes = IndexHelper.DEFAULT_INDEXES; - for (int i = 0; i < indexes.size(); i++) { + int i = 1, size = IndexHelper.DEFAULT_INDEXES.size(); + for (Index index : IndexHelper.DEFAULT_INDEXES) { try { - Index index = indexes.get(i); - logger.info("[{}|{}] Creating database index on {}.", i + 1, indexes.size(), index); - indexHelper.create(index); + logger.info("[{}|{}] Creating database index on {}.", i++, size, index); + indexHelper.create(index, mode == Mode.partial && IndexHelper.DEFAULT_PARTIAL_INDEXES.contains(index)); } catch (SQLException e) { throw new ExecutionException("Failed to create database indexes.", e); } diff --git a/citydb-cli/src/main/java/org/citydb/cli/index/DropIndexCommand.java b/citydb-cli/src/main/java/org/citydb/cli/index/DropIndexCommand.java index 0bf60322..05b8ec5a 100644 --- a/citydb-cli/src/main/java/org/citydb/cli/index/DropIndexCommand.java +++ b/citydb-cli/src/main/java/org/citydb/cli/index/DropIndexCommand.java @@ -31,7 +31,6 @@ import picocli.CommandLine; import java.sql.SQLException; -import java.util.List; @CommandLine.Command( name = "drop", @@ -46,11 +45,10 @@ public Integer call() throws ExecutionException { logger.info("Dropping database indexes."); - List indexes = IndexHelper.DEFAULT_INDEXES; - for (int i = 0; i < indexes.size(); i++) { + int i = 1, size = IndexHelper.DEFAULT_INDEXES.size(); + for (Index index : IndexHelper.DEFAULT_INDEXES) { try { - Index index = indexes.get(i); - logger.info("[{}|{}] Dropping database index on {}.", i + 1, indexes.size(), index); + logger.info("[{}|{}] Dropping database index on {}.", i++, size, index); indexHelper.drop(index); } catch (SQLException e) { throw new ExecutionException("Failed to drop database indexes.", e); diff --git a/citydb-cli/src/main/java/org/citydb/cli/index/IndexStatusCommand.java b/citydb-cli/src/main/java/org/citydb/cli/index/IndexStatusCommand.java index 3c8702ab..438170c3 100644 --- a/citydb-cli/src/main/java/org/citydb/cli/index/IndexStatusCommand.java +++ b/citydb-cli/src/main/java/org/citydb/cli/index/IndexStatusCommand.java @@ -31,7 +31,6 @@ import picocli.CommandLine; import java.sql.SQLException; -import java.util.List; @CommandLine.Command( name = "status", @@ -47,11 +46,10 @@ public Integer call() throws ExecutionException { helper.logIndexStatus(Level.INFO, databaseManager.getAdapter()); logger.info("Indexes list:"); - List indexes = IndexHelper.DEFAULT_INDEXES; - for (int i = 0; i < indexes.size(); i++) { + int i = 1, size = IndexHelper.DEFAULT_INDEXES.size(); + for (Index index : IndexHelper.DEFAULT_INDEXES) { try { - Index index = indexes.get(i); - logger.info("[{}|{}] Database index on {}: {}", i + 1, indexes.size(), index, + logger.info("[{}|{}] Database index on {}: {}", i++, size, index, indexHelper.exists(index) ? "on" : "off"); } catch (SQLException e) { throw new ExecutionException("Failed to query status of database indexes.", e); diff --git a/citydb-cli/src/main/java/org/citydb/cli/util/CommandHelper.java b/citydb-cli/src/main/java/org/citydb/cli/util/CommandHelper.java index 6f81239a..b7497956 100644 --- a/citydb-cli/src/main/java/org/citydb/cli/util/CommandHelper.java +++ b/citydb-cli/src/main/java/org/citydb/cli/util/CommandHelper.java @@ -49,7 +49,6 @@ import java.nio.file.Path; import java.sql.SQLException; import java.util.Collection; -import java.util.List; import java.util.function.Consumer; public class CommandHelper { @@ -142,11 +141,10 @@ public String getFormattedSql(SqlObject object, DatabaseAdapter adapter) { public void createIndexes(DatabaseAdapter adapter) throws ExecutionException { try { IndexHelper indexHelper = adapter.getSchemaAdapter().getIndexHelper(); - List indexes = IndexHelper.DEFAULT_INDEXES; - for (int i = 0; i < indexes.size(); i++) { - Index index = indexes.get(i); - logger.debug("Creating database index {} of {} on {}.", i + 1, indexes.size(), index); - indexHelper.create(index); + int i = 1, size = IndexHelper.DEFAULT_INDEXES.size(); + for (Index index : IndexHelper.DEFAULT_INDEXES) { + logger.debug("Creating database index {} of {} on {}.", i++, size, index); + indexHelper.create(index, IndexHelper.DEFAULT_PARTIAL_INDEXES.contains(index)); } } catch (SQLException e) { throw new ExecutionException("Failed to create database indexes.", e); @@ -156,10 +154,9 @@ public void createIndexes(DatabaseAdapter adapter) throws ExecutionException { public void dropIndexes(DatabaseAdapter adapter) throws ExecutionException { try { IndexHelper indexHelper = adapter.getSchemaAdapter().getIndexHelper(); - List indexes = IndexHelper.DEFAULT_INDEXES; - for (int i = 0; i < indexes.size(); i++) { - Index index = indexes.get(i); - logger.debug("Dropping database index {} of {} on {}.", i + 1, indexes.size(), index); + int i = 1, size = IndexHelper.DEFAULT_INDEXES.size(); + for (Index index : IndexHelper.DEFAULT_INDEXES) { + logger.debug("Dropping database index {} of {} on {}.", i++, size, index); indexHelper.drop(index); } } catch (SQLException e) { diff --git a/citydb-database-postgres/src/main/java/org/citydb/database/postgres/SchemaAdapter.java b/citydb-database-postgres/src/main/java/org/citydb/database/postgres/SchemaAdapter.java index f1b66778..14115eb7 100644 --- a/citydb-database-postgres/src/main/java/org/citydb/database/postgres/SchemaAdapter.java +++ b/citydb-database-postgres/src/main/java/org/citydb/database/postgres/SchemaAdapter.java @@ -183,11 +183,19 @@ public Select getRecursiveLodQuery(Set lods, boolean requireAll, int sea } @Override - public String getCreateIndex(Index index) { - return "create index if not exists " + index.getName() + + public String getCreateIndex(Index index, boolean ignoreNulls) { + String stmt = "create index if not exists " + index.getName() + " on " + adapter.getConnectionDetails().getSchema() + "." + index.getTable().getName() + (index.getType() == Index.Type.SPATIAL ? " using gist " : " ") + "(" + String.join(", ", index.getColumns()) + ")"; + + if (ignoreNulls) { + stmt += " where " + index.getColumns().stream() + .map(column -> column + " is not null") + .collect(Collectors.joining(" and ")); + } + + return stmt; } @Override diff --git a/citydb-database/src/main/java/org/citydb/database/adapter/SchemaAdapter.java b/citydb-database/src/main/java/org/citydb/database/adapter/SchemaAdapter.java index dde301fd..e3ce9d15 100644 --- a/citydb-database/src/main/java/org/citydb/database/adapter/SchemaAdapter.java +++ b/citydb-database/src/main/java/org/citydb/database/adapter/SchemaAdapter.java @@ -63,7 +63,7 @@ protected SchemaAdapter(DatabaseAdapter adapter) { public abstract Select getRecursiveLodQuery(Set lods, boolean requireAll, int searchDepth, Table table); - public abstract String getCreateIndex(Index index); + public abstract String getCreateIndex(Index index, boolean ignoreNulls); public abstract String getDropIndex(Index index); @@ -94,4 +94,8 @@ public SqlHelper getSqlHelper() { public IndexHelper getIndexHelper() { return indexHelper; } + + public String getCreateIndex(Index index) { + return getCreateIndex(index, false); + } } diff --git a/citydb-database/src/main/java/org/citydb/database/util/IndexHelper.java b/citydb-database/src/main/java/org/citydb/database/util/IndexHelper.java index 5834b312..0fe13089 100644 --- a/citydb-database/src/main/java/org/citydb/database/util/IndexHelper.java +++ b/citydb-database/src/main/java/org/citydb/database/util/IndexHelper.java @@ -28,12 +28,11 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; +import java.util.*; +import java.util.function.Function; public class IndexHelper { - public static final List DEFAULT_INDEXES = List.of( + public static final Set DEFAULT_INDEXES = new LinkedHashSet<>(List.of( Index.FEATURE_OBJECTID, Index.FEATURE_IDENTIFIER, Index.FEATURE_ENVELOPE, @@ -48,14 +47,23 @@ public class IndexHelper { Index.PROPERTY_VAL_LOD, Index.PROPERTY_VAL_STRING, Index.PROPERTY_VAL_UOM, - Index.PROPERTY_VAL_URI); + Index.PROPERTY_VAL_URI)); - public static final List DEFAULT_NORMAL_INDEXES = DEFAULT_INDEXES.stream() + public static final Set DEFAULT_PARTIAL_INDEXES = new LinkedHashSet<>(List.of( + Index.PROPERTY_VAL_TIMESTAMP, + Index.PROPERTY_VAL_DOUBLE, + Index.PROPERTY_VAL_INT, + Index.PROPERTY_VAL_LOD, + Index.PROPERTY_VAL_STRING, + Index.PROPERTY_VAL_UOM, + Index.PROPERTY_VAL_URI)); + + public static final Set DEFAULT_NORMAL_INDEXES = new LinkedHashSet<>(DEFAULT_INDEXES.stream() .filter(index -> index.getType() == Index.Type.NORMAL) - .toList(); - public static final List DEFAULT_SPATIAL_INDEXES = DEFAULT_INDEXES.stream() + .toList()); + public static final Set DEFAULT_SPATIAL_INDEXES = new LinkedHashSet<>(DEFAULT_INDEXES.stream() .filter(index -> index.getType() == Index.Type.SPATIAL) - .toList(); + .toList()); public enum Status { ON, @@ -75,22 +83,35 @@ public static IndexHelper newInstance(DatabaseAdapter adapter) { } public void create(Index index) throws SQLException { + create(index, false); + } + + public void create(Index index, boolean ignoreNulls) throws SQLException { try (Connection connection = adapter.getPool().getConnection(true)) { if (!exists(index, connection)) { try (Statement stmt = createStatement(connection)) { - stmt.executeUpdate(adapter.getSchemaAdapter().getCreateIndex(index)); + stmt.executeUpdate(adapter.getSchemaAdapter().getCreateIndex(index, ignoreNulls)); } } } } public void createAll(Index... indexes) throws SQLException { - createAll(Arrays.asList(indexes)); + createAll(Arrays.asList(indexes), index -> false); + } + + public void createAll(Function ignoreNulls, Index... indexes) throws SQLException { + createAll(Arrays.asList(indexes), ignoreNulls); } public void createAll(Collection indexes) throws SQLException { + createAll(indexes, index -> false); + } + + public void createAll(Collection indexes, Function ignoreNulls) throws SQLException { for (Index index : indexes) { - create(index); + Boolean result = ignoreNulls != null ? ignoreNulls.apply(index) : null; + create(index, result != null ? result : false); } }