diff --git a/core/utils/metadata-utils/src/main/java/datawave/query/util/AllFieldMetadataHelper.java b/core/utils/metadata-utils/src/main/java/datawave/query/util/AllFieldMetadataHelper.java index a4b0edd1165..16795fb4666 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/query/util/AllFieldMetadataHelper.java +++ b/core/utils/metadata-utils/src/main/java/datawave/query/util/AllFieldMetadataHelper.java @@ -1108,6 +1108,36 @@ public Multimap loadIndexedFields() throws TableNotFoundException return Multimaps.unmodifiableMultimap(fields); } + /** + * Fetch the Set of all fields marked as being hidden, {@link ColumnFamilyConstants#COLF_H}. Returns a multimap of datatype to field + * + * @return a multimap of datatypes to hidden fields + * @throws TableNotFoundException + * if no table exists + */ + @Cacheable(value = "loadHiddenFields", key = "{#root.target.auths,#root.target.metadataTableName}", cacheManager = "metadataHelperCacheManager", + sync = true) + public Multimap loadHiddenFields() throws TableNotFoundException { + log.debug("cache fault for loadHiddenFields({}, {})", this.auths, this.metadataTableName); + + if (log.isTraceEnabled()) { + log.trace("loadHiddenFields from table: {}", metadataTableName); + } + + Multimap fields = HashMultimap.create(); + + try (Scanner bs = ScannerHelper.createScanner(accumuloClient, metadataTableName, auths)) { + bs.setRange(new Range()); + bs.fetchColumnFamily(ColumnFamilyConstants.COLF_H); + + for (Entry entry : bs) { + fields.put(getDatatype(entry.getKey()), entry.getKey().getRow().toString()); + } + } + + return Multimaps.unmodifiableMultimap(fields); + } + /** * Fetch the Set of all fields marked as being reverse indexed, {@link ColumnFamilyConstants#COLF_RI}. Returns a multimap of datatype to field * diff --git a/core/utils/metadata-utils/src/main/java/datawave/query/util/MetadataHelper.java b/core/utils/metadata-utils/src/main/java/datawave/query/util/MetadataHelper.java index 50d946112c9..aab7639abf2 100644 --- a/core/utils/metadata-utils/src/main/java/datawave/query/util/MetadataHelper.java +++ b/core/utils/metadata-utils/src/main/java/datawave/query/util/MetadataHelper.java @@ -717,6 +717,30 @@ public boolean isTokenized(String fieldName, Set ingestTypeFilter) throw } } + /** + * Get the set of hidden fields for the provided ingest type filter. A null or empty filter indicates all hidden fields should be returned. + * + * @param ingestTypeFilter + * the ingest type filter + * @return the set of hidden fields given the provided ingest type filter + * @throws TableNotFoundException + * if the table does not exist + */ + public Set getHiddenFields(Set ingestTypeFilter) throws TableNotFoundException { + + Multimap hiddenFields = this.allFieldMetadataHelper.loadHiddenFields(); + + Set fields = new HashSet<>(); + if (ingestTypeFilter == null || ingestTypeFilter.isEmpty()) { + fields.addAll(hiddenFields.values()); + } else { + for (String datatype : ingestTypeFilter) { + fields.addAll(hiddenFields.get(datatype)); + } + } + return Collections.unmodifiableSet(fields); + } + /** * Returns a Set of all TextNormalizers in use by any type in Accumulo * @@ -2155,5 +2179,4 @@ public static IteratorSetting getCQRegexFilter(String regex) { RegExFilter.setRegexs(cqRegex, null, null, regex, null, false); return cqRegex; } - } diff --git a/core/utils/metadata-utils/src/test/java/datawave/query/util/MetadataHelperTest.java b/core/utils/metadata-utils/src/test/java/datawave/query/util/MetadataHelperTest.java index 00c16c98af5..57dce9bb8e5 100644 --- a/core/utils/metadata-utils/src/test/java/datawave/query/util/MetadataHelperTest.java +++ b/core/utils/metadata-utils/src/test/java/datawave/query/util/MetadataHelperTest.java @@ -1,6 +1,7 @@ package datawave.query.util; import static datawave.data.ColumnFamilyConstants.COLF_F; +import static datawave.data.ColumnFamilyConstants.COLF_H; import static datawave.query.util.TestUtils.createDateFrequencyMap; import static org.apache.accumulo.core.iterators.LongCombiner.VAR_LEN_ENCODER; @@ -104,6 +105,12 @@ private void givenMutation(String row, String columnFamily, String columnQualifi givenMutation(mutation); } + private void givenHiddenField(String row, String datatype) { + Mutation mutation = new Mutation(row); + mutation.put(COLF_H, new Text(datatype), new Value()); + givenMutation(mutation); + } + private void givenNonAggregatedFrequencyRows(String row, Text colf, String datatype, String startDate, String endDate, long count) { Mutation mutation = new Mutation(row); Value value = new Value(VAR_LEN_ENCODER.encode(count)); @@ -392,4 +399,20 @@ void testMixedEntryFormats() { Assertions.assertEquals(DateHelper.parse("20200103"), helper.getEarliestOccurrenceOfFieldWithType("NAME", "maze", accumuloClient, null)); } } + + /** + * Test against a table with hidden entries. + */ + @Test + void testHiddenEntry() throws TableNotFoundException { + givenHiddenField("NAME", "csv"); + givenHiddenField("NAME", "wiki"); + givenHiddenField("EVENT_DATE", "maze"); + writeMutations(); + + Assertions.assertTrue(helper.getHiddenFields(Set.of("csv")).contains("NAME")); + Assertions.assertTrue(helper.getHiddenFields(Set.of()).contains("EVENT_DATE")); + Assertions.assertFalse(helper.getHiddenFields(Set.of("foo")).contains("NAME")); + Assertions.assertFalse(helper.getHiddenFields(Set.of()).contains("FOO")); + } } diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitor.java index e4473a7210a..9307c295b52 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitor.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitor.java @@ -43,6 +43,7 @@ public class FieldMissingFromSchemaVisitor extends ShortCircuitBaseVisitor { private final Set allFieldsForDatatypes; // All fields for the specified datatypes pulled from MetadataHelper private final Set specialFields; private final Set datatypeFilter; + private final Set allHiddenFields; public FieldMissingFromSchemaVisitor(MetadataHelper helper, Set datatypeFilter, Set specialFields) { this.helper = helper; @@ -53,6 +54,7 @@ public FieldMissingFromSchemaVisitor(MetadataHelper helper, Set datatype datatypeFilter = Collections.emptySet(); } this.allFieldsForDatatypes = this.helper.getAllFields(datatypeFilter); + this.allHiddenFields = this.helper.getHiddenFields(datatypeFilter); } catch (TableNotFoundException e) { log.error(e); throw new RuntimeException("Unable to get metadata", e); @@ -94,7 +96,7 @@ protected Object findMissingFields(JexlNode node, Object data) { for (ASTIdentifier identifier : identifiers) { String fieldName = JexlASTHelper.deconstructIdentifier(identifier); - if (!this.allFieldsForDatatypes.contains(fieldName) && !specialFields.contains(fieldName)) { + if (!this.allFieldsForDatatypes.contains(fieldName) && !specialFields.contains(fieldName) || this.allHiddenFields.contains(fieldName)) { nonExistentFieldNames.add(fieldName); } } @@ -152,7 +154,7 @@ public Object visit(ASTFunctionNode node, Object data) { // deconstruct the identifier final String testFieldName = JexlASTHelper.deconstructIdentifier(fieldName); // changed to allow _ANYFIELD_ in functions - if (!this.allFieldsForDatatypes.contains(testFieldName) && !specialFields.contains(fieldName)) { + if (!this.allFieldsForDatatypes.contains(testFieldName) && !specialFields.contains(fieldName) || this.allHiddenFields.contains(fieldName)) { nonExistentFieldNames.add(testFieldName); } } diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitorTest.java index 232dbdb8ffc..ab54fbe35a4 100644 --- a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitorTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitorTest.java @@ -23,7 +23,6 @@ public class FieldMissingFromSchemaVisitorTest { // Special fields required by visitor. private Set specialFields = Sets.newHashSet(ANY_FIELD, NO_FIELD); - private MockMetadataHelper helper; @Before diff --git a/warehouse/query-core/src/test/java/datawave/query/planner/ScanHintRulesTest.java b/warehouse/query-core/src/test/java/datawave/query/planner/ScanHintRulesTest.java index f5546975d38..59f7a227fad 100644 --- a/warehouse/query-core/src/test/java/datawave/query/planner/ScanHintRulesTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/planner/ScanHintRulesTest.java @@ -58,6 +58,7 @@ public void setup() { private void setupMetadata() throws TableNotFoundException, InstantiationException, IllegalAccessException { expect(metadataHelper.getIndexedFields(EasyMock.anyObject())).andReturn(Collections.emptySet()).anyTimes(); + expect(metadataHelper.getHiddenFields(EasyMock.anyObject())).andReturn(Collections.emptySet()).anyTimes(); expect(metadataHelper.getReverseIndexedFields(EasyMock.anyObject())).andReturn(Collections.emptySet()).anyTimes(); expect(metadataHelper.getIndexOnlyFields(EasyMock.anyObject())).andReturn(Collections.emptySet()).anyTimes(); expect(metadataHelper.getAllFields(EasyMock.anyObject())).andReturn(Collections.singleton("A")).anyTimes(); diff --git a/warehouse/query-core/src/test/java/datawave/query/rules/FieldExistenceRuleTest.java b/warehouse/query-core/src/test/java/datawave/query/rules/FieldExistenceRuleTest.java index cd14cfe6106..bb75aefb07d 100644 --- a/warehouse/query-core/src/test/java/datawave/query/rules/FieldExistenceRuleTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/rules/FieldExistenceRuleTest.java @@ -14,8 +14,9 @@ public class FieldExistenceRuleTest extends ShardQueryRuleTest { - private static final Set ALL_FIELDS = Set.of("FOO", "BAR", "BAT"); + private static final Set ALL_FIELDS = Set.of("FOO", "BAR", "BAT", "HIDDEN_FIELD"); private static final String ANYFIELD = "_ANYFIELD_"; + private static final Set hiddenFields = Set.of("HIDDEN_FIELD"); private static final MockMetadataHelper defaultMetadataHelper = new MockMetadataHelper(); private final Set fieldExceptions = new HashSet<>(); @@ -23,6 +24,7 @@ public class FieldExistenceRuleTest extends ShardQueryRuleTest { @BeforeAll public static void beforeClass() throws Exception { defaultMetadataHelper.addFields(ALL_FIELDS); + defaultMetadataHelper.setHiddenFields(hiddenFields); } @BeforeEach @@ -52,6 +54,16 @@ public void testNonExistentFields() throws Exception { assertResult(); } + /** + * Test a query with a hidden field. + */ + @Test + public void testHiddenFields() throws Exception { + givenQuery("FOO == 'abc' || BAR =~ 'abc' || filter:includeRegex(SHENANIGANS, '45.8') || HIDDEN_FIELD == 'aa'"); + expectMessage("Fields not found in data dictionary: SHENANIGANS, HIDDEN_FIELD"); + assertResult(); + } + /** * Test a query that has a non-existent field that is a special field. */ diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java b/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java index 9fb68f87eef..ab09211e098 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java @@ -46,6 +46,7 @@ public class MockMetadataHelper extends MetadataHelper { protected final Metadata metadata = new Metadata(); private Set indexOnlyFields = new HashSet<>(); + private Set hiddenFields = new HashSet<>(); private Set expansionFields = new HashSet<>(); private Set contentFields = new HashSet<>(); private Set riFields = new HashSet<>(); @@ -179,6 +180,11 @@ public Set getIndexOnlyFields(Set ingestTypeFilter) throws Table return indexOnlyFields; } + @Override + public Set getHiddenFields(Set ingestTypeFilter) throws TableNotFoundException { + return hiddenFields; + } + @Override public QueryModel getQueryModel(String modelTableName, String modelName, Collection unevaluatedFields) throws TableNotFoundException { return models.get(modelName); @@ -405,6 +411,10 @@ public void setIndexOnlyFields(Set indexOnlyFields) { this.indexOnlyFields = indexOnlyFields; } + public void setHiddenFields(Set hiddenFields) { + this.hiddenFields = hiddenFields; + } + public void setNormalizedFields(Set normalizedFields) { getMetadata().normalizedFields = normalizedFields; }