Make FieldExistenceRule check for hidden fields

core/utils/metadata-utils/src/main/java/datawave/query/util/AllFieldMetadataHelper.java

@@ -1108,6 +1108,36 @@ public Multimap<String,String> loadIndexedFields() throws TableNotFoundException
         return Multimaps.unmodifiableMultimap(fields);
     }
 
+    /**
+     * Fetch the Set of all fields marked as being hidden, {@link ColumnFamilyConstants#COLF_H}. Returns a multimap of datatype to field
+     *
+     * @return a multimap of datatypes to hidden fields
+     * @throws TableNotFoundException
+     *             if no table exists
+     */
+    @Cacheable(value = "loadHiddenFields", key = "{#root.target.auths,#root.target.metadataTableName}", cacheManager = "metadataHelperCacheManager",
+                    sync = true)
+    public Multimap<String,String> loadHiddenFields() throws TableNotFoundException {
+        log.debug("cache fault for loadHiddenFields({}, {})", this.auths, this.metadataTableName);
+
+        if (log.isTraceEnabled()) {
+            log.trace("loadHiddenFields from table: {}", metadataTableName);
+        }
+
+        Multimap<String,String> fields = HashMultimap.create();
+
+        try (Scanner bs = ScannerHelper.createScanner(accumuloClient, metadataTableName, auths)) {
+            bs.setRange(new Range());
+            bs.fetchColumnFamily(ColumnFamilyConstants.COLF_H);
+
+            for (Entry<Key,Value> entry : bs) {
+                fields.put(getDatatype(entry.getKey()), entry.getKey().getRow().toString());
+            }
+        }
+
+        return Multimaps.unmodifiableMultimap(fields);
+    }
+
     /**
      * Fetch the Set of all fields marked as being reverse indexed, {@link ColumnFamilyConstants#COLF_RI}. Returns a multimap of datatype to field
      *

core/utils/metadata-utils/src/main/java/datawave/query/util/MetadataHelper.java

@@ -717,6 +717,30 @@ public boolean isTokenized(String fieldName, Set<String> ingestTypeFilter) throw
         }
     }
 
+    /**
+     * Get the set of hidden fields for the provided ingest type filter. A null or empty filter indicates all hidden fields should be returned.
+     *
+     * @param ingestTypeFilter
+     *            the ingest type filter
+     * @return the set of hidden fields given the provided ingest type filter
+     * @throws TableNotFoundException
+     *             if the table does not exist
+     */
+    public Set<String> getHiddenFields(Set<String> ingestTypeFilter) throws TableNotFoundException {
+
+        Multimap<String,String> hiddenFields = this.allFieldMetadataHelper.loadHiddenFields();
+
+        Set<String> fields = new HashSet<>();
+        if (ingestTypeFilter == null || ingestTypeFilter.isEmpty()) {
+            fields.addAll(hiddenFields.values());
+        } else {
+            for (String datatype : ingestTypeFilter) {
+                fields.addAll(hiddenFields.get(datatype));
+            }
+        }
+        return Collections.unmodifiableSet(fields);
+    }
+
     /**
      * Returns a Set of all TextNormalizers in use by any type in Accumulo
      *
@@ -2155,5 +2179,4 @@ public static IteratorSetting getCQRegexFilter(String regex) {
         RegExFilter.setRegexs(cqRegex, null, null, regex, null, false);
         return cqRegex;
     }
-
 }

core/utils/metadata-utils/src/test/java/datawave/query/util/MetadataHelperTest.java

@@ -1,6 +1,7 @@
 package datawave.query.util;
 
 import static datawave.data.ColumnFamilyConstants.COLF_F;
+import static datawave.data.ColumnFamilyConstants.COLF_H;
 import static datawave.query.util.TestUtils.createDateFrequencyMap;
 import static org.apache.accumulo.core.iterators.LongCombiner.VAR_LEN_ENCODER;
 
@@ -104,6 +105,12 @@ private void givenMutation(String row, String columnFamily, String columnQualifi
         givenMutation(mutation);
     }
 
+    private void givenHiddenField(String row, String datatype) {
+        Mutation mutation = new Mutation(row);
+        mutation.put(COLF_H, new Text(datatype), new Value());
+        givenMutation(mutation);
+    }
+
     private void givenNonAggregatedFrequencyRows(String row, Text colf, String datatype, String startDate, String endDate, long count) {
         Mutation mutation = new Mutation(row);
         Value value = new Value(VAR_LEN_ENCODER.encode(count));
@@ -392,4 +399,20 @@ void testMixedEntryFormats() {
             Assertions.assertEquals(DateHelper.parse("20200103"), helper.getEarliestOccurrenceOfFieldWithType("NAME", "maze", accumuloClient, null));
         }
     }
+
+    /**
+     * Test against a table with hidden entries.
+     */
+    @Test
+    void testHiddenEntry() throws TableNotFoundException {
+        givenHiddenField("NAME", "csv");
+        givenHiddenField("NAME", "wiki");
+        givenHiddenField("EVENT_DATE", "maze");
+        writeMutations();
+
+        Assertions.assertTrue(helper.getHiddenFields(Set.of("csv")).contains("NAME"));
+        Assertions.assertTrue(helper.getHiddenFields(Set.of()).contains("EVENT_DATE"));
+        Assertions.assertFalse(helper.getHiddenFields(Set.of("foo")).contains("NAME"));
+        Assertions.assertFalse(helper.getHiddenFields(Set.of()).contains("FOO"));
+    }
 }

warehouse/query-core/src/main/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitor.java

@@ -43,6 +43,7 @@ public class FieldMissingFromSchemaVisitor extends ShortCircuitBaseVisitor {
     private final Set<String> allFieldsForDatatypes; // All fields for the specified datatypes pulled from MetadataHelper
     private final Set<String> specialFields;
     private final Set<String> datatypeFilter;
+    private final Set<String> allHiddenFields;
 
     public FieldMissingFromSchemaVisitor(MetadataHelper helper, Set<String> datatypeFilter, Set<String> specialFields) {
         this.helper = helper;
@@ -53,6 +54,7 @@ public FieldMissingFromSchemaVisitor(MetadataHelper helper, Set<String> datatype
                 datatypeFilter = Collections.emptySet();
             }
             this.allFieldsForDatatypes = this.helper.getAllFields(datatypeFilter);
+            this.allHiddenFields = this.helper.getHiddenFields(datatypeFilter);
         } catch (TableNotFoundException e) {
             log.error(e);
             throw new RuntimeException("Unable to get metadata", e);
@@ -94,7 +96,7 @@ protected Object findMissingFields(JexlNode node, Object data) {
 
         for (ASTIdentifier identifier : identifiers) {
             String fieldName = JexlASTHelper.deconstructIdentifier(identifier);
-            if (!this.allFieldsForDatatypes.contains(fieldName) && !specialFields.contains(fieldName)) {
+            if (!this.allFieldsForDatatypes.contains(fieldName) && !specialFields.contains(fieldName) || this.allHiddenFields.contains(fieldName)) {
                 nonExistentFieldNames.add(fieldName);
             }
         }
@@ -152,7 +154,7 @@ public Object visit(ASTFunctionNode node, Object data) {
                 // deconstruct the identifier
                 final String testFieldName = JexlASTHelper.deconstructIdentifier(fieldName);
                 // changed to allow _ANYFIELD_ in functions
-                if (!this.allFieldsForDatatypes.contains(testFieldName) && !specialFields.contains(fieldName)) {
+                if (!this.allFieldsForDatatypes.contains(testFieldName) && !specialFields.contains(fieldName) || this.allHiddenFields.contains(fieldName)) {
                     nonExistentFieldNames.add(testFieldName);
                 }
             }

warehouse/query-core/src/test/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitorTest.java

@@ -23,7 +23,6 @@ public class FieldMissingFromSchemaVisitorTest {
 
     // Special fields required by visitor.
     private Set<String> specialFields = Sets.newHashSet(ANY_FIELD, NO_FIELD);
-
     private MockMetadataHelper helper;
 
     @Before

warehouse/query-core/src/test/java/datawave/query/planner/ScanHintRulesTest.java

@@ -58,6 +58,7 @@ public void setup() {
 
     private void setupMetadata() throws TableNotFoundException, InstantiationException, IllegalAccessException {
         expect(metadataHelper.getIndexedFields(EasyMock.anyObject())).andReturn(Collections.emptySet()).anyTimes();
+        expect(metadataHelper.getHiddenFields(EasyMock.anyObject())).andReturn(Collections.emptySet()).anyTimes();
         expect(metadataHelper.getReverseIndexedFields(EasyMock.anyObject())).andReturn(Collections.emptySet()).anyTimes();
         expect(metadataHelper.getIndexOnlyFields(EasyMock.anyObject())).andReturn(Collections.emptySet()).anyTimes();
         expect(metadataHelper.getAllFields(EasyMock.anyObject())).andReturn(Collections.singleton("A")).anyTimes();

warehouse/query-core/src/test/java/datawave/query/rules/FieldExistenceRuleTest.java

@@ -14,15 +14,17 @@
 
 public class FieldExistenceRuleTest extends ShardQueryRuleTest {
 
-    private static final Set<String> ALL_FIELDS = Set.of("FOO", "BAR", "BAT");
+    private static final Set<String> ALL_FIELDS = Set.of("FOO", "BAR", "BAT", "HIDDEN_FIELD");
     private static final String ANYFIELD = "_ANYFIELD_";
+    private static final Set<String> hiddenFields = Set.of("HIDDEN_FIELD");
     private static final MockMetadataHelper defaultMetadataHelper = new MockMetadataHelper();
 
     private final Set<String> fieldExceptions = new HashSet<>();
 
     @BeforeAll
     public static void beforeClass() throws Exception {
         defaultMetadataHelper.addFields(ALL_FIELDS);
+        defaultMetadataHelper.setHiddenFields(hiddenFields);
     }
 
     @BeforeEach
@@ -52,6 +54,16 @@ public void testNonExistentFields() throws Exception {
         assertResult();
     }
 
+    /**
+     * Test a query with a hidden field.
+     */
+    @Test
+    public void testHiddenFields() throws Exception {
+        givenQuery("FOO == 'abc' || BAR =~ 'abc' || filter:includeRegex(SHENANIGANS, '45.8') || HIDDEN_FIELD == 'aa'");
+        expectMessage("Fields not found in data dictionary: SHENANIGANS, HIDDEN_FIELD");
+        assertResult();
+    }
+
     /**
      * Test a query that has a non-existent field that is a special field.
      */

warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java

@@ -46,6 +46,7 @@
 public class MockMetadataHelper extends MetadataHelper {
     protected final Metadata metadata = new Metadata();
     private Set<String> indexOnlyFields = new HashSet<>();
+    private Set<String> hiddenFields = new HashSet<>();
     private Set<String> expansionFields = new HashSet<>();
     private Set<String> contentFields = new HashSet<>();
     private Set<String> riFields = new HashSet<>();
@@ -179,6 +180,11 @@ public Set<String> getIndexOnlyFields(Set<String> ingestTypeFilter) throws Table
         return indexOnlyFields;
     }
 
+    @Override
+    public Set<String> getHiddenFields(Set<String> ingestTypeFilter) throws TableNotFoundException {
+        return hiddenFields;
+    }
+
     @Override
     public QueryModel getQueryModel(String modelTableName, String modelName, Collection<String> unevaluatedFields) throws TableNotFoundException {
         return models.get(modelName);
@@ -405,6 +411,10 @@ public void setIndexOnlyFields(Set<String> indexOnlyFields) {
         this.indexOnlyFields = indexOnlyFields;
     }
 
+    public void setHiddenFields(Set<String> hiddenFields) {
+        this.hiddenFields = hiddenFields;
+    }
+
     public void setNormalizedFields(Set<String> normalizedFields) {
         getMetadata().normalizedFields = normalizedFields;
     }