Make FieldExistenceRule check for hidden fields

core/utils/metadata-utils/src/main/java/datawave/query/util/MetadataHelper.java

@@ -717,6 +717,28 @@ public boolean isTokenized(String fieldName, Set<String> ingestTypeFilter) throw
         }
     }
 
+    /**
+     * Determines whether a field has been hidden by looking for the h column in the metadata table
+     *
+     * @param fieldName
+     *            the field name
+     * @param ingestTypeFilter
+     *            the ingest type filter
+     * @return true if the field is hidden for the provided ingest types
+     */
+    public boolean isHidden(String fieldName, Set<String> ingestTypeFilter) {
+        Preconditions.checkNotNull(fieldName);
+        Preconditions.checkNotNull(ingestTypeFilter);
+
+        Entry<String,Entry<String,Set<String>>> entry = Maps.immutableEntry(metadataTableName, Maps.immutableEntry(fieldName, ingestTypeFilter));
+
+        try {
+            return this.allFieldMetadataHelper.isIndexed(ColumnFamilyConstants.COLF_H, entry);
+        } catch (InstantiationException | ExecutionException | TableNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      * Returns a Set of all TextNormalizers in use by any type in Accumulo
      *
@@ -2156,4 +2178,7 @@ public static IteratorSetting getCQRegexFilter(String regex) {
         return cqRegex;
     }
 
+    public AccumuloClient getAccumuloClient() {
+        return this.accumuloClient;
+    }
 }

core/utils/metadata-utils/src/test/java/datawave/query/util/MetadataHelperTest.java

@@ -1,6 +1,7 @@
 package datawave.query.util;
 
 import static datawave.data.ColumnFamilyConstants.COLF_F;
+import static datawave.data.ColumnFamilyConstants.COLF_H;
 import static datawave.query.util.TestUtils.createDateFrequencyMap;
 import static org.apache.accumulo.core.iterators.LongCombiner.VAR_LEN_ENCODER;
 
@@ -104,6 +105,12 @@ private void givenMutation(String row, String columnFamily, String columnQualifi
         givenMutation(mutation);
     }
 
+    private void givenHiddenField(String row, String datatype) {
+        Mutation mutation = new Mutation(row);
+        mutation.put(COLF_H, new Text(datatype), new Value());
+        givenMutation(mutation);
+    }
+
     private void givenNonAggregatedFrequencyRows(String row, Text colf, String datatype, String startDate, String endDate, long count) {
         Mutation mutation = new Mutation(row);
         Value value = new Value(VAR_LEN_ENCODER.encode(count));
@@ -392,4 +399,20 @@ void testMixedEntryFormats() {
             Assertions.assertEquals(DateHelper.parse("20200103"), helper.getEarliestOccurrenceOfFieldWithType("NAME", "maze", accumuloClient, null));
         }
     }
+
+    /**
+     * Test against a table with hidden entries.
+     */
+    @Test
+    void testHiddenEntry() throws TableNotFoundException {
+        givenHiddenField("NAME", "csv");
+        givenHiddenField("NAME", "wiki");
+        givenHiddenField("EVENT_DATE", "maze");
+        writeMutations();
+
+        Assertions.assertTrue(helper.isHidden("NAME", Set.of("csv")));
+        Assertions.assertTrue(helper.isHidden("EVENT_DATE", Set.of()));
+        Assertions.assertFalse(helper.isHidden("NAME", Set.of("foo")));
+        Assertions.assertFalse(helper.isHidden("FOO", Set.of()));
+    }
 }

warehouse/query-core/src/main/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitor.java

@@ -94,7 +94,7 @@ protected Object findMissingFields(JexlNode node, Object data) {
 
         for (ASTIdentifier identifier : identifiers) {
             String fieldName = JexlASTHelper.deconstructIdentifier(identifier);
-            if (!this.allFieldsForDatatypes.contains(fieldName) && !specialFields.contains(fieldName)) {
+            if ((!this.allFieldsForDatatypes.contains(fieldName) && !specialFields.contains(fieldName)) || helper.isHidden(fieldName, this.datatypeFilter)) {
                 nonExistentFieldNames.add(fieldName);
             }
         }
@@ -152,7 +152,8 @@ public Object visit(ASTFunctionNode node, Object data) {
                 // deconstruct the identifier
                 final String testFieldName = JexlASTHelper.deconstructIdentifier(fieldName);
                 // changed to allow _ANYFIELD_ in functions
-                if (!this.allFieldsForDatatypes.contains(testFieldName) && !specialFields.contains(fieldName)) {
+                if ((!this.allFieldsForDatatypes.contains(testFieldName) && !specialFields.contains(fieldName))
+                                || this.helper.isHidden(fieldName, this.datatypeFilter)) {
                     nonExistentFieldNames.add(testFieldName);
                 }
             }

warehouse/query-core/src/test/java/datawave/query/jexl/visitors/FieldMissingFromSchemaVisitorTest.java

@@ -7,6 +7,11 @@
 import java.util.Collections;
 import java.util.Set;
 
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.security.Authorizations;
 import org.apache.commons.jexl3.parser.ASTJexlScript;
 import org.apache.commons.jexl3.parser.ParseException;
 import org.junit.Before;
@@ -16,19 +21,26 @@
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 
+import datawave.accumulo.inmemory.InMemoryAccumuloClient;
+import datawave.accumulo.inmemory.InMemoryInstance;
 import datawave.query.jexl.JexlASTHelper;
 import datawave.query.util.MockMetadataHelper;
+import datawave.util.TableName;
 
 public class FieldMissingFromSchemaVisitorTest {
 
     // Special fields required by visitor.
     private Set<String> specialFields = Sets.newHashSet(ANY_FIELD, NO_FIELD);
-
+    private static final String[] AUTHS = {"FOO", "BAR"};
+    private static final Set<Authorizations> AUTHS_SET = Collections.singleton(new Authorizations(AUTHS));
     private MockMetadataHelper helper;
 
     @Before
-    public void before() {
-        helper = new MockMetadataHelper();
+    public void before() throws AccumuloException, TableExistsException, AccumuloSecurityException {
+        helper = new MockMetadataHelper(AUTHS_SET, AUTHS_SET, AUTHS_SET, getClient());
+        if (!helper.getAccumuloClient().tableOperations().exists(TableName.METADATA)) {
+            helper.getAccumuloClient().tableOperations().create(TableName.METADATA);
+        }
     }
 
     /**
@@ -314,4 +326,12 @@ private void checkEmptyDatatypeFilter(Set<String> expected, ASTJexlScript script
         Set<String> actual = FieldMissingFromSchemaVisitor.getNonExistentFields(helper, script, Collections.emptySet(), specialFields);
         assertEquals(expected, actual);
     }
+
+    private static AccumuloClient getClient() {
+        try {
+            return new InMemoryAccumuloClient("root", new InMemoryInstance());
+        } catch (AccumuloSecurityException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }

warehouse/query-core/src/test/java/datawave/query/rules/FieldExistenceRuleTest.java

@@ -4,19 +4,27 @@
 import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.security.Authorizations;
 import org.easymock.EasyMock;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
+import datawave.accumulo.inmemory.InMemoryAccumuloClient;
+import datawave.accumulo.inmemory.InMemoryInstance;
 import datawave.query.util.MetadataHelper;
 import datawave.query.util.MockMetadataHelper;
+import datawave.util.TableName;
 
 public class FieldExistenceRuleTest extends ShardQueryRuleTest {
 
     private static final Set<String> ALL_FIELDS = Set.of("FOO", "BAR", "BAT");
     private static final String ANYFIELD = "_ANYFIELD_";
-    private static final MockMetadataHelper defaultMetadataHelper = new MockMetadataHelper();
+    private static final String[] AUTHS = {"FOO", "BAR"};
+    private static final Set<Authorizations> AUTHS_SET = Collections.singleton(new Authorizations(AUTHS));
+    private static final MockMetadataHelper defaultMetadataHelper = new MockMetadataHelper(AUTHS_SET, AUTHS_SET, AUTHS_SET, getClient());
 
     private final Set<String> fieldExceptions = new HashSet<>();
 
@@ -30,6 +38,9 @@ public void setUp() throws Exception {
         givenRuleName(RULE_NAME);
         givenMetadataHelper(defaultMetadataHelper);
         expectRuleName(RULE_NAME);
+        if (!defaultMetadataHelper.getAccumuloClient().tableOperations().exists(TableName.METADATA)) {
+            defaultMetadataHelper.getAccumuloClient().tableOperations().create(TableName.METADATA);
+        }
     }
 
     /**
@@ -95,4 +106,12 @@ protected ShardQueryRule getNewRule() {
         rule.setSpecialFields(fieldExceptions);
         return rule;
     }
+
+    private static AccumuloClient getClient() {
+        try {
+            return new InMemoryAccumuloClient("root", new InMemoryInstance());
+        } catch (AccumuloSecurityException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }

warehouse/query-core/src/test/java/datawave/query/util/MockMetadataHelper.java

@@ -73,6 +73,11 @@ public MockMetadataHelper() {
                         Collections.emptySet());
     }
 
+    public MockMetadataHelper(Set<Authorizations> allMetadataAuths, Set<Authorizations> auths, Set<Authorizations> fullUserAuths, AccumuloClient client) {
+        super(createAllFieldMetadataHelperWithCertainAuths(client, allMetadataAuths, auths), allMetadataAuths, client, TableName.METADATA, auths,
+                        fullUserAuths);
+    }
+
     private static AllFieldMetadataHelper createAllFieldMetadataHelper(AccumuloClient client) {
         final Set<Authorizations> allMetadataAuths = Collections.emptySet();
         final Set<Authorizations> auths = Collections.emptySet();
@@ -82,6 +87,14 @@ private static AllFieldMetadataHelper createAllFieldMetadataHelper(AccumuloClien
 
     }
 
+    private static AllFieldMetadataHelper createAllFieldMetadataHelperWithCertainAuths(AccumuloClient client, Set<Authorizations> allMetadataAuths,
+                    Set<Authorizations> auths) {
+        TypeMetadataHelper tmh = new TypeMetadataHelper(Maps.newHashMap(), allMetadataAuths, client, TableName.METADATA, auths, false);
+        CompositeMetadataHelper cmh = new CompositeMetadataHelper(client, TableName.METADATA, auths);
+        return new AllFieldMetadataHelper(tmh, cmh, client, TableName.METADATA, auths, allMetadataAuths);
+
+    }
+
     private static AccumuloClient getClient() {
         try {
             return new InMemoryAccumuloClient("root", new InMemoryInstance());