Merge pull request #271 from InseeFr/feature/check_hierarchy

Feature/check hierarchy
InseeFr · Aug 11, 2023 · c2ef063 · c2ef063
2 parents 7319ad0 + 3c0b831
commit c2ef063
Show file tree

Hide file tree

Showing 15 changed files with 1,074 additions and 43 deletions.
diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml
@@ -1,4 +1,4 @@
-name: Trevas CI
+name: Trevas Tests
 
 on:
   pull_request:

diff --git a/docs/docs/user-guide/coverage/data-validation-operators.mdx b/docs/docs/user-guide/coverage/data-validation-operators.mdx
@@ -9,5 +9,5 @@ custom_edit_url: null
 | Name            | Symbol          |     Supported      |
 | --------------- | :-------------- | :----------------: |
 | Check datapoint | check_datapoint | :heavy_check_mark: |
-| Check hierarchy | check_hierarchy |        :x:         |
+| Check hierarchy | check_hierarchy | :heavy_check_mark: |
 | Check           | check           | :heavy_check_mark: |
diff --git a/...s-plugin-content-docs/current/user-guide/coverage/data-validation-operators.mdx b/...s-plugin-content-docs/current/user-guide/coverage/data-validation-operators.mdx
@@ -9,5 +9,5 @@ custom_edit_url: null
 | Nom                              | Symbole         |     Supported      |
 | -------------------------------- | :-------------- | :----------------: |
 | Validation des points de données | check_datapoint | :heavy_check_mark: |
-| Validation de la hiérarchie      | check_hierarchy |        :x:         |
+| Validation de la hiérarchie      | check_hierarchy | :heavy_check_mark: |
 | Validation                       | check           | :heavy_check_mark: |
diff --git a/...s-plugin-content-docs/current/user-guide/coverage/data-validation-operators.mdx b/...s-plugin-content-docs/current/user-guide/coverage/data-validation-operators.mdx
@@ -9,5 +9,5 @@ custom_edit_url: null
 | Navn              | Symbol          |     Supported      |
 | ----------------- | :-------------- | :----------------: |
 | Sjekk datapunktet | check_datapoint | :heavy_check_mark: |
-| Sjekk hierarki    | check_hierarchy |        :x:         |
+| Sjekk hierarki    | check_hierarchy | :heavy_check_mark: |
 | Sjekk             | check           | :heavy_check_mark: |
diff --git a/...s-plugin-content-docs/current/user-guide/coverage/data-validation-operators.mdx b/...s-plugin-content-docs/current/user-guide/coverage/data-validation-operators.mdx
@@ -9,5 +9,5 @@ custom_edit_url: null
 | Name            | Symbol          |     Supported      |
 | --------------- | :-------------- | :----------------: |
 | Check datapoint | check_datapoint | :heavy_check_mark: |
-| Check hierarchy | check_hierarchy |        :x:         |
+| Check hierarchy | check_hierarchy | :heavy_check_mark: |
 | Check           | check           | :heavy_check_mark: |
diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlScriptEngine.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlScriptEngine.java
@@ -110,6 +110,9 @@ public static Positioned fromContext(ParseTree tree) {
     }
 
     public static Positioned fromTokens(Token from, Token to) {
+        if (to == null) {
+            to = from;
+        }
         var position = new Positioned.Position(
                 from.getLine() - 1,
                 to.getLine() - 1,

diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/processors/InMemoryProcessingEngine.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/processors/InMemoryProcessingEngine.java
@@ -2,17 +2,7 @@
 
 import fr.insee.vtl.engine.utils.KeyExtractor;
 import fr.insee.vtl.engine.utils.MapCollector;
-import fr.insee.vtl.model.AggregationExpression;
-import fr.insee.vtl.model.Analytics;
-import fr.insee.vtl.model.DataPointRuleset;
-import fr.insee.vtl.model.Dataset;
-import fr.insee.vtl.model.DatasetExpression;
-import fr.insee.vtl.model.InMemoryDataset;
-import fr.insee.vtl.model.Positioned;
-import fr.insee.vtl.model.ProcessingEngine;
-import fr.insee.vtl.model.ProcessingEngineFactory;
-import fr.insee.vtl.model.ResolvableExpression;
-import fr.insee.vtl.model.Structured;
+import fr.insee.vtl.model.*;
 
 import javax.script.ScriptEngine;
 import java.util.ArrayList;
@@ -313,6 +303,13 @@ public DatasetExpression executeValidationSimple(DatasetExpression dsE, Resolvab
         throw new UnsupportedOperationException();
     }
 
+    @Override
+    public DatasetExpression executeHierarchicalValidation(DatasetExpression dsE, HierarchicalRuleset hr,
+                                                     String componentID, String validationMode,
+                                                     String inputMode, String validationOutput, Positioned pos) {
+        throw new UnsupportedOperationException();
+    }
+
     /**
      * Returns a structure with the common identifiers only once.
      */

diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AssignmentVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/AssignmentVisitor.java
@@ -5,16 +5,16 @@
 import fr.insee.vtl.engine.exceptions.VtlRuntimeException;
 import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor;
 import fr.insee.vtl.model.*;
+import fr.insee.vtl.model.exceptions.VtlScriptException;
 import fr.insee.vtl.parser.VtlBaseVisitor;
 import fr.insee.vtl.parser.VtlParser;
 import org.antlr.v4.runtime.tree.TerminalNode;
 
 import javax.script.Bindings;
 import javax.script.ScriptContext;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import javax.script.ScriptException;
+import javax.script.SimpleBindings;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
@@ -139,4 +139,139 @@ public Object visitDefDatapointRuleset(VtlParser.DefDatapointRulesetContext ctx)
         bindings.put(rulesetName, dataPointRuleset);
         return dataPointRuleset;
     }
+
+    // TODO: handle when clause (expr ctx)
+    @Override
+    public Object visitDefHierarchical(VtlParser.DefHierarchicalContext ctx) {
+        var pos = fromContext(ctx);
+        String rulesetName = ctx.rulesetID().getText();
+
+        // Mix variables and valuedomain. Information useless for now, find use case to do so
+        String variable = ctx.hierRuleSignature().IDENTIFIER().getText();
+
+        Set<Class<?>> erCodeTypes = ctx.ruleClauseHierarchical().ruleItemHierarchical().stream().map(c -> {
+            VtlParser.ErCodeContext erCodeContext = c.erCode();
+            if (null == erCodeContext) return Object.class;
+            return expressionVisitor.visit(c.erCode()).getType();
+        }).collect(Collectors.toSet());
+        List<Class<?>> filteredErCodeTypes = erCodeTypes.stream().filter(t -> !t.equals(Object.class)).collect(Collectors.toList());
+        if (filteredErCodeTypes.size() > 1) {
+            throw new VtlRuntimeException(
+                    new InvalidArgumentException("Error codes of rules have different types", pos)
+            );
+        }
+        Class<?> erCodeType = filteredErCodeTypes.isEmpty() ? String.class : filteredErCodeTypes.iterator().next();
+
+        Set<Class<?>> erLevelTypes = ctx.ruleClauseHierarchical().ruleItemHierarchical().stream().map(c -> {
+            VtlParser.ErLevelContext erLevelContext = c.erLevel();
+            if (null == erLevelContext) return Object.class;
+            return expressionVisitor.visit(c.erLevel()).getType();
+        }).collect(Collectors.toSet());
+        List<Class<?>> filteredErLevelTypes = erLevelTypes.stream().filter(t -> !t.equals(Object.class)).collect(Collectors.toList());
+        if (filteredErLevelTypes.size() > 1) {
+            throw new VtlRuntimeException(
+                    new InvalidArgumentException("Error levels of rules have different types", pos)
+            );
+        }
+        Class<?> erLevelType = filteredErLevelTypes.isEmpty() ? Long.class : filteredErLevelTypes.iterator().next();
+
+        AtomicInteger index = new AtomicInteger();
+        List<HierarchicalRule> rules = ctx.ruleClauseHierarchical().ruleItemHierarchical()
+                .stream()
+                .map(r -> {
+                    TerminalNode identifier = r.IDENTIFIER();
+                    int i = index.getAndIncrement() + 1;
+                    String ruleName = null != identifier ? identifier.getText() : rulesetName + "_" + i;
+
+                    List<String> codeItems = new ArrayList<>();
+                    VtlParser.CodeItemRelationContext codeItemRelationContext = r.codeItemRelation();
+                    String valueDomainValue = codeItemRelationContext.valueDomainValue().IDENTIFIER().getText();
+                    codeItems.add(valueDomainValue);
+
+                    VtlParser.ComparisonOperandContext comparisonOperandContext = codeItemRelationContext.comparisonOperand();
+
+                    StringBuilder codeItemExpressionBuilder = new StringBuilder();
+                    codeItemRelationContext.codeItemRelationClause()
+                            .forEach(circ -> {
+                                TerminalNode minus = circ.MINUS();
+                                String rightCodeItem = circ.rightCodeItem.getText();
+                                codeItems.add(rightCodeItem);
+                                if (minus != null)
+                                    codeItemExpressionBuilder.append(" -").append(rightCodeItem);
+                                // plus value or plus null & minus null mean plus
+                                codeItemExpressionBuilder.append(" +").append(rightCodeItem);
+                            });
+
+                    String rightExpressionToEval = codeItemExpressionBuilder.toString();
+                    String expressionToEval = "bool_var := " +
+                            valueDomainValue + " " +
+                            comparisonOperandContext.getText() + " " +
+                            rightExpressionToEval + ";";
+
+                    ResolvableExpression leftExpression = ResolvableExpression.withType(Double.class)
+                            .withPosition(pos)
+                            .using(context -> {
+                                Bindings bindings = new SimpleBindings(context);
+                                bindings.forEach((k, v) -> engine.getContext().setAttribute(k, v, ScriptContext.ENGINE_SCOPE));
+                                try {
+                                    engine.eval("left := " + valueDomainValue + ";");
+                                    Object left = engine.getContext().getAttribute("left");
+                                    engine.getContext().removeAttribute("left", ScriptContext.ENGINE_SCOPE);
+                                    bindings.keySet().forEach(k -> engine.getContext().removeAttribute(k, ScriptContext.ENGINE_SCOPE));
+                                    if (left.getClass().isAssignableFrom(Double.class)) {
+                                        return (Double) left;
+                                    }
+                                    return ((Long) left).doubleValue();
+                                } catch (ScriptException e) {
+                                    throw new VtlRuntimeException(new VtlScriptException(
+                                            "right hierarchical rule has to return long or double", pos));
+                                }
+                            });
+
+                    ResolvableExpression rightExpression = ResolvableExpression.withType(Double.class)
+                            .withPosition(pos)
+                            .using(context -> {
+                                Bindings bindings = new SimpleBindings(context);
+                                bindings.forEach((k, v) -> engine.getContext().setAttribute(k, v, ScriptContext.ENGINE_SCOPE));
+                                try {
+                                    engine.eval("right := " + rightExpressionToEval + ";");
+                                    Object right = engine.getContext().getAttribute("right");
+                                    engine.getContext().removeAttribute("right", ScriptContext.ENGINE_SCOPE);
+                                    bindings.keySet().forEach(k -> engine.getContext().removeAttribute(k, ScriptContext.ENGINE_SCOPE));
+                                    if (right.getClass().isAssignableFrom(Double.class)) {
+                                        return (Double) right;
+                                    }
+                                    return ((Long) right).doubleValue();
+                                } catch (ScriptException e) {
+                                    throw new VtlRuntimeException(new VtlScriptException(
+                                            "right hierarchical rule has to return long or double", pos));
+                                }
+                            });
+
+                    ResolvableExpression expression = ResolvableExpression.withType(Boolean.class)
+                            .withPosition(pos)
+                            .using(context -> {
+                                Bindings bindings = new SimpleBindings(context);
+                                bindings.forEach((k, v) -> engine.getContext().setAttribute(k, v, ScriptContext.ENGINE_SCOPE));
+                                try {
+                                    engine.eval(expressionToEval);
+                                    Boolean boolVar = (Boolean) engine.getContext().getAttribute("bool_var");
+                                    engine.getContext().removeAttribute("bool_var", ScriptContext.ENGINE_SCOPE);
+                                    bindings.keySet().forEach(k -> engine.getContext().removeAttribute(k, ScriptContext.ENGINE_SCOPE));
+                                    return boolVar;
+                                } catch (ScriptException e) {
+                                    throw new VtlRuntimeException(new VtlScriptException(
+                                            "hierarchical rule has to return boolean", pos));
+                                }
+                            });
+
+                    ResolvableExpression errorCodeExpression = null != r.erCode() ? expressionVisitor.visit(r.erCode()) : null;
+                    ResolvableExpression errorLevelExpression = null != r.erLevel() ? expressionVisitor.visit(r.erLevel()) : null;
+                    return new HierarchicalRule(ruleName, valueDomainValue, expression, leftExpression, rightExpression, codeItems, errorCodeExpression, errorLevelExpression);
+                }).collect(Collectors.toList());
+        HierarchicalRuleset hr = new HierarchicalRuleset(rules, variable, erCodeType, erLevelType);
+        Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE);
+        bindings.put(rulesetName, hr);
+        return hr;
+    }
 }
diff --git a/...in/java/fr/insee/vtl/engine/visitors/expression/functions/ValidationFunctionsVisitor.java b/...in/java/fr/insee/vtl/engine/visitors/expression/functions/ValidationFunctionsVisitor.java
@@ -5,12 +5,7 @@
 import fr.insee.vtl.engine.exceptions.UndefinedVariableException;
 import fr.insee.vtl.engine.exceptions.VtlRuntimeException;
 import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor;
-import fr.insee.vtl.model.DataPointRuleset;
-import fr.insee.vtl.model.Dataset;
-import fr.insee.vtl.model.DatasetExpression;
-import fr.insee.vtl.model.ProcessingEngine;
-import fr.insee.vtl.model.ResolvableExpression;
-import fr.insee.vtl.model.Structured;
+import fr.insee.vtl.model.*;
 import fr.insee.vtl.parser.VtlBaseVisitor;
 import fr.insee.vtl.parser.VtlParser;
 
@@ -140,8 +135,62 @@ public ResolvableExpression visitValidationSimple(VtlParser.ValidationSimpleCont
         return processingEngine.executeValidationSimple(dsExpression, erCodeExpression, erLevelExpression, imbalanceExpression, output, pos);
     }
 
+    // TODO: handle other IDs than componentID? build unique ID tuples to calculate
+    @Override
+    public ResolvableExpression visitValidateHRruleset(VtlParser.ValidateHRrulesetContext ctx) {
+        var pos = fromContext(ctx);
+        DatasetExpression dsExpression = (DatasetExpression) assertTypeExpression(expressionVisitor.visit(ctx.expr()),
+                Dataset.class, ctx.expr());
+        String datasetName = ctx.expr().getText();
+        String hrName = ctx.hrName.getText();
+        Object hrObject = engine.getContext().getAttribute((hrName));
+        if (!(hrObject instanceof HierarchicalRuleset))
+            throw new VtlRuntimeException(new UndefinedVariableException(hrName, pos));
+        HierarchicalRuleset hr = (HierarchicalRuleset) hrObject;
+
+        // Check that dsE is a monomeasure<number> dataset
+        Structured.DataStructure dataStructure = dsExpression.getDataStructure();
+        List<Structured.Component> measures = dataStructure.getMeasures();
+        if (measures.size() != 1) {
+            throw new VtlRuntimeException(
+                    new InvalidArgumentException("Dataset " + datasetName +
+                            " is not monomeasure", fromContext(ctx))
+            );
+        }
+        List<Class<?>> supportedClasses = new ArrayList<>(Arrays.asList(Double.class, Long.class));
+        if (!supportedClasses.contains(measures.get(0).getType())) {
+            throw new VtlRuntimeException(
+                    new InvalidArgumentException("Dataset " + datasetName +
+                            " measure " + measures.get(0).getName() + " has to have number type", fromContext(ctx))
+            );
+        }
+
+        // check if hr componentID is in ds structure
+        String componentID = ctx.componentID().getText();
+        if (!dataStructure.containsKey(componentID)) {
+            throw new VtlRuntimeException(
+                    new InvalidArgumentException("ComponentID " + componentID +
+                            " not contained in dataset " + datasetName, fromContext(ctx))
+            );
+        }
+        String validationMode = getValidationMode(ctx.validationMode());
+        String inputMode = getInputMode(ctx.inputMode());
+        String validationOutput = getValidationOutput(ctx.validationOutput());
+        return processingEngine.executeHierarchicalValidation(dsExpression, hr, componentID, validationMode, inputMode, validationOutput, pos);
+    }
+
     private String getValidationOutput(VtlParser.ValidationOutputContext voc) {
         if (null == voc) return null;
         return voc.getText();
     }
+
+    private String getValidationMode(VtlParser.ValidationModeContext vmc) {
+        if (null == vmc) return null;
+        return vmc.getText();
+    }
+
+    private String getInputMode(VtlParser.InputModeContext imc) {
+        if (null == imc) return null;
+        return imc.getText();
+    }
 }
diff --git a/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/AssignmentTest.java b/vtl-engine/src/test/java/fr/insee/vtl/engine/visitors/AssignmentTest.java
@@ -3,15 +3,12 @@
 import fr.insee.vtl.engine.samples.DatasetSamples;
 import fr.insee.vtl.model.DataPointRuleset;
 import fr.insee.vtl.model.Dataset;
+import fr.insee.vtl.model.HierarchicalRuleset;
 import fr.insee.vtl.model.exceptions.VtlScriptException;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-import javax.script.Bindings;
-import javax.script.ScriptContext;
-import javax.script.ScriptEngine;
-import javax.script.ScriptEngineManager;
-import javax.script.ScriptException;
+import javax.script.*;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -115,4 +112,26 @@ public void testMembership() throws ScriptException {
                 .hasMessage("column baaaddd not found in ds");
     }
 
+    @Test
+    public void checkHierarchy() throws ScriptException {
+
+        String hierarchicalRulesetDef = "define hierarchical ruleset HR_1 (variable rule Me_1) is \n" +
+                "R010 : A = J + K + L errorlevel 5;\n" +
+                "R020 : B = M + N + O errorlevel 5;\n" +
+                "R030 : C = P + Q errorcode \"XX\" errorlevel 5;\n" +
+                "R040 : D = R + S errorlevel 1;\n" +
+                "R050 : E = T + U + V errorlevel 0;\n" +
+                "R060 : F = Y + W + Z errorlevel 7;\n" +
+                "R070 : G = B + C;\n" +
+                "R080 : H = D + E errorlevel 0;\n" +
+                "R090 : I = D + G errorcode \"YY\" errorlevel 0;\n" +
+                "R100 : M >= N errorlevel 5;\n" +
+                "R110 : M <= G errorlevel 5\n" +
+                "end hierarchical ruleset; \n";
+
+        engine.eval(hierarchicalRulesetDef);
+        HierarchicalRuleset hr1 = (HierarchicalRuleset) engine.getContext().getAttribute("HR_1");
+        assertThat(hr1.getRules()).hasSize(11);
+    }
+
 }