Validation / Add Shacl validator

services/src/main/java/org/fao/geonet/api/records/ShaclValidationApi.java

@@ -0,0 +1,192 @@
+/*
+ * Copyright (C) 2001-2016 Food and Agriculture Organization of the
+ * United Nations (FAO-UN), United Nations World Food Programme (WFP)
+ * and United Nations Environment Programme (UNEP)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+ *
+ * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+ * Rome - Italy. email: [email protected]
+ */
+
+package org.fao.geonet.api.records;
+
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.ExampleObject;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import java.util.List;
+import javax.servlet.http.HttpServletRequest;
+import jeeves.server.context.ServiceContext;
+import jeeves.services.ReadWriteController;
+import org.fao.geonet.api.ApiParams;
+import static org.fao.geonet.api.ApiParams.API_CLASS_RECORD_OPS;
+import static org.fao.geonet.api.ApiParams.API_CLASS_RECORD_TAG;
+import static org.fao.geonet.api.ApiParams.API_PARAM_RECORD_UUID;
+import org.fao.geonet.api.ApiUtils;
+import org.fao.geonet.api.tools.i18n.LanguageUtils;
+import org.fao.geonet.domain.AbstractMetadata;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestHeader;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+@RequestMapping(value = {"/{portal}/api/records"})
+@Tag(name = API_CLASS_RECORD_TAG, description = API_CLASS_RECORD_OPS)
+@Controller("shaclValidationApi")
+@PreAuthorize("hasAuthority('Editor')")
+@ReadWriteController
+public class ShaclValidationApi {
+
+    @Autowired
+    LanguageUtils languageUtils;
+
+    @Autowired
+    ShaclValidationService shaclValidationService;
+
+    @io.swagger.v3.oas.annotations.Operation(
+        summary = "Validate a record using [SHACL](https://www.w3.org/TR/shacl/)",
+        description = "User MUST be able to edit the record to validate it.\n" +
+            "\n" +
+            "Use a testsuite (preferred) OR define one or more SHACL shapes to validate the record.\n" +
+            "Validation is done using the [JENA library](https://jena.apache.org/documentation/shacl/)."
+    )
+    @RequestMapping(
+        value = "/{metadataUuid}/validate/shacl",
+        method = {
+            RequestMethod.GET,
+        },
+        produces = {
+            MediaType.APPLICATION_XML_VALUE,
+            MediaType.APPLICATION_JSON_VALUE,
+            "application/ld+json",
+            "text/turtle",
+            "application/rdf+xml"
+        }
+    )
+    @ResponseStatus(HttpStatus.OK)
+    @PreAuthorize("hasAuthority('Editor')")
+    @ApiResponses(value = {@ApiResponse(responseCode = "201", description = "Validation report."),
+        @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_CAN_EDIT)})
+    public @ResponseBody
+    String validateRecordUsingShacl(
+        @Parameter(description = API_PARAM_RECORD_UUID, required = true)
+        @PathVariable
+        String metadataUuid,
+        @Parameter(description = "Formatter to validate",
+            examples = {
+                @ExampleObject(
+                    name = "DCAT",
+                    value = "dcat"
+                ),
+                @ExampleObject(
+                    name = "EU DCAT AP",
+                    value = "eu-dcat-ap"
+                ),
+                @ExampleObject(
+                    name = "EU DCAT HVD",
+                    value = "eu-dcat-ap-hvd"
+                )
+            },
+            required = false)
+        @RequestParam(required = false, defaultValue = "dcat")
+        String formatter,
+        @Parameter(description = "SHACL testsuite to use",
+            examples = {
+                @ExampleObject(
+                    name = "EU DCAT AP 3.0.1 - Base Zero (no background knowledge)",
+                    value = "EU DCAT AP 3.0.1 - Base Zero (no background knowledge)"
+                ),
+                @ExampleObject(
+                    name = "EU DCAT AP 3.0.1 - Ranges Zero (no background knowledge)",
+                    value = "EU DCAT AP 3.0.1 - Ranges Zero (no background knowledge)"
+                ),
+                @ExampleObject(
+                    name = "EU DCAT AP 3.0.1 - Full (no background knowledge)",
+                    value = "EU DCAT AP 3.0.1 - Full (no background knowledge)"
+                )
+            },
+            required = false)
+        @RequestParam(required = false) String testsuite,
+        @Parameter(description = "SHACL shapes to use", required = false)
+        @RequestParam(required = false) List<String> shapeModel,
+        @Parameter(description = "Save validation status. When set to true, the validation status will be saved in the database with a validation type set as `shacl-{formatter}-{testsuite_or_shapeshash}`.",
+            required = false)
+        @RequestParam(required = false, defaultValue = "false") boolean isSavingValidationStatus,
+        HttpServletRequest request,
+        @Parameter(hidden = true)
+        @RequestHeader(value = HttpHeaders.ACCEPT, defaultValue = MediaType.APPLICATION_XML_VALUE)
+        String acceptHeader) throws Exception {
+        AbstractMetadata metadata = ApiUtils.canEditRecord(metadataUuid, request);
+
+        ServiceContext context = ApiUtils.createServiceContext(request);
+        return shaclValidationService.validate(formatter, metadata, testsuite, shapeModel, context, acceptHeader, isSavingValidationStatus);
+    }
+
+
+    @io.swagger.v3.oas.annotations.Operation(
+        summary = "Get available SHACL testsuites",
+        description = "Returns a list of available SHACL testsuites (configured in `config-shacl-validator.xml`). " +
+            "A testsuite is a set of SHACL shapes. " +
+            "Rules are common for all schemas and a proper formatter MUST be used to validate metadata (eg. use `eu-dcat-ap` formatter to apply `eu-dcat-ap-300` testsuite). "
+    )
+    @RequestMapping(
+        value = "/{metadataUuid}/validate/shacl/testsuites",
+        method = RequestMethod.GET,
+        produces = {
+            MediaType.APPLICATION_JSON_VALUE
+        }
+    )
+    @ResponseStatus(HttpStatus.OK)
+    @PreAuthorize("hasAuthority('Editor')")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "List of SHACL testsuites."),
+        @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_CAN_EDIT)})
+    public @ResponseBody
+    List<String> getShaclTestsuites() throws Exception {
+        return shaclValidationService.getShaclValidationTestsuites();
+    }
+
+
+    @io.swagger.v3.oas.annotations.Operation(
+        summary = "Get available SHACL shapes",
+        description = "Returns a list of available SHACL shapes (files with .ttl extension in the shacl directory). " +
+            "Rules are common for all schemas and a proper formatter MUST be used to validate metadata (eg. use `eu-dcat-ap` formatter to apply `eu-dcat-ap-300` testsuite). "
+    )
+    @RequestMapping(
+        value = "/{metadataUuid}/validate/shacl/testsuites/shapes",
+        method = RequestMethod.GET,
+        produces = {
+            MediaType.APPLICATION_JSON_VALUE
+        }
+    )
+    @ResponseStatus(HttpStatus.OK)
+    @PreAuthorize("hasAuthority('Editor')")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "List of SHACL rules."),
+        @ApiResponse(responseCode = "403", description = ApiParams.API_RESPONSE_NOT_ALLOWED_CAN_EDIT)})
+    public @ResponseBody
+    List<String> getShaclShapes() throws Exception {
+        return shaclValidationService.getShaclValidationFiles();
+    }
+}

services/src/main/java/org/fao/geonet/api/records/ShaclValidationService.java

@@ -0,0 +1,201 @@
+package org.fao.geonet.api.records;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.Resource;
+import jeeves.server.context.ServiceContext;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.jena.graph.Node;
+import org.apache.jena.graph.compose.MultiUnion;
+import org.apache.jena.rdf.model.Model;
+import org.apache.jena.rdf.model.ModelFactory;
+import org.apache.jena.reasoner.Reasoner;
+import org.apache.jena.reasoner.ReasonerRegistry;
+import org.apache.jena.riot.Lang;
+import org.apache.jena.riot.RDFDataMgr;
+import org.apache.jena.shacl.ShaclValidator;
+import org.apache.jena.shacl.Shapes;
+import org.apache.jena.shacl.ValidationReport;
+import org.apache.jena.shacl.lib.ShLib;
+import org.fao.geonet.api.records.formatters.FormatType;
+import org.fao.geonet.api.records.formatters.FormatterApi;
+import org.fao.geonet.api.records.formatters.FormatterWidth;
+import org.fao.geonet.api.records.formatters.cache.Key;
+import org.fao.geonet.constants.Geonet;
+import org.fao.geonet.domain.AbstractMetadata;
+import org.fao.geonet.domain.MetadataValidation;
+import org.fao.geonet.domain.MetadataValidationId;
+import org.fao.geonet.domain.MetadataValidationStatus;
+import org.fao.geonet.kernel.GeonetworkDataDirectory;
+import org.fao.geonet.kernel.datamanager.IMetadataIndexer;
+import org.fao.geonet.repository.MetadataValidationRepository;
+import org.fao.geonet.utils.Log;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ShaclValidationService {
+
+    private static final Map<String, Lang> OUTPUT_MAP = Map.of(
+        MediaType.APPLICATION_JSON_VALUE, Lang.JSONLD,
+        "application/ld+json", Lang.JSONLD,
+        MediaType.APPLICATION_XML_VALUE, Lang.RDFXML,
+        "application/rdf+xml", Lang.RDFXML,
+        "text/turtle", Lang.TURTLE,
+        "text/n3", Lang.N3,
+        "application/n-triples", Lang.NTRIPLES
+    );
+
+    @Resource(name = "shaclValidatorTestsuites")
+    private Map<String, String[]> testsuites;
+
+    @Autowired
+    MetadataValidationRepository metadataValidationRepository;
+
+    @Autowired
+    private IMetadataIndexer metadataIndexer;
+
+    @Autowired
+    private GeonetworkDataDirectory dataDirectory;
+
+    public String convertMetadataToRdf(AbstractMetadata metadata, String formatter, ServiceContext context) {
+        try {
+            Key key = new Key(metadata.getId(), "eng", FormatType.xml, formatter, true, FormatterWidth._100);
+            byte[] data = new FormatterApi().new FormatMetadata(context, key, null).call().data;
+            return new String(data, StandardCharsets.UTF_8);
+        } catch (Exception e) {
+            return "Error loading metadata: " + e.getMessage();
+        }
+    }
+
+    public List<String> getShaclValidationFiles() {
+        Path shaclRulesFolder = dataDirectory.getConfigDir().resolve("shacl");
+        try (Stream<Path> paths = Files.walk(shaclRulesFolder)) {
+            return paths.filter(path -> path.toString().endsWith(".ttl"))
+                .map(path -> shaclRulesFolder.relativize(path).toString())
+                .collect(Collectors.toList());
+        } catch (Exception e) {
+            return List.of();
+        }
+    }
+
+    public List<String> getShaclValidationTestsuites() {
+        return testsuites.keySet().stream()
+            .collect(Collectors.toList());
+    }
+
+    public String validate(String formatter, AbstractMetadata metadata,
+                           String testsuite, List<String> shaclShapes,
+                           ServiceContext context,
+                           String outputFormat, boolean isSavingValidationStatus) {
+        String rdfToValidate = convertMetadataToRdf(metadata, formatter, context);
+
+        shaclShapes = getShaclShapes(testsuite, shaclShapes);
+        Shapes shapes = parseShapesFromFiles(shaclShapes);
+        Reasoner reasoner = configureReasoner(shapes.getImports());
+
+        Model dataModel = ModelFactory.createDefaultModel();
+        try (StringReader reader = new StringReader(rdfToValidate)) {
+            RDFDataMgr.read(dataModel, reader, null, Lang.RDFXML);
+        } catch (Exception e) {
+            return buildStatusResponse("Document is not valid RDF/XML: " + e.getMessage(), false);
+        }
+
+        long violationCount = 0;
+        String validationReportKey = buildValidationReportKey(formatter, testsuite, shaclShapes);
+
+        Model infModel = ModelFactory.createInfModel(reasoner, dataModel);
+        ValidationReport report = ShaclValidator.get().validate(shapes, infModel.getGraph());
+
+        if (!report.conforms()) {
+            violationCount = report.getEntries().stream()
+                .filter(e -> e.severity().level().getURI().equals("http://www.w3.org/ns/shacl#Violation"))
+                .count();
+
+            if (isSavingValidationStatus) {
+                saveValidationStatus(metadata, validationReportKey, violationCount);
+            }
+
+            ShLib.printReport(report);
+            StringWriter writer = new StringWriter();
+            RDFDataMgr.write(writer, report.getModel(), OUTPUT_MAP.getOrDefault(outputFormat, Lang.RDFXML));
+            return writer.toString();
+        }
+
+        if (isSavingValidationStatus) {
+            saveValidationStatus(metadata, validationReportKey, violationCount);
+        }
+
+        return buildStatusResponse(String.format("Document in format %s is valid according to testsuite %s.", formatter, testsuite), true);
+    }
+
+    private List<String> getShaclShapes(String testsuite, List<String> shaclShapes) {
+        List<String> testSuiteShapes = testsuite == null ? List.of() : List.of(testsuites.get(testsuite));
+        if (!testSuiteShapes.isEmpty()){
+            shaclShapes = testSuiteShapes;
+        }
+        return shaclShapes;
+    }
+
+    private Shapes parseShapesFromFiles(List<String> shaclFiles) {
+        MultiUnion shapesGraph = new MultiUnion();
+        for (String shaclFile : shaclFiles) {
+            Path shaclPath = dataDirectory.getConfigDir().resolve("shacl").resolve(shaclFile);
+            if (!Files.exists(shaclPath)) {
+                throw new IllegalArgumentException("SHACL shape file not found: " + shaclPath);
+            }
+            shapesGraph.addGraph(RDFDataMgr.loadGraph(shaclPath.toString()));
+        }
+        return Shapes.parse(shapesGraph);
+    }
+
+    private Reasoner configureReasoner(Collection<Node> imports) {
+        Reasoner reasoner = ReasonerRegistry.getRDFSReasoner();
+        Model combinedOntologyModel = ModelFactory.createDefaultModel();
+
+        for (Node importedShape : imports) {
+            Model model = ModelFactory.createOntologyModel();
+            combinedOntologyModel.add(model.read(importedShape.getURI().toString()));
+        }
+        return reasoner.bindSchema(combinedOntologyModel);
+    }
+
+    private static String buildStatusResponse(String message, boolean isValid) {
+        return String.format("{\"valid\": %s, \"message\": \"%s\"}", isValid, message);
+    }
+
+    private static String buildValidationReportKey(String formatter, String testsuite, List<String> shaclShapes) {
+        return String.format("shacl-%s-%s",
+            formatter,
+            StringUtils.isNotEmpty(testsuite) ? testsuite :
+                // Use a hash of the shapes to create a unique key
+                // Shorten to 8 first characters
+                DigestUtils.sha256Hex(shaclShapes.stream().sorted().collect(Collectors.joining())).substring(0, 8)
+        );
+    }
+
+    private void saveValidationStatus(AbstractMetadata metadata, String validationReportKey, long violationCount) {
+        MetadataValidation validation = new MetadataValidation().setId(new MetadataValidationId(metadata.getId(), validationReportKey))
+            .setStatus(violationCount > 0 ? MetadataValidationStatus.INVALID : MetadataValidationStatus.VALID)
+            .setRequired(true)
+            .setNumTests(1)
+            .setNumFailures((int) violationCount);
+        metadataValidationRepository.save(validation);
+
+        try {
+            metadataIndexer.indexMetadata(List.of(metadata.getId() + ""));
+        } catch (Exception e) {
+            Log.debug(Geonet.DATA_MANAGER, "Exception while indexing metadata after SHACL validation: {}", e.getMessage(), e);
+        }
+    }
+}