Merge branch 'develop'

Author: DaniFdezAlvarez

setup.py

@@ -8,12 +8,12 @@ def read(file_path):
 setup(
   name = 'shexer',
   packages = find_packages(exclude=["*.local_code.*"]), # this must be the same as the name above
-  version = '2.6.5.1',
+  version = '2.7.0',
   description = 'Automatic schema extraction for RDF graphs',
   author = 'Daniel Fernandez-Alvarez',
   author_email = 'danifdezalvarez@gmail.com',
   url = 'https://github.com/DaniFdezAlvarez/shexer',
-  download_url = 'https://github.com/DaniFdezAlvarez/shexer/archive/2.6.5.1.tar.gz',
+  download_url = 'https://github.com/DaniFdezAlvarez/shexer/archive/2.7.0.tar.gz',
   keywords = ['testing', 'shexer', 'shexerp3', "rdf", "shex", "shacl", "schema"],
   long_description = read('README.md'),
   long_description_content_type='text/markdown',

shexer/consts.py

@@ -50,5 +50,8 @@
             "http://xmlns.com/foaf/0.1/": "foaf"
             }
 
-# WESO-SHAPES-ONTO
-FREQ_PROP = "http://weso.es/shexer/ontology/ratio_property_usage"
+# WESO-SHAPES-ONTO  # todo !
+FREQ_PROP = "http://weso.es/shexer/ontology/ratio_property_usage"
+EXTRA_INFO_PROP = "http://www.w3.org/2000/01/rdf-schema#comment"
+EXAMPLE_CONFORMANCE_PROP = "http://weso.es/shexer/ontology/conformant_example"
+ABSOLUTE_COUNT_PROP = "http://weso.es/shexer/ontology/total_conforming_instances"

shexer/core/shexing/class_shexer.py

@@ -1,6 +1,6 @@
 import json
 
-from shexer.consts import RDF_TYPE, SHAPES_DEFAULT_NAMESPACE, FREQ_PROP
+from shexer.consts import RDF_TYPE, SHAPES_DEFAULT_NAMESPACE, FREQ_PROP, ABSOLUTE_COUNT_PROP, EXTRA_INFO_PROP
 from shexer.core.shexing.strategy.direct_shexing_strategy import DirectShexingStrategy
 from shexer.core.shexing.strategy.direct_and_inverse_shexing_strategy import DirectAndInverseShexingStrategy
 from shexer.utils.target_elements import determine_original_target_nodes_if_needed
@@ -18,8 +18,9 @@ def __init__(self, class_counts_dict, class_profile_dict=None, class_profile_jso
                  allow_opt_cardinality=True, disable_exact_cardinality=False,
                  shapes_namespace=SHAPES_DEFAULT_NAMESPACE, inverse_paths=False,
                  decimals=-1, instances_report_mode=RATIO_INSTANCES, detect_minimal_iri=False,
-                 class_min_iris_dict=None, allow_redundant_or=False, shape_names_dict=None, frequency_property=FREQ_PROP,
-                 comments_to_annotations=False):
+                 class_min_iris_dict=None, allow_redundant_or=False, shape_names_dict=None,
+                 frequency_property=FREQ_PROP, comments_to_annotations=False, extra_info_property=EXTRA_INFO_PROP,
+                 absolute_count_property=ABSOLUTE_COUNT_PROP):
         self._class_counts_dict = class_counts_dict
         self._class_profile_dict = class_profile_dict if class_profile_dict is not None else self._load_class_profile_dict_from_file(
             class_profile_json_file)
@@ -45,6 +46,8 @@ def __init__(self, class_counts_dict, class_profile_dict=None, class_profile_jso
         self._shape_names_dict = shape_names_dict if shape_names_dict is not None else {}
         self._frequency_property = frequency_property
         self._comments_to_annotations = comments_to_annotations
+        self._extra_info_property = extra_info_property
+        self._absolute_count_property = absolute_count_property
 
         self._original_target_nodes = determine_original_target_nodes_if_needed(remove_empty_shapes=remove_empty_shapes,
                                                                                 original_target_classes=original_target_classes,

shexer/core/shexing/class_shexer_fed_sources.py

@@ -1,5 +1,5 @@
 from shexer.core.shexing.class_shexer import ClassShexer
-from shexer.consts import RDF_TYPE, SHAPES_DEFAULT_NAMESPACE, FREQ_PROP, RATIO_INSTANCES
+from shexer.consts import RDF_TYPE, SHAPES_DEFAULT_NAMESPACE, FREQ_PROP, RATIO_INSTANCES, ABSOLUTE_COUNT_PROP, EXTRA_INFO_PROP
 from shexer.core.instances.pconsts import FEDERATION_TAG_MARK
 
 _COMMENT_FED_PROPERTY = "# Constraint only observed in {}"
@@ -15,14 +15,15 @@ def __init__(self, class_counts_dict, class_profile_dict=None, class_profile_jso
                  allow_opt_cardinality=True, disable_exact_cardinality=False, shapes_namespace=SHAPES_DEFAULT_NAMESPACE,
                  inverse_paths=False, decimals=-1, instances_report_mode=RATIO_INSTANCES, detect_minimal_iri=False,
                  class_min_iris_dict=None, allow_redundant_or=False, fed_sources=None, shape_names_dict=None,
-                 frequency_property=FREQ_PROP, comments_to_annotations=False):
+                 frequency_property=FREQ_PROP, comments_to_annotations=False, extra_info_property=EXTRA_INFO_PROP,
+                 absolute_counts_prop=ABSOLUTE_COUNT_PROP):
         super().__init__(class_counts_dict, class_profile_dict, class_profile_json_file, remove_empty_shapes,
                          original_target_classes, original_shape_map, discard_useless_constraints_with_positive_closure,
                          keep_less_specific, all_compliant_mode, instantiation_property, disable_or_statements,
                          disable_comments, namespaces_dict, tolerance_to_keep_similar_rules, allow_opt_cardinality,
                          disable_exact_cardinality, shapes_namespace, inverse_paths, decimals, instances_report_mode,
                          detect_minimal_iri, class_min_iris_dict, allow_redundant_or, shape_names_dict,
-                         frequency_property, comments_to_annotations)
+                         frequency_property, comments_to_annotations, extra_info_property, absolute_counts_prop)
         self._fed_sources = fed_sources
 
     def shex_classes(self, acceptance_threshold=0,

shexer/core/shexing/strategy/abstract_shexing_strategy.py

@@ -39,7 +39,9 @@ def __init__(self, class_shexer):
                                                                  disable_comments=self._disable_comments,
                                                                  frequency_property=self._class_shexer._frequency_property,
                                                                  namespaces_dict=self._namespaces_dict,
-                                                                 comments_to_annotations=class_shexer._comments_to_annotations)
+                                                                 comments_to_annotations=class_shexer._comments_to_annotations,
+                                                                 extra_info_property=self._class_shexer._extra_info_property,
+                                                                 absolute_count_property=self._class_shexer._absolute_count_property)
 
 
     def yield_base_shapes(self, acceptance_threshold):

shexer/io/shacl/formater/shacl_serializer.py

@@ -2,12 +2,14 @@
 from shexer.model.shape import STARTING_CHAR_FOR_SHAPE_NAME
 from rdflib import Graph, Namespace, URIRef, RDF, BNode, XSD, Literal
 from shexer.model.statement import POSITIVE_CLOSURE, KLEENE_CLOSURE, OPT_CARDINALITY
-from shexer.utils.literal import XSD_NAMESPACE, LANG_STRING_TYPE
+from shexer.utils.literal import XSD_NAMESPACE, LANG_STRING_TYPE, decide_literal_type
 from shexer.model.const_elem_types import IRI_ELEM_TYPE, LITERAL_ELEM_TYPE, DOT_ELEM_TYPE, BNODE_ELEM_TYPE
 from shexer.io.wikidata import wikidata_annotation
 from wlighter import TURTLE_FORMAT
-from shexer.model.node_selector import NodeSelectorSparql, NodeSelectorNoSparql
+from shexer.model.node_selector import NodeSelectorSparql
 from shexer.utils.log import log_msg
+from shexer.consts import RATIO_INSTANCES, EXAMPLE_CONFORMANCE_PROP, ABSOLUTE_COUNT_PROP, EXTRA_INFO_PROP, FREQ_PROP, \
+    SHAPE_EXAMPLES, ALL_EXAMPLES, MIXED_INSTANCES, ABSOLUTE_INSTANCES, CONSTRAINT_EXAMPLES
 
 _EXPECTED_SHAPE_BEGINING = STARTING_CHAR_FOR_SHAPE_NAME + "<"
 _EXPECTED_SHAPE_ENDING = ">"
@@ -56,7 +58,11 @@ class ShaclSerializer(object):
 
     def __init__(self, target_file, shapes_list, namespaces_dict=None, string_return=False,
                  instantiation_property_str=RDF_TYPE_STR, wikidata_annotation=False,
-                 detect_minimal_iri=False, shape_example_features=None, shape_map=None, verbose=False):
+                 detect_minimal_iri=False, shape_example_features=None, shape_map=None, verbose=False,
+                 instances_report_mode=RATIO_INSTANCES, examples_mode=None, inverse_paths=False,
+                 example_constraint_prop=EXAMPLE_CONFORMANCE_PROP, comments_to_annotations=False,
+                 absolute_counts_prop=ABSOLUTE_COUNT_PROP, extra_info_prop=EXTRA_INFO_PROP,
+                 frequency_prop=FREQ_PROP):
         self._target_file = target_file
         self._namespaces_dict = namespaces_dict if namespaces_dict is not None else {}
         self._shapes_list = shapes_list
@@ -68,6 +74,15 @@ def __init__(self, target_file, shapes_list, namespaces_dict=None, string_return
         self._shape_map = shape_map
         self._verbose = verbose
 
+        self._instances_report_mode = instances_report_mode
+        self._examples_mode = examples_mode
+        self._inverse_paths = inverse_paths
+        self._example_constraint_prop = URIRef(example_constraint_prop)
+        self._generate_annotations = comments_to_annotations
+        self._absolute_counts_prop = URIRef(absolute_counts_prop)
+        self._extra_info_prop = URIRef(extra_info_prop)
+        self._frequency_prop = URIRef(frequency_prop)
+
         self._g_shapes = Graph()
 
         # self._uri_dict = {}
@@ -128,8 +143,26 @@ def _add_shape(self, shape):
                               shape=shape)
         self._add_shape_constraints(shape=shape,
                                     r_shape_uri=r_shape_uri)
+        self._add_shape_annotations(shape=shape,
+                                    r_shape_uri=r_shape_uri)
 
-
+    def _add_shape_annotations(self, shape, r_shape_uri):
+        if not self._generate_annotations:
+            return
+        if self._examples_mode in [SHAPE_EXAMPLES, ALL_EXAMPLES]:
+            self._add_shape_example(shape, r_shape_uri)
+        if self._instances_report_mode in [MIXED_INSTANCES, ABSOLUTE_INSTANCES]:
+            self._add_shape_counts(shape, r_shape_uri)
+
+    def _add_shape_example(self, shape, r_shape_uri):
+        self._add_triple(r_shape_uri,
+                         self._example_constraint_prop,
+                         URIRef(self._shape_example_features.shape_example(shape_id=shape.class_uri)))
+
+    def _add_shape_counts(self, shape, r_shape_uri):
+        self._add_triple(r_shape_uri,
+                         self._absolute_counts_prop,
+                         Literal(shape.n_instances))
     def _add_target_class(self, shape, r_shape_uri):
         if self._shape_map is None:
             if shape.class_uri is not None:
@@ -152,18 +185,58 @@ def _literal_iri_pattern(self, shape):
     def _add_shape_constraints(self, shape, r_shape_uri):
         for a_statement in shape.yield_statements():
             self._add_constraint(statement=a_statement,
-                                 r_shape_uri=r_shape_uri)
+                                 r_shape_uri=r_shape_uri,
+                                 shape=shape)
 
     def _is_instantiation_property(self, str_property):
         return str_property == self._instantiation_property_str
 
-    def _add_constraint(self, statement, r_shape_uri):
+    def _add_constraint(self, statement, r_shape_uri, shape):
+        r_constraint_node = self._generate_bnode()
         if self._is_instantiation_property(statement.st_property):
             self._add_instantiation_constraint(statement=statement,
-                                               r_shape_uri=r_shape_uri)
+                                               r_shape_uri=r_shape_uri,
+                                               r_constraint_node=r_constraint_node)
         else:
             self._add_regular_constraint(statement=statement,
-                                         r_shape_uri=r_shape_uri)
+                                         r_shape_uri=r_shape_uri,
+                                         r_constraint_node=r_constraint_node,
+                                         shape=shape)
+
+        if not self._generate_annotations:
+            return
+        if self._instances_report_mode in [MIXED_INSTANCES, RATIO_INSTANCES]:
+            self._add_constraint_ratio_annotation(statement, r_constraint_node)
+        if self._instances_report_mode in [MIXED_INSTANCES, ABSOLUTE_INSTANCES]:
+            self._add_constraint_absolutes_annotation(statement, r_constraint_node)
+
+    def _add_constraint_ratio_annotation(self, statement, r_constraint_node):
+        self._add_triple(r_constraint_node,
+                         self._frequency_prop,
+                         Literal(statement.probability, datatype=XSD.decimal))
+
+    def _add_constraint_absolutes_annotation(self, statement, r_constraint_node):
+        self._add_triple(r_constraint_node,
+                         self._absolute_counts_prop,
+                         Literal(statement.n_occurences))
+
+    def _add_constraint_example(self, statement, r_constraint_node, shape):
+        triple_obj = self._get_example_of_triple_obj(statement, shape)
+        self._add_triple(r_constraint_node,
+                         self._example_constraint_prop,
+                         triple_obj)
+
+    def _get_example_of_triple_obj(self, statement, shape):
+        if self._inverse_paths:
+            candidate = self._shape_example_features.get_constraint_example(shape_id=shape.class_uri,
+                                                                            prop=statement.st_property,
+                                                                            inverse=statement.is_inverse)
+        else:
+            candidate = self._shape_example_features.get_constraint_example(shape_id=shape.class_uri,
+                                                                            prop=statement.st_property)
+        if candidate.startswith("http://"):
+            return URIRef(candidate)
+        return Literal(candidate, datatype=decide_literal_type(candidate))
 
     def _add_exactly_one_cardinality(self, r_constraint_node):
         self._add_min_occurs(r_constraint_node=r_constraint_node,
@@ -178,8 +251,7 @@ def _add_in_instance(self, r_constraint_node, statement):
         self._add_triple(list_seed_node, RDF.first, target_node)
         self._add_triple(list_seed_node, RDF.rest, RDF.nil)
 
-    def _add_instantiation_constraint(self, statement, r_shape_uri):
-        r_constraint_node = self._generate_bnode()
+    def _add_instantiation_constraint(self, statement, r_shape_uri, r_constraint_node):
         self._add_bnode_property(r_shape_uri=r_shape_uri,
                                  r_constraint_node=r_constraint_node)
         self._add_direct_path(statement=statement,
@@ -188,8 +260,7 @@ def _add_instantiation_constraint(self, statement, r_shape_uri):
         self._add_in_instance(statement=statement,
                               r_constraint_node=r_constraint_node)
 
-    def _add_regular_constraint(self, statement, r_shape_uri):
-        r_constraint_node = self._generate_bnode()
+    def _add_regular_constraint(self, statement, r_shape_uri, r_constraint_node, shape):
         self._add_bnode_property(r_shape_uri=r_shape_uri,
                                  r_constraint_node=r_constraint_node)
         self._add_node_type(statement=statement,
@@ -198,6 +269,17 @@ def _add_regular_constraint(self, statement, r_shape_uri):
                               r_constraint_node=r_constraint_node)
         self._add_path(statement=statement,
                        r_constraint_node=r_constraint_node)
+        if self._generate_annotations:
+            self._add_statement_comments(statement=statement,
+                                         r_constraint_node=r_constraint_node)
+            if self._examples_mode in [ALL_EXAMPLES, CONSTRAINT_EXAMPLES]:
+                self._add_constraint_example(statement, r_constraint_node, shape)
+
+    def _add_statement_comments(self, statement, r_constraint_node):
+        for a_comment in statement.comments:
+            self._add_triple(r_constraint_node,
+                             self._extra_info_prop,
+                             Literal(a_comment))
 
     def _add_path(self, statement, r_constraint_node):
         if not statement.is_inverse:

shexer/io/shex/formater/consts.py

@@ -6,4 +6,5 @@
 POSITIVE_CLOSURE = "+"
 KLEENE_CLOSURE = "*"
 OPT_CARDINALITY = "?"
-SHAPE_LINK_CHAR = "@"
+SHAPE_LINK_CHAR = "@"
+ANNOTATION_BEGIN = "//"