QLever OOM on SPARQL UPDATE

[ddeboer]

This SPARQL UPDATE query crashes QLever, without anything being logged, so probably an OOM?

  PREFIX schema: <http://schema.org/>

  WITH <https://demo.netwerkdigitaalerfgoed.nl/registry/registrations>
  DELETE {
    <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> ?p ?o .
    ?dataset schema:subjectOf <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> .
    ?dataset ?dp ?do .
  }
  WHERE {
    {
      <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> ?p ?o .
    } UNION {
      <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> schema:about ?dataset .
      ?dataset schema:subjectOf <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> .
      ?dataset ?dp ?do .
    }
  };

  INSERT DATA {
    GRAPH <https://demo.netwerkdigitaalerfgoed.nl/registry/registrations> {
      <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> <http://schema.org/datePosted> "2021-11-09T10:20:26.157Z"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
<https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://schema.org/EntryPoint> .
<https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> <http://schema.org/encoding> <http://schema.org> .
<https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> <http://schema.org/about> <https://lit.hosting.deventit.net/AtlantisPubliek/data/dataset/LOD+Archiefbeschrijvingen> .
<https://lit.hosting.deventit.net/AtlantisPubliek/data/dataset/LOD+Archiefbeschrijvingen> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://schema.org/Dataset> .
<https://lit.hosting.deventit.net/AtlantisPubliek/data/dataset/LOD+Archiefbeschrijvingen> <http://schema.org/subjectOf> <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> .
<https://lit.hosting.deventit.net/AtlantisPubliek/data/dataset/LOD+Archiefbeschrijvingen> <http://schema.org/dateRead> "2025-10-30T09:51:06.190Z"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
<https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> <http://schema.org/dateRead> "2025-10-30T09:51:06.190Z"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
<https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> <http://schema.org/status> "200"^^<http://www.w3.org/2001/XMLSchema#integer> .
    }
  }

A simplified query works but is quite slow:

DELETE WHERE {
    GRAPH <https://demo.netwerkdigitaalerfgoed.nl/registry/registrations> {
        <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> ?p
                ?o .
    }
}

[
  {
    "delta-triples": {
      "after": {
        "deleted": 7890462,
        "inserted": 80956,
        "total": 7971418
      },
      "before": {
        "deleted": 7890462,
        "inserted": 80956,
        "total": 7971418
      },
      "difference": {
        "deleted": 0,
        "inserted": 0,
        "total": 0
      },
      "operation": {
        "deleted": 0,
        "inserted": 0,
        "total": 0
      }
    },
    "located-triples": {
      "OPS": {
        "blocks-affected": 36,
        "blocks-total": 37
      },
      "OSP": {
        "blocks-affected": 36,
        "blocks-total": 37
      },
      "POS": {
        "blocks-affected": 51,
        "blocks-total": 54
      },
      "PSO": {
        "blocks-affected": 29,
        "blocks-total": 54
      },
      "SOP": {
        "blocks-affected": 13,
        "blocks-total": 34
      },
      "SPO": {
        "blocks-affected": 13,
        "blocks-total": 34
      }
    },
    "runtimeInformation": {
      "meta": {
        "time_query_planning": 0
      },
      "query_execution_tree": {
        "cache_status": "computed",
        "children": [],
        "column_names": [
          "?p",
          "?o"
        ],
        "description": "IndexScan <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> ?p ?o",
        "details": null,
        "estimated_column_multiplicities": [
          1.0,
          1.0
        ],
        "estimated_operation_cost": 29631,
        "estimated_size": 29631,
        "estimated_total_cost": 29631,
        "operation_time": 27,
        "original_operation_time": 0,
        "original_total_time": 0,
        "result_cols": 2,
        "result_rows": 0,
        "status": "fully materialized",
        "total_time": 27
      }
    },
    "status": "OK",
    "time": {
      "execution": {
        "acquiringDeltaTriplesWriteLock": 0,
        "diskWriteback": 1190,
        "processUpdateImpl": {
          "clearCache": 1,
          "deleteTriples": 0,
          "insertTriples": 0,
          "materializeResult": 27,
          "preparation": 0,
          "total": 28
        },
        "snapshotCreation": 5081,
        "total": 6299
      },
      "planning": 0,
      "snapshot": 0,
      "total": 6299,
      "waitingForUpdateThread": 0
    },
    "update": "PREFIX schema: <http://schema.org/>\n\nDELETE WHERE {\n    GRAPH <https://demo.netwerkdigitaalerfgoed.nl/registry/registrations> {\n        <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> ?p\n                ?o .\n    }\n}",
    "warnings": [
      "SPARQL 1.1 Update for QLever is experimental."
    ]
  },
  {
    "delta-triples": {
      "after": {
        "deleted": 7890462,
        "inserted": 80956,
        "total": 7971418
      },
      "before": {
        "deleted": 7890462,
        "inserted": 80956,
        "total": 7971418
      },
      "difference": {
        "deleted": 0,
        "inserted": 0,
        "total": 0
      },
      "operation": {
        "deleted": 0,
        "inserted": 0,
        "total": 0
      }
    },
    "located-triples": {
      "OPS": {
        "blocks-affected": 36,
        "blocks-total": 37
      },
      "OSP": {
        "blocks-affected": 36,
        "blocks-total": 37
      },
      "POS": {
        "blocks-affected": 51,
        "blocks-total": 54
      },
      "PSO": {
        "blocks-affected": 29,
        "blocks-total": 54
      },
      "SOP": {
        "blocks-affected": 13,
        "blocks-total": 34
      },
      "SPO": {
        "blocks-affected": 13,
        "blocks-total": 34
      }
    },
    "runtimeInformation": {
      "meta": {
        "time_query_planning": 7391
      },
      "query_execution_tree": {
        "cache_status": "computed",
        "children": [],
        "column_names": [
          "?p",
          "?o"
        ],
        "description": "IndexScan <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> ?p ?o",
        "details": null,
        "estimated_column_multiplicities": [
          1.0,
          1.0
        ],
        "estimated_operation_cost": 29631,
        "estimated_size": 29631,
        "estimated_total_cost": 29631,
        "operation_time": 29,
        "original_operation_time": 0,
        "original_total_time": 0,
        "result_cols": 2,
        "result_rows": 0,
        "status": "fully materialized",
        "total_time": 29
      }
    },
    "status": "OK",
    "time": {
      "execution": {
        "acquiringDeltaTriplesWriteLock": 0,
        "diskWriteback": 1233,
        "processUpdateImpl": {
          "clearCache": 0,
          "deleteTriples": 0,
          "insertTriples": 0,
          "materializeResult": 29,
          "preparation": 3,
          "total": 32
        },
        "snapshotCreation": 5108,
        "total": 6373
      },
      "planning": 5,
      "snapshot": 1084,
      "total": 7462,
      "waitingForUpdateThread": 0
    },
    "update": "DELETE WHERE {\n    GRAPH <https://demo.netwerkdigitaalerfgoed.nl/registry/registrations> {\n        <https://raw.githubusercontent.com/netwerk-digitaal-erfgoed/dataset-register-entries/main/lit/catalog.rdf> ?p\n                ?o .\n    }\n}",
    "warnings": [
      "SPARQL 1.1 Update for QLever is experimental."
    ]
  }
]

[hannahbast]

@ddeboer Can you provide the dataset and a simpler query that produces the same behavior? Without that information, it's hard to say something meaningful

[ddeboer]

We had even the simplest SPARQL UPDATE query fail on our endpoint https://datasetregister.netwerkdigitaalerfgoed.nl/sparql. Throwing away the .update-triples file, restarting the server and re-inserting the lost data file was the only solution to get the endpoint working again.

Is there a known problem in QLever with --persist-updates datasets over time? Will the .update-triples file grow indefinitely?

[RobinTF]

@ddeboer Well it should "grow indefinitely" if more data is being added without being removed, we currently don't have a mechanism to incorporate everything into a clean index (which is being worked on but will require some time until it is production ready).
But since your problems went away after deleting the file and readding all the entries by hand that's probably not what you're experiencing. Any chance you're frequently adding new entries (as in IRIs that were not present in the original data) and removing them again? In this case it could be that the temporary vocabulary keeps growing and growing out of proportion because we currently don't have a garbage collection mechanism for that yet.

[hannahbast]

@ddeboer It's hard to say something meaningful if you don't tell us exactly what you did with which data

[ddeboer]

I suspect this has to do with this being a dynamic SPARQL store with lots of SPARQL updates every day. Our SPARQL updates look like:

1. Completely replacing named graphs

      CLEAR GRAPH <${graph}>;
      
      INSERT DATA {
        GRAPH <${graph}> {
          ${triples}
        }
      }`;

2. DELETE/INSERT triples inside a named graph

      PREFIX schema: <http://schema.org/>
      
      WITH <${this.graphIri}>
      DELETE {
        ?dataset schema:contentRating ?rating .
        ?rating ?p ?o .
      }
      WHERE {
        BIND(<${datasetUri}> as ?dataset)
        ?dataset schema:contentRating ?rating .
        ?rating ?p ?o .
      };

      INSERT DATA {
        GRAPH <${this.graphIri}> {
          <${datasetUri.toString()}> schema:contentRating [
            schema:bestRating ${rating.bestRating} ;
            schema:worstRating ${rating.worstRating} ;
            schema:ratingValue ${rating.score} ;
            schema:ratingExplanation "${rating.explanation}" ;
          ]
        }
      }

and:

      PREFIX schema: <http://schema.org/>
      
      WITH <${this.graphIri}>
      DELETE {
        <${registration.url}> ?p ?o .
        ?dataset schema:subjectOf <${registration.url}> .
        ?dataset ?dp ?do .
      }
      WHERE {
        {
          <${registration.url}> ?p ?o .
        } UNION {
          <${registration.url}> schema:about ?dataset .
          ?dataset schema:subjectOf <${registration.url}> .
          ?dataset ?dp ?do .
        }
      };
      
      INSERT DATA {
        GRAPH <${this.graphIri}> {
          ${triples}
        }
      }`;

(Please don't mind the TypeScript variable interpolations.)