Skip to content

Commit cb4bb0c

Browse files
vbyndychvbyndych
authored
REL-1197: Rdf export/import from neo4j (#3890)
2 parents 3251229 + ed38ea6 commit cb4bb0c

File tree

4 files changed

+137
-49
lines changed

4 files changed

+137
-49
lines changed

‎helpers/data/class.GenerisAdapterRdf.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ private function addResource(Graph $graph, core_kernel_classes_Resource $resourc
145145
) {
146146
continue;
147147
}
148-
$graph->add($triple->subject, $triple->predicate, $triple->object);
148+
$graph->addResource($triple->subject, $triple->predicate, $triple->object);
149149
} else {
150150
if ($this->isSerializedFile($triple->object)) {
151151
continue;

‎migrations/Version202309111518342234_tao.php

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
<?php
2+
3+
/*
4+
* This program is free software; you can redistribute it and/or
5+
* modify it under the terms of the GNU General Public License
6+
* as published by the Free Software Foundation; under version 2
7+
* of the License (non-upgradable).
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with this program; if not, write to the Free Software
16+
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17+
*
18+
* Copyright (c) 2023(original work) Open Assessment Technologies SA;
19+
*/
20+
21+
declare(strict_types=1);
22+
23+
namespace oat\tao\migrations;
24+
25+
use Doctrine\DBAL\Schema\Schema;
26+
use oat\tao\scripts\SyncModels;
27+
use oat\tao\scripts\tools\migrations\AbstractMigration;
28+
29+
/**
30+
* phpcs:disable Squiz.Classes.ValidClassName
31+
*/
32+
final class Version202309111518342234_tao extends AbstractMigration
33+
{
34+
public function getDescription(): string
35+
{
36+
return 'Update Ontology models';
37+
}
38+
39+
public function up(Schema $schema): void
40+
{
41+
$this->addReport(
42+
$this->propagate(new SyncModels())([])
43+
);
44+
}
45+
46+
public function down(Schema $schema): void
47+
{
48+
$this->throwIrreversibleMigrationException(
49+
'The models should be updated via `SyncModels` script after reverting their RDF definitions.'
50+
);
51+
}
52+
}

‎scripts/tools/MigrateSqlToNeo4j.php

Lines changed: 44 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
use oat\oatbox\extension\script\ScriptAction;
3333
use oat\oatbox\reporting\Report;
3434
use oat\tao\model\TaoOntology;
35+
use WikibaseSolutions\CypherDSL\Query;
3536

3637
/**
3738
* php -dmemory_limit=1G index.php 'oat\tao\scripts\tools\MigrateSqlToNeo4j' -u -i -s 10000 -n 10000 -vvv
@@ -242,12 +243,12 @@ public function extractDataFromSqlStorage(int $chunkSize): \Generator
242243
*/
243244
public function loadNTripleToNeo4j($neo4j, string $nTriple, int $neo4jChunkSize): void
244245
{
245-
$nTriple = $this->escapeTriple($nTriple);
246-
247-
$result = $neo4j->run(<<<CYPHER
248-
CALL n10s.rdf.import.inline('${nTriple}',"N-Triples",{commitSize:${neo4jChunkSize}}) YIELD terminationStatus, extraInfo
246+
$result = $neo4j->run(
247+
<<<CYPHER
248+
CALL n10s.rdf.import.inline(\$nTriple,"N-Triples",{commitSize:${neo4jChunkSize}}) YIELD terminationStatus, extraInfo
249249
RETURN terminationStatus, extraInfo
250-
CYPHER
250+
CYPHER,
251+
['nTriple' => $nTriple]
251252
);
252253

253254
$responseMessage = $result->first();
@@ -263,31 +264,6 @@ public function loadNTripleToNeo4j($neo4j, string $nTriple, int $neo4jChunkSize)
263264
$this->logInfo('Chunk of triples successfully loaded.');
264265
}
265266

266-
public function escapeTriple(string $nTriple): string
267-
{
268-
$escapeCharacters = [
269-
'\\\\' => '\\\\\\\\', //Escape double slash
270-
'\"' => '\\\\"', // Escaped slash in escaped double quote
271-
'\n' => '\\\\n', // Escaped slash in EOL
272-
'\r' => '\\\\r', // Escaped slash in carriage return
273-
'\t' => '\\\\t', // Escaped slash in horizontal tab
274-
"'" => "\'", //Escape single quote
275-
];
276-
277-
$escapeList = [];
278-
foreach ($escapeCharacters as $needle => $replacement) {
279-
if (strpos($nTriple, $needle) !== false) {
280-
$escapeList[$needle] = $replacement;
281-
}
282-
}
283-
284-
if (!empty($escapeList)) {
285-
$nTriple = str_replace(array_keys($escapeList), array_values($escapeList), $nTriple);
286-
}
287-
288-
return $nTriple;
289-
}
290-
291267
protected function provideOptions(): array
292268
{
293269
return [
@@ -354,10 +330,48 @@ protected function run(): Report
354330
foreach ($nTripleList as $nTriple) {
355331
$this->loadNTripleToNeo4j($neo4j, $nTriple, $neo4jChunkSize);
356332
}
333+
334+
$this->addSystemLabel($neo4j, $sqlChunkSize, $neo4jChunkSize);
357335
} catch (\Throwable $e) {
358336
return Report::createError($e->getMessage());
359337
}
360338

361339
return Report::createSuccess('Data transfer finished successfully.');
362340
}
341+
342+
private function addSystemLabel($neo4j, int $sqlChunkSize, int $neo4jChunkSize)
343+
{
344+
$sql = $this->getSqlAdapter();
345+
$nonSystemModelId = \core_kernel_persistence_smoothsql_SmoothModel::DEFAULT_WRITABLE_MODEL;
346+
347+
/** @var \Doctrine\DBAL\ForwardCompatibility\Result $idResult */
348+
$result = $sql->query(<<<SQL
349+
SELECT subject
350+
FROM statements
351+
WHERE modelid <> {$nonSystemModelId}
352+
GROUP BY subject;
353+
SQL);
354+
355+
$subjectList = [];
356+
while ($r = $result->fetchColumn()) {
357+
$subjectList[] = $r;
358+
359+
if (count($subjectList) >= $neo4jChunkSize) {
360+
$systemNode = Query::node('Resource');
361+
$query = Query::new()->match($systemNode)
362+
->where($systemNode->property('uri')->in($subjectList))
363+
->set($systemNode->labeled('System'));
364+
$neo4j->runStatement($query);
365+
$subjectList = [];
366+
}
367+
}
368+
369+
if (!empty($subjectList)) {
370+
$systemNode = Query::node('Resource');
371+
$query = Query::new()->match($systemNode)
372+
->where($systemNode->property('uri')->in($subjectList))
373+
->set($systemNode->labeled('System'));
374+
$neo4j->run($query->build());
375+
}
376+
}
363377
}

‎scripts/update/OntologyUpdater.php

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -23,34 +23,25 @@
2323
namespace oat\tao\scripts\update;
2424

2525
use AppendIterator;
26-
use oat\generis\model\kernel\persistence\file\FileModel;
27-
use oat\generis\model\data\ModelManager;
28-
use helpers_RdfDiff;
29-
use core_kernel_persistence_smoothsql_SmoothModel;
30-
use common_persistence_SqlPersistence;
3126
use common_ext_ExtensionsManager;
32-
use core_kernel_persistence_smoothsql_SmoothIterator;
27+
use helpers_RdfDiff;
28+
use oat\generis\model\data\Model;
29+
use oat\generis\model\data\ModelManager;
30+
use oat\generis\model\GenerisRdf;
31+
use oat\generis\model\kernel\persistence\file\FileModel;
3332
use oat\tao\model\extension\ExtensionModel;
33+
use oat\tao\model\user\TaoRoles;
3434

3535
class OntologyUpdater
3636
{
3737
public static function syncModels()
3838
{
3939
$currentModel = ModelManager::getModel();
40-
$modelIds = array_diff($currentModel->getReadableModels(), ['1']);
41-
42-
$persistence = common_persistence_SqlPersistence::getPersistence('default');
4340

44-
$smoothIterator = new core_kernel_persistence_smoothsql_SmoothIterator($persistence, $modelIds);
45-
46-
$nominalModel = new AppendIterator();
47-
foreach (common_ext_ExtensionsManager::singleton()->getInstalledExtensions() as $ext) {
48-
$nominalModel->append(new ExtensionModel($ext));
49-
}
50-
$langModel = \tao_models_classes_LanguageService::singleton()->getLanguageDefinition();
51-
$nominalModel->append($langModel);
41+
$existingTriples = self::getCurrentTriples($currentModel);
42+
$nominalTriples = self::getNominalTriples();
5243

53-
$diff = helpers_RdfDiff::create($smoothIterator, $nominalModel);
44+
$diff = helpers_RdfDiff::create($existingTriples, $nominalTriples);
5445
self::logDiff($diff);
5546

5647
$diff->applyTo($currentModel);
@@ -82,4 +73,35 @@ protected static function logDiff(\helpers_RdfDiff $diff)
8273
FileModel::toFile($path . DIRECTORY_SEPARATOR . 'add.rdf', $diff->getTriplesToAdd());
8374
FileModel::toFile($path . DIRECTORY_SEPARATOR . 'remove.rdf', $diff->getTriplesToRemove());
8475
}
76+
77+
public static function getNominalTriples(): \Traversable
78+
{
79+
$nominalModel = new AppendIterator();
80+
foreach (common_ext_ExtensionsManager::singleton()->getInstalledExtensions() as $ext) {
81+
$nominalModel->append(new ExtensionModel($ext));
82+
}
83+
$langModel = \tao_models_classes_LanguageService::singleton()->getLanguageDefinition();
84+
$nominalModel->append($langModel);
85+
return $nominalModel;
86+
}
87+
88+
public static function getCurrentTriples(Model $currentModel): \Traversable
89+
{
90+
return new \CallbackFilterIterator(
91+
$currentModel->getRdfInterface()->getIterator(),
92+
function (\core_kernel_classes_Triple $item) {
93+
/**
94+
* Those includes generated with a script and created in non-system space, so we ignore them.
95+
* @see \tao_install_ExtensionInstaller::installManagementRole
96+
*/
97+
$isAutomaticIncludeRole = $item->subject === TaoRoles::GLOBAL_MANAGER
98+
&& $item->predicate === GenerisRdf::PROPERTY_ROLE_INCLUDESROLE;
99+
100+
// GrantAccess field added to entities in non-system space and also should be ignored for now.
101+
$isGrantAccess = $item->predicate === 'http://www.tao.lu/Ontologies/taoFuncACL.rdf#GrantAccess';
102+
103+
return !$isGrantAccess && !$isAutomaticIncludeRole;
104+
}
105+
);
106+
}
85107
}

0 commit comments

Comments
 (0)