diff --git a/helpers/data/class.GenerisAdapterRdf.php b/helpers/data/class.GenerisAdapterRdf.php index 154a4d465f2..0e8efd38324 100644 --- a/helpers/data/class.GenerisAdapterRdf.php +++ b/helpers/data/class.GenerisAdapterRdf.php @@ -145,7 +145,7 @@ private function addResource(Graph $graph, core_kernel_classes_Resource $resourc ) { continue; } - $graph->add($triple->subject, $triple->predicate, $triple->object); + $graph->addResource($triple->subject, $triple->predicate, $triple->object); } else { if ($this->isSerializedFile($triple->object)) { continue; diff --git a/migrations/Version202309111518342234_tao.php b/migrations/Version202309111518342234_tao.php new file mode 100644 index 00000000000..6b5116bcef5 --- /dev/null +++ b/migrations/Version202309111518342234_tao.php @@ -0,0 +1,52 @@ +addReport( + $this->propagate(new SyncModels())([]) + ); + } + + public function down(Schema $schema): void + { + $this->throwIrreversibleMigrationException( + 'The models should be updated via `SyncModels` script after reverting their RDF definitions.' + ); + } +} diff --git a/scripts/tools/MigrateSqlToNeo4j.php b/scripts/tools/MigrateSqlToNeo4j.php index 3179ec56431..f04f15618dd 100644 --- a/scripts/tools/MigrateSqlToNeo4j.php +++ b/scripts/tools/MigrateSqlToNeo4j.php @@ -32,6 +32,7 @@ use oat\oatbox\extension\script\ScriptAction; use oat\oatbox\reporting\Report; use oat\tao\model\TaoOntology; +use WikibaseSolutions\CypherDSL\Query; /** * php -dmemory_limit=1G index.php 'oat\tao\scripts\tools\MigrateSqlToNeo4j' -u -i -s 10000 -n 10000 -vvv @@ -242,12 +243,11 @@ public function extractDataFromSqlStorage(int $chunkSize): \Generator */ public function loadNTripleToNeo4j($neo4j, string $nTriple, int $neo4jChunkSize): void { - $nTriple = $this->escapeTriple($nTriple); - $result = $neo4j->run(<< $nTriple] ); $responseMessage = $result->first(); @@ -263,31 +263,6 @@ public function loadNTripleToNeo4j($neo4j, string $nTriple, int $neo4jChunkSize) $this->logInfo('Chunk of triples successfully loaded.'); } - public function escapeTriple(string $nTriple): string - { - $escapeCharacters = [ - '\\\\' => '\\\\\\\\', //Escape double slash - '\"' => '\\\\"', // Escaped slash in escaped double quote - '\n' => '\\\\n', // Escaped slash in EOL - '\r' => '\\\\r', // Escaped slash in carriage return - '\t' => '\\\\t', // Escaped slash in horizontal tab - "'" => "\'", //Escape single quote - ]; - - $escapeList = []; - foreach ($escapeCharacters as $needle => $replacement) { - if (strpos($nTriple, $needle) !== false) { - $escapeList[$needle] = $replacement; - } - } - - if (!empty($escapeList)) { - $nTriple = str_replace(array_keys($escapeList), array_values($escapeList), $nTriple); - } - - return $nTriple; - } - protected function provideOptions(): array { return [ @@ -354,10 +329,47 @@ protected function run(): Report foreach ($nTripleList as $nTriple) { $this->loadNTripleToNeo4j($neo4j, $nTriple, $neo4jChunkSize); } + + $this->addSystemLabel($neo4j, $sqlChunkSize, $neo4jChunkSize); } catch (\Throwable $e) { return Report::createError($e->getMessage()); } return Report::createSuccess('Data transfer finished successfully.'); } + + private function addSystemLabel($neo4j, int $sqlChunkSize, int $neo4jChunkSize) + { + $sql = $this->getSqlAdapter(); + + /** @var \Doctrine\DBAL\ForwardCompatibility\Result $idResult */ + $result = $sql->query(<<<'SQL' + SELECT subject + FROM statements + WHERE modelid <> 1 + GROUP BY subject; +SQL); + + $subjectList = []; + while ($r = $result->fetchColumn()) { + $subjectList[] = $r; + + if (count($subjectList) >= $neo4jChunkSize) { + $systemNode = Query::node('Resource'); + $query = Query::new()->match($systemNode) + ->where($systemNode->property('uri')->in($subjectList)) + ->set($systemNode->labeled('System')); + $neo4j->runStatement($query); + $subjectList = []; + } + } + + if (!empty($subjectList)) { + $systemNode = Query::node('Resource'); + $query = Query::new()->match($systemNode) + ->where($systemNode->property('uri')->in($subjectList)) + ->set($systemNode->labeled('System')); + $neo4j->run($query->build()); + } + } } diff --git a/scripts/update/OntologyUpdater.php b/scripts/update/OntologyUpdater.php index 62d996ac834..ee757ca3bd4 100644 --- a/scripts/update/OntologyUpdater.php +++ b/scripts/update/OntologyUpdater.php @@ -23,13 +23,11 @@ namespace oat\tao\scripts\update; use AppendIterator; -use oat\generis\model\kernel\persistence\file\FileModel; -use oat\generis\model\data\ModelManager; -use helpers_RdfDiff; -use core_kernel_persistence_smoothsql_SmoothModel; -use common_persistence_SqlPersistence; use common_ext_ExtensionsManager; -use core_kernel_persistence_smoothsql_SmoothIterator; +use helpers_RdfDiff; +use oat\generis\model\data\Model; +use oat\generis\model\data\ModelManager; +use oat\generis\model\kernel\persistence\file\FileModel; use oat\tao\model\extension\ExtensionModel; class OntologyUpdater @@ -37,20 +35,11 @@ class OntologyUpdater public static function syncModels() { $currentModel = ModelManager::getModel(); - $modelIds = array_diff($currentModel->getReadableModels(), ['1']); - - $persistence = common_persistence_SqlPersistence::getPersistence('default'); - $smoothIterator = new core_kernel_persistence_smoothsql_SmoothIterator($persistence, $modelIds); - - $nominalModel = new AppendIterator(); - foreach (common_ext_ExtensionsManager::singleton()->getInstalledExtensions() as $ext) { - $nominalModel->append(new ExtensionModel($ext)); - } - $langModel = \tao_models_classes_LanguageService::singleton()->getLanguageDefinition(); - $nominalModel->append($langModel); + $existingTriples = self::getCurrentTriples($currentModel); + $nominalTriples = self::getNominalTriples(); - $diff = helpers_RdfDiff::create($smoothIterator, $nominalModel); + $diff = helpers_RdfDiff::create($existingTriples, $nominalTriples); self::logDiff($diff); $diff->applyTo($currentModel); @@ -82,4 +71,30 @@ protected static function logDiff(\helpers_RdfDiff $diff) FileModel::toFile($path . DIRECTORY_SEPARATOR . 'add.rdf', $diff->getTriplesToAdd()); FileModel::toFile($path . DIRECTORY_SEPARATOR . 'remove.rdf', $diff->getTriplesToRemove()); } + + public static function getNominalTriples(): \Traversable + { + $nominalModel = new AppendIterator(); + foreach (common_ext_ExtensionsManager::singleton()->getInstalledExtensions() as $ext) { + $nominalModel->append(new ExtensionModel($ext)); + } + $langModel = \tao_models_classes_LanguageService::singleton()->getLanguageDefinition(); + $nominalModel->append($langModel); + return $nominalModel; + } + + public static function getCurrentTriples(Model $currentModel): \Traversable + { + return new \CallbackFilterIterator( + $currentModel->getRdfInterface()->getIterator(), + function (\core_kernel_classes_Triple $item) { + $isAutomaticIncludeRole = $item->subject === 'http://www.tao.lu/Ontologies/TAO.rdf#GlobalManagerRole' + && $item->predicate === 'http://www.tao.lu/Ontologies/generis.rdf#includesRole'; + $isGrantAccess = $item->predicate === 'http://www.tao.lu/Ontologies/taoFuncACL.rdf#GrantAccess'; + + + return !$isGrantAccess && !$isAutomaticIncludeRole; + } + ); + } }