Merge "Revert^2 "Implement new usage types for statement with qualifiers and references""

Author: jenkins-bot

client/includes/Changes/AffectedPagesFinder.php

@@ -141,9 +141,20 @@ public function getChangedAspects( EntityChange $change ) {
 			$aspects = array_merge( $aspects, $aliasAspects );
 		}
 
-		if ( $diffAspects->getStatementChanges() !== [] ) {
-			$statementAspects = $this->getChangedStatementAspects(
-				$diffAspects->getStatementChanges()
+		// If a statement mainsnak changes, we need to tell any pages which have STATEMENT_USAGE or STATEMENT_WITH_QUAL_OR_REF_USAGE
+		// This is regardless of whether it also includes a reference/qualifer change, in addition to the mainSnak change
+		if ( $diffAspects->getStatementChangesExcludingQualOrRefOnly() !== [] ) {
+			$statementAspects = $this->getChangedStatementAspects( //tells STATEMENT_USAGE and STATEMENT_WITH_QUAL_OR_REF_USAGE
+				$diffAspects->getStatementChangesExcludingQualOrRefOnly()
+			);
+			$aspects = array_merge( $aspects, $statementAspects );
+		}
+
+		// If it is a qualifer or ref only change (no mainsnak change), we don't need to tell those with STATEMENT_USAGE, but those with
+		// STATEMENT_WITH_QUAL_OR_REF_USAGE
+		if ( $diffAspects->getStatementChangesQualOrRefOnly() !== [] ) {
+			$statementAspects = $this->getChangedStatementAspectsQualOrRefOnlyChange( //only tells STATEMENT_WITH_QUAL_OR_REF_USAGE
+				$diffAspects->getStatementChangesQualOrRefOnly()
 			);
 			$aspects = array_merge( $aspects, $statementAspects );
 		}
@@ -164,15 +175,39 @@ public function getChangedAspects( EntityChange $change ) {
 	 * @param string[] $diff
 	 *
 	 * @return string[]
+	 *
+	 * If a statement changes, we need to tell any pages which have STATEMENT_USAGE or STATEMENT_WITH_QUAL_OR_REF_USAGE
 	 */
 	private function getChangedStatementAspects( array $diff ) {
 		$aspects = [];
 
 		foreach ( $diff as $propertyId ) {
 			$aspects[] = EntityUsage::makeAspectKey( EntityUsage::STATEMENT_USAGE, $propertyId );
+			$aspects[] = EntityUsage::makeAspectKey( EntityUsage::STATEMENT_WITH_QUAL_OR_REF_USAGE, $propertyId );
 		}
 
 		$aspects[] = EntityUsage::makeAspectKey( EntityUsage::STATEMENT_USAGE );
+		$aspects[] = EntityUsage::makeAspectKey( EntityUsage::STATEMENT_WITH_QUAL_OR_REF_USAGE );
+
+		return $aspects;
+	}
+
+	/**
+	 * @param string[] $diff
+	 *
+	 * @return string[]
+	 *
+	 * If any statement's qual/ref changes, we need to tell any pages which have STATEMENT_WITH_QUAL_OR_REF_USAGE,
+	 * but do not need to tell those with STATEMENT_USAGE
+	 */
+	private function getChangedStatementAspectsQualOrRefOnlyChange( array $diff ) {
+		$aspects = [];
+
+		foreach ( $diff as $propertyId ) {
+			$aspects[] = EntityUsage::makeAspectKey( EntityUsage::STATEMENT_WITH_QUAL_OR_REF_USAGE, $propertyId );
+		}
+
+		$aspects[] = EntityUsage::makeAspectKey( EntityUsage::STATEMENT_WITH_QUAL_OR_REF_USAGE );
 
 		return $aspects;
 	}

client/includes/DataAccess/Scribunto/WikibaseEntityLibrary.php

@@ -161,7 +161,7 @@ public function getUsageAccumulator(): UsageAccumulator {
 	}
 
 	/**
-	 * Add a statement usage (called once specific statements are accessed).
+	 * Add a statement usage without qualifiers and/or references (called once specific statements are accessed).
 	 *
 	 * @param string $entityId The Entity from which the statements were accessed.
 	 * @param string $propertyId Property id of the statements accessed.
@@ -170,6 +170,16 @@ public function addStatementUsage( string $entityId, string $propertyId ): void
 		$this->getImplementation()->addStatementUsage( $entityId, $propertyId );
 	}
 
+	/**
+	 * Add a statement usage with qualifiers and/or references as well as mainsnak (called once specific statements are accessed).
+	 *
+	 * @param string $entityId The Entity from which the statements were accessed.
+	 * @param string $propertyId Property id of the statements accessed.
+	 */
+	public function addStatementWithQualOrRefUsage( string $entityId, string $propertyId ): void {
+		$this->getImplementation()->addStatementWithQualOrRefUsage( $entityId, $propertyId );
+	}
+
 	/**
 	 * Add a label usage (called once specific labels are accessed).
 	 *
@@ -232,6 +242,7 @@ public function register() {
 			'formatStatements' => [ $this, 'formatStatements' ],
 			'formatPropertyValues' => [ $this, 'formatPropertyValues' ],
 			'addStatementUsage' => [ $this, 'addStatementUsage' ],
+			'addStatementWithQualOrRefUsage' => [ $this, 'addStatementWithQualOrRefUsage' ],
 			'addLabelUsage' => [ $this, 'addLabelUsage' ],
 			'addDescriptionUsage' => [ $this, 'addDescriptionUsage' ],
 			'addTitleOrSiteLinksUsage' => [ $this, 'addTitleOrSiteLinksUsage' ],

client/includes/DataAccess/Scribunto/WikibaseLibrary.php

@@ -23,6 +23,7 @@
 use Wikibase\DataModel\Entity\EntityId;
 use Wikibase\DataModel\Entity\EntityIdParser;
 use Wikibase\DataModel\Entity\EntityIdParsingException;
+use Wikibase\DataModel\Entity\NumericPropertyId;
 use Wikibase\DataModel\Entity\PropertyId;
 use Wikibase\DataModel\Services\Lookup\EntityAccessLimitException;
 use Wikibase\DataModel\Services\Lookup\EntityRetrievingClosestReferencedEntityIdLookup;
@@ -352,6 +353,7 @@ public function register() {
 			'entityExists' => [ $this, 'entityExists' ],
 			'getBadges' => [ $this, 'getBadges' ],
 			'getEntityStatements' => [ $this, 'getEntityStatements' ],
+			'addStatementWithQualOrRefUsage' => [ $this, 'addStatementWithQualOrRefUsage' ],
 			'getEntityUrl' => [ $this, 'getEntityUrl' ],
 			'renderSnak' => [ $this, 'renderSnak' ],
 			'formatValue' => [ $this, 'formatValue' ],
@@ -495,6 +497,20 @@ public function getEntityStatements( string $prefixedEntityId, string $propertyI
 		return [ $statements ];
 	}
 
+	/**
+	 * Add a statement usage (called once specific statements are accessed) when the qualifiers and/or references are also accessed.
+	 *
+	 * @param string $entityId The Entity from which the statements were accessed.
+	 * @param string $propertyId Property id of the statements accessed.
+	 */
+	public function addStatementWithQualOrRefUsage( string $entityId, string $propertyId ): void {
+		$entityId = trim( $entityId );
+		$entityId = $this->entityIdParser->parse( $entityId );
+		$propertyId = new NumericPropertyId( $propertyId );
+
+		$this->usageAccumulator->addStatementWithQualOrRefUsage( $entityId, $propertyId );
+	}
+
 	/**
 	 * Wrapper for getEntityId in WikibaseLanguageIndependentLuaBindings
 	 */

client/includes/DataAccess/Scribunto/WikibaseLuaEntityBindings.php

@@ -96,12 +96,27 @@ public function formatStatements(
 	 * @param string $propertyId Property id of the statements accessed.
 	 */
 	public function addStatementUsage( string $entityId, string $propertyId ): void {
+		$entityId = trim( $entityId );
 		$entityId = $this->entityIdParser->parse( $entityId );
 		$propertyId = new NumericPropertyId( $propertyId );
 
 		$this->usageAccumulator->addStatementUsage( $entityId, $propertyId );
 	}
 
+	/**
+	 * Add a statement usage (called once specific statements are accessed) when the qualifiers and/or references are also accessed.
+	 *
+	 * @param string $entityId The Entity from which the statements were accessed.
+	 * @param string $propertyId Property id of the statements accessed.
+	 */
+	public function addStatementWithQualOrRefUsage( string $entityId, string $propertyId ): void {
+		$entityId = trim( $entityId );
+		$entityId = $this->entityIdParser->parse( $entityId );
+		$propertyId = new NumericPropertyId( $propertyId );
+
+		$this->usageAccumulator->addStatementWithQualOrRefUsage( $entityId, $propertyId );
+	}
+
 	/**
 	 * Add a label usage (called once specific labels are accessed).
 	 *

client/includes/DataAccess/Scribunto/mw.wikibase.entity.lua

@@ -49,14 +49,28 @@ local function isValidPropertyId( propertyId )
 	return type( propertyId ) == 'string' and propertyId:match( '^P[1-9]%d*$' )
 end
 
--- Log access to claims of entity
+-- This should only ever change during the clone operation of getEntityStatements
+local qualOrRefUsageTrackingEnabled = true
+
+-- Log access to claims of entity with qualifiers and/or references
+--
+-- @param {string} entityId
+-- @param {string} propertyId
+local function addStatementWithQualOrRefUsage( entityId, propertyId )
+	if isValidPropertyId( propertyId ) and qualOrRefUsageTrackingEnabled == true then
+		-- Only attempt to track the usage if we have a valid property id.
+		php.addStatementWithQualOrRefUsage( entityId, propertyId )
+	end
+end
+
+-- Log access to claims of entity mainsnak only
 --
 -- @param {string} entityId
 -- @param {string} propertyId
 local function addStatementUsage( entityId, propertyId )
 	if isValidPropertyId( propertyId ) then
 		-- Only attempt to track the usage if we have a valid property id.
-		php.addStatementUsage( entityId, propertyId ) --add an argument here??
+		php.addStatementUsage( entityId, propertyId )
 	end
 end
 
@@ -104,11 +118,84 @@ local function maskEntityTable( entity, tableName, usageFunc )
 	setmetatable( entity[tableName], pseudoTableMetatable )
 end
 
+-- Function to mask a statement's subtables in order to log access
+-- Code for logging based on: http://www.lua.org/pil/13.4.4.html
+--
+-- @param {table} statement
+-- @param {string} tableName
+-- @param {function} usageFunc
+-- @param {string} entityId
+-- @param {string} propertyId
+local function maskStatementTable( statement, tableName, usageFunc, entityId, propertyId )
+	if statement[tableName] == nil then
+		return
+	end
+
+	local actualStatementTable = statement[tableName]
+	statement[tableName] = {}
+
+	local function logNext( _, key )
+		local k, v = next( actualStatementTable, key )
+		if k ~= nil then
+			usageFunc( entityId, propertyId )
+		end
+		return k, v
+	end
+
+	local function ipairsfunc( _ , i )
+    		i = i + 1
+    		if actualStatementTable[i] ~= nil then
+    			usageFunc( entityId, propertyId )
+    			return i, actualStatementTable[i]
+    		end
+    		return -- no nil to match default ipairs()
+    	end
+
+	local pseudoTableMetatable = {
+		__index = function( _, key )
+		-- note: we do not specify string here like in maskEntityTable, because
+		-- sometimes the key is e.g. P1 e.g. for qualifier
+		-- sometimes the key is a number e.g. for references
+			usageFunc( entityId, propertyId )
+			return actualStatementTable[key]
+		end,
+
+		__newindex = function (table,key,value)
+			if (table[key] == actualStatementTable[key]) then
+				return
+				--note: this seems to happen in getBestStatements but the values match so no update needed
+			else
+				error( 'Statement table cannot be modified', 2 )
+			end
+		end,
+
+		__pairs = function( _ )
+			return logNext, {}, nil
+		end,
+
+		__ipairs = function( _ )
+			return ipairsfunc, {}, 0
+		end,
+	}
+
+	 setmetatable( statement[tableName], pseudoTableMetatable )
+end
+
 -- Function to mask an entity's subtables in order to log access and prevent modifications
 --
 -- @param {table} entity
 local function maskEntityTables( entity )
-	maskEntityTable( entity, 'claims', addStatementUsage )
+	-- within claim we need to mask subtables for quals and refs too so they can be tracked
+	if entity.claims then
+		for propertyId, statements in pairs(entity.claims) do         -- for each property e.g. P1
+			for _, statement in ipairs(statements) do
+				maskStatementTable(statement, "qualifiers", addStatementWithQualOrRefUsage, entity.id, propertyId)
+				maskStatementTable(statement, "references", addStatementWithQualOrRefUsage, entity.id, propertyId)
+			end
+		end
+	end
+
+ 	maskEntityTable( entity, 'claims', addStatementUsage )
 	maskEntityTable( entity, 'labels', php.addLabelUsage )
 	maskEntityTable( entity, 'sitelinks', php.addTitleOrSiteLinksUsage )
 	maskEntityTable( entity, 'descriptions', php.addDescriptionUsage )
@@ -266,8 +353,15 @@ local function getEntityStatements( entity, propertyLabelOrId, funcName )
 	end
 
 	if propertyId and entity.claims[propertyId] then
+
+		qualOrRefUsageTrackingEnabled = false
 		-- Create a deep copy of the table to prevent unexpected value changes (T270851).
-		return mw.clone(entity.claims[propertyId])
+		-- The clone operation is considered a read of qualifiers and references so we need to
+		-- switch off tracking just during the clone operation
+		local clone = mw.clone(entity.claims[propertyId])
+		qualOrRefUsageTrackingEnabled = true
+
+		return clone
 	end
 
 	return {}