feat(dav): introduce paginate with custom headers

apps/dav/appinfo/info.xml

@@ -10,7 +10,7 @@
 	<name>WebDAV</name>
 	<summary>WebDAV endpoint</summary>
 	<description>WebDAV endpoint</description>
-	<version>1.32.0</version>
+	<version>1.33.0</version>
 	<licence>agpl</licence>
 	<author>owncloud.org</author>
 	<namespace>DAV</namespace>
@@ -27,6 +27,7 @@
 		<job>OCA\DAV\BackgroundJob\CleanupDirectLinksJob</job>
 		<job>OCA\DAV\BackgroundJob\UpdateCalendarResourcesRoomsBackgroundJob</job>
 		<job>OCA\DAV\BackgroundJob\CleanupInvitationTokenJob</job>
+		<job>OCA\DAV\BackgroundJob\CleanupPaginateCacheJob</job>
 		<job>OCA\DAV\BackgroundJob\EventReminderJob</job>
 		<job>OCA\DAV\BackgroundJob\CalendarRetentionJob</job>
 		<job>OCA\DAV\BackgroundJob\PruneOutdatedSyncTokensJob</job>

apps/dav/composer/composer/autoload_classmap.php

@@ -16,6 +16,7 @@
     'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => $baseDir . '/../lib/BackgroundJob/CalendarRetentionJob.php',
     'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => $baseDir . '/../lib/BackgroundJob/CleanupDirectLinksJob.php',
     'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
+    'OCA\\DAV\\BackgroundJob\\CleanupPaginateCacheJob' => $baseDir . '/../lib/BackgroundJob/CleanupPaginateCacheJob.php',
     'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => $baseDir . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php',
     'OCA\\DAV\\BackgroundJob\\EventReminderJob' => $baseDir . '/../lib/BackgroundJob/EventReminderJob.php',
     'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
@@ -328,6 +329,7 @@
     'OCA\\DAV\\Migration\\Version1008Date20181105110300' => $baseDir . '/../lib/Migration/Version1008Date20181105110300.php',
     'OCA\\DAV\\Migration\\Version1008Date20181105112049' => $baseDir . '/../lib/Migration/Version1008Date20181105112049.php',
     'OCA\\DAV\\Migration\\Version1008Date20181114084440' => $baseDir . '/../lib/Migration/Version1008Date20181114084440.php',
+    'OCA\\DAV\\Migration\\Version1009Date20181108161232' => $baseDir . '/../lib/Migration/Version1009Date20181108161232.php',
     'OCA\\DAV\\Migration\\Version1011Date20190725113607' => $baseDir . '/../lib/Migration/Version1011Date20190725113607.php',
     'OCA\\DAV\\Migration\\Version1011Date20190806104428' => $baseDir . '/../lib/Migration/Version1011Date20190806104428.php',
     'OCA\\DAV\\Migration\\Version1012Date20190808122342' => $baseDir . '/../lib/Migration/Version1012Date20190808122342.php',
@@ -341,6 +343,10 @@
     'OCA\\DAV\\Migration\\Version1029Date20231004091403' => $baseDir . '/../lib/Migration/Version1029Date20231004091403.php',
     'OCA\\DAV\\Migration\\Version1030Date20240205103243' => $baseDir . '/../lib/Migration/Version1030Date20240205103243.php',
     'OCA\\DAV\\Migration\\Version1031Date20240610134258' => $baseDir . '/../lib/Migration/Version1031Date20240610134258.php',
+    'OCA\\DAV\\Migration\\Version1032Date20241011093632' => $baseDir . '/../lib/Migration/Version1032Date20241011093632.php',
+    'OCA\\DAV\\Paginate\\LimitedCopyIterator' => $baseDir . '/../lib/Paginate/LimitedCopyIterator.php',
+    'OCA\\DAV\\Paginate\\PaginateCache' => $baseDir . '/../lib/Paginate/PaginateCache.php',
+    'OCA\\DAV\\Paginate\\PaginatePlugin' => $baseDir . '/../lib/Paginate/PaginatePlugin.php',
     'OCA\\DAV\\Profiler\\ProfilerPlugin' => $baseDir . '/../lib/Profiler/ProfilerPlugin.php',
     'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningNode.php',
     'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => $baseDir . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php',

apps/dav/composer/composer/autoload_static.php

@@ -31,6 +31,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CalendarRetentionJob.php',
         'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupDirectLinksJob.php',
         'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php',
+        'OCA\\DAV\\BackgroundJob\\CleanupPaginateCacheJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupPaginateCacheJob.php',
         'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => __DIR__ . '/..' . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php',
         'OCA\\DAV\\BackgroundJob\\EventReminderJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/EventReminderJob.php',
         'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php',
@@ -343,6 +344,7 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Migration\\Version1008Date20181105110300' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181105110300.php',
         'OCA\\DAV\\Migration\\Version1008Date20181105112049' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181105112049.php',
         'OCA\\DAV\\Migration\\Version1008Date20181114084440' => __DIR__ . '/..' . '/../lib/Migration/Version1008Date20181114084440.php',
+        'OCA\\DAV\\Migration\\Version1009Date20181108161232' => __DIR__ . '/..' . '/../lib/Migration/Version1009Date20181108161232.php',
         'OCA\\DAV\\Migration\\Version1011Date20190725113607' => __DIR__ . '/..' . '/../lib/Migration/Version1011Date20190725113607.php',
         'OCA\\DAV\\Migration\\Version1011Date20190806104428' => __DIR__ . '/..' . '/../lib/Migration/Version1011Date20190806104428.php',
         'OCA\\DAV\\Migration\\Version1012Date20190808122342' => __DIR__ . '/..' . '/../lib/Migration/Version1012Date20190808122342.php',
@@ -356,6 +358,10 @@ class ComposerStaticInitDAV
         'OCA\\DAV\\Migration\\Version1029Date20231004091403' => __DIR__ . '/..' . '/../lib/Migration/Version1029Date20231004091403.php',
         'OCA\\DAV\\Migration\\Version1030Date20240205103243' => __DIR__ . '/..' . '/../lib/Migration/Version1030Date20240205103243.php',
         'OCA\\DAV\\Migration\\Version1031Date20240610134258' => __DIR__ . '/..' . '/../lib/Migration/Version1031Date20240610134258.php',
+        'OCA\\DAV\\Migration\\Version1032Date20241011093632' => __DIR__ . '/..' . '/../lib/Migration/Version1032Date20241011093632.php',
+        'OCA\\DAV\\Paginate\\LimitedCopyIterator' => __DIR__ . '/..' . '/../lib/Paginate/LimitedCopyIterator.php',
+        'OCA\\DAV\\Paginate\\PaginateCache' => __DIR__ . '/..' . '/../lib/Paginate/PaginateCache.php',
+        'OCA\\DAV\\Paginate\\PaginatePlugin' => __DIR__ . '/..' . '/../lib/Paginate/PaginatePlugin.php',
         'OCA\\DAV\\Profiler\\ProfilerPlugin' => __DIR__ . '/..' . '/../lib/Profiler/ProfilerPlugin.php',
         'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningNode' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningNode.php',
         'OCA\\DAV\\Provisioning\\Apple\\AppleProvisioningPlugin' => __DIR__ . '/..' . '/../lib/Provisioning/Apple/AppleProvisioningPlugin.php',

apps/dav/lib/BackgroundJob/CleanupPaginateCacheJob.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\DAV\BackgroundJob;
+
+use OCA\DAV\Paginate\PaginateCache;
+use OCP\BackgroundJob\Job;
+
+class CleanupPaginateCacheJob extends Job {
+	/** @var PaginateCache */
+	private $cache;
+
+	public function __construct(PaginateCache $cache) {
+		$this->cache = $cache;
+	}
+
+	public function run($argument): void {
+		$this->cache->cleanup();
+	}
+}

apps/dav/lib/Migration/Version1032Date20241011093632.php

@@ -0,0 +1,63 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\DAV\Migration;
+
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version1032Date20241011093632 extends SimpleMigrationStep {
+	public function name(): string {
+		return 'Add dav_page_cache table';
+	}
+
+	public function description(): string {
+		return 'Add table to cache webdav multistatus responses for pagination purpose';
+	}
+
+	public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options) {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if (!$schema->hasTable('dav_page_cache')) {
+			$table = $schema->createTable('dav_page_cache');
+
+			$table->addColumn('id', Types::BIGINT, [
+				'autoincrement' => true
+			]);
+			$table->addColumn('url_hash', Types::STRING, [
+				'notnull' => true,
+				'length' => 32,
+			]);
+			$table->addColumn('token', Types::STRING, [
+				'notnull' => true,
+				'length' => 32
+			]);
+			$table->addColumn('result_index', Types::INTEGER, [
+				'notnull' => true
+			]);
+			$table->addColumn('result_value', Types::TEXT, [
+				'notnull' => false,
+			]);
+			$table->addColumn('insert_time', Types::DATETIME, [
+				'notnull' => true,
+			]);
+
+			$table->setPrimaryKey(['id'], 'dav_page_cache_id_index');
+			$table->addIndex(['token', 'url_hash'], 'dav_page_cache_token_url');
+			$table->addUniqueIndex(['token', 'url_hash', 'result_index'], 'dav_page_cache_url_index');
+			$table->addIndex(['result_index'], 'dav_page_cache_index');
+			$table->addIndex(['insert_time'], 'dav_page_cache_time');
+		}
+
+		return $schema;
+	}
+}

apps/dav/lib/Paginate/LimitedCopyIterator.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\DAV\Paginate;
+
+/**
+ * Save a copy of the first X items into a separate iterator
+ *
+ * this allows us to pass the iterator to the cache while keeping a copy
+ * of the first X items
+ */
+class LimitedCopyIterator extends \AppendIterator {
+	/** @var array */
+	private array $copy = [];
+
+	public function __construct(\Traversable $iterator, int $count) {
+		parent::__construct();
+
+		if (!$iterator instanceof \Iterator) {
+			$iterator = new \IteratorIterator($iterator);
+		}
+		$iterator = new \NoRewindIterator($iterator);
+
+		while ($iterator->valid() && count($this->copy) < $count) {
+			$this->copy[] = $iterator->current();
+			$iterator->next();
+		}
+
+		$this->append($this->getFirstItems());
+		$this->append($iterator);
+	}
+
+	public function getFirstItems(): \Iterator {
+		return new \ArrayIterator($this->copy);
+	}
+}

apps/dav/lib/Paginate/PaginateCache.php

@@ -0,0 +1,95 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\DAV\Paginate;
+
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IDBConnection;
+use OCP\Security\ISecureRandom;
+
+class PaginateCache {
+	public const TTL = '1 hour';
+
+	public function __construct(
+		private IDBConnection $database,
+		private ISecureRandom $random,
+		private ITimeFactory $timeFactory,
+	) {
+	}
+
+	/**
+	 * @param string $uri
+	 * @param \Iterator $items
+	 * @return array{'token': string, 'count': int}
+	 */
+	public function store(string $uri, \Iterator $items): array {
+		$token = $this->random->generate(32);
+		$now = $this->timeFactory->getDateTime();
+
+		$query = $this->database->getQueryBuilder();
+		$query->insert('dav_page_cache')
+			->values([
+				'url_hash' => $query->createNamedParameter(md5($uri), IQueryBuilder::PARAM_STR),
+				'token' => $query->createNamedParameter($token, IQueryBuilder::PARAM_STR),
+				'insert_time' => $query->createNamedParameter($now, IQueryBuilder::PARAM_DATETIME_MUTABLE),
+				'result_index' => $query->createParameter('index'),
+				'result_value' => $query->createParameter('value'),
+			]);
+
+		$count = 0;
+		foreach ($items as $item) {
+			$value = serialize($item);
+			$query->setParameter('index', $count, IQueryBuilder::PARAM_INT);
+			$query->setParameter('value', $value);
+			$query->executeStatement();
+			$count++;
+		}
+
+		return ['token' => $token, 'count' => $count];
+	}
+
+	/**
+	 * @param string $url
+	 * @param string $token
+	 * @param int $offset
+	 * @param int $count
+	 * @return array|\Traversable
+	 */
+	public function get(string $url, string $token, int $offset, int $count) {
+		$query = $this->database->getQueryBuilder();
+		$query->select(['result_value'])
+			->from('dav_page_cache')
+			->where($query->expr()->eq('token', $query->createNamedParameter($token)))
+			->andWhere($query->expr()->eq('url_hash', $query->createNamedParameter(md5($url))))
+			->andWhere($query->expr()->gte('result_index', $query->createNamedParameter($offset, IQueryBuilder::PARAM_INT)))
+			->andWhere($query->expr()->lt('result_index', $query->createNamedParameter($offset + $count, IQueryBuilder::PARAM_INT)));
+
+		$result = $query->executeQuery();
+		return array_map(function (string $entry) {
+			return unserialize($entry);
+		}, $result->fetchAll(\PDO::FETCH_COLUMN));
+	}
+
+	public function cleanup(): void {
+		$now = $this->timeFactory->getDateTime();
+		$minDate = $now->sub(\DateInterval::createFromDateString(self::TTL));
+
+		$query = $this->database->getQueryBuilder();
+		$query->delete('dav_page_cache')
+			->where($query->expr()->lt('insert_time', $query->createNamedParameter($minDate, IQueryBuilder::PARAM_DATETIME_MUTABLE)));
+		$query->executeStatement();
+	}
+
+	public function clear(): void {
+		$query = $this->database->getQueryBuilder();
+		$query->delete('dav_page_cache');
+		$query->executeStatement();
+	}
+}

apps/dav/lib/Paginate/PaginatePlugin.php

@@ -0,0 +1,93 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2018 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+namespace OCA\DAV\Paginate;
+
+use Sabre\DAV\Server;
+use Sabre\DAV\ServerPlugin;
+use Sabre\HTTP\RequestInterface;
+use Sabre\HTTP\ResponseInterface;
+
+class PaginatePlugin extends ServerPlugin {
+	public const PAGINATE_HEADER = 'X-NC-Paginate';
+	public const PAGINATE_TOTAL_HEADER = 'X-NC-Paginate-Total';
+	public const PAGINATE_TOKEN_HEADER = 'X-NC-Paginate-Token';
+	public const PAGINATE_OFFSET_HEADER = 'X-NC-Paginate-Offset';
+	public const PAGINATE_COUNT_HEADER = 'X-NC-Paginate-Count';
+
+	/** @var Server */
+	private $server;
+
+	public function __construct(
+		private PaginateCache $cache,
+		private int $pageSize = 100,
+	) {
+	}
+
+	public function initialize(Server $server): void {
+		$this->server = $server;
+		$server->on('beforeMultiStatus', [$this, 'onMultiStatus']);
+		$server->on('method:SEARCH', [$this, 'onMethod'], 1);
+		$server->on('method:PROPFIND', [$this, 'onMethod'], 1);
+		$server->on('method:REPORT', [$this, 'onMethod'], 1);
+	}
+
+	public function getFeatures(): array {
+		return ['nc-paginate'];
+	}
+
+	public function onMultiStatus(&$fileProperties): void {
+		$request = $this->server->httpRequest;
+		if (is_array($fileProperties)) {
+			$fileProperties = new \ArrayIterator($fileProperties);
+		}
+		if (
+			$request->hasHeader(self::PAGINATE_HEADER) &&
+			!$request->hasHeader(self::PAGINATE_TOKEN_HEADER)
+		) {
+			$url = $request->getUrl();
+
+			$pageSize = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize;
+			$copyIterator = new LimitedCopyIterator($fileProperties, $pageSize);
+			['token' => $token, 'count' => $count] = $this->cache->store($url, $copyIterator);
+
+			$fileProperties = $copyIterator->getFirstItems();
+			$this->server->httpResponse->addHeader(self::PAGINATE_HEADER, 'true');
+			$this->server->httpResponse->addHeader(self::PAGINATE_TOKEN_HEADER, $token);
+			$this->server->httpResponse->addHeader(self::PAGINATE_TOTAL_HEADER, (string)$count);
+		}
+	}
+
+	public function onMethod(RequestInterface $request, ResponseInterface $response) {
+		if (
+			$request->hasHeader(self::PAGINATE_TOKEN_HEADER) &&
+			$request->hasHeader(self::PAGINATE_OFFSET_HEADER)
+		) {
+			$url = $this->server->httpRequest->getUrl();
+			$token = $request->getHeader(self::PAGINATE_TOKEN_HEADER);
+			$offset = (int)$request->getHeader(self::PAGINATE_OFFSET_HEADER);
+			$count = (int)$request->getHeader(self::PAGINATE_COUNT_HEADER) ?: $this->pageSize;
+
+			$items = $this->cache->get($url, $token, $offset, $count);
+
+			$response->setStatus(207);
+			$response->addHeader(self::PAGINATE_HEADER, 'true');
+			$response->setHeader('Content-Type', 'application/xml; charset=utf-8');
+			$response->setHeader('Vary', 'Brief,Prefer');
+
+			$prefer = $this->server->getHTTPPrefer();
+			$minimal = $prefer['return'] === 'minimal';
+
+			$data = $this->server->generateMultiStatus($items, $minimal);
+			$response->setBody($data);
+
+			return false;
+		}
+	}
+}