From 0b2dd29a76e167262b2fb642974218880d863f61 Mon Sep 17 00:00:00 2001 From: Florian Proksch Date: Wed, 9 Oct 2024 11:44:01 +0200 Subject: [PATCH 1/3] chore: allow for customization of local dynamodb endpoint --- tests/DynamoDbTestCase.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/DynamoDbTestCase.php b/tests/DynamoDbTestCase.php index de64b7a..fc81254 100644 --- a/tests/DynamoDbTestCase.php +++ b/tests/DynamoDbTestCase.php @@ -38,7 +38,7 @@ protected function getEnvironmentSetUp($app) 'secret' => 'secret', ], 'region' => 'test', - 'endpoint' => 'http://localhost:3000', + 'endpoint' => env('DYNAMODB_LOCAL_ENDPOINT', 'http://localhost:3000'), ]); } From 3a1ee4462d17963d7df8a046e8e7e398d893c116 Mon Sep 17 00:00:00 2001 From: Florian Proksch Date: Wed, 9 Oct 2024 11:47:56 +0200 Subject: [PATCH 2/3] feat: pagination using laravel's CursorPaginator contract Mimic the classic query builder's pagination functions to wrap the result in a CursorPaginator. Instead of the built-in functionality to calculate the cursor based on the last returned item, we simply lean on the `lastEvaluatedKey` result from dynamo. --- src/DynamoDbCursorPaginator.php | 68 ++++++++++++++++++++ src/DynamoDbQueryBuilder.php | 31 +++++++++ tests/DynamoDbCursorPaginatorTest.php | 92 +++++++++++++++++++++++++++ 3 files changed, 191 insertions(+) create mode 100644 src/DynamoDbCursorPaginator.php create mode 100644 tests/DynamoDbCursorPaginatorTest.php diff --git a/src/DynamoDbCursorPaginator.php b/src/DynamoDbCursorPaginator.php new file mode 100644 index 0000000..5c41e87 --- /dev/null +++ b/src/DynamoDbCursorPaginator.php @@ -0,0 +1,68 @@ +hasMore = (boolean) $lastEvaluatedKey; + } + + /** + * Return a cursor to the previous page. + * + * Since DynamoDB cannot page back we just always return null. + * + * @return null + */ + public function previousCursor() + { + return null; + } + + /** + * Return a cursor to the next page. + * + * This is largely cloning the base method but then just returning a cursor holding the `lastEvaluatedKey` instead. + * + * @return Cursor|null + */ + public function nextCursor() + { + if ((is_null($this->cursor) && ! $this->hasMore) || + (! is_null($this->cursor) && $this->cursor->pointsToNextItems() && ! $this->hasMore)) { + return null; + } + + if ($this->items->isEmpty()) { + return null; + } + + // The Cursor implementation expects the parameters to only be 1 level deep, so we JSON-Encode the lastEvaluatedKey, + // since it can hold both a PK and an SK. + return new Cursor(['lastEvaluatedKey' => json_encode(DynamoDb::unmarshalItem($this->lastEvaluatedKey))]); + } + + /** + * Get the instance as an array. + * + * We patch in the first_page_url here, since we can always just jump back to the first page by not passing a start + * key to the query. + * + * @return array|string[] + */ + public function toArray() + { + return array_merge(parent::toArray(), [ + 'first_page_url' => $this->url(null), + ]); + } +} diff --git a/src/DynamoDbQueryBuilder.php b/src/DynamoDbQueryBuilder.php index 54e584b..40a5664 100644 --- a/src/DynamoDbQueryBuilder.php +++ b/src/DynamoDbQueryBuilder.php @@ -7,9 +7,12 @@ use BaoPham\DynamoDb\Facades\DynamoDb; use BaoPham\DynamoDb\H; use Closure; +use Illuminate\Contracts\Pagination\CursorPaginator; use Illuminate\Contracts\Support\Arrayable; use Illuminate\Database\Eloquent\ModelNotFoundException; use Illuminate\Database\Eloquent\Scope; +use Illuminate\Pagination\Cursor; +use Illuminate\Pagination\Paginator; use Illuminate\Support\Arr; class DynamoDbQueryBuilder @@ -444,6 +447,34 @@ public function chunk($chunkSize, callable $callback) return true; } + public function paginate($perPage = 15, $columns = [], $cursorName = 'cursor', $cursor = null): CursorPaginator + { + if (! $cursor instanceof Cursor) { + $cursor = is_string($cursor) + ? Cursor::fromEncoded($cursor) + : DynamoDbCursorPaginator::resolveCurrentCursor($cursorName, $cursor); + } + + $this->limit($perPage); + + if ($cursor && $cursor->pointsToNextItems()) { + $this->afterKey(json_decode($cursor->parameter('lastEvaluatedKey'))); + } + + $items = $this->get($columns); + + return new DynamoDbCursorPaginator( + $items, + $perPage, + $cursor, + [ + 'path' => Paginator::resolveCurrentPath(), + 'cursorName' => $cursorName, + ], + $this->lastEvaluatedKey, + ); + } + /** * @param $id * @param array $columns diff --git a/tests/DynamoDbCursorPaginatorTest.php b/tests/DynamoDbCursorPaginatorTest.php new file mode 100644 index 0000000..c3037df --- /dev/null +++ b/tests/DynamoDbCursorPaginatorTest.php @@ -0,0 +1,92 @@ +assertInstanceOf(DynamoDbCursorPaginator::class, $paginator); + } + + public function testPageSizeLimit() + { + $this->seedMultiple(3); + $paginator = TestModel::paginate(1); + + $this->assertCount(1, $paginator->items()); + $this->assertTrue($paginator->hasMorePages()); + } + + public function testNextPage() + { + $this->seed(['id' => ['S' => 'ONE']]); + $this->seed(['id' => ['S' => 'TWO']]); + $this->seed(['id' => ['S' => 'THREE']]); + + $paginator = TestModel::paginate(1); + $this->assertCount(1, $paginator->items()); + $this->assertEquals('ONE', $paginator->items()[0]->id); + $this->assertTrue($paginator->hasMorePages()); + + $nextPaginator = TestModel::paginate(cursor: $paginator->nextCursor()); + + $items = $nextPaginator->items(); + $this->assertCount(2, $items); + $this->assertEquals('TWO', $items[0]->id); + $this->assertFalse($nextPaginator->hasMorePages()); + } + + public function seed($attributes = []) + { + $item = [ + 'id' => ['S' => Str::random(36)], + 'name' => ['S' => Str::random(36)], + 'description' => ['S' => Str::random(256)], + 'count' => ['N' => rand()], + 'author' => ['S' => Str::random()], + ]; + + $item = array_merge($item, $attributes); + + $this->getClient()->putItem([ + 'TableName' => $this->testModel->getTable(), + 'Item' => $item, + ]); + + return $item; + } + public function seedMultiple($amount = 1) + { + for ($i = 0; $i < $amount; $i++) { + $this->seed(); + } + } +} + +// phpcs:disable PSR1.Classes.ClassDeclaration.MultipleClasses +class TestModel extends \BaoPham\DynamoDb\DynamoDbModel +{ + protected $fillable = ['name', 'description', 'count']; + + protected $table = 'test_model'; + + protected $connection = 'test'; + + public $timestamps = true; + + protected $dynamoDbIndexKeys = [ + 'count_index' => [ + 'hash' => 'count', + ], + ]; +} +// phpcs:enable PSR1.Classes.ClassDeclaration.MultipleClasses From 7c11d4c3256dff95a2f9a00d075dbc035c2b2f25 Mon Sep 17 00:00:00 2001 From: Florian Proksch Date: Wed, 9 Oct 2024 12:24:34 +0200 Subject: [PATCH 3/3] chore: update documentation --- README.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 82a557b..dc4f232 100644 --- a/README.md +++ b/README.md @@ -176,21 +176,28 @@ $model->first(); #### Pagination Unfortunately, offset of how many records to skip does not make sense for DynamoDb. -Instead, provide the last result of the previous query as the starting point for the next query. +However, you can use the `paginate` function on the query builder to get a CursorPaginator, which will +handle passing the last evaluated key around: -**Examples:** +```PHP +$paginator = $model->paginate(); +``` + +The paginator implements laravel's pagination interface, so it supports automatic link generation as usual. +It will also try to extract the cursor from the request parameters automatically, so if you use the rendered +pagination links or resource collections, pagination should be seamless! + +See https://laravel.com/docs/11.x/pagination#cursor-pagination and +https://laravel.com/docs/11.x/pagination#cursor-paginator-instance-methods for details. -For query such as: +If you need to manage the last evaluated key manually, you can use the `after` and `afterKey` methods to provide +a model or the raw key respectively: ```php $query = $model->where('count', 10)->limit(2); $items = $query->all(); $last = $items->last(); -``` - -Take the last item of this query result as the next "offset": -```php $nextPage = $query->after($last)->limit(2)->all(); // or $nextPage = $query->afterKey($items->lastKey())->limit(2)->all();