Merge pull request #183 from marcoshuck/feature/list_tasks

List tasks: Cursor-based pagination

Author: marcoshuck

Makefile

@@ -33,10 +33,10 @@ test-html:
 	go tool cover -html=coverage.out -o=coverage.html
 
 ci/compose-up:
-	docker-compose -f ./deployments/local/ci.docker-compose.yml up -d --build
+	docker compose -f ./deployments/local/ci.docker-compose.yml up -d --build
 
 ci/compose-down:
-	docker-compose -f ./deployments/local/ci.docker-compose.yml down --rmi all
+	docker compose -f ./deployments/local/ci.docker-compose.yml down --rmi all
 
 test-e2e:
 	npx playwright test

api/tasks/v1/tasks.pb.go

@@ -135,7 +135,6 @@ message ListTasksRequest {
   reserved "parent";
 
   int32 page_size = 2 [
-    (buf.validate.field).int32.gt = 0,
     (google.api.field_behavior) = OPTIONAL
   ];
   string page_token = 3 [

api/tasks/v1/tasks.proto

@@ -1,4 +1,3 @@
-version: "3.9"
 services:
   app:
     image: "ghcr.io/marcoshuck/todo/app:latest"

deployments/local/ci.docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.9"
 services:
   app:
     image: "marcoshuck/todo-app"

deployments/local/docker-compose.yml

@@ -0,0 +1,22 @@
+package serializer
+
+import (
+	"encoding/base64"
+	"time"
+)
+
+// EncodePageToken converts the given timestamp into a base64-encoded page token.
+func EncodePageToken(t time.Time) string {
+	return base64.URLEncoding.EncodeToString([]byte(t.Format(time.RFC3339)))
+}
+
+// DecodePageToken converts the given token into a valid timestamp.
+// It returns an error if the token is not a valid representation of a base64-encoded
+// timestamp.
+func DecodePageToken(token string) (time.Time, error) {
+	b, err := base64.URLEncoding.DecodeString(token)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return time.Parse(time.RFC3339, string(b))
+}

internal/serializer/page_token.go

@@ -0,0 +1,36 @@
+package serializer
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestEncodePageToken(t *testing.T) {
+	now := time.Now()
+	token := EncodePageToken(now)
+	assert.NotNil(t, token)
+	var expected string
+	assert.IsType(t, expected, token)
+	assert.NotEmpty(t, token)
+}
+
+func TestDecodePageToken(t *testing.T) {
+	now := time.Now()
+	token := EncodePageToken(now)
+
+	out, err := DecodePageToken(token)
+	assert.NoError(t, err)
+	assert.Equal(t, now.Year(), out.Year())
+	assert.Equal(t, now.Month(), out.Month())
+	assert.Equal(t, now.Day(), out.Day())
+	assert.Equal(t, now.Hour(), out.Hour())
+	assert.Equal(t, now.Minute(), out.Minute())
+	assert.Equal(t, now.Second(), out.Second())
+}
+
+func TestDecodePageToken_InvalidToken(t *testing.T) {
+	out, err := DecodePageToken("1234")
+	assert.Error(t, err)
+	assert.Zero(t, out)
+}

internal/serializer/page_token_test.go

@@ -5,12 +5,14 @@ import (
 	"errors"
 	tasksv1 "github.com/marcoshuck/todo/api/tasks/v1"
 	"github.com/marcoshuck/todo/internal/domain"
+	"github.com/marcoshuck/todo/internal/serializer"
 	"go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"gorm.io/gorm"
+	"time"
 )
 
 // tasks implements tasksv1.TasksWriterServiceServer.
@@ -48,18 +50,41 @@ func (svc *tasks) ListTasks(ctx context.Context, request *tasksv1.ListTasksReque
 	svc.logger.Debug("Getting task list", zap.Int32("page_size", request.GetPageSize()), zap.String("page_token", request.GetPageToken()))
 	span := trace.SpanFromContext(ctx)
 	defer span.End()
+
 	span.AddEvent("Getting tasks from the database")
+	if request.GetPageSize() == 0 {
+		request.PageSize = 50
+	}
+
+	query := svc.db.Model(&domain.Task{}).WithContext(ctx)
+	if len(request.GetPageToken()) > 0 {
+		updatedAt, err := serializer.DecodePageToken(request.GetPageToken())
+		if err == nil {
+			svc.logger.Debug("Getting records older than page token", zap.Time("updated_at", updatedAt))
+			query = query.Where("updated_at < ?", updatedAt)
+		}
+	}
+	query = query.Limit(int(request.GetPageSize() + 1)).Order("updated_at DESC")
+
 	var out []domain.Task
-	err := svc.db.Model(&domain.Task{}).WithContext(ctx).Find(&out).Error
+	err := query.Find(&out).Error
 	if err != nil {
 		svc.logger.Error("Failed to list tasks", zap.Error(err))
 		span.RecordError(err)
 		return nil, status.Errorf(codes.Unavailable, "failed to get task: %v", err)
 	}
+
+	var nextPageToken string
+	if len(out) > int(request.GetPageSize()) {
+		nextPageToken = serializer.EncodePageToken(out[request.GetPageSize()-1].UpdatedAt)
+		svc.logger.Debug("Generating next page token", zap.Int32("page_size", request.GetPageSize()), zap.Int("count", len(out)), zap.String("page_token", nextPageToken), zap.Uint("last_element_id", out[request.GetPageSize()].ID))
+		out = out[:request.GetPageSize()]
+	}
+
 	svc.logger.Debug("Returning task list", zap.Int32("page_size", request.GetPageSize()), zap.String("page_token", request.GetPageToken()), zap.Int("count", len(out)))
 	res := tasksv1.ListTasksResponse{
 		Tasks:         make([]*tasksv1.Task, len(out)),
-		NextPageToken: "",
+		NextPageToken: nextPageToken,
 	}
 	for i, task := range out {
 		res.Tasks[i] = task.API()
@@ -77,6 +102,9 @@ func (svc *tasks) CreateTask(ctx context.Context, request *tasksv1.CreateTaskReq
 	svc.logger.Debug("Filling out task information")
 	span.AddEvent("Parsing task from API request")
 	task.FromAPI(request.GetTask())
+	now := time.Now()
+	task.CreatedAt = now
+	task.UpdatedAt = now
 	span.AddEvent("Persisting task in the database")
 	svc.logger.Debug("Persisting task in the database", zap.String("task.title", request.GetTask().GetTitle()))
 	err := svc.db.Model(&domain.Task{}).WithContext(ctx).Create(&task).Error

internal/service/tasks.go

@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	"fmt"
 	tasksv1 "github.com/marcoshuck/todo/api/tasks/v1"
 	"github.com/marcoshuck/todo/internal/domain"
 	"github.com/stretchr/testify/suite"
@@ -13,6 +14,7 @@ import (
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"testing"
+	"time"
 )
 
 func TestTasksServiceSuite(t *testing.T) {
@@ -24,6 +26,7 @@ type TasksServiceTestSuite struct {
 	db     *gorm.DB
 	writer tasksv1.TasksWriterServiceServer
 	reader tasksv1.TasksReaderServiceServer
+	logger *zap.Logger
 }
 
 func (suite *TasksServiceTestSuite) SetupSuite() {
@@ -32,12 +35,17 @@ func (suite *TasksServiceTestSuite) SetupSuite() {
 
 func (suite *TasksServiceTestSuite) SetupTest() {
 	var err error
+
+	suite.logger, err = zap.NewDevelopment()
+	suite.Require().NoError(err)
+
 	suite.db, err = gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
 	suite.Require().NoError(err)
+	suite.db = suite.db.Debug()
 	suite.Require().NoError(suite.db.Migrator().AutoMigrate(&domain.Task{}))
 
-	suite.writer = NewTasksWriter(suite.db, zap.NewNop(), noop.NewMeterProvider().Meter(""))
-	suite.reader = NewTasksReader(suite.db, zap.NewNop(), noop.NewMeterProvider().Meter(""))
+	suite.writer = NewTasksWriter(suite.db, suite.logger, noop.NewMeterProvider().Meter(""))
+	suite.reader = NewTasksReader(suite.db, suite.logger, noop.NewMeterProvider().Meter(""))
 }
 
 func (suite *TasksServiceTestSuite) TearDownTest() {
@@ -63,6 +71,8 @@ func (suite *TasksServiceTestSuite) TestCreate_Success() {
 	suite.Assert().NoError(err)
 	suite.Assert().NotNil(res)
 	suite.Assert().Equal(title, res.GetTitle())
+	suite.Assert().NotZero(res.GetCreateTime().AsTime())
+	suite.Assert().NotZero(res.GetUpdateTime().AsTime())
 
 	var after int64
 	suite.Require().NoError(suite.db.Model(&domain.Task{}).Count(&after).Error)
@@ -108,25 +118,38 @@ func (suite *TasksServiceTestSuite) TestList_Empty() {
 func (suite *TasksServiceTestSuite) TestList_Success() {
 	ctx := context.Background()
 
-	for i := 0; i < 10; i++ {
-		_, err := suite.writer.CreateTask(ctx, &tasksv1.CreateTaskRequest{
-			Task: &tasksv1.Task{
-				Title: "A test",
+	list := make([]domain.Task, 0, 10)
+	for i := 1; i <= 10; i++ {
+		list = append(list, domain.Task{
+			Model: gorm.Model{
+				CreatedAt: time.Now().Add(-time.Duration(i) * time.Hour),
+				UpdatedAt: time.Now().Add(-time.Duration(i) * time.Hour),
 			},
+			Title: fmt.Sprintf("%s %d", suite.T().Name(), i),
 		})
-		suite.Require().NoError(err)
 	}
+	suite.Require().NoError(suite.db.Create(list).Error)
 
 	var expected int64
 	suite.Require().NoError(suite.db.Model(&domain.Task{}).Count(&expected).Error)
 
 	response, err := suite.reader.ListTasks(ctx, &tasksv1.ListTasksRequest{
-		PageSize:  0,
+		PageSize:  5,
 		PageToken: "",
 	})
 	suite.Assert().NoError(err)
 	suite.Assert().NotEmpty(response.GetTasks())
-	suite.Assert().Len(response.GetTasks(), int(expected))
+	suite.Assert().Len(response.GetTasks(), 5)
+	suite.Assert().NotEmpty(response.GetNextPageToken())
+
+	response, err = suite.reader.ListTasks(ctx, &tasksv1.ListTasksRequest{
+		PageSize:  5,
+		PageToken: response.GetNextPageToken(),
+	})
+	suite.Assert().NoError(err)
+	suite.Assert().NotEmpty(response.GetTasks())
+	suite.Assert().Len(response.GetTasks(), 5)
+	suite.Assert().Empty(response.GetNextPageToken())
 }
 
 func (suite *TasksServiceTestSuite) TestDelete_NotFound() {

internal/service/tasks_test.go

@@ -1,5 +1,6 @@
 import {expect, test} from '@playwright/test';
-import {createTask, deleteTask, getTask, undeleteTask, updateTask} from "./tasks.utils";
+import {createTask, deleteTask, getTask, listTasks, undeleteTask, updateTask} from "./tasks.utils";
+import {Task} from "../api/tasks/v1/tasks_pb";
 
 
 test('POST /v1/tasks', async ({request}) => {
@@ -35,8 +36,30 @@ test('GET /v1/tasks/:id', async ({request}) => {
     expect(output.title).toEqual(expected.title);
     expect(output.description).toEqual(expected.description);
     expect(output.id).toEqual(expected.id);
-    expect(output.createTime).toEqual(expected.createTime);
-    expect(output.updateTime).toEqual(expected.updateTime);
+})
+
+test('GET /v1/tasks', async ({request}) => {
+    let input = {
+        title: 'An awesome task',
+        description: 'An awesome description for an awesome task',
+    };
+
+    const list: Task[] = [];
+
+    for (let i = 0; i < 10; i++) {
+        let task = await createTask(request, input);
+        list.push(task);
+        await new Promise(r => setTimeout(r, 1000));
+    }
+
+    let response = await listTasks(request, 5, undefined);
+
+    expect(response.data.nextPageToken).not.toBe('');
+
+
+    for (const task of list) {
+        await deleteTask(request, task.id);
+    }
 })
 
 test('DELETE /v1/tasks/:id', async ({request}) => {

tests/tasks.spec.ts

@@ -1,5 +1,5 @@
 import {APIRequestContext, expect} from "@playwright/test";
-import {Task} from '../api/tasks/v1/tasks_pb';
+import {ListTasksResponse, Task} from '../api/tasks/v1/tasks_pb';
 
 export async function createTask(request: APIRequestContext, input: any): Promise<Task> {
     // Send the request and wait for the response.
@@ -16,16 +16,31 @@ export async function createTask(request: APIRequestContext, input: any): Promis
 }
 
 export async function getTask(request: APIRequestContext, id: bigint) {
-    const response = await request.get(`/v1/tasks/${id}`)
+    const response = await request.get(`/v1/tasks/${id}`);
     const body = await response.body();
     return {
         response: response,
         data: Task.fromJsonString(body.toString()),
     }
 }
 
+export async function listTasks(request: APIRequestContext, size: number, nextPageToken: string | undefined) {
+    const response = await request.get(`/v1/tasks`, {
+        params: {
+            "page_size": size,
+            "page_token": nextPageToken,
+        },
+    });
+    const body = await response.body();
+
+    return {
+        response: response,
+        data: ListTasksResponse.fromJsonString(body.toString()),
+    }
+}
+
 export async function deleteTask(request: APIRequestContext, id: bigint): Promise<void> {
-    const response = await request.delete(`/v1/tasks/${id}`)
+    const response = await request.delete(`/v1/tasks/${id}`);
     expect(response.ok()).toBeTruthy();
 }
 
@@ -34,14 +49,14 @@ export async function undeleteTask(request: APIRequestContext, id: bigint): Prom
         data: {
             id: Number(id),
         }
-    })
+    });
     expect(response.ok()).toBeTruthy();
 }
 
 export async function updateTask(request: APIRequestContext, id: bigint, payload: any): Promise<Task> {
     const response = await request.patch(`/v1/tasks/${id}`, {
         data: payload,
-    })
+    });
     // Read the body
     const body = await response.body();
     return Task.fromJsonString(body.toString());