From 91f2684f02be1034c8476d9acbadbdb47d6015b3 Mon Sep 17 00:00:00 2001 From: Connor van Spronssen Date: Mon, 13 Jan 2025 07:17:57 +0100 Subject: [PATCH 1/7] feat: Include assets in the initial sync call --- cmd/cs_sdk/functions/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cs_sdk/functions/sync.go b/cmd/cs_sdk/functions/sync.go index 8dfe7a8..fdee968 100644 --- a/cmd/cs_sdk/functions/sync.go +++ b/cmd/cs_sdk/functions/sync.go @@ -124,7 +124,7 @@ func getSyncData(paginationToken string, reset bool, syncToken string) (map[stri data, err = cs_sdk.Request(path, "GET", nil, false) } else if err != nil || reset { logger.Info("Initialising a fresh sync") - path := "stacks/sync?init=true&type=entry_published,entry_unpublished,entry_deleted" + path := "stacks/sync?init=true&type=entry_published,entry_unpublished,entry_deleted,asset_published,asset_unpublished,asset_deleted" data, err = cs_sdk.Request(path, "GET", nil, false) } else { logger.Info("Syncing data using an existing sync token") From 3893557cedac13ca2bc504783d295f361fd8041a Mon Sep 17 00:00:00 2001 From: Connor van Spronssen Date: Mon, 13 Jan 2025 07:27:39 +0100 Subject: [PATCH 2/7] feat: Add logic to filter out assets during the sync --- cmd/cs_sdk/functions/sync.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cmd/cs_sdk/functions/sync.go b/cmd/cs_sdk/functions/sync.go index fdee968..ae2312f 100644 --- a/cmd/cs_sdk/functions/sync.go +++ b/cmd/cs_sdk/functions/sync.go @@ -51,6 +51,12 @@ func Sync(reset bool) error { return err } + err = addAllAssets(data) + + if err != nil { + return err + } + var hasPaginationToken bool paginationToken, hasPaginationToken = data["pagination_token"].(string) @@ -86,6 +92,30 @@ func getNewSyncToken(data map[string]interface{}) (string, error) { return newSyncToken, nil } +func addAllAssets(data map[string]interface{}) error { + items, hasItems := data["items"].([]interface{}) + + if !hasItems { + return errors.New("sync data has no items") + } + + itemCount := len(items) + + for idx, item := range items { + logger.Info("Fetching item data (%d/%d)", (idx + 1), itemCount) + + item := item.(map[string]interface{}) + + if item["content_type_uid"].(string) != "sys_assets" { + continue + } + + fmt.Println(item) + } + + return nil +} + func addAllRoutes(data map[string]interface{}, routes *map[string]structs.Route) error { err := addSyncRoutes(data, routes) From 30219c79b35dfe3dcdffd6ac73369fd00b9118f2 Mon Sep 17 00:00:00 2001 From: Connor van Spronssen Date: Mon, 13 Jan 2025 07:31:31 +0100 Subject: [PATCH 3/7] feat: Add a migration to create an "assets" table --- cmd/migrate_db/migrations/000004_create_assets_table.down.sql | 1 + cmd/migrate_db/migrations/000004_create_assets_table.up.sql | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 cmd/migrate_db/migrations/000004_create_assets_table.down.sql create mode 100644 cmd/migrate_db/migrations/000004_create_assets_table.up.sql diff --git a/cmd/migrate_db/migrations/000004_create_assets_table.down.sql b/cmd/migrate_db/migrations/000004_create_assets_table.down.sql new file mode 100644 index 0000000..2d59c03 --- /dev/null +++ b/cmd/migrate_db/migrations/000004_create_assets_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS assets; diff --git a/cmd/migrate_db/migrations/000004_create_assets_table.up.sql b/cmd/migrate_db/migrations/000004_create_assets_table.up.sql new file mode 100644 index 0000000..f9b532a --- /dev/null +++ b/cmd/migrate_db/migrations/000004_create_assets_table.up.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS assets( + id varchar(48) NOT NULL PRIMARY KEY UNIQUE, + uid varchar(32) NOT NULL +); From e998bc573b4d1a3410cf7e63a1064b7c867e0823 Mon Sep 17 00:00:00 2001 From: Connor van Spronssen Date: Mon, 13 Jan 2025 07:50:31 +0100 Subject: [PATCH 4/7] feat: Add logic to filter out assets during the sync --- cmd/cs_sdk/functions/sync.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/cs_sdk/functions/sync.go b/cmd/cs_sdk/functions/sync.go index ae2312f..648087e 100644 --- a/cmd/cs_sdk/functions/sync.go +++ b/cmd/cs_sdk/functions/sync.go @@ -182,6 +182,11 @@ func addSyncRoutes(data map[string]interface{}, routes *map[string]structs.Route logger.Info("Fetching item data (%d/%d)", (idx + 1), itemCount) item := item.(map[string]interface{}) + + if item["content_type_uid"].(string) == "sys_assets" { + continue + } + data := item["data"].(map[string]interface{}) publishDetails, hasPublishDetails := data["publish_details"].(map[string]interface{}) From 3b375c9d3d3cf7f4256bcba7b7b29014e6733ad6 Mon Sep 17 00:00:00 2001 From: Connor van Spronssen Date: Mon, 13 Jan 2025 08:16:24 +0100 Subject: [PATCH 5/7] feat: Add some fields to the new assets table --- .../migrations/000004_create_assets_table.up.sql | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/migrate_db/migrations/000004_create_assets_table.up.sql b/cmd/migrate_db/migrations/000004_create_assets_table.up.sql index f9b532a..177d2e3 100644 --- a/cmd/migrate_db/migrations/000004_create_assets_table.up.sql +++ b/cmd/migrate_db/migrations/000004_create_assets_table.up.sql @@ -1,4 +1,12 @@ CREATE TABLE IF NOT EXISTS assets( id varchar(48) NOT NULL PRIMARY KEY UNIQUE, - uid varchar(32) NOT NULL + uid varchar(32) NOT NULL, + title varchar(255) NOT NULL, + content_type varchar(255) NOT NULL, + locale varchar(16) NOT NULL, + slug varchar(255) NOT NULL, + url varchar(255) NOT NULL, + parent varchar(64), + updated_at timestamp NOT NULL, + published boolean NOT NULL DEFAULT false ); From 0ff61abd85a89e71e97c2002b78233f9ee0833f2 Mon Sep 17 00:00:00 2001 From: Connor van Spronssen Date: Mon, 13 Jan 2025 08:16:51 +0100 Subject: [PATCH 6/7] feat: Add a function to set asset data in the database --- cmd/database/assets/set-asset.go | 65 ++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 cmd/database/assets/set-asset.go diff --git a/cmd/database/assets/set-asset.go b/cmd/database/assets/set-asset.go new file mode 100644 index 0000000..6090fce --- /dev/null +++ b/cmd/database/assets/set-asset.go @@ -0,0 +1,65 @@ +package assets + +import ( + "github.com/Dobefu/csb/cmd/cs_sdk/structs" + "github.com/Dobefu/csb/cmd/cs_sdk/utils" + "github.com/Dobefu/csb/cmd/database/query" + db_structs "github.com/Dobefu/csb/cmd/database/structs" +) + +func SetAsset( + route structs.Route, +) error { + err := query.Upsert("assets", getAssetValues(route)) + + if err != nil { + return err + } + + return nil +} + +func getAssetValues(route structs.Route) []db_structs.QueryValue { + return []db_structs.QueryValue{ + { + Name: "id", + Value: utils.GenerateId(route), + }, + { + Name: "uid", + Value: route.Uid, + }, + { + Name: "title", + Value: route.Title, + }, + { + Name: "content_type", + Value: route.ContentType, + }, + { + Name: "locale", + Value: route.Locale, + }, + { + Name: "slug", + Value: route.Slug, + }, + { + Name: "url", + Value: route.Url, + }, + { + Name: "parent", + Value: route.Parent, + }, + { + Name: "updated_at", + Value: route.UpdatedAt, + }, + { + Name: "published", + Value: route.Published, + }, + } +} From 85837cc5bc8d59baa844d54251a79e984132b818 Mon Sep 17 00:00:00 2001 From: Connor van Spronssen Date: Mon, 13 Jan 2025 08:17:29 +0100 Subject: [PATCH 7/7] feat: Add logic to the sync to save assets to the database --- cmd/cs_sdk/functions/sync.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/cmd/cs_sdk/functions/sync.go b/cmd/cs_sdk/functions/sync.go index 648087e..afac84d 100644 --- a/cmd/cs_sdk/functions/sync.go +++ b/cmd/cs_sdk/functions/sync.go @@ -10,6 +10,7 @@ import ( "github.com/Dobefu/csb/cmd/cs_sdk" "github.com/Dobefu/csb/cmd/cs_sdk/structs" "github.com/Dobefu/csb/cmd/cs_sdk/utils" + "github.com/Dobefu/csb/cmd/database/assets" "github.com/Dobefu/csb/cmd/database/query" db_routes "github.com/Dobefu/csb/cmd/database/routes" "github.com/Dobefu/csb/cmd/database/state" @@ -111,6 +112,36 @@ func addAllAssets(data map[string]interface{}) error { } fmt.Println(item) + assetData := item["data"].(map[string]interface{}) + + publishDetails, hasPublishDetails := assetData["publish_details"].(map[string]interface{}) + + if !hasPublishDetails { + publishDetails = map[string]interface{}{ + "locale": "", + } + } + + parentUid, hasParentUid := assetData["parent_uid"].(string) + + if !hasParentUid { + parentUid = "" + } + fmt.Println(assetData) + err := assets.SetAsset(structs.Route{ + Uid: assetData["uid"].(string), + Title: assetData["title"].(string), + ContentType: assetData["content_type"].(string), + Locale: publishDetails["locale"].(string), + Url: assetData["url"].(string), + Parent: parentUid, + UpdatedAt: getUpdatedAt(assetData), + Published: item["type"].(string) == "asset_published", + }) + + if err != nil { + return err + } } return nil