From 8beb858c40163a8006701b0bcf340d2cef0d9fd0 Mon Sep 17 00:00:00 2001 From: rook1e Date: Sun, 25 Aug 2024 18:40:34 +0800 Subject: [PATCH 1/4] feat(#5): create groups and import feeds into them, based on opml --- api/group.go | 5 +- frontend/src/lib/api/group.ts | 12 ++- frontend/src/lib/opml.ts | 59 +++++++++--- .../routes/(authed)/feeds/ActionAdd.svelte | 8 +- .../routes/(authed)/feeds/ActionOPML.svelte | 95 +++++++++++-------- frontend/src/routes/+layout.svelte | 2 +- model/feed.go | 4 +- repo/feed.go | 6 +- repo/repo.go | 43 +++++++++ server/feed.go | 3 - server/group.go | 11 ++- server/group_form.go | 4 + 12 files changed, 178 insertions(+), 74 deletions(-) diff --git a/api/group.go b/api/group.go index b67af19..b7ded72 100644 --- a/api/group.go +++ b/api/group.go @@ -33,11 +33,12 @@ func (f groupAPI) Create(c echo.Context) error { return err } - if err := f.srv.Create(c.Request().Context(), &req); err != nil { + resp, err := f.srv.Create(c.Request().Context(), &req) + if err != nil { return err } - return c.NoContent(http.StatusCreated) + return c.JSON(http.StatusCreated, resp) } func (f groupAPI) Update(c echo.Context) error { diff --git a/frontend/src/lib/api/group.ts b/frontend/src/lib/api/group.ts index 4aa94bc..eaf2f97 100644 --- a/frontend/src/lib/api/group.ts +++ b/frontend/src/lib/api/group.ts @@ -7,11 +7,13 @@ export async function allGroups() { } export async function createGroup(name: string) { - return await api.post('groups', { - json: { - name: name - } - }); + return await api + .post('groups', { + json: { + name: name + } + }) + .json<{ id: number }>(); } export async function updateGroup(id: number, name: string) { diff --git a/frontend/src/lib/opml.ts b/frontend/src/lib/opml.ts index 15dc772..60811b8 100644 --- a/frontend/src/lib/opml.ts +++ b/frontend/src/lib/opml.ts @@ -1,18 +1,55 @@ export function parse(content: string) { - const feeds: { name: string; link: string }[] = []; + type feedT = { + name: string; + link: string; + }; + type groupT = { + name: string; + feeds: feedT[]; + }; + const groups = new Map(); + const defaultGroup = { name: 'Default', feeds: [] }; + groups.set('Default', defaultGroup); + + function dfs(parentGroup: groupT | null, node: Element) { + if (node.tagName !== 'outline') { + return; + } + if (node.getAttribute('type')?.toLowerCase() == 'rss') { + if (!parentGroup) { + parentGroup = defaultGroup; + } + parentGroup.feeds.push({ + name: node.getAttribute('title') || node.getAttribute('text') || '', + link: node.getAttribute('xmlUrl') || node.getAttribute('htmlUrl') || '' + }); + return; + } + if (!node.children.length) { + return; + } + const nodeName = node.getAttribute('text') || node.getAttribute('title') || ''; + const name = parentGroup ? parentGroup.name + '/' + nodeName : nodeName; + let curGroup = groups.get(name); + if (!curGroup) { + curGroup = { name: name, feeds: [] }; + groups.set(name, curGroup); + } + for (const n of node.children) { + dfs(curGroup, n); + } + } + const xmlDoc = new DOMParser().parseFromString(content, 'text/xml'); - const outlines = xmlDoc.getElementsByTagName('outline'); - - for (let i = 0; i < outlines.length; i++) { - const outline = outlines.item(i); - if (!outline) continue; - const link = outline.getAttribute('xmlUrl') || outline.getAttribute('htmlUrl') || ''; - if (!link) continue; - const name = outline.getAttribute('title') || outline.getAttribute('text') || ''; - feeds.push({ name, link }); + const body = xmlDoc.getElementsByTagName('body')[0]; + if (!body) { + return []; + } + for (const n of body.children) { + dfs(null, n); } - return feeds; + return Array.from(groups.values()); } export function dump(data: { name: string; feeds: { name: string; link: string }[] }[]) { diff --git a/frontend/src/routes/(authed)/feeds/ActionAdd.svelte b/frontend/src/routes/(authed)/feeds/ActionAdd.svelte index 442d30c..db83255 100644 --- a/frontend/src/routes/(authed)/feeds/ActionAdd.svelte +++ b/frontend/src/routes/(authed)/feeds/ActionAdd.svelte @@ -106,6 +106,9 @@ }} required /> +

+ The existing feed with the same link will be override. +

@@ -121,11 +124,6 @@ }} required /> - {#if formData.name} -

- The existing feed with the same link will be renamed as {formData.name}. -

- {/if}
diff --git a/frontend/src/routes/(authed)/feeds/ActionOPML.svelte b/frontend/src/routes/(authed)/feeds/ActionOPML.svelte index e96b4eb..76ffc95 100644 --- a/frontend/src/routes/(authed)/feeds/ActionOPML.svelte +++ b/frontend/src/routes/(authed)/feeds/ActionOPML.svelte @@ -1,7 +1,6 @@ - + Import or Export Feeds @@ -87,25 +108,6 @@
-
- - { - return { value: v.id, label: v.name }; - })} - onSelectedChange={(v) => v && (opmlGroup.id = v.value)} - > - - - - - {#each groups as g} - {g.name} - {/each} - - -
- {#if parsedOpmlFeeds.length > 0} + {#if parsedGroupFeeds.length > 0}
-

Parsed out {parsedOpmlFeeds.length} feeds

+

Parsed successfully.

-
    - {#each parsedOpmlFeeds as feed, index} -
  • {index + 1}. {feed.name} {feed.link}
    • - {/each} -
+ {#each parsedGroupFeeds as group} +
+ {group.name} +
+
    + {#each group.feeds as feed} +
  • {feed.name}, {feed.link}
    • + {/each} +
+ {/each}
{/if} - +
+

Note:

+

+ 1. Feeds will be imported into the corresponding group, which will be created + automatically if it does not exist. +

+

2. The existing feed with the same link will be override.

+
+
diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte index 6ee8d3a..588419a 100644 --- a/frontend/src/routes/+layout.svelte +++ b/frontend/src/routes/+layout.svelte @@ -11,7 +11,7 @@ - + diff --git a/model/feed.go b/model/feed.go index e17d90c..fdd9c56 100644 --- a/model/feed.go +++ b/model/feed.go @@ -16,10 +16,10 @@ type Feed struct { ID uint `gorm:"primarykey"` CreatedAt time.Time UpdatedAt time.Time - DeletedAt soft_delete.DeletedAt + DeletedAt soft_delete.DeletedAt `gorm:"uniqueIndex:idx_link"` Name *string `gorm:"name;not null"` - Link *string `gorm:"link;not null"` // FIX: unique index? + Link *string `gorm:"link;not null;uniqueIndex:idx_link"` // LastBuild is the last time the content of the feed changed LastBuild *time.Time `gorm:"last_build"` // Failure is the reason of failure. If it is not null or empty, the fetch processor diff --git a/repo/feed.go b/repo/feed.go index 1fa710f..29c674e 100644 --- a/repo/feed.go +++ b/repo/feed.go @@ -6,6 +6,7 @@ import ( "github.com/0x2e/fusion/model" "gorm.io/gorm" + "gorm.io/gorm/clause" ) func NewFeed(db *gorm.DB) *Feed { @@ -47,7 +48,10 @@ func (f Feed) Get(id uint) (*model.Feed, error) { } func (f Feed) Create(data []*model.Feed) error { - return f.db.Create(data).Error + return f.db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "link"}, {Name: "deleted_at"}}, + DoUpdates: clause.AssignmentColumns([]string{"name", "link"}), + }).Create(data).Error } func (f Feed) Update(id uint, feed *model.Feed) error { diff --git a/repo/repo.go b/repo/repo.go index 53b7e88..5c9e0c6 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -2,6 +2,7 @@ package repo import ( "errors" + "log" "github.com/0x2e/fusion/conf" "github.com/0x2e/fusion/model" @@ -27,6 +28,48 @@ func Init() { } func migrage() { + // The verison after v0.8.7 will add a unique index to Feed.Link. + // We must delete any duplicate feeds before AutoMigrate applies the + // new unique constraint. + err := DB.Transaction(func(tx *gorm.DB) error { + // query duplicate feeds + dupFeeds := make([]model.Feed, 0) + err := tx.Model(&model.Feed{}).Where( + "link IN (?)", + tx.Model(&model.Feed{}).Select("link").Group("link"). + Having("count(link) > 1"), + ).Order("link, id").Find(&dupFeeds).Error + if err != nil { + return err + } + + // filter out feeds that will be deleted. + // we've queried with order, so the first one is the one we should keep. + distinct := map[string]uint{} + deleteIDs := make([]uint, 0, len(dupFeeds)) + for _, f := range dupFeeds { + if _, ok := distinct[*f.Link]; !ok { + distinct[*f.Link] = f.ID + continue + } + deleteIDs = append(deleteIDs, f.ID) + log.Println("delete duplicate feed: ", f.ID, *f.Name, *f.Link) + } + + if len(deleteIDs) > 0 { + // **hard** delete duplicate feeds and their items + err = tx.Where("id IN ?", deleteIDs).Unscoped().Delete(&model.Feed{}).Error + if err != nil { + return err + } + return tx.Where("feed_id IN ?", deleteIDs).Unscoped().Delete(&model.Item{}).Error + } + return nil + }) + if err != nil { + panic(err) + } + // FIX: gorm not auto drop index and change 'not null' if err := DB.AutoMigrate(&model.Feed{}, &model.Group{}, &model.Item{}); err != nil { panic(err) diff --git a/server/feed.go b/server/feed.go index 8283980..5b7dc2d 100644 --- a/server/feed.go +++ b/server/feed.go @@ -93,9 +93,6 @@ func (f Feed) Create(ctx context.Context, req *ReqFeedCreate) error { } if err := f.repo.Create(feeds); err != nil { - if errors.Is(err, repo.ErrDuplicatedKey) { - err = NewBizError(err, http.StatusBadRequest, "link is not allowed to be the same as other feeds") - } return err } diff --git a/server/group.go b/server/group.go index 518fd88..b99de7e 100644 --- a/server/group.go +++ b/server/group.go @@ -46,15 +46,18 @@ func (g Group) All(ctx context.Context) (*RespGroupAll, error) { }, nil } -func (g Group) Create(ctx context.Context, req *ReqGroupCreate) error { +func (g Group) Create(ctx context.Context, req *ReqGroupCreate) (*RespGroupCreate, error) { newGroup := &model.Group{ Name: req.Name, } err := g.repo.Create(newGroup) - if errors.Is(err, repo.ErrDuplicatedKey) { - err = NewBizError(err, http.StatusBadRequest, "name is not allowed to be the same as other groups") + if err != nil { + if errors.Is(err, repo.ErrDuplicatedKey) { + err = NewBizError(err, http.StatusBadRequest, "name is not allowed to be the same as other groups") + } + return nil, err } - return err + return &RespGroupCreate{ID: newGroup.ID}, nil } func (g Group) Update(ctx context.Context, req *ReqGroupUpdate) error { diff --git a/server/group_form.go b/server/group_form.go index d1a4b2f..01a31e5 100644 --- a/server/group_form.go +++ b/server/group_form.go @@ -13,6 +13,10 @@ type ReqGroupCreate struct { Name *string `json:"name" validate:"required"` } +type RespGroupCreate struct { + ID uint `json:"id"` +} + type ReqGroupUpdate struct { ID uint `param:"id" validate:"required"` Name *string `json:"name" validate:"required"` From 8ba841d08af47b7790b978498993be4f9befe88a Mon Sep 17 00:00:00 2001 From: rook1e Date: Sun, 25 Aug 2024 19:58:40 +0800 Subject: [PATCH 2/4] fix: custom migration should skip the first launch --- repo/repo.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/repo/repo.go b/repo/repo.go index 5c9e0c6..ed4d5a5 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -3,6 +3,7 @@ package repo import ( "errors" "log" + "strings" "github.com/0x2e/fusion/conf" "github.com/0x2e/fusion/model" @@ -32,6 +33,11 @@ func migrage() { // We must delete any duplicate feeds before AutoMigrate applies the // new unique constraint. err := DB.Transaction(func(tx *gorm.DB) error { + // skip if it's the first launch + if !tableExist(&model.Feed{}) || !tableExist(&model.Feed{}) { + return nil + } + // query duplicate feeds dupFeeds := make([]model.Feed, 0) err := tx.Model(&model.Feed{}).Where( @@ -82,6 +88,17 @@ func migrage() { } } +func tableExist(table interface{}) bool { + err := DB.Model(table).First(table, "id = 1").Error + if err != nil { + if strings.Contains(err.Error(), "no such table") { + return false + } + panic(err) + } + return true +} + func registerCallback() { if err := DB.Callback().Query().After("*").Register("convert_error", func(db *gorm.DB) { if errors.Is(db.Error, gorm.ErrRecordNotFound) { From fcb0417d3d3226d54006a1a0d5fc4cebdf38e320 Mon Sep 17 00:00:00 2001 From: rook1e Date: Sun, 25 Aug 2024 20:12:16 +0800 Subject: [PATCH 3/4] refactor: add some notes --- frontend/src/routes/(authed)/feeds/ActionOPML.svelte | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/routes/(authed)/feeds/ActionOPML.svelte b/frontend/src/routes/(authed)/feeds/ActionOPML.svelte index 76ffc95..34ef4ee 100644 --- a/frontend/src/routes/(authed)/feeds/ActionOPML.svelte +++ b/frontend/src/routes/(authed)/feeds/ActionOPML.svelte @@ -144,7 +144,11 @@ 1. Feeds will be imported into the corresponding group, which will be created automatically if it does not exist.

-

2. The existing feed with the same link will be override.

+

+ 2. Multidimensional group will be flattened to a one-dimensional structure, using a + naming convention like 'a/b/c'. +

+

3. The existing feed with the same link will be override.

From 692ef37fb197f39fa8ab049a8f83be11d92e9a45 Mon Sep 17 00:00:00 2001 From: rook1e Date: Sun, 25 Aug 2024 20:42:11 +0800 Subject: [PATCH 4/4] fix: update group_id of feed as well --- repo/feed.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/feed.go b/repo/feed.go index 29c674e..1d710fb 100644 --- a/repo/feed.go +++ b/repo/feed.go @@ -50,7 +50,7 @@ func (f Feed) Get(id uint) (*model.Feed, error) { func (f Feed) Create(data []*model.Feed) error { return f.db.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "link"}, {Name: "deleted_at"}}, - DoUpdates: clause.AssignmentColumns([]string{"name", "link"}), + DoUpdates: clause.AssignmentColumns([]string{"name", "link", "group_id"}), }).Create(data).Error }