news: stricter db pagination limits (fixes #9205)

src/app/news/news-list.component.ts

@@ -67,6 +67,7 @@ export class NewsListComponent implements OnInit, OnChanges, AfterViewInit, OnDe
   }
 
   ngOnChanges() {
+    this.isLoadingMore = false;
     let isLatest = true;
     this.replyObject = {};
     this.items.forEach(item => {
@@ -81,7 +82,7 @@ export class NewsListComponent implements OnInit, OnChanges, AfterViewInit, OnDe
       }
     });
     this.displayedItems = this.replyObject[this.replyViewing._id];
-    this.loadPagedItems(true);
+    this.updateDisplayedItems(true);
     if (this.replyViewing._id !== 'root') {
       this.replyViewing = this.items.find(item => item._id === this.replyViewing._id);
     }
@@ -149,7 +150,7 @@ export class NewsListComponent implements OnInit, OnChanges, AfterViewInit, OnDe
     const news = this.items.find(item => item._id === newsId) || { _id: 'root' };
     this.replyViewing = news;
     this.displayedItems = this.replyObject[news._id];
-    this.loadPagedItems(true);
+    this.updateDisplayedItems(true);
     this.isMainPostShared = this.replyViewing._id === 'root' || this.newsService.postSharedWithCommunity(this.replyViewing);
     this.showMainPostShare = !this.replyViewing.doc || !this.replyViewing.doc.replyTo ||
       (
@@ -285,27 +286,38 @@ export class NewsListComponent implements OnInit, OnChanges, AfterViewInit, OnDe
     };
   }
 
-  loadPagedItems(initial = true) {
-    let pageSize = this.pageSize;
-    if (initial) {
+  loadMoreItems() {
+    if (this.isLoadingMore) {
+      return;
+    }
+    this.updateDisplayedItems();
+  }
+
+  private updateDisplayedItems(reset = false) {
+    const news = this.getCurrentItems();
+    if (reset) {
       this.displayedItems = [];
       this.nextStartIndex = 0;
-      // Take maximum so if fewer posts than page size adding a post doesn't add a "Load More" button
-      pageSize = Math.max(this.pageEnd[this.replyViewing._id] || this.pageSize, this.pageSize);
     }
-    const news = this.getCurrentItems();
+    const pageSize = reset ? Math.max(this.pageEnd[this.replyViewing._id] || this.pageSize, this.pageSize) : this.pageSize;
     const { items, endIndex, hasMore } = this.paginateItems(news, this.nextStartIndex, pageSize);
 
-    this.displayedItems = [ ...this.displayedItems, ...items ];
+    this.displayedItems = reset ? items : [ ...this.displayedItems, ...items ];
     this.pageEnd[this.replyViewing._id] = this.displayedItems.length;
     this.nextStartIndex = endIndex;
-    this.hasMoreNews = hasMore;
-    this.isLoadingMore = false;
     this.totalReplies = news.length;
-  }
 
-  loadMoreItems() {
-    this.isLoadingMore = true;
-    this.loadPagedItems(false);
+    const reachedLocalEnd = endIndex >= news.length;
+    const shouldFetchRemote = reachedLocalEnd && !hasMore && this.newsService.canLoadMore();
+
+    if (shouldFetchRemote) {
+      this.isLoadingMore = true;
+      this.newsService.loadMoreNews();
+    } else {
+      this.isLoadingMore = false;
+    }
+
+    this.hasMoreNews = hasMore || shouldFetchRemote || this.newsService.canLoadMore();
   }
+
 }

src/app/news/news.service.ts

@@ -19,6 +19,12 @@ export class NewsService {
   imgUrlPrefix = environment.couchAddress;
   newsUpdated$ = new Subject<any[]>();
   currentOptions: { selectors: any, viewId: string } = { selectors: {}, viewId: '' };
+  private currentBookmark: string | undefined;
+  private hasMoreResults = true;
+  private isLoading = false;
+  private pageSize = 50;
+  private accumulatedNews: any[] = [];
+  private avatarCache: Record<string, any> = {};
 
   constructor(
     private couchService: CouchService,
@@ -29,25 +35,22 @@ export class NewsService {
 
   requestNews({ selectors, viewId } = this.currentOptions) {
     this.currentOptions = { selectors, viewId };
-    this.couchService.findAll(this.dbName, findDocuments(selectors, 0, [ { 'time': 'desc' } ])).pipe(
-      switchMap((newsItems: any[]) =>
-        this.couchService.findAttachmentsByIds(this.collectAttachmentIds(newsItems)).pipe(
-          map((attachments: any[]) => ({
-            newsItems,
-            avatarMap: new Map<string, any>(attachments.map((attachment: any) => [ attachment._id, attachment ]))
-          }))
-        )
-      )
-    ).subscribe(({ newsItems, avatarMap }) => {
-      this.newsUpdated$.next(newsItems.map((item: any) => (
-        { doc: item, sharedDate: this.findShareDate(item, viewId), avatar: this.findAvatar(item.user, avatarMap), _id: item._id }
-      )));
-    });
+    this.resetPagination();
+    this.fetchNewsPage(true);
+  }
+
+  loadMoreNews() {
+    if (!this.canLoadMore()) { return; }
+    this.fetchNewsPage(false);
+  }
+
+  canLoadMore(): boolean {
+    return this.hasMoreResults && !this.isLoading;
   }
 
-  findAvatar(user: any, attachments: Map<string, any>) {
+  findAvatar(user: any, attachments: Record<string, any>) {
     const attachmentId = `${user._id}@${user.planetCode}`;
-    const attachment = attachments.get(attachmentId);
+    const attachment = attachments ? attachments[attachmentId] : undefined;
     const extractFilename = (object) => Object.keys(object._attachments)[0];
     return attachment ?
       `${this.imgUrlPrefix}/attachments/${attachmentId}/${extractFilename(attachment)}` :
@@ -71,6 +74,75 @@ export class NewsService {
     return Array.from(ids);
   }
 
+  private resetPagination() {
+    this.currentBookmark = undefined;
+    this.hasMoreResults = true;
+    this.isLoading = false;
+    this.accumulatedNews = [];
+    this.avatarCache = {};
+  }
+
+  private fetchNewsPage(reset: boolean) {
+    const { selectors } = this.currentOptions;
+    const baseQuery = findDocuments(selectors, 0, [ { 'time': 'desc' } ], this.pageSize);
+    const query: any = { ...baseQuery };
+    if (!reset && this.currentBookmark) {
+      query.bookmark = this.currentBookmark;
+    }
+    this.isLoading = true;
+    this.couchService.post(`${this.dbName}/_find`, query).subscribe({
+      next: (response: any) => {
+        const docs = response?.docs ?? [];
+        const bookmark = response?.bookmark;
+        this.isLoading = false;
+        this.currentBookmark = bookmark;
+        if (reset) {
+          this.accumulatedNews = [];
+        }
+        if (docs.length === 0) {
+          this.hasMoreResults = false;
+          this.emitNews();
+          return;
+        }
+        this.hasMoreResults = docs.length === this.pageSize && !!bookmark;
+        this.accumulatedNews = [ ...this.accumulatedNews, ...docs ];
+        this.emitNews();
+        this.loadMissingAttachments(docs);
+      },
+      error: () => {
+        this.isLoading = false;
+        this.hasMoreResults = false;
+      }
+    });
+  }
+
+  private loadMissingAttachments(newsItems: any[]) {
+    const missingIds = this.collectAttachmentIds(newsItems).filter(id => !(id in this.avatarCache));
+    if (missingIds.length === 0) {
+      return;
+    }
+    this.couchService.findAttachmentsByIds(missingIds).subscribe((attachments: any[]) => {
+      if (!attachments || attachments.length === 0) { return; }
+      attachments.forEach((attachment: any) => {
+        if (attachment && attachment._id) {
+          this.avatarCache[attachment._id] = attachment;
+        }
+      });
+      this.emitNews();
+    });
+  }
+
+  private emitNews() {
+    const viewId = this.currentOptions.viewId;
+    const formatted = this.accumulatedNews.map((item: any) => ({
+      doc: item,
+      sharedDate: this.findShareDate(item, viewId),
+      avatar: this.findAvatar(item.user, this.avatarCache),
+      _id: item._id
+    }));
+    this.newsUpdated$.next([ ...formatted ]);
+  }
+
   postNews(post, successMessage = $localize`Thank you for submitting your message`, isMessageEdit = true) {
     const { configuration } = this.stateService;
     const message = typeof post.message === 'string' ? post.message : post.message.text;