news: stricter db pagination limits (fixes #9205)

src/app/news/news-list.component.ts

@@ -1,7 +1,7 @@
 import { Component, Input, OnInit, OnChanges, EventEmitter, Output, AfterViewInit, ViewChild, OnDestroy } from '@angular/core';
 import { MatLegacyDialog as MatDialog, MatLegacyDialogRef as MatDialogRef } from '@angular/material/legacy-dialog';
 import { ActivatedRoute, Router } from '@angular/router';
-import { forkJoin } from 'rxjs';
+import { forkJoin, Subscription } from 'rxjs';
 import { DialogsFormService } from '../shared/dialogs/dialogs-form.service';
 import { DialogsLoadingService } from '../shared/dialogs/dialogs-loading.service';
 import { NewsService } from './news.service';
@@ -51,6 +51,9 @@ export class NewsListComponent implements OnInit, OnChanges, AfterViewInit, OnDe
   pageIndex = 0;
   pageSizeOptions = [ 5, 10, 25, 50 ];
   totalItems = 0;
+  private loadingCompleteSubscription: Subscription;
+  private newsByIdSubscription?: Subscription;
+  private currentVoiceId = 'root';
 
   constructor(
     private dialog: MatDialog,
@@ -68,9 +71,15 @@ export class NewsListComponent implements OnInit, OnChanges, AfterViewInit, OnDe
     });
 
     this.initNews();
+
+    // Subscribe to loading completion to reset isLoadingMore flag
+    this.loadingCompleteSubscription = this.newsService.loadingComplete$.subscribe(() => {
+      this.isLoadingMore = false;
+    });
   }
 
   ngOnChanges() {
+    this.isLoadingMore = false;
     let isLatest = true;
     this.replyObject = {};
     this.items.forEach(item => {
@@ -99,6 +108,13 @@ export class NewsListComponent implements OnInit, OnChanges, AfterViewInit, OnDe
     if (this.observer) {
       this.observer.disconnect();
     }
+    if (this.loadingCompleteSubscription) {
+      this.loadingCompleteSubscription.unsubscribe();
+    }
+    if (this.newsByIdSubscription) {
+      this.newsByIdSubscription.unsubscribe();
+      this.newsByIdSubscription = undefined;
+    }
   }
 
   setupObserver() {
@@ -121,6 +137,48 @@ export class NewsListComponent implements OnInit, OnChanges, AfterViewInit, OnDe
 
   initNews() {
     const newVoiceId = this.route.firstChild?.snapshot.paramMap.get('id') || 'root';
+    this.currentVoiceId = newVoiceId;
+
+    // Check if voice is in the current items
+    if (newVoiceId !== 'root' && !this.items.find(item => item._id === newVoiceId)) {
+      if (this.newsByIdSubscription) {
+        this.newsByIdSubscription.unsubscribe();
+        this.newsByIdSubscription = undefined;
+      }
+      const requestVoiceId = newVoiceId;
+      this.newsByIdSubscription = this.newsService.getNewsById(newVoiceId).subscribe({
+        next: (newsItem) => {
+          if (this.currentVoiceId !== requestVoiceId) {
+            return;
+          }
+          if (!this.items.find(item => item._id === newsItem._id)) {
+            this.items = [ newsItem, ...this.items ];
+            this.ngOnChanges();
+          }
+          this.filterNewsToShow(newVoiceId);
+          if (this.newsByIdSubscription) {
+            this.newsByIdSubscription.unsubscribe();
+            this.newsByIdSubscription = undefined;
+          }
+        },
+        error: () => {
+          if (this.currentVoiceId === requestVoiceId) {
+            this.filterNewsToShow('root');
+          }
+          if (this.newsByIdSubscription) {
+            this.newsByIdSubscription.unsubscribe();
+            this.newsByIdSubscription = undefined;
+          }
+        },
+        complete: () => {
+          if (this.currentVoiceId === requestVoiceId) {
+            this.newsByIdSubscription = undefined;
+          }
+        }
+      });
+      return;
+    }
+
     this.filterNewsToShow(newVoiceId);
   }
 
@@ -312,10 +370,19 @@ export class NewsListComponent implements OnInit, OnChanges, AfterViewInit, OnDe
       const news = this.getCurrentItems();
       this.totalItems = news.length;
       const start = this.pageIndex * this.pageSize;
+      const end = start + this.pageSize;
       const { items } = this.paginateItems(news, start, this.pageSize);
       this.displayedItems = items;
+
+      const shouldFetchRemote = news.length >= this.pageSize && end >= news.length && this.newsService.canLoadMore();
+      if (shouldFetchRemote && !this.isLoadingMore) {
+        this.isLoadingMore = true;
+        this.newsService.loadMoreNews();
+      } else {
+        this.isLoadingMore = false;
+      }
+
       this.hasMoreNews = false;
-      this.isLoadingMore = false;
     } else {
       let pageSize = this.pageSize;
       if (initial) {
@@ -330,13 +397,25 @@ export class NewsListComponent implements OnInit, OnChanges, AfterViewInit, OnDe
       this.displayedItems = [ ...this.displayedItems, ...items ];
       this.pageEnd[this.replyViewing._id] = this.displayedItems.length;
       this.nextStartIndex = endIndex;
-      this.hasMoreNews = hasMore;
-      this.isLoadingMore = false;
       this.totalReplies = news.length;
+
+      const shouldFetchRemote = endIndex >= news.length && !hasMore && this.newsService.canLoadMore();
+
+      if (shouldFetchRemote) {
+        this.isLoadingMore = true;
+        this.newsService.loadMoreNews();
+      } else {
+        this.isLoadingMore = false;
+      }
+
+      this.hasMoreNews = hasMore || shouldFetchRemote || this.newsService.canLoadMore();
     }
   }
 
   loadMoreItems() {
+    if (this.isLoadingMore) {
+      return;
+    }
     this.isLoadingMore = true;
     this.loadPagedItems(false);
   }

src/app/news/news.service.ts

@@ -18,7 +18,14 @@ export class NewsService {
   dbName = 'news';
   imgUrlPrefix = environment.couchAddress;
   newsUpdated$ = new Subject<any[]>();
+  loadingComplete$ = new Subject<{ success: boolean }>();
   currentOptions: { selectors: any, viewId: string } = { selectors: {}, viewId: '' };
+  private currentBookmark: string | undefined;
+  private hasMoreResults = true;
+  private isLoading = false;
+  private pageSize = 50;
+  private accumulatedNews: any[] = [];
+  private avatarCache: Record<string, any> = {};
 
   constructor(
     private couchService: CouchService,
@@ -29,25 +36,22 @@ export class NewsService {
 
   requestNews({ selectors, viewId } = this.currentOptions) {
     this.currentOptions = { selectors, viewId };
-    this.couchService.findAll(this.dbName, findDocuments(selectors, 0, [ { 'time': 'desc' } ])).pipe(
-      switchMap((newsItems: any[]) =>
-        this.couchService.findAttachmentsByIds(this.collectAttachmentIds(newsItems)).pipe(
-          map((attachments: any[]) => ({
-            newsItems,
-            avatarMap: new Map<string, any>(attachments.map((attachment: any) => [ attachment._id, attachment ]))
-          }))
-        )
-      )
-    ).subscribe(({ newsItems, avatarMap }) => {
-      this.newsUpdated$.next(newsItems.map((item: any) => (
-        { doc: item, sharedDate: this.findShareDate(item, viewId), avatar: this.findAvatar(item.user, avatarMap), _id: item._id }
-      )));
-    });
+    this.resetPagination();
+    this.fetchNewsPage(true);
+  }
+
+  loadMoreNews() {
+    if (!this.canLoadMore()) { return; }
+    this.fetchNewsPage(false);
   }
 
-  findAvatar(user: any, attachments: Map<string, any>) {
+  canLoadMore(): boolean {
+    return this.hasMoreResults && !this.isLoading;
+  }
+
+  findAvatar(user: any, attachments: Record<string, any>) {
     const attachmentId = `${user._id}@${user.planetCode}`;
-    const attachment = attachments.get(attachmentId);
+    const attachment = attachments ? attachments[attachmentId] : undefined;
     const extractFilename = (object) => Object.keys(object._attachments)[0];
     return attachment ?
       `${this.imgUrlPrefix}/attachments/${attachmentId}/${extractFilename(attachment)}` :
@@ -71,6 +75,75 @@ export class NewsService {
     return Array.from(ids);
   }
 
+  private resetPagination() {
+    this.currentBookmark = undefined;
+    this.hasMoreResults = true;
+    this.isLoading = false;
+    this.accumulatedNews = [];
+    this.avatarCache = {};
+  }
+
+  private fetchNewsPage(reset: boolean) {
+    const query: any = { ...findDocuments(this.currentOptions.selectors, 0, [ { 'time': 'desc' } ], this.pageSize) };
+    if (!reset && this.currentBookmark) {
+      query.bookmark = this.currentBookmark;
+    }
+    this.isLoading = true;
+    this.couchService.post(`${this.dbName}/_find`, query).subscribe({
+      next: (response: any) => {
+        const docs = response?.docs ?? [];
+        const bookmark = response?.bookmark;
+        this.isLoading = false;
+        this.currentBookmark = bookmark;
+        if (reset) {
+          this.accumulatedNews = [];
+        }
+        if (docs.length === 0) {
+          this.hasMoreResults = false;
+          this.emitNews();
+          this.loadingComplete$.next({ success: true });
+          return;
+        }
+        this.hasMoreResults = docs.length === this.pageSize && !!bookmark;
+        this.accumulatedNews = [ ...this.accumulatedNews, ...docs ];
+        this.emitNews();
+        this.loadMissingAttachments(docs);
+        this.loadingComplete$.next({ success: true });
+      },
+      error: () => {
+        this.isLoading = false;
+        this.loadingComplete$.next({ success: false });
+      }
+    });
+  }
+
+  private loadMissingAttachments(newsItems: any[]) {
+    const missingIds = this.collectAttachmentIds(newsItems).filter(id => !(id in this.avatarCache));
+    if (missingIds.length === 0) {
+      return;
+    }
+    this.couchService.findAttachmentsByIds(missingIds).subscribe((attachments: any[]) => {
+      if (!attachments || attachments.length === 0) { return; }
+      attachments.forEach((attachment: any) => {
+        if (attachment && attachment._id) {
+          this.avatarCache[attachment._id] = attachment;
+        }
+      });
+      this.emitNews();
+    });
+  }
+
+  private emitNews() {
+    const viewId = this.currentOptions.viewId;
+    const formatted = this.accumulatedNews.map((item: any) => ({
+      doc: item,
+      sharedDate: this.findShareDate(item, viewId),
+      avatar: this.findAvatar(item.user, this.avatarCache),
+      _id: item._id
+    }));
+    this.newsUpdated$.next([ ...formatted ]);
+  }
+
   postNews(post, successMessage = $localize`Thank you for submitting your message`, isMessageEdit = true) {
     const { configuration } = this.stateService;
     const message = typeof post.message === 'string' ? post.message : post.message.text;
@@ -146,4 +219,28 @@ export class NewsService {
     return post && post.doc && (post.doc.viewIn || []).some(({ _id }) => _id === planetAndParentId(this.stateService.configuration));
   }
 
+  getNewsById(newsId: string) {
+    return this.couchService.get(`${this.dbName}/${newsId}`).pipe(
+      switchMap((newsItem: any) => {
+        const attachmentIds = this.collectAttachmentIds([ newsItem ]);
+        return this.couchService.findAttachmentsByIds(attachmentIds).pipe(
+          map((attachments: any[]) => {
+            const avatarMap: Record<string, any> = {};
+            attachments.forEach(attachment => {
+              if (attachment && attachment._id) {
+                avatarMap[attachment._id] = attachment;
+              }
+            });
+            return {
+              doc: newsItem,
+              sharedDate: this.findShareDate(newsItem, this.currentOptions.viewId),
+              avatar: this.findAvatar(newsItem.user, avatarMap),
+              _id: newsItem._id
+            };
+          })
+        );
+      })
+    );
+  }
+
 }