perf: throttle watch updates (gokcehan#2311)

* perf: throttle watch updates

* Fix file not exist error in the UI

* Use separate timeout for each file

Author: joelim-work

nav.go

@@ -175,7 +175,10 @@ func readdir(path string) ([]*file, error) {
 
 	files := make([]*file, 0, len(names))
 	for _, fname := range names {
-		files = append(files, newFile(filepath.Join(path, fname)))
+		file := newFile(filepath.Join(path, fname))
+		if !os.IsNotExist(file.err) {
+			files = append(files, file)
+		}
 	}
 
 	return files, err

watch.go

@@ -10,29 +10,27 @@ import (
 )
 
 type watch struct {
-	watcher     *fsnotify.Watcher
-	events      <-chan fsnotify.Event
-	quit        chan struct{}
-	loads       map[string]bool
-	loadTimer   *time.Timer
-	updates     map[string]bool
-	updateTimer *time.Timer
-	dirChan     chan<- *dir
-	fileChan    chan<- *file
-	delChan     chan<- string
+	watcher  *fsnotify.Watcher
+	events   <-chan fsnotify.Event
+	quit     chan struct{}
+	pending  map[watchUpdate]bool
+	timeout  chan watchUpdate
+	dirChan  chan<- *dir
+	fileChan chan<- *file
+	delChan  chan<- string
 }
 
 func newWatch(dirChan chan<- *dir, fileChan chan<- *file, delChan chan<- string) *watch {
-	return &watch{
-		quit:        make(chan struct{}),
-		loads:       make(map[string]bool),
-		loadTimer:   time.NewTimer(0),
-		updates:     make(map[string]bool),
-		updateTimer: time.NewTimer(0),
-		dirChan:     dirChan,
-		fileChan:    fileChan,
-		delChan:     delChan,
+	watch := &watch{
+		quit:     make(chan struct{}),
+		pending:  make(map[watchUpdate]bool),
+		timeout:  make(chan watchUpdate, 1024),
+		dirChan:  dirChan,
+		fileChan: fileChan,
+		delChan:  delChan,
 	}
+
+	return watch
 }
 
 func (watch *watch) start() {
@@ -83,17 +81,17 @@ func (watch *watch) loop() {
 		case ev := <-watch.events:
 			if ev.Has(fsnotify.Create) {
 				for _, path := range watch.getSameDirs(filepath.Dir(ev.Name)) {
-					watch.addLoad(path)
-					watch.addUpdate(path)
+					watch.addUpdate(watchUpdate{"dir", path})
+					watch.addUpdate(watchUpdate{"file", path})
 				}
 			}
 
 			if ev.Has(fsnotify.Remove) || ev.Has(fsnotify.Rename) {
 				dir, file := filepath.Split(ev.Name)
 				for _, path := range watch.getSameDirs(dir) {
 					watch.delChan <- filepath.Join(path, file)
-					watch.addLoad(path)
-					watch.addUpdate(path)
+					watch.addUpdate(watchUpdate{"dir", path})
+					watch.addUpdate(watchUpdate{"file", path})
 				}
 			}
 
@@ -108,47 +106,53 @@ func (watch *watch) loop() {
 
 				dir, file := filepath.Split(ev.Name)
 				for _, path := range watch.getSameDirs(dir) {
-					watch.addUpdate(filepath.Join(path, file))
+					watch.addUpdate(watchUpdate{"file", filepath.Join(path, file)})
 				}
 			}
-		case <-watch.loadTimer.C:
-			for path := range watch.loads {
-				if _, err := os.Lstat(path); err != nil {
-					continue
-				}
-				dir := newDir(path)
-				dir.sort()
-				watch.dirChan <- dir
-			}
-			clear(watch.loads)
-		case <-watch.updateTimer.C:
-			for path := range watch.updates {
-				if _, err := os.Lstat(path); err != nil {
-					continue
-				}
-				watch.fileChan <- newFile(path)
+		case update := <-watch.timeout:
+			if watch.pending[update] {
+				watch.processUpdate(update)
+				time.AfterFunc(100*time.Millisecond, func() { watch.timeout <- update })
+				watch.pending[update] = false
+			} else {
+				delete(watch.pending, update)
 			}
-			clear(watch.updates)
 		case <-watch.quit:
 			return
 		}
 	}
 }
 
-func (watch *watch) addLoad(path string) {
-	if len(watch.loads) == 0 {
-		watch.loadTimer.Stop()
-		watch.loadTimer.Reset(10 * time.Millisecond)
+type watchUpdate struct {
+	kind string
+	path string
+}
+
+func (watch *watch) addUpdate(update watchUpdate) {
+	// process an update immediately if is the first time, otherwise store it
+	// and process only after a timeout to reduce the number of actual loads
+	if _, ok := watch.pending[update]; !ok {
+		watch.processUpdate(update)
+		time.AfterFunc(100*time.Millisecond, func() { watch.timeout <- update })
+		watch.pending[update] = false
+	} else {
+		watch.pending[update] = true
 	}
-	watch.loads[path] = true
 }
 
-func (watch *watch) addUpdate(path string) {
-	if len(watch.updates) == 0 {
-		watch.updateTimer.Stop()
-		watch.updateTimer.Reset(10 * time.Millisecond)
+func (watch *watch) processUpdate(update watchUpdate) {
+	switch update.kind {
+	case "dir":
+		if _, err := os.Lstat(update.path); err == nil {
+			dir := newDir(update.path)
+			dir.sort()
+			watch.dirChan <- dir
+		}
+	case "file":
+		if _, err := os.Lstat(update.path); err == nil {
+			watch.fileChan <- newFile(update.path)
+		}
 	}
-	watch.updates[path] = true
 }
 
 // Hacky workaround since fsnotify reports changes for only one path if a