reopen an index if Lucene closed it for any reason

closes #5521

Author: rnewson

nouveau/src/main/java/org/apache/couchdb/nouveau/core/Index.java

@@ -129,6 +129,12 @@ public synchronized boolean isDeleteOnClose() {
         return deleteOnClose;
     }
 
+    public final boolean isOpen() {
+        return doIsOpen();
+    }
+
+    protected abstract boolean doIsOpen();
+
     public synchronized void setDeleteOnClose(final boolean deleteOnClose) {
         this.deleteOnClose = deleteOnClose;
     }

nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java

@@ -119,11 +119,21 @@ public <R> R with(final String name, final IndexFunction<Index, R> indexFun)
             // CachedData pattern from ReentrantReadWriteLock javadoc
             holder.lock.readLock().lock();
 
-            // Load if not already loaded.
-            if (holder.state == HolderState.NOT_LOADED) {
+            // Load if not already loaded or remove if Lucene closed the index elsewhere.
+            if (holder.state == HolderState.NOT_LOADED
+                    || (holder.state == HolderState.LOADED && !holder.index.isOpen())) {
                 holder.lock.readLock().unlock();
                 holder.lock.writeLock().lock();
                 try {
+                    if (holder.state == HolderState.LOADED && !holder.index.isOpen()) {
+                        LOGGER.info("removing closed index {}", name);
+                        holder.state = HolderState.UNLOADED;
+                        holder.index = null;
+                        synchronized (cache) {
+                            cache.remove(name, holder);
+                        }
+                        continue retry;
+                    }
                     if (holder.state == HolderState.NOT_LOADED) {
                         holder.index = load(name);
                         holder.commitFuture = this.schedulerExecutorService.scheduleWithFixedDelay(

nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java

@@ -190,6 +190,11 @@ public void doClose() throws IOException {
                 });
     }
 
+    @Override
+    public boolean doIsOpen() {
+        return writer.isOpen();
+    }
+
     @Override
     public SearchResults doSearch(final SearchRequest request) throws IOException {
         final Query query = parse(request);

nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java

@@ -0,0 +1,85 @@
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.apache.couchdb.nouveau.core;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.nio.file.Path;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.couchdb.nouveau.api.IndexDefinition;
+import org.apache.couchdb.nouveau.api.SearchRequest;
+import org.apache.couchdb.nouveau.lucene9.ParallelSearcherFactory;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+public class IndexManagerTest {
+
+    private static IndexManager manager;
+    private static ScheduledExecutorService executorService;
+
+    @BeforeAll
+    public static void setupManager(@TempDir Path path) throws Exception {
+        executorService = Executors.newScheduledThreadPool(2);
+
+        manager = new IndexManager();
+        manager.setRootDir(path);
+        manager.setObjectMapper(new ObjectMapper());
+        manager.setCommitIntervalSeconds(10);
+        manager.setScheduledExecutorService(executorService);
+        manager.setSearcherFactory(new ParallelSearcherFactory(ForkJoinPool.commonPool()));
+        manager.start();
+    }
+
+    @AfterAll
+    public static void cleanup() throws Exception {
+        executorService.shutdownNow();
+        executorService.awaitTermination(5, TimeUnit.SECONDS);
+        manager.stop();
+    }
+
+    @Test
+    public void managerReturnsUsableIndex() throws Exception {
+        final IndexDefinition indexDefinition = new IndexDefinition();
+        indexDefinition.setDefaultAnalyzer("standard");
+        manager.create("foo", indexDefinition);
+        var searchRequest = new SearchRequest();
+        searchRequest.setQuery("*:*");
+        var searchResults = manager.with("foo", (index) -> index.search(searchRequest));
+        assertThat(searchResults.getTotalHits()).isEqualTo(0);
+    }
+
+    @Test
+    public void managerReopensAClosedIndex() throws Exception {
+        final IndexDefinition indexDefinition = new IndexDefinition();
+        indexDefinition.setDefaultAnalyzer("standard");
+
+        manager.create("bar", indexDefinition);
+
+        manager.with("bar", (index) -> {
+            index.close();
+            return null;
+        });
+
+        final boolean isOpen = manager.with("bar", (index) -> {
+            return index.isOpen();
+        });
+        assertThat(isOpen);
+    }
+}