HDDS-12134. Implement Snapshot Cache lock for OM Bootstrap

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -4494,6 +4494,16 @@
     </description>
   </property>
 
+  <property>
+    <name>ozone.om.snapshot.cache.lock.timeout</name>
+    <value>5s</value>
+    <tag>OZONE, OM</tag>
+    <description>
+      Wait Timeout for snapshot cache lock.
+      Uses milliseconds by default when no time unit is specified.
+    </description>
+  </property>
+
   <property>
     <name>ozone.om.snapshot.load.native.lib</name>
     <value>true</value>

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -599,6 +599,12 @@ private OMConfigKeys() {
   public static final long
       OZONE_OM_SNAPSHOT_DIFF_CLEANUP_SERVICE_RUN_INTERVAL_DEFAULT
       = TimeUnit.MINUTES.toMillis(1);
+  public static final String
+      OZONE_OM_SNAPSHOT_CACHE_LOCK_TIMEOUT
+      = "ozone.om.snapshot.cache.lock.timeout";
+  public static final long
+      OZONE_OM_SNAPSHOT_CACHE_LOCK_TIMEOUT_DEFAULT
+      = TimeUnit.SECONDS.toMillis(5);
   public static final long
       OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL_DEFAULT
       = TimeUnit.MINUTES.toMillis(1);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java

@@ -88,6 +88,8 @@
 import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_LOCK_TIMEOUT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_LOCK_TIMEOUT_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_CACHE_MAX_SIZE_DEFAULT;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES;
@@ -282,8 +284,12 @@ public OmSnapshotManager(OzoneManager ozoneManager) {
         .getTimeDuration(OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL,
             OZONE_OM_SNAPSHOT_CACHE_CLEANUP_SERVICE_RUN_INTERVAL_DEFAULT,
             TimeUnit.MILLISECONDS);
+    long cacheLockTimeout = ozoneManager.getConfiguration()
+        .getTimeDuration(OZONE_OM_SNAPSHOT_CACHE_LOCK_TIMEOUT,
+            OZONE_OM_SNAPSHOT_CACHE_LOCK_TIMEOUT_DEFAULT,
+            TimeUnit.MILLISECONDS);
     this.snapshotCache = new SnapshotCache(loader, softCacheSize, ozoneManager.getMetrics(),
-        cacheCleanupServiceInterval);
+        cacheCleanupServiceInterval, cacheLockTimeout);
 
     this.snapshotDiffManager = new SnapshotDiffManager(snapshotDiffDb, differ,
         ozoneManager, snapDiffJobCf, snapDiffReportCf,

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/ReferenceCounted.java

@@ -18,7 +18,9 @@
 package org.apache.hadoop.ozone.om.snapshot;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
 
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -51,10 +53,10 @@ public class ReferenceCounted<T>
   /**
    * Parent instance whose callback will be triggered upon this RC closure.
    */
-  private final ReferenceCountedCallback parentWithCallback;
+  private final ReferenceCountedCallback<T> parentWithCallback;
 
   public ReferenceCounted(T obj, boolean disableCounter,
-      ReferenceCountedCallback parentWithCallback) {
+      ReferenceCountedCallback<T> parentWithCallback) {
     // A param to allow disabling ref counting to reduce active DB
     //  access penalties due to AtomicLong operations.
     this.obj = obj;
@@ -126,9 +128,7 @@ public long decrementRefCount() {
       Preconditions.checkState(newValTotal >= 0L,
           "Total reference count underflow");
     }
-    if (refCount.get() == 0) {
-      this.parentWithCallback.callback(this);
-    }
+    this.parentWithCallback.callback(this);
     return refCount.get();
   }
 
@@ -161,4 +161,8 @@ public void close() {
     // so it is eligible to be used with try-with-resources.
     decrementRefCount();
   }
+
+  public Map<Long, Long> getThreadCntMap() {
+    return ImmutableMap.copyOf(threadMap);
+  }
 }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/ReferenceCountedCallback.java

@@ -20,6 +20,6 @@
 /**
  * Callback interface for ReferenceCounted.
  */
-public interface ReferenceCountedCallback {
-  void callback(ReferenceCounted referenceCounted);
+public interface ReferenceCountedCallback<T> {
+  void callback(ReferenceCounted<T> referenceCounted);
 }