curvefs/metaserver: fix trash bugs

curvefs/src/metaserver/copyset/copyset_node.cpp

@@ -170,7 +170,7 @@ bool CopysetNode::Start() {
         LOG(ERROR) << "Fail to init raft node, copyset: " << name_;
         return false;
     }
-
+    metaStore_->LoadDeletedInodes();
     LOG(INFO) << "Run copyset success, copyset: " << name_;
     return true;
 }

curvefs/src/metaserver/inode_manager.cpp

@@ -391,11 +391,6 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request,
         }
     }
 
-    if (needAddTrash) {
-        trash_->Add(old.inodeid(), old.dtime());
-        --(*type2InodeNum_)[old.type()];
-    }
-
     const S3ChunkInfoMap &map2add = request.s3chunkinfoadd();
     const S3ChunkInfoList *list2add;
     VLOG(9) << "UpdateInode inode " << old.inodeid() << " map2add size "
@@ -446,10 +441,27 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request,
             return MetaStatusCode::STORAGE_INTERNAL_ERROR;
         }
     }
+
+    if (needAddTrash) {
+        trash_->Add(old.inodeid(), old.dtime(), false);
+        --(*type2InodeNum_)[old.type()];
+    }
+
     VLOG(9) << "UpdateInode success, " << request.ShortDebugString();
     return MetaStatusCode::OK;
 }
 
+void InodeManager::LoadDeletedInodes() {
+    std::list<std::string> items;
+    inodeStorage_->LoadDeletedInodes(&items);
+    VLOG(3) << "build trash items size: " << items.size();
+    std::vector<std::string> names;
+    for (auto iter : items) {
+        curve::common::SplitString(iter , ":", &names);
+        trash_->Add(std::stoull(names[names.size() - 1 ]), 0, true);
+    }
+}
+
 MetaStatusCode InodeManager::GetOrModifyS3ChunkInfo(
     uint32_t fsId, uint64_t inodeId, const S3ChunkInfoMap& map2add,
     const S3ChunkInfoMap& map2del, bool returnS3ChunkInfoMap,

curvefs/src/metaserver/inode_manager.h

@@ -96,6 +96,8 @@ class InodeManager {
     MetaStatusCode UpdateInode(const UpdateInodeRequest& request,
                                int64_t logIndex);
 
+    void LoadDeletedInodes();
+
     MetaStatusCode GetOrModifyS3ChunkInfo(
         uint32_t fsId, uint64_t inodeId, const S3ChunkInfoMap& map2add,
         const S3ChunkInfoMap& map2del, bool returnS3ChunkInfoMap,

curvefs/src/metaserver/inode_storage.cpp

@@ -67,6 +67,7 @@ InodeStorage::InodeStorage(std::shared_ptr<KVStorage> kvStorage,
                            uint64_t nInode)
     : kvStorage_(std::move(kvStorage)),
       table4Inode_(nameGenerator->GetInodeTableName()),
+      table4DelInode_(nameGenerator->GetDelInodeTableName()),
       table4S3ChunkInfo_(nameGenerator->GetS3ChunkInfoTableName()),
       table4VolumeExtent_(nameGenerator->GetVolumeExtentTableName()),
       table4InodeAuxInfo_(nameGenerator->GetInodeAuxInfoTableName()),
@@ -185,9 +186,74 @@ MetaStatusCode InodeStorage::Insert(const Inode& inode, int64_t logIndex) {
     return MetaStatusCode::STORAGE_INTERNAL_ERROR;
 }
 
+MetaStatusCode InodeStorage::AddDeletedInode(const Key4Inode& keyInode) {
+    WriteLockGuard lg(rwLock_);
+    std::string skey = conv_.SerializeToString(keyInode);
+    VLOG(9) << "update deleting key, " << keyInode.inodeId;
+    const char* step = "Begin transaction";
+    std::shared_ptr<storage::StorageTransaction> txn;
+    txn = kvStorage_->BeginTransaction();
+    if (txn == nullptr) {
+        LOG(ERROR) << "Begin transaction failed";
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+    auto rc = txn->HSet(table4DelInode_, skey, google::protobuf::Empty());
+    step = "insert inode ";
+    if (rc.ok()) {
+        rc = txn->Commit();
+        step = "commit";
+    }
+    if (rc.ok()) {
+        VLOG(9) << "set deleting key ok";
+        return MetaStatusCode::OK;
+    }
+    LOG(ERROR) << step << "failed, status = " << rc.ToString();
+    if (!txn->Rollback().ok()) {
+        LOG(ERROR) << "Rollback delete inode transaction failed, status = "
+                   << rc.ToString();
+    }
+    return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+}
+
+MetaStatusCode InodeStorage::RemoveDeletedInode(const Key4Inode& key) {
+    WriteLockGuard lg(rwLock_);
+    std::string skey = conv_.SerializeToString(key);
+    VLOG(9) << "clear deleting key start, " << skey;
+    std::shared_ptr<storage::StorageTransaction> txn = nullptr;
+    const char* step = "Begin transaction";
+    txn = kvStorage_->BeginTransaction();
+    if (txn == nullptr) {
+        LOG(ERROR) << "Begin transaction failed";
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+    step = "Delete inode from transaction";
+    auto s = txn->HDel(table4DelInode_, skey);
+    if (s.ok()) {
+        step = "Delete inode";
+        s = txn->Commit();
+    }
+    if (s.ok()) {
+        VLOG(9) << "clear deleting key ok, " << skey;
+        return MetaStatusCode::OK;
+    }
+    LOG(ERROR) << step << " failed, status = " << s.ToString();
+    if (!txn->Rollback().ok()) {
+        LOG(ERROR) << "Rollback delete inode transaction failed, status = "
+                   << s.ToString();
+    }
+    return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+}
+
+void InodeStorage::LoadDeletedInodes(std::list<std::string> * inodes) {
+    VLOG(6) << "load deleted key start with: " << table4DelInode_;
+    kvStorage_->GetPrefix(inodes, table4DelInode_);
+    VLOG(6) << "oad deleted over";
+}
+
 MetaStatusCode InodeStorage::Get(const Key4Inode& key, Inode* inode) {
     ReadLockGuard lg(rwLock_);
     std::string skey = conv_.SerializeToString(key);
+
     Status s = kvStorage_->HGet(table4Inode_, skey, inode);
     if (s.ok()) {
         return MetaStatusCode::OK;
@@ -471,7 +537,6 @@ MetaStatusCode InodeStorage::Clear() {
     // because if we fail stop, we will replay
     // raft logs and clear it again
     WriteLockGuard lg(rwLock_);
-
     Status s = kvStorage_->HClear(table4Inode_);
     if (!s.ok()) {
         LOG(ERROR) << "InodeStorage clear inode table failed, status = "
@@ -492,7 +557,6 @@ MetaStatusCode InodeStorage::Clear() {
             << s.ToString();
         return MetaStatusCode::STORAGE_INTERNAL_ERROR;
     }
-
     s = kvStorage_->HClear(table4InodeAuxInfo_);
     if (!s.ok()) {
         LOG(ERROR)

curvefs/src/metaserver/inode_storage.h

@@ -76,6 +76,18 @@ class InodeStorage {
      */
     MetaStatusCode Insert(const Inode& inode, int64_t logIndex);
 
+    /**
+     * @brief update deleting inode key in storage
+     * @param[in] inode: the inode want to update
+     * @param[in] logIndex: the index of raft log
+     * @return
+     */
+    MetaStatusCode AddDeletedInode(const Key4Inode& inode);
+
+    MetaStatusCode RemoveDeletedInode(const Key4Inode& key);
+
+    void LoadDeletedInodes(std::list<std::string> * inodes);
+
     /**
      * @brief get inode from storage
      * @param[in] key: the key of inode want to get
@@ -237,6 +249,7 @@ class InodeStorage {
     RWLock rwLock_;
     std::shared_ptr<KVStorage> kvStorage_;
     std::string table4Inode_;
+    std::string table4DelInode_;
     std::string table4S3ChunkInfo_;
     std::string table4VolumeExtent_;
     std::string table4InodeAuxInfo_;

curvefs/src/metaserver/metastore.cpp

@@ -25,6 +25,7 @@
 #include <glog/logging.h>
 #include <sys/types.h>
 
+#include <bitset>
 #include <memory>
 #include <thread>  // NOLINT
 #include <unordered_map>
@@ -60,6 +61,7 @@ using KVStorage = ::curvefs::metaserver::storage::KVStorage;
 using Key4S3ChunkInfoList = ::curvefs::metaserver::storage::Key4S3ChunkInfoList;
 
 using ::curvefs::metaserver::storage::MemoryStorage;
+using ::curvefs::metaserver::storage::NameGenerator;
 using ::curvefs::metaserver::storage::RocksDBStorage;
 using ::curvefs::metaserver::storage::StorageOptions;
 
@@ -147,6 +149,7 @@ bool MetaStoreImpl::Load(const std::string &pathname) {
     }
 
     startCompacts();
+
     return true;
 }
 
@@ -866,5 +869,17 @@ bool MetaStoreImpl::InitStorage() {
     return kvStorage_->Open();
 }
 
+void MetaStoreImpl::LoadDeletedInodes() {
+    VLOG(6) << "load deleted inodes start.";
+    WriteLockGuard writeLockGuard(rwLock_);
+    MetaStatusCode status;
+    for (auto it = partitionMap_.begin(); it != partitionMap_.end(); it++) {
+        uint32_t partitionId = it->second->GetPartitionId();
+        VLOG(6) << "load deleted inodes, partitionId: " << partitionId;
+        it->second->LoadDeletedInodes();
+    }
+    VLOG(6) << "load deleted inodes end.";
+}
+
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/metastore.h

@@ -117,6 +117,7 @@ class MetaStore {
     virtual bool SaveData(const std::string& dir,
                           std::vector<std::string>* files) = 0;
     virtual bool Clear() = 0;
+    virtual void LoadDeletedInodes() {}
     virtual bool Destroy() = 0;
     virtual MetaStatusCode CreatePartition(
         const CreatePartitionRequest* request,
@@ -223,6 +224,7 @@ class MetaStoreImpl : public MetaStore {
                   std::vector<std::string>* files) override;
     bool Clear() override;
     bool Destroy() override;
+    void LoadDeletedInodes() override;
 
     MetaStatusCode CreatePartition(const CreatePartitionRequest* request,
                                    CreatePartitionResponse* response,

curvefs/src/metaserver/partition.cpp

@@ -339,6 +339,10 @@ MetaStatusCode Partition::UpdateInode(const UpdateInodeRequest& request,
     return ret;
 }
 
+void Partition::LoadDeletedInodes() {
+    inodeManager_->LoadDeletedInodes();
+}
+
 MetaStatusCode Partition::GetOrModifyS3ChunkInfo(
     uint32_t fsId, uint64_t inodeId, const S3ChunkInfoMap& map2add,
     const S3ChunkInfoMap& map2del, bool returnS3ChunkInfoMap,

curvefs/src/metaserver/partition.h

@@ -98,6 +98,8 @@ class Partition {
 
     MetaStatusCode GetInode(uint32_t fsId, uint64_t inodeId, Inode* inode);
 
+    void LoadDeletedInodes();
+
     MetaStatusCode GetInodeAttr(uint32_t fsId, uint64_t inodeId,
                                 InodeAttr* attr);
 

curvefs/src/metaserver/storage/converter.cpp

@@ -23,6 +23,7 @@
 #include <inttypes.h>
 #include <glog/logging.h>
 
+#include <bitset>
 #include <cstring>
 #include <string>
 #include <vector>
@@ -54,6 +55,7 @@ static bool CompareType(const std::string& str, KEY_TYPE keyType) {
 
 NameGenerator::NameGenerator(uint32_t partitionId)
     : tableName4Inode_(Format(kTypeInode, partitionId)),
+      tableName4DelInode_(FormatDel(kTypeInode, partitionId)),
       tableName4DeallocatableIndoe_(
           Format(kTypeDeallocatableInode, partitionId)),
       tableName4DeallocatableBlockGroup_(
@@ -71,6 +73,10 @@ std::string NameGenerator::GetInodeTableName() const {
     return tableName4Inode_;
 }
 
+std::string NameGenerator::GetDelInodeTableName() const {
+    return tableName4DelInode_;
+}
+
 std::string NameGenerator::GetDeallocatableInodeTableName() const {
     return tableName4DeallocatableIndoe_;
 }
@@ -123,6 +129,13 @@ std::string NameGenerator::Format(KEY_TYPE type, uint32_t partitionId) {
     return absl::StrCat(type, kDelimiter, absl::string_view(buf, sizeof(buf)));
 }
 
+std::string NameGenerator::FormatDel(KEY_TYPE type, uint32_t partitionId) {
+    char buf[sizeof(partitionId)];
+    std::memcpy(buf, reinterpret_cast<char*>(&partitionId), sizeof(buf));
+    return absl::StrCat(DELETEPREFIXINKV, type,
+      kDelimiter, absl::string_view(buf, sizeof(buf)));
+}
+
 Key4Inode::Key4Inode() : fsId(0), inodeId(0) {}
 
 Key4Inode::Key4Inode(uint32_t fsId, uint64_t inodeId)

curvefs/src/metaserver/storage/converter.h

@@ -34,9 +34,10 @@ namespace curvefs {
 namespace metaserver {
 
 class MetaStoreFStream;
-
 namespace storage {
 
+#define DELETEPREFIXINKV "deleting:"
+
 enum KEY_TYPE : unsigned char {
     kTypeInode = 1,
     kTypeS3ChunkInfo = 2,
@@ -61,6 +62,8 @@ class NameGenerator {
 
     std::string GetInodeTableName() const;
 
+    std::string GetDelInodeTableName() const;
+
     std::string GetDeallocatableInodeTableName() const;
 
     std::string GetS3ChunkInfoTableName() const;
@@ -87,9 +90,12 @@ class NameGenerator {
 
  private:
     std::string Format(KEY_TYPE type, uint32_t partitionId);
+    std::string FormatDel(KEY_TYPE type, uint32_t partitionId);
+
 
  private:
     std::string tableName4Inode_;
+    std::string tableName4DelInode_;
     std::string tableName4DeallocatableIndoe_;
     std::string tableName4DeallocatableBlockGroup_;
     std::string tableName4S3ChunkInfo_;