diff --git a/curvefs/src/metaserver/copyset/copyset_node.cpp b/curvefs/src/metaserver/copyset/copyset_node.cpp index 449886204f..a235a7d886 100644 --- a/curvefs/src/metaserver/copyset/copyset_node.cpp +++ b/curvefs/src/metaserver/copyset/copyset_node.cpp @@ -170,7 +170,7 @@ bool CopysetNode::Start() { LOG(ERROR) << "Fail to init raft node, copyset: " << name_; return false; } - + metaStore_->LoadDeletedInodes(); LOG(INFO) << "Run copyset success, copyset: " << name_; return true; } diff --git a/curvefs/src/metaserver/inode_manager.cpp b/curvefs/src/metaserver/inode_manager.cpp index f1b2d87c14..a1ca93fdd9 100644 --- a/curvefs/src/metaserver/inode_manager.cpp +++ b/curvefs/src/metaserver/inode_manager.cpp @@ -391,11 +391,6 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, } } - if (needAddTrash) { - trash_->Add(old.inodeid(), old.dtime()); - --(*type2InodeNum_)[old.type()]; - } - const S3ChunkInfoMap &map2add = request.s3chunkinfoadd(); const S3ChunkInfoList *list2add; VLOG(9) << "UpdateInode inode " << old.inodeid() << " map2add size " @@ -446,10 +441,27 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request, return MetaStatusCode::STORAGE_INTERNAL_ERROR; } } + + if (needAddTrash) { + trash_->Add(old.inodeid(), old.dtime(), false); + --(*type2InodeNum_)[old.type()]; + } + VLOG(9) << "UpdateInode success, " << request.ShortDebugString(); return MetaStatusCode::OK; } +void InodeManager::LoadDeletedInodes() { + std::map items; + inodeStorage_->LoadDeletedInodes(&items); + VLOG(3) << "build trash items size: " << items.size(); + std::vector names; + for (auto iter : items) { + curve::common::SplitString(iter.first , ":", &names); + trash_->Add(std::stoull(names[names.size() - 1 ]), iter.second, true); + } +} + MetaStatusCode InodeManager::GetOrModifyS3ChunkInfo( uint32_t fsId, uint64_t inodeId, const S3ChunkInfoMap& map2add, const S3ChunkInfoMap& map2del, bool returnS3ChunkInfoMap, diff --git a/curvefs/src/metaserver/inode_manager.h b/curvefs/src/metaserver/inode_manager.h index 54256029e6..81ba1a4516 100644 --- a/curvefs/src/metaserver/inode_manager.h +++ b/curvefs/src/metaserver/inode_manager.h @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -96,6 +97,8 @@ class InodeManager { MetaStatusCode UpdateInode(const UpdateInodeRequest& request, int64_t logIndex); + void LoadDeletedInodes(); + MetaStatusCode GetOrModifyS3ChunkInfo( uint32_t fsId, uint64_t inodeId, const S3ChunkInfoMap& map2add, const S3ChunkInfoMap& map2del, bool returnS3ChunkInfoMap, diff --git a/curvefs/src/metaserver/inode_storage.cpp b/curvefs/src/metaserver/inode_storage.cpp index 8e5da8a913..a593240ded 100644 --- a/curvefs/src/metaserver/inode_storage.cpp +++ b/curvefs/src/metaserver/inode_storage.cpp @@ -57,6 +57,7 @@ using ::curvefs::metaserver::storage::Prefix4ChunkIndexS3ChunkInfoList; using ::curvefs::metaserver::storage::Prefix4InodeS3ChunkInfoList; using ::curvefs::metaserver::storage::Prefix4InodeVolumeExtent; using ::curvefs::metaserver::storage::Status; +using curvefs::metaserver::Time; const char* InodeStorage::kInodeCountKey("count"); @@ -67,6 +68,7 @@ InodeStorage::InodeStorage(std::shared_ptr kvStorage, uint64_t nInode) : kvStorage_(std::move(kvStorage)), table4Inode_(nameGenerator->GetInodeTableName()), + table4DelInode_(nameGenerator->GetDelInodeTableName()), table4S3ChunkInfo_(nameGenerator->GetS3ChunkInfoTableName()), table4VolumeExtent_(nameGenerator->GetVolumeExtentTableName()), table4InodeAuxInfo_(nameGenerator->GetInodeAuxInfoTableName()), @@ -185,9 +187,78 @@ MetaStatusCode InodeStorage::Insert(const Inode& inode, int64_t logIndex) { return MetaStatusCode::STORAGE_INTERNAL_ERROR; } +MetaStatusCode InodeStorage::AddDeletedInode( + const Key4Inode& keyInode, uint64_t dtime) { + WriteLockGuard lg(rwLock_); + std::string skey = conv_.SerializeToString(keyInode); + VLOG(9) << "update deleting key, " << keyInode.inodeId << ", " << dtime; + const char* step = "Begin transaction"; + std::shared_ptr txn; + txn = kvStorage_->BeginTransaction(); + if (txn == nullptr) { + LOG(ERROR) << "Begin transaction failed"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + Time dtimeInfo; + dtimeInfo.set_sec(dtime); + dtimeInfo.set_nsec(0); + auto rc = txn->HSet(table4DelInode_, skey, dtimeInfo); + step = "insert inode "; + if (rc.ok()) { + rc = txn->Commit(); + step = "commit"; + } + if (rc.ok()) { + VLOG(9) << "set deleting key ok"; + return MetaStatusCode::OK; + } + LOG(ERROR) << step << "failed, status = " << rc.ToString(); + if (!txn->Rollback().ok()) { + LOG(ERROR) << "Rollback delete inode transaction failed, status = " + << rc.ToString(); + } + return MetaStatusCode::STORAGE_INTERNAL_ERROR; +} + +MetaStatusCode InodeStorage::RemoveDeletedInode(const Key4Inode& key) { + WriteLockGuard lg(rwLock_); + std::string skey = conv_.SerializeToString(key); + VLOG(9) << "clear deleting key start, " << skey; + std::shared_ptr txn = nullptr; + const char* step = "Begin transaction"; + txn = kvStorage_->BeginTransaction(); + if (txn == nullptr) { + LOG(ERROR) << "Begin transaction failed"; + return MetaStatusCode::STORAGE_INTERNAL_ERROR; + } + step = "Delete inode from transaction"; + auto s = txn->HDel(table4DelInode_, skey); + if (s.ok()) { + step = "Delete inode"; + s = txn->Commit(); + } + if (s.ok()) { + VLOG(9) << "clear deleting key ok, " << skey; + return MetaStatusCode::OK; + } + LOG(ERROR) << step << " failed, status = " << s.ToString(); + if (!txn->Rollback().ok()) { + LOG(ERROR) << "Rollback delete inode transaction failed, status = " + << s.ToString(); + } + return MetaStatusCode::STORAGE_INTERNAL_ERROR; +} + +void InodeStorage::LoadDeletedInodes(std::map * inodes) { + VLOG(6) << "load deleted key start with: " << table4DelInode_; + kvStorage_->GetPrefix(inodes, table4DelInode_); + VLOG(6) << "oad deleted over"; +} + MetaStatusCode InodeStorage::Get(const Key4Inode& key, Inode* inode) { ReadLockGuard lg(rwLock_); std::string skey = conv_.SerializeToString(key); + Status s = kvStorage_->HGet(table4Inode_, skey, inode); if (s.ok()) { return MetaStatusCode::OK; @@ -471,7 +542,6 @@ MetaStatusCode InodeStorage::Clear() { // because if we fail stop, we will replay // raft logs and clear it again WriteLockGuard lg(rwLock_); - Status s = kvStorage_->HClear(table4Inode_); if (!s.ok()) { LOG(ERROR) << "InodeStorage clear inode table failed, status = " @@ -492,7 +562,6 @@ MetaStatusCode InodeStorage::Clear() { << s.ToString(); return MetaStatusCode::STORAGE_INTERNAL_ERROR; } - s = kvStorage_->HClear(table4InodeAuxInfo_); if (!s.ok()) { LOG(ERROR) diff --git a/curvefs/src/metaserver/inode_storage.h b/curvefs/src/metaserver/inode_storage.h index 38ad3c5f56..a7654c5963 100644 --- a/curvefs/src/metaserver/inode_storage.h +++ b/curvefs/src/metaserver/inode_storage.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -76,6 +77,18 @@ class InodeStorage { */ MetaStatusCode Insert(const Inode& inode, int64_t logIndex); + /** + * @brief update deleting inode key in storage + * @param[in] inode: the inode want to update + * @param[in] logIndex: the index of raft log + * @return + */ + MetaStatusCode AddDeletedInode(const Key4Inode& inode, uint64_t dtime); + + MetaStatusCode RemoveDeletedInode(const Key4Inode& key); + + void LoadDeletedInodes(std::map * inodes); + /** * @brief get inode from storage * @param[in] key: the key of inode want to get @@ -237,6 +250,7 @@ class InodeStorage { RWLock rwLock_; std::shared_ptr kvStorage_; std::string table4Inode_; + std::string table4DelInode_; std::string table4S3ChunkInfo_; std::string table4VolumeExtent_; std::string table4InodeAuxInfo_; diff --git a/curvefs/src/metaserver/metastore.cpp b/curvefs/src/metaserver/metastore.cpp index 81a289390a..f102275a38 100644 --- a/curvefs/src/metaserver/metastore.cpp +++ b/curvefs/src/metaserver/metastore.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include // NOLINT #include @@ -60,6 +61,7 @@ using KVStorage = ::curvefs::metaserver::storage::KVStorage; using Key4S3ChunkInfoList = ::curvefs::metaserver::storage::Key4S3ChunkInfoList; using ::curvefs::metaserver::storage::MemoryStorage; +using ::curvefs::metaserver::storage::NameGenerator; using ::curvefs::metaserver::storage::RocksDBStorage; using ::curvefs::metaserver::storage::StorageOptions; @@ -147,6 +149,7 @@ bool MetaStoreImpl::Load(const std::string &pathname) { } startCompacts(); + return true; } @@ -933,5 +936,17 @@ bool MetaStoreImpl::InitStorage() { return kvStorage_->Open(); } +void MetaStoreImpl::LoadDeletedInodes() { + VLOG(6) << "load deleted inodes start."; + WriteLockGuard writeLockGuard(rwLock_); + MetaStatusCode status; + for (auto it = partitionMap_.begin(); it != partitionMap_.end(); it++) { + uint32_t partitionId = it->second->GetPartitionId(); + VLOG(6) << "load deleted inodes, partitionId: " << partitionId; + it->second->LoadDeletedInodes(); + } + VLOG(6) << "load deleted inodes end."; +} + } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/metastore.h b/curvefs/src/metaserver/metastore.h index 9d906a62dc..4babaf657e 100644 --- a/curvefs/src/metaserver/metastore.h +++ b/curvefs/src/metaserver/metastore.h @@ -117,6 +117,7 @@ class MetaStore { virtual bool SaveData(const std::string& dir, std::vector* files) = 0; virtual bool Clear() = 0; + virtual void LoadDeletedInodes() {} virtual bool Destroy() = 0; virtual MetaStatusCode CreatePartition( const CreatePartitionRequest* request, @@ -236,6 +237,7 @@ class MetaStoreImpl : public MetaStore { std::vector* files) override; bool Clear() override; bool Destroy() override; + void LoadDeletedInodes() override; MetaStatusCode CreatePartition(const CreatePartitionRequest* request, CreatePartitionResponse* response, diff --git a/curvefs/src/metaserver/partition.cpp b/curvefs/src/metaserver/partition.cpp index 3ec98b8b18..c6e230428d 100644 --- a/curvefs/src/metaserver/partition.cpp +++ b/curvefs/src/metaserver/partition.cpp @@ -376,6 +376,10 @@ MetaStatusCode Partition::UpdateInode(const UpdateInodeRequest& request, return ret; } +void Partition::LoadDeletedInodes() { + inodeManager_->LoadDeletedInodes(); +} + MetaStatusCode Partition::GetOrModifyS3ChunkInfo( uint32_t fsId, uint64_t inodeId, const S3ChunkInfoMap& map2add, const S3ChunkInfoMap& map2del, bool returnS3ChunkInfoMap, diff --git a/curvefs/src/metaserver/partition.h b/curvefs/src/metaserver/partition.h index 30bf94255a..2ee1d2b1d1 100644 --- a/curvefs/src/metaserver/partition.h +++ b/curvefs/src/metaserver/partition.h @@ -109,6 +109,8 @@ class Partition { MetaStatusCode GetInode(uint32_t fsId, uint64_t inodeId, Inode* inode); + void LoadDeletedInodes(); + MetaStatusCode GetInodeAttr(uint32_t fsId, uint64_t inodeId, InodeAttr* attr); diff --git a/curvefs/src/metaserver/storage/converter.cpp b/curvefs/src/metaserver/storage/converter.cpp index f5ce0328ac..cab13e9abf 100644 --- a/curvefs/src/metaserver/storage/converter.cpp +++ b/curvefs/src/metaserver/storage/converter.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -57,6 +58,7 @@ static bool CompareType(const std::string& str, KEY_TYPE keyType) { NameGenerator::NameGenerator(uint32_t partitionId) : tableName4Inode_(Format(kTypeInode, partitionId)), + tableName4DelInode_(Format(kTypeDelInode, partitionId)), tableName4DeallocatableIndoe_( Format(kTypeDeallocatableInode, partitionId)), tableName4DeallocatableBlockGroup_( @@ -76,6 +78,10 @@ std::string NameGenerator::GetInodeTableName() const { return tableName4Inode_; } +std::string NameGenerator::GetDelInodeTableName() const { + return tableName4DelInode_; +} + std::string NameGenerator::GetDeallocatableInodeTableName() const { return tableName4DeallocatableIndoe_; } diff --git a/curvefs/src/metaserver/storage/converter.h b/curvefs/src/metaserver/storage/converter.h index ea045271e0..20743dd5fd 100644 --- a/curvefs/src/metaserver/storage/converter.h +++ b/curvefs/src/metaserver/storage/converter.h @@ -34,7 +34,6 @@ namespace curvefs { namespace metaserver { class MetaStoreFStream; - namespace storage { enum KEY_TYPE : unsigned char { @@ -52,7 +51,8 @@ enum KEY_TYPE : unsigned char { kTypeInodeCount = 11, kTypeDentryCount = 12, kTypeTxLock = 13, - kTypeTxWrite = 14 + kTypeTxWrite = 14, + kTypeDelInode = 15 }; // NOTE: you must generate all table name by NameGenerator class for @@ -64,6 +64,8 @@ class NameGenerator { std::string GetInodeTableName() const; + std::string GetDelInodeTableName() const; + std::string GetDeallocatableInodeTableName() const; std::string GetS3ChunkInfoTableName() const; @@ -99,6 +101,7 @@ class NameGenerator { private: std::string tableName4Inode_; + std::string tableName4DelInode_; std::string tableName4DeallocatableIndoe_; std::string tableName4DeallocatableBlockGroup_; std::string tableName4S3ChunkInfo_; diff --git a/curvefs/src/metaserver/storage/rocksdb_storage.cpp b/curvefs/src/metaserver/storage/rocksdb_storage.cpp index a94fed47c6..1c902f9c77 100644 --- a/curvefs/src/metaserver/storage/rocksdb_storage.cpp +++ b/curvefs/src/metaserver/storage/rocksdb_storage.cpp @@ -26,6 +26,7 @@ #include #include +#include "src/common/string_util.h" #include "src/common/timeutility.h" #include "curvefs/src/metaserver/storage/utils.h" #include "curvefs/src/metaserver/storage/storage.h" @@ -248,6 +249,7 @@ Status RocksDBStorage::Get(const std::string& name, ROCKSDB_NAMESPACE::Status s; std::string svalue; std::string ikey = ToInternalKey(name, key, type); + VLOG(9) << "Get key: " << ikey << ", " << options_.dataDir; auto handle = GetColumnFamilyHandle(type); { RocksDBPerfGuard guard(OP_GET); @@ -257,6 +259,7 @@ Status RocksDBStorage::Get(const std::string& name, if (s.ok() && !value->ParseFromString(svalue)) { return Status::ParsedFailed(); } + return ToStorageStatus(s); } @@ -273,6 +276,7 @@ Status RocksDBStorage::Set(const std::string& name, auto handle = GetColumnFamilyHandle(type); std::string ikey = ToInternalKey(name, key, type); + VLOG(9) << "set key: " << ikey << ", " << options_.dataDir; RocksDBPerfGuard guard(OP_PUT); ROCKSDB_NAMESPACE::Status s = InTransaction_ ? txn_->Put(handle, ikey, svalue) : @@ -289,6 +293,7 @@ Status RocksDBStorage::Del(const std::string& name, std::string ikey = ToInternalKey(name, key, type); auto handle = GetColumnFamilyHandle(type); + VLOG(9) << "del key: " << ikey << ", " << options_.dataDir; RocksDBPerfGuard guard(OP_DELETE); ROCKSDB_NAMESPACE::Status s = InTransaction_ ? txn_->Delete(handle, ikey) : @@ -547,6 +552,31 @@ bool RocksDBStorage::Recover(const std::string& dir) { return true; } +void RocksDBStorage::GetPrefix( + std::map* item, const std::string prefix) { + std::string sprefix = absl::StrCat("0", ":", prefix); + VLOG(3) << "load deleted inodes from: " << options_.dataDir + << ", " << sprefix << ", " << prefix; + int counts = 0; + rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions()); + curvefs::metaserver::Time time; + for (it->Seek(sprefix); it->Valid() && + it->key().starts_with(sprefix); it->Next()) { + std::string key = it->key().ToString(); + if (!time.ParseFromString(it->value().ToString())) { + return; + } + VLOG(9) << "key: " << key << ", " << key.size() << ", " + << it->value().ToString() << ", " << time.sec(); + item->emplace(key, time.sec()); + counts++; + } + delete it; + VLOG(3) << "load deleted inodes end, size is: " << item->size() + << ", " << counts << ", " << options_.dataDir; +} + + } // namespace storage } // namespace metaserver } // namespace curvefs diff --git a/curvefs/src/metaserver/storage/rocksdb_storage.h b/curvefs/src/metaserver/storage/rocksdb_storage.h index 2432da6988..1eccc2ec35 100644 --- a/curvefs/src/metaserver/storage/rocksdb_storage.h +++ b/curvefs/src/metaserver/storage/rocksdb_storage.h @@ -23,7 +23,9 @@ #ifndef CURVEFS_SRC_METASERVER_STORAGE_ROCKSDB_STORAGE_H_ #define CURVEFS_SRC_METASERVER_STORAGE_ROCKSDB_STORAGE_H_ +#include #include +#include #include #include #include @@ -40,10 +42,10 @@ #include "rocksdb/utilities/transaction_db.h" #include "rocksdb/utilities/table_properties_collectors.h" #include "src/common/concurrent/rw_lock.h" +#include "curvefs/src/metaserver/storage/converter.h" #include "curvefs/src/metaserver/storage/utils.h" #include "curvefs/src/metaserver/storage/storage.h" #include "curvefs/src/metaserver/storage/rocksdb_perf.h" -#include "curvefs/src/metaserver/storage/converter.h" namespace curvefs { namespace metaserver { @@ -90,6 +92,9 @@ class RocksDBStorage : public KVStorage, public StorageTransaction { STORAGE_TYPE Type() override; +void GetPrefix( + std::map* item, const std::string prefix) override; + StorageOptions GetStorageOptions() const override; // unordered diff --git a/curvefs/src/metaserver/storage/storage.h b/curvefs/src/metaserver/storage/storage.h index 97cef01fca..c7513e23eb 100644 --- a/curvefs/src/metaserver/storage/storage.h +++ b/curvefs/src/metaserver/storage/storage.h @@ -23,7 +23,9 @@ #ifndef CURVEFS_SRC_METASERVER_STORAGE_STORAGE_H_ #define CURVEFS_SRC_METASERVER_STORAGE_STORAGE_H_ +#include #include +#include #include #include @@ -51,6 +53,9 @@ class BaseStorage { const std::string& key, const ValueType& value) = 0; + virtual void GetPrefix(std::map* item, + const std::string prefix) {} + virtual Status HDel(const std::string& name, const std::string& key) = 0; virtual std::shared_ptr HGetAll(const std::string& name) = 0; diff --git a/curvefs/src/metaserver/trash.cpp b/curvefs/src/metaserver/trash.cpp index c0f01376a3..28aa77fbcf 100644 --- a/curvefs/src/metaserver/trash.cpp +++ b/curvefs/src/metaserver/trash.cpp @@ -56,6 +56,7 @@ void TrashImpl::Init(const TrashOption &option) { mdsClient_ = option.mdsClient; copysetNode_ = copyset::CopysetNodeManager::GetInstance(). GetSharedCopysetNode(poolId_, copysetId_); + isStop_ = false; } @@ -63,20 +64,25 @@ void TrashImpl::StopScan() { isStop_ = true; } -bool TrashImpl::IsStop() { - return isStop_; -} - -void TrashImpl::Add(uint64_t inodeId, uint64_t dtime) { +void TrashImpl::Add(uint64_t inodeId, uint64_t dtime, bool deleted) { if (isStop_) { return; } + LockGuard lg(itemsMutex_); - trashItems_[inodeId] = dtime; - VLOG(6) << "Add Trash Item success, fsId = " << fsId_ - << ", partitionId = " << partitionId_ + trashItems_.emplace(inodeId, dtime); + + VLOG(6) << "Add trash item success" << ", inodeId = " << inodeId - << ", dtime = " << dtime; + << ", dtime = " << dtime + << ", deleted = " << deleted; + if (!deleted) { + inodeStorage_->AddDeletedInode(Key4Inode(fsId_, inodeId), dtime); + } +} + +bool TrashImpl::IsStop() { + return isStop_; } void TrashImpl::Remove(uint64_t inodeId) { @@ -84,7 +90,10 @@ void TrashImpl::Remove(uint64_t inodeId) { return; } LockGuard lg(itemsMutex_); - trashItems_.erase(inodeId); + auto it = trashItems_.find(inodeId); + if (it != trashItems_.end()) { + trashItems_.erase(it); + } VLOG(6) << "Remove Trash Item success, fsId = " << fsId_ << ", partitionId = " << partitionId_ << ", inodeId = " << inodeId; @@ -100,7 +109,7 @@ void TrashImpl::ScanTrash() { return; } - std::unordered_map temp; + std::map temp; { LockGuard lgItems(itemsMutex_); trashItems_.swap(temp); @@ -116,24 +125,30 @@ void TrashImpl::ScanTrash() { LOG(ERROR) << "DeleteInodeAndData fail, fsId = " << fsId_ << ", inodeId = " << it->first << ", ret = " << MetaStatusCode_Name(ret); + RemoveDeletedInode(it->first); it++; continue; } LOG(INFO) << "Trash delete inode, fsId = " << fsId_ << ", partitionId = " << partitionId_ << ", inodeId = " << it->first; + RemoveDeletedInode(it->first); it = temp.erase(it); } else { it++; } } - { LockGuard lgItems(itemsMutex_); - trashItems_.insert(temp.begin(), temp.end()); + trashItems_.swap(temp); } } +void TrashImpl::RemoveDeletedInode(uint64_t inodeId) { + VLOG(9) << "RemoveDeletedInode: " << fsId_ << ", " << inodeId; + inodeStorage_->RemoveDeletedInode(Key4Inode(fsId_, inodeId)); +} + bool TrashImpl::NeedDelete(uint64_t dtime) { // for compatibility, if fs recycleTimeHour is 0, use old trash logic // if fs recycleTimeHour is 0, use trash wait until expiredAfterSec diff --git a/curvefs/src/metaserver/trash.h b/curvefs/src/metaserver/trash.h index dacb67df34..adc4f18a8a 100644 --- a/curvefs/src/metaserver/trash.h +++ b/curvefs/src/metaserver/trash.h @@ -25,6 +25,7 @@ #include #include +#include #include #include @@ -73,7 +74,8 @@ class Trash { virtual void Init(const TrashOption &option) = 0; - virtual void Add(uint64_t inodeId, uint64_t dtime) = 0; + virtual void Add(uint64_t inodeId, + uint64_t dtime, bool deleted = false) = 0; virtual void Remove(uint64_t inodeId) = 0; @@ -98,7 +100,7 @@ class TrashImpl : public Trash { void Init(const TrashOption &option) override; - void Add(uint64_t inodeId, uint64_t dtime) override; + void Add(uint64_t inodeId, uint64_t dtime, bool deleted = false) override; void Remove(uint64_t inodeId) override; @@ -124,6 +126,8 @@ class TrashImpl : public Trash { MetaStatusCode DeleteInode(uint64_t inodeId); + void RemoveDeletedInode(uint64_t inodeId); + private: std::shared_ptr inodeStorage_; std::shared_ptr s3Adaptor_; @@ -136,7 +140,8 @@ class TrashImpl : public Trash { CopysetId copysetId_; PartitionId partitionId_; - std::unordered_map trashItems_; + std::map trashItems_; + mutable Mutex itemsMutex_; TrashOption options_; diff --git a/curvefs/src/metaserver/trash_manager.h b/curvefs/src/metaserver/trash_manager.h index 221b4629d6..0e2d6ee2f6 100644 --- a/curvefs/src/metaserver/trash_manager.h +++ b/curvefs/src/metaserver/trash_manager.h @@ -27,6 +27,8 @@ #include #include +#include "src/common/string_util.h" + #include "src/common/concurrent/concurrent.h" #include "curvefs/src/metaserver/trash.h" @@ -73,6 +75,7 @@ class TrashManager { InterruptibleSleeper sleeper_; std::map> trashs_; + curve::common::RWLock rwLock_; }; diff --git a/curvefs/test/metaserver/inode_storage_test.cpp b/curvefs/test/metaserver/inode_storage_test.cpp index 4ea0bc3f54..4eab5c03c4 100644 --- a/curvefs/test/metaserver/inode_storage_test.cpp +++ b/curvefs/test/metaserver/inode_storage_test.cpp @@ -1033,5 +1033,26 @@ TEST_F(InodeStorageTest, Test_UpdateDeallocatableBlockGroup) { ASSERT_EQ(1, deallocatableBlockGroupVec.size()); } +TEST_F(InodeStorageTest, test_deleting_key) { + InodeStorage storage(kvStorage_, nameGenerator_, 0); + ASSERT_TRUE(storage.Init()); + Inode inode1 = GenInode(1, 1); + Inode inode2 = GenInode(2, 2); + + // insert + ASSERT_EQ(storage.Insert(inode1, logIndex_++), MetaStatusCode::OK); + ASSERT_EQ(storage.Insert(inode2, logIndex_++), MetaStatusCode::OK); + + ASSERT_EQ(storage.AddDeletedInode(Key4Inode( + inode1.fsid(), inode1.inodeid()), 1), MetaStatusCode::OK); + ASSERT_EQ(storage.RemoveDeletedInode(Key4Inode( + inode1.fsid(), inode1.inodeid())), MetaStatusCode::OK); + + ASSERT_EQ(storage.AddDeletedInode(Key4Inode( + inode2.fsid(), inode2.inodeid()), 1), MetaStatusCode::OK); + ASSERT_EQ(storage.RemoveDeletedInode(Key4Inode( + inode2.fsid(), inode2.inodeid())), MetaStatusCode::OK); +} + } // namespace metaserver } // namespace curvefs diff --git a/curvefs/test/metaserver/trash_test.cpp b/curvefs/test/metaserver/trash_test.cpp index 36a4de1eca..1145cfaa4b 100644 --- a/curvefs/test/metaserver/trash_test.cpp +++ b/curvefs/test/metaserver/trash_test.cpp @@ -219,9 +219,9 @@ TEST_F(TestTrash, testAdd3ItemAndDelete) { })); uint64_t dtime = curve::common::TimeUtility::GetTimeofDaySec(); - trash1->Add(1, dtime - 6); - trash1->Add(2, dtime - 2); - trash2->Add(1, dtime); + trash1->Add(1, dtime - 6, false); + trash1->Add(2, dtime - 2, false); + trash2->Add(1, dtime, false); std::this_thread::sleep_for(std::chrono::seconds(5));