Skip to content

Commit

Permalink
fix: Temporary fix for loading plain lists
Browse files Browse the repository at this point in the history
**The Issue**

We do not serialize lists in PLAIN format well today. We're working on
getting it right, but existing wire format simply doesn't work.

**The (temporary) fix**

This PR attempts to load lists as usual, but upon failure it reverts to
loading a plain list format. This is not a perfect solution, and in some
extreme edge cases it could fail. But.. it's temporary.
  • Loading branch information
chakaz committed Oct 10, 2024
1 parent 4785767 commit 4959bef
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 6 deletions.
9 changes: 9 additions & 0 deletions src/server/list_family_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,15 @@ TEST_F(ListFamilyTest, LRem) {
ASSERT_THAT(Run({"lrem", "nexists", "0", "elem"}), IntArg(0));
}

TEST_F(ListFamilyTest, DumpRestorePlain) {
const string kValue(10'000, '#');
EXPECT_EQ(CheckedInt({"LPUSH", kKey1, kValue}), 1);
auto buffer = Run({"DUMP", kKey1}).GetBuf();
EXPECT_EQ(Run({"RESTORE", kKey2, "0", ToSV(buffer)}), "OK");
EXPECT_EQ(CheckedInt({"LLEN", kKey2}), 1);
EXPECT_EQ(Run({"LRANGE", kKey2, "0", "1"}), kValue);
}

TEST_F(ListFamilyTest, LTrim) {
Run({"rpush", kKey1, "a", "b", "c", "d"});
ASSERT_EQ(Run({"ltrim", kKey1, "-2", "-1"}), "OK");
Expand Down
21 changes: 15 additions & 6 deletions src/server/rdb_load.cc
Original file line number Diff line number Diff line change
Expand Up @@ -722,13 +722,19 @@ void RdbLoaderBase::OpaqueObjLoader::CreateList(const LoadTrace* ltrace) {
if (ec_)
return false;

uint8_t* lp = nullptr;

auto load_plain = [&]() {
lp = (uint8_t*)zmalloc(sv.size());
::memcpy(lp, (uint8_t*)sv.data(), sv.size());
quicklistAppendPlainNode(ql, lp, sv.size());
};

if (container == QUICKLIST_NODE_CONTAINER_PLAIN) {
quicklistAppendPlainNode(ql, (uint8_t*)sv.data(), sv.size());
load_plain();
return true;
}

uint8_t* lp = nullptr;

if (rdb_type_ == RDB_TYPE_LIST_QUICKLIST_2) {
uint8_t* src = (uint8_t*)sv.data();
if (!lpValidateIntegrity(src, sv.size(), 0, nullptr, nullptr)) {
Expand All @@ -747,10 +753,13 @@ void RdbLoaderBase::OpaqueObjLoader::CreateList(const LoadTrace* ltrace) {
lp = lpNew(sv.size());
if (!ziplistValidateIntegrity((uint8_t*)sv.data(), sv.size(), 1,
ziplistEntryConvertAndValidate, &lp)) {
LOG(ERROR) << "Ziplist integrity check failed: " << sv.size();
zfree(lp);
ec_ = RdbError(errc::rdb_file_corrupted);
return false;
// TODO: Revert loading plain format here once we fix serialization size.
// LOG(ERROR) << "Ziplist integrity check failed: " << sv.size();
// ec_ = RdbError(errc::rdb_file_corrupted);
// return false;
load_plain();
return true;
}

/* Silently skip empty ziplists, if we'll end up with empty quicklist we'll fail later. */
Expand Down

0 comments on commit 4959bef

Please sign in to comment.