Skip to content

Commit c7da455

Browse files
mm304321141mm304321141
committed
Refactor marked_for_compaction in order to improve management (#142)
1 parent 6fa21b4 commit c7da455

17 files changed

+171
-46
lines changed

‎db/builder.cc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,6 @@ Status BuildTable(
209209
blob_meta->prop.purpose = kEssenceSst;
210210
blob_meta->prop.flags |= TablePropertyCache::kNoRangeDeletions;
211211
status = blob_builder->Finish(&blob_meta->prop, nullptr);
212-
blob_meta->marked_for_compaction = blob_builder->NeedCompact();
213212
TableProperties& tp = *separate_helper.current_prop;
214213
if (status.ok()) {
215214
blob_meta->fd.file_size = blob_builder->FileSize();
@@ -422,7 +421,8 @@ Status BuildTable(
422421
if (s.ok() && !empty) {
423422
uint64_t file_size = builder->FileSize();
424423
sst_meta()->fd.file_size = file_size;
425-
sst_meta()->marked_for_compaction = builder->NeedCompact();
424+
sst_meta()->marked_for_compaction =
425+
builder->NeedCompact() ? FileMetaData::kMarkedFromTableBuilder : 0;
426426
sst_meta()->prop.num_entries = builder->NumEntries();
427427
assert(sst_meta()->fd.GetFileSize() > 0);
428428
// refresh now that builder is finished

‎db/compaction.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ void ProcessFileMetaData(const char* job_info, FileMetaData* meta,
196196

197197
if (tp->num_range_deletions > 0 && mopt->optimize_range_deletion &&
198198
!iopt->enable_lazy_compaction) {
199-
meta->marked_for_compaction = true;
199+
meta->marked_for_compaction |= FileMetaData::kMarkedFromRangeDeletion;
200200
}
201201
if (iopt->ttl_extractor_factory != nullptr) {
202202
GetCompactionTimePoint(tp->user_collected_properties,

‎db/compaction.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ struct CompactionWorkerResult {
181181

182182
// use UserProperties["User.Collected.Transient.Stat"] to reduce complexity
183183
// std::string stat_one;
184-
bool marked_for_compaction;
184+
uint8_t marked_for_compaction;
185185
};
186186
std::vector<FileInfo> files;
187187
std::string stat_all;

‎db/compaction_dispatcher.cc

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -931,7 +931,8 @@ std::string RemoteCompactionDispatcher::Worker::DoCompaction(Slice data) {
931931
}
932932
}
933933
if (s.ok()) {
934-
meta.marked_for_compaction = builder->NeedCompact();
934+
meta.marked_for_compaction =
935+
builder->NeedCompact() ? FileMetaData::kMarkedFromTableBuilder : 0;
935936
meta.prop.num_entries = builder->NumEntries();
936937
for (auto& pair : dependence) {
937938
meta.prop.dependence.emplace_back(Dependence{pair.first, pair.second});
@@ -976,7 +977,7 @@ std::string RemoteCompactionDispatcher::Worker::DoCompaction(Slice data) {
976977
file_info.smallest_seqno = meta.fd.smallest_seqno;
977978
file_info.largest_seqno = meta.fd.largest_seqno;
978979
file_info.file_size = meta.fd.file_size;
979-
file_info.marked_for_compaction |= builder->NeedCompact();
980+
file_info.marked_for_compaction = meta.marked_for_compaction;
980981
result.files.emplace_back(file_info);
981982
}
982983
meta = FileMetaData();

‎db/compaction_job.cc

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,18 @@ const char* GetCompactionReasonString(CompactionReason compaction_reason) {
108108
return "FIFOTtl";
109109
case CompactionReason::kManualCompaction:
110110
return "ManualCompaction";
111-
case CompactionReason::kFilesMarkedForCompaction:
112-
return "FilesMarkedForCompaction";
111+
case CompactionReason::kFilesMarkedFromUser:
112+
return "FilesMarkedFromUser";
113+
case CompactionReason::kFilesMarkedFromTableBuilder:
114+
return "FilesMarkedFromTableBuilder";
115+
case CompactionReason::kFilesMarkedFromRangeDeletion:
116+
return "FilesMarkedFromRangeDeletion";
117+
case CompactionReason::kFilesMarkedFromTTL:
118+
return "FilesMarkedFromTTL";
119+
case CompactionReason::kFilesMarkedFromFileSystem:
120+
return "FilesMarkedFromFileSystem";
121+
case CompactionReason::kFilesMarkedFromUpdateBlob:
122+
return "FilesMarkedFromUpdateBlob";
113123
case CompactionReason::kBottommostFiles:
114124
return "BottommostFiles";
115125
case CompactionReason::kFlush:
@@ -2428,7 +2438,9 @@ Status CompactionJob::FinishCompactionOutputFile(
24282438
}
24292439
}
24302440
if (s.ok()) {
2431-
meta->marked_for_compaction = sub_compact->builder->NeedCompact();
2441+
meta->marked_for_compaction = sub_compact->builder->NeedCompact()
2442+
? FileMetaData::kMarkedFromTableBuilder
2443+
: 0;
24322444
meta->prop.num_entries = sub_compact->builder->NumEntries();
24332445
for (auto& pair : dependence) {
24342446
meta->prop.dependence.emplace_back(Dependence{pair.first, pair.second});
@@ -2558,7 +2570,6 @@ Status CompactionJob::FinishCompactionOutputBlob(
25582570
auto meta = &sub_compact->current_blob_output()->meta;
25592571
assert(meta != nullptr);
25602572
if (s.ok()) {
2561-
meta->marked_for_compaction = sub_compact->blob_builder->NeedCompact();
25622573
meta->prop.num_entries = sub_compact->blob_builder->NumEntries();
25632574
meta->prop.inheritance = InheritanceTreeToSet(inheritance_tree);
25642575
assert(std::is_sorted(meta->prop.inheritance.begin(),

‎db/compaction_picker.cc

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@ void AssignUserKey(std::string& key, const Slice& ikey) {
156156
Slice ukey = ExtractUserKey(ikey);
157157
key.assign(ukey.data(), ukey.size());
158158
};
159+
159160
} // anonymous namespace
160161

161162
bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
@@ -259,6 +260,29 @@ CompactionPicker::CompactionPicker(TableCache* table_cache,
259260

260261
CompactionPicker::~CompactionPicker() {}
261262

263+
CompactionReason CompactionPicker::ConvertCompactionReason(
264+
uint8_t marked, CompactionReason default_reason) {
265+
if (marked & FileMetaData::kMarkedFromUser) {
266+
return CompactionReason::kFilesMarkedFromUser;
267+
}
268+
if (marked & FileMetaData::kMarkedFromTableBuilder) {
269+
return CompactionReason::kFilesMarkedFromTableBuilder;
270+
}
271+
if (marked & FileMetaData::kMarkedFromRangeDeletion) {
272+
return CompactionReason::kFilesMarkedFromRangeDeletion;
273+
}
274+
if (marked & FileMetaData::kMarkedFromTTL) {
275+
return CompactionReason::kFilesMarkedFromTTL;
276+
}
277+
if (marked & FileMetaData::kMarkedFromFileSystem) {
278+
return CompactionReason::kFilesMarkedFromFileSystem;
279+
}
280+
if (marked & FileMetaData::kMarkedFromUpdateBlob) {
281+
return CompactionReason::kFilesMarkedFromUpdateBlob;
282+
}
283+
return default_reason;
284+
}
285+
262286
double CompactionPicker::GetQ(std::vector<double>::const_iterator b,
263287
std::vector<double>::const_iterator e, size_t g) {
264288
double S = std::accumulate(b, e, 0.0);
@@ -2030,8 +2054,12 @@ void CompactionPicker::PickFilesMarkedForCompaction(
20302054
// files as being_compacted, but didn't call ComputeCompactionScore()
20312055
assert(!level_file.second->being_compacted);
20322056
*start_level = level_file.first;
2033-
*output_level =
2034-
(*start_level == 0) ? vstorage->base_level() : *start_level + 1;
2057+
if (level_file.second->is_output_to_parent_level()) {
2058+
*output_level =
2059+
(*start_level == 0) ? vstorage->base_level() : *start_level + 1;
2060+
} else {
2061+
*output_level = level_file.first;
2062+
}
20352063

20362064
if (*start_level == 0 && !level0_compactions_in_progress()->empty()) {
20372065
return false;
@@ -2229,7 +2257,9 @@ void LevelCompactionBuilder::SetupInitialFiles() {
22292257
cf_name_, vstorage_, &start_level_, &output_level_,
22302258
&start_level_inputs_);
22312259
if (!start_level_inputs_.empty()) {
2232-
compaction_reason_ = CompactionReason::kFilesMarkedForCompaction;
2260+
compaction_reason_ = CompactionPicker::ConvertCompactionReason(
2261+
start_level_inputs_.files.front()->marked_for_compaction,
2262+
CompactionReason::kUnknown);
22332263
return;
22342264
}
22352265

@@ -2252,7 +2282,9 @@ void LevelCompactionBuilder::SetupInitialFiles() {
22522282
start_level_inputs_.clear();
22532283
} else {
22542284
assert(!start_level_inputs_.empty());
2255-
compaction_reason_ = CompactionReason::kBottommostFiles;
2285+
compaction_reason_ = CompactionPicker::ConvertCompactionReason(
2286+
start_level_inputs_.files.front()->marked_for_compaction,
2287+
CompactionReason::kBottommostFiles);
22562288
return;
22572289
}
22582290

‎db/compaction_picker.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ class CompactionPicker {
8080
bool skip_composite;
8181
};
8282

83+
static CompactionReason ConvertCompactionReason(
84+
uint8_t marked, CompactionReason default_reason);
85+
8386
static double GetQ(std::vector<double>::const_iterator b,
8487
std::vector<double>::const_iterator e, size_t g);
8588

‎db/compaction_picker_universal.cc

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1097,6 +1097,7 @@ Compaction* UniversalCompactionPicker::PickDeleteTriggeredCompaction(
10971097
CompactionInputFiles start_level_inputs;
10981098
int output_level;
10991099
std::vector<CompactionInputFiles> inputs;
1100+
uint8_t marked = 0;
11001101

11011102
if (vstorage->num_levels() == 1) {
11021103
// This is single level universal. Since we're basically trying to reclaim
@@ -1111,6 +1112,7 @@ Compaction* UniversalCompactionPicker::PickDeleteTriggeredCompaction(
11111112
for (FileMetaData* f : vstorage->LevelFiles(0)) {
11121113
if (f->marked_for_compaction && !f->being_compacted) {
11131114
compact = true;
1115+
marked = f->marked_for_compaction;
11141116
}
11151117
if (compact) {
11161118
start_level_inputs.files.push_back(f);
@@ -1132,6 +1134,7 @@ Compaction* UniversalCompactionPicker::PickDeleteTriggeredCompaction(
11321134
if (start_level_inputs.empty()) {
11331135
return nullptr;
11341136
}
1137+
marked = start_level_inputs.files.front()->marked_for_compaction;
11351138

11361139
// Pick the first non-empty level after the start_level
11371140
for (output_level = start_level + 1; output_level < vstorage->num_levels();
@@ -1216,7 +1219,8 @@ Compaction* UniversalCompactionPicker::PickDeleteTriggeredCompaction(
12161219
params.manual_compaction = true;
12171220
params.score = score;
12181221
params.compaction_type = compaction_type;
1219-
params.compaction_reason = CompactionReason::kFilesMarkedForCompaction;
1222+
params.compaction_reason =
1223+
ConvertCompactionReason(marked, CompactionReason::kUnknown);
12201224

12211225
return new Compaction(std::move(params));
12221226
}

‎db/db_impl.cc

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -945,15 +945,18 @@ void DBImpl::ScheduleTtlGC() {
945945
continue;
946946
}
947947
++total_count;
948-
old_mark_count += meta->marked_for_compaction;
948+
bool marked =
949+
!!(meta->marked_for_compaction & FileMetaData::kMarkedFromTTL);
950+
old_mark_count += marked;
949951
TEST_SYNC_POINT("DBImpl:Exist-SST");
950-
if (!meta->marked_for_compaction &&
952+
if (!marked &&
951953
should_marked_for_compacted(
952954
l, meta->fd.GetNumber(), meta->prop.earliest_time_begin_compact,
953955
meta->prop.latest_time_end_compact, now)) {
954-
meta->marked_for_compaction = true;
956+
meta->marked_for_compaction |= FileMetaData::kMarkedFromTTL;
957+
marked = true;
955958
}
956-
if (meta->marked_for_compaction) {
959+
if (marked) {
957960
new_mark_count++;
958961
TEST_SYNC_POINT("DBImpl:ScheduleTtlGC-mark");
959962
}
@@ -1104,13 +1107,15 @@ void DBImpl::ScheduleZNSGC() {
11041107
continue;
11051108
}
11061109
++total_count;
1107-
old_mark_count += meta->marked_for_compaction;
1110+
bool marked = !!(meta->marked_for_compaction &
1111+
FileMetaData::kMarkedFromFileSystem);
1112+
old_mark_count += marked;
11081113
TEST_SYNC_POINT("DBImpl:Exist-SST");
1109-
if (!meta->marked_for_compaction &&
1110-
mark_for_gc.count(meta->fd.GetNumber()) > 0) {
1111-
meta->marked_for_compaction = true;
1114+
if (!marked && mark_for_gc.count(meta->fd.GetNumber()) > 0) {
1115+
meta->marked_for_compaction |= FileMetaData::kMarkedFromFileSystem;
1116+
marked = true;
11121117
}
1113-
if (meta->marked_for_compaction) {
1118+
if (marked) {
11141119
new_mark_count++;
11151120
TEST_SYNC_POINT("DBImpl:ScheduleZNSGC-mark");
11161121
}

‎db/db_impl_experimental.cc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ Status DBImpl::SuggestCompactRange(ColumnFamilyHandle* column_family,
4747
level, begin == nullptr ? nullptr : &start_key,
4848
end == nullptr ? nullptr : &end_key, &inputs);
4949
for (auto f : inputs) {
50-
f->marked_for_compaction = true;
50+
f->marked_for_compaction |= FileMetaData::kMarkedFromUser;
5151
}
5252
}
5353
// Since we have some more files to compact, we should also recompute
@@ -68,7 +68,7 @@ Status DBImpl::SuggestCompactColumnFamily(ColumnFamilyHandle* column_family) {
6868
auto vstorage = cfd->current()->storage_info();
6969
for (int level = -1; level < vstorage->num_non_empty_levels(); ++level) {
7070
for (auto f : vstorage->LevelFiles(level)) {
71-
f->marked_for_compaction = true;
71+
f->marked_for_compaction |= FileMetaData::kMarkedFromUser;
7272
}
7373
}
7474
// Since we have some more files to compact, we should also recompute

‎db/db_test.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4871,7 +4871,7 @@ TEST_F(DBTest, SuggestCompactRangeTest) {
48714871
"Compaction::Compaction::Start", [&](void* arg) {
48724872
Compaction* c = (Compaction*)arg;
48734873
ASSERT_EQ(c->compaction_reason(),
4874-
CompactionReason::kFilesMarkedForCompaction);
4874+
CompactionReason::kFilesMarkedFromUser);
48754875
ASSERT_FALSE(c->is_manual_compaction());
48764876
});
48774877
// disable trival move make current test happy

‎db/map_builder.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ struct RangeWithDepend {
135135
include[0] = true;
136136
include[1] = true;
137137
has_delete_range = false;
138-
marked_for_compaction = f->marked_for_compaction;
138+
marked_for_compaction = !!f->marked_for_compaction;
139139
stable = false;
140140
dependence.emplace_back(MapSstElement::LinkTarget{f->fd.GetNumber(), 0});
141141
}

‎db/version_builder.cc

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,7 @@ class VersionBuilder::Rep {
391391
item.f = f;
392392
}
393393
item.f->gc_status = FileMetaData::kGarbageCollectionPermitted;
394+
item.f->marked_for_compaction &= FileMetaData::kMarkedFromUser;
394395
}
395396
break;
396397
case FileMetaData::kGarbageCollectionCandidate:
@@ -421,7 +422,8 @@ class VersionBuilder::Rep {
421422
old_file_queue.pop();
422423
}
423424
while (!old_file_queue.empty()) {
424-
dependence_map[old_file_queue.top()].f->marked_for_compaction = true;
425+
dependence_map[old_file_queue.top()].f->marked_for_compaction |=
426+
FileMetaData::kMarkedFromUpdateBlob;
425427
old_file_queue.pop();
426428
}
427429
}

‎db/version_edit.cc

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,28 @@
1717
#include "util/string_util.h"
1818
#include "util/sync_point.h"
1919

20+
namespace {
21+
uint8_t reverse_byte(uint8_t v) {
22+
// normal
23+
// return static_cast<uint8_t>(((v & (1 << 0)) << 7) | ((v & (1 << 1)) << 6) |
24+
// ((v & (1 << 2)) << 5) | ((v & (1 << 3)) << 4) |
25+
// ((v & (1 << 4)) << 3) | ((v & (1 << 5)) << 2) |
26+
// ((v & (1 << 6)) << 1) | ((v & (1 << 7)) << 0));
27+
28+
// hack
29+
// return ((v * 0x0802LU & 0x22110LU) | (v * 0x8020LU & 0x88440LU)) *
30+
// 0x10101LU >>
31+
// 16;
32+
33+
// lookup table
34+
constexpr static uint8_t lookup[16] = {
35+
0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe,
36+
0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf,
37+
};
38+
return (lookup[v & 0b1111] << 4) | lookup[v >> 4];
39+
}
40+
} // namespace
41+
2042
namespace TERARKDB_NAMESPACE {
2143

2244
// Tag numbers for serialized VersionEdit. These numbers are written to
@@ -174,7 +196,7 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
174196
}
175197
if (f.marked_for_compaction) {
176198
PutVarint32(dst, CustomTag::kNeedCompaction);
177-
char p = static_cast<char>(1);
199+
char p = static_cast<char>(reverse_byte(f.marked_for_compaction));
178200
PutLengthPrefixedSlice(dst, Slice(&p, 1));
179201
}
180202
if (has_min_log_number_to_keep_ && !min_log_num_written) {
@@ -305,7 +327,8 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
305327
if (field.size() != 1) {
306328
return "need_compaction field wrong size";
307329
}
308-
f.marked_for_compaction = (field[0] == 1);
330+
f.marked_for_compaction =
331+
reverse_byte(static_cast<uint8_t>(field[0]));
309332
break;
310333
case kMinLogNumberToKeepHack:
311334
// This is a hack to encode kMinLogNumberToKeep in a

0 commit comments

Comments
 (0)