From 6ee6634bbe10af307b4f66eb775bfafbd52a62cb Mon Sep 17 00:00:00 2001 From: chenbt Date: Mon, 14 Jul 2025 11:38:09 +0800 Subject: [PATCH 1/3] fix: compact loop error --- .gitignore | 2 +- conf/pika.conf | 34 ++++++++++++++-------------------- include/pika_server.h | 1 + src/pika_server.cc | 19 +++++++++++++++---- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index ab567194a1..ea63c73681 100644 --- a/.gitignore +++ b/.gitignore @@ -62,7 +62,7 @@ src/build_version.cc #build -build/ +build*/ buildtrees deps pkg diff --git a/conf/pika.conf b/conf/pika.conf index 4f51f9cdbd..ba466faff5 100644 --- a/conf/pika.conf +++ b/conf/pika.conf @@ -681,30 +681,24 @@ wash-data: true # obd-compact https://github.com/OpenAtomFoundation/pika/issues/2255 compaction-strategy : obd-compact -# For OBD_Compact +################################## Configure For OBD_Compact ##################### # According to the number of sst files in rocksdb, # compact every `compact-every-num-of-files` file. compact-every-num-of-files : 10 -# For OBD_Compact -# In another search, if the file creation time is -# greater than `force-compact-file-age-seconds`, -# a compaction of the upper and lower boundaries -# of the file will be performed at the same time -# `compact-every-num-of-files` -1 -force-compact-file-age-seconds : 300 +# Extends the time a file must exist (20 minutes) before triggering forced compaction +force-compact-file-age-seconds : 1200 -# For OBD_Compact -# According to the number of sst files in rocksdb, -# compact every `compact-every-num-of-files` file. -force-compact-min-delete-ratio : 10 +# The forced deletion ratio threshold - compaction will be triggered when +# the deletion markers percentage in SST files reaches this value +force-compact-min-delete-ratio : 50 -# For OBD_Compact -# According to the number of sst files in rocksdb, -# compact every `compact-every-num-of-files` file. -dont-compact-sst-created-in-seconds : 20 +# Files created within the last 5 minutes will not be compacted +dont-compact-sst-created-in-seconds : 300 -# For OBD_Compact -# According to the number of sst files in rocksdb, -# compact every `compact-every-num-of-files` file. -best-delete-min-ratio : 10 \ No newline at end of file +# The optimal deletion ratio - defines the best threshold percentage +# for deletion markers in SST files to optimize compaction efficiency +best-delete-min-ratio : 30 + +# 定时触发事件间隔 +obd-compact-interval \ No newline at end of file diff --git a/include/pika_server.h b/include/pika_server.h index df75229188..0bc3fc4e67 100644 --- a/include/pika_server.h +++ b/include/pika_server.h @@ -550,6 +550,7 @@ class PikaServer : public pstd::noncopyable { */ bool have_scheduled_crontask_ = false; struct timeval last_check_compact_time_; + struct timeval last_strategy_compact_time_; /* * ResumeDB used diff --git a/src/pika_server.cc b/src/pika_server.cc index b205f3e34b..49c032dc59 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -46,6 +46,7 @@ PikaServer::PikaServer() : exit_(false), slow_cmd_thread_pool_flag_(g_pika_conf->slow_cmd_pool()), last_check_compact_time_({0, 0}), + last_strategy_compact_time_({0, 0}), last_check_resume_time_({0, 0}), repl_state_(PIKA_REPL_NO_CONNECT), role_(PIKA_ROLE_SINGLE) { @@ -1230,10 +1231,20 @@ void PikaServer::AutoCompactRange() { } } - if (g_pika_conf->compaction_strategy() == PikaConf::FullCompact) { - DoSameThingEveryDB(TaskType::kCompactAll); - } else if (g_pika_conf->compaction_strategy() == PikaConf::OldestOrBestDeleteRatioSstCompact) { - DoSameThingEveryDB(TaskType::kCompactOldestOrBestDeleteRatioSst); + + struct timeval strategy_now; + gettimeofday(&strategy_now, nullptr); + // 处理基于策略的压缩，增加时间间隔控制，间隔时间（2小时 = 7200秒） + if (last_strategy_compact_time_.tv_sec == 0 || + strategy_now.tv_sec - last_strategy_compact_time_.tv_sec >= 7200) { + + // 更新上次策略压缩时间 + gettimeofday(&last_strategy_compact_time_, nullptr); + + if (g_pika_conf->compaction_strategy() == PikaConf::OldestOrBestDeleteRatioSstCompact) { + LOG(INFO) << "[Strategy]schedule OldestOrBestDeleteRatioSst, interval: 2 hours"; + DoSameThingEveryDB(TaskType::kCompactOldestOrBestDeleteRatioSst); + } } } From 1630a3952fa48f2e1aaec289689547830d7bf541 Mon Sep 17 00:00:00 2001 From: chenbt Date: Mon, 14 Jul 2025 17:02:51 +0800 Subject: [PATCH 2/3] fix: compact loop error --- conf/pika.conf | 5 +++-- include/pika_conf.h | 10 ++++++++++ src/pika_admin.cc | 14 ++++++++++++++ src/pika_conf.cc | 10 +++++++++- src/pika_server.cc | 11 +++++++---- src/storage/include/storage/storage.h | 1 + 6 files changed, 44 insertions(+), 7 deletions(-) diff --git a/conf/pika.conf b/conf/pika.conf index ba466faff5..4a68be3544 100644 --- a/conf/pika.conf +++ b/conf/pika.conf @@ -700,5 +700,6 @@ dont-compact-sst-created-in-seconds : 300 # for deletion markers in SST files to optimize compaction efficiency best-delete-min-ratio : 30 -# 定时触发事件间隔 -obd-compact-interval \ No newline at end of file +# Trigger the obd-compact task periodically according to `obd-compact-interval` +# obd-compact-interval >= 600. +obd-compact-interval : 600 \ No newline at end of file diff --git a/include/pika_conf.h b/include/pika_conf.h index ecd42e0f91..1c1b6b4861 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -143,6 +143,10 @@ class PikaConf : public pstd::BaseConf { std::shared_lock l(rwlock_); return best_delete_min_ratio_; } + int obd_compact_interval () { + std::shared_lock l(rwlock_); + return obd_compact_interval_; + } CompactionStrategy compaction_strategy() { std::shared_lock l(rwlock_); return compaction_strategy_; @@ -984,6 +988,11 @@ class PikaConf : public pstd::BaseConf { ConfigRewrite(); } + void SetObdCompactInterval(const int value) { + std::unique_lock l(rwlock_); + obd_compact_interval_ = value < 600 ? 600 : value; + } + size_t GetUnfinishedFullSyncCount() { std::shared_lock l(rwlock_); return internal_used_unfinished_full_sync_.size(); @@ -1038,6 +1047,7 @@ class PikaConf : public pstd::BaseConf { int force_compact_min_delete_ratio_; int dont_compact_sst_created_in_seconds_; int best_delete_min_ratio_; + int obd_compact_interval_; CompactionStrategy compaction_strategy_; int64_t resume_check_interval_ = 60; // seconds diff --git a/src/pika_admin.cc b/src/pika_admin.cc index 3275e6cb7b..7e3dd548eb 100644 --- a/src/pika_admin.cc +++ b/src/pika_admin.cc @@ -2308,6 +2308,12 @@ void ConfigCmd::ConfigGet(std::string& ret) { EncodeNumber(&config_body, g_pika_conf->db_statistics_level()); } + if (pstd::stringmatch(pattern.data(), "obd-compact-interval", 1)) { + elements += 2; + EncodeString(&config_body, "obd-compact-interval"); + EncodeNumber(&config_body, g_pika_conf->obd_compact_interval()); + } + std::stringstream resp; resp << "*" << std::to_string(elements) << "\r\n" << config_body; ret = resp.str(); @@ -2370,6 +2376,7 @@ void ConfigCmd::ConfigSet(std::shared_ptr db) { "zset-cache-field-num-per-key", "cache-lfu-decay-time", "max-conn-rbuf-size", + "obd-compact-interval", }); res_.AppendStringVector(replyVt); return; @@ -3092,6 +3099,13 @@ void ConfigCmd::ConfigSet(std::shared_ptr db) { } g_pika_conf->SetMaxConnRbufSize(static_cast(ival)); res_.AppendStringRaw("+OK\r\n"); + } else if (set_item == "obd-compact-interval") { + if (pstd::string2int(value.data(), value.size(), &ival) == 0 || ival < 0) { + res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'obd-compact-interval'\r\n"); + return; + } + g_pika_conf->SetObdCompactInterval(ival); + res_.AppendStringRaw("+OK\r\n"); } else { res_.AppendStringRaw("-ERR Unsupported CONFIG parameter: " + set_item + "\r\n"); } diff --git a/src/pika_conf.cc b/src/pika_conf.cc index 0469a9d498..7d4ee9fa2c 100644 --- a/src/pika_conf.cc +++ b/src/pika_conf.cc @@ -334,6 +334,11 @@ int PikaConf::Load() { if (best_delete_min_ratio_ < 10) { best_delete_min_ratio_ = 10; } + + GetConfInt("obd-compact-interval", &obd_compact_interval_); + if (obd_compact_interval_ < 600) { + obd_compact_interval_ = 600; + } std::string cs_; GetConfStr("compaction-strategy", &cs_); @@ -882,7 +887,10 @@ int PikaConf::ConfigRewrite() { if (best_delete_min_ratio_ < 10) { best_delete_min_ratio_ = 10; } - + SetConfInt("obd-compact-interval", obd_compact_interval_); + if (obd_compact_interval_ < 600) { + obd_compact_interval_ = 600; + } std::string cs_; SetConfStr("compaction-strategy", cs_); if (cs_ == "full-compact") { diff --git a/src/pika_server.cc b/src/pika_server.cc index 49c032dc59..30835cbacf 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1234,15 +1234,17 @@ void PikaServer::AutoCompactRange() { struct timeval strategy_now; gettimeofday(&strategy_now, nullptr); - // 处理基于策略的压缩，增加时间间隔控制，间隔时间（2小时 = 7200秒） - if (last_strategy_compact_time_.tv_sec == 0 || - strategy_now.tv_sec - last_strategy_compact_time_.tv_sec >= 7200) { + if (last_strategy_compact_time_.tv_sec == 0){ + gettimeofday(&last_strategy_compact_time_, nullptr); + } + if (last_strategy_compact_time_.tv_sec > 0 && + strategy_now.tv_sec - last_strategy_compact_time_.tv_sec >= g_pika_conf->obd_compact_interval()) { // 更新上次策略压缩时间 gettimeofday(&last_strategy_compact_time_, nullptr); if (g_pika_conf->compaction_strategy() == PikaConf::OldestOrBestDeleteRatioSstCompact) { - LOG(INFO) << "[Strategy]schedule OldestOrBestDeleteRatioSst, interval: 2 hours"; + LOG(INFO) << "[Compaction]schedule OldestOrBestDeleteRatioSstCompact, interval: " << g_pika_conf->obd_compact_interval() << " seconds"; DoSameThingEveryDB(TaskType::kCompactOldestOrBestDeleteRatioSst); } } @@ -1511,6 +1513,7 @@ void PikaServer::InitStorageOptions() { // For Storage compaction storage_options_.compact_param_.best_delete_min_ratio_ = g_pika_conf->best_delete_min_ratio(); + storage_options_.compact_param_.obd_compact_interval_ = g_pika_conf->obd_compact_interval(); storage_options_.compact_param_.dont_compact_sst_created_in_seconds_ = g_pika_conf->dont_compact_sst_created_in_seconds(); storage_options_.compact_param_.force_compact_file_age_seconds_ = g_pika_conf->force_compact_file_age_seconds(); storage_options_.compact_param_.force_compact_min_delete_ratio_ = g_pika_conf->force_compact_min_delete_ratio(); diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index dd41b3ea94..dba4c4e06a 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -81,6 +81,7 @@ struct StorageOptions { int force_compact_min_delete_ratio_; int dont_compact_sst_created_in_seconds_; int best_delete_min_ratio_; + int obd_compact_interval_; }; CompactParam compact_param_; Status ResetOptions(const OptionType& option_type, const std::unordered_map& options_map); From 78e990cc7eeb3ac89dd3193945dca88ab749ff36 Mon Sep 17 00:00:00 2001 From: chenbt Date: Mon, 14 Jul 2025 17:16:04 +0800 Subject: [PATCH 3/3] fix: compact loop error --- .gitignore | 2 +- src/pika_server.cc | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index ea63c73681..ab567194a1 100644 --- a/.gitignore +++ b/.gitignore @@ -62,7 +62,7 @@ src/build_version.cc #build -build*/ +build/ buildtrees deps pkg diff --git a/src/pika_server.cc b/src/pika_server.cc index 30835cbacf..2a6efae563 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1239,10 +1239,7 @@ void PikaServer::AutoCompactRange() { } if (last_strategy_compact_time_.tv_sec > 0 && strategy_now.tv_sec - last_strategy_compact_time_.tv_sec >= g_pika_conf->obd_compact_interval()) { - - // 更新上次策略压缩时间 gettimeofday(&last_strategy_compact_time_, nullptr); - if (g_pika_conf->compaction_strategy() == PikaConf::OldestOrBestDeleteRatioSstCompact) { LOG(INFO) << "[Compaction]schedule OldestOrBestDeleteRatioSstCompact, interval: " << g_pika_conf->obd_compact_interval() << " seconds"; DoSameThingEveryDB(TaskType::kCompactOldestOrBestDeleteRatioSst);