Skip to content

Commit 8aafefa

Browse files
chejingebrother-jin
chejinge
and
brother-jin
authored
feat:new big key count (OpenAtomFoundation#3009)
Co-authored-by: chejinge <[email protected]>
1 parent 19ded5a commit 8aafefa

8 files changed

Lines changed: 130 additions & 0 deletions

File tree

‎src/pika_admin.cc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1406,6 +1406,7 @@ void InfoCmd::InfoData(std::string& info) {
14061406
uint64_t total_table_reader_usage = 0;
14071407
uint64_t memtable_usage = 0;
14081408
uint64_t table_reader_usage = 0;
1409+
uint64_t total_big_key_count = 0;
14091410
std::shared_lock db_rwl(g_pika_server->dbs_rw_);
14101411
for (const auto& db_item : g_pika_server->dbs_) {
14111412
if (!db_item.second) {
@@ -1417,6 +1418,9 @@ void InfoCmd::InfoData(std::string& info) {
14171418
db_item.second->storage()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_CUR_SIZE_ALL_MEM_TABLES, &memtable_usage);
14181419
db_item.second->storage()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_ESTIMATE_TABLE_READER_MEM, &table_reader_usage);
14191420
db_item.second->storage()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS, &background_errors);
1421+
1422+
uint64_t big_key_count = db_item.second->storage()->GetBigKeyStatistics(db_item.first, "bigkey_property");
1423+
total_big_key_count += big_key_count;
14201424
db_item.second->DBUnlockShared();
14211425
total_memtable_usage += memtable_usage;
14221426
total_table_reader_usage += table_reader_usage;
@@ -1436,6 +1440,7 @@ void InfoCmd::InfoData(std::string& info) {
14361440
tmp_stream << "db_tablereader_usage:" << total_table_reader_usage << "\r\n";
14371441
tmp_stream << "db_fatal:" << (total_background_errors != 0 ? "1" : "0") << "\r\n";
14381442
tmp_stream << "db_fatal_msg:" << (total_background_errors != 0 ? db_fatal_msg_stream.str() : "nullptr") << "\r\n";
1443+
tmp_stream << "big_key_count:" << total_big_key_count << "\r\n";
14391444

14401445
info.append(tmp_stream.str());
14411446
}

‎src/storage/include/storage/storage.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1100,6 +1100,7 @@ class Storage {
11001100
Status EnableAutoCompaction(const OptionType& option_type,
11011101
const std::string& db_type, const std::unordered_map<std::string, std::string>& options);
11021102
void GetRocksDBInfo(std::string& info);
1103+
uint64_t GetBigKeyStatistics(const std::string& db_type, const std::string& property);
11031104

11041105
private:
11051106
std::unique_ptr<RedisStrings> strings_db_;

‎src/storage/src/redis.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <memory>
1010
#include <string>
1111
#include <vector>
12+
#include <iostream>
1213

1314
#include "rocksdb/db.h"
1415
#include "rocksdb/slice.h"
@@ -115,6 +116,21 @@ class Redis {
115116
Status SetSmallCompactionDurationThreshold(uint64_t small_compaction_duration_threshold);
116117
std::vector<rocksdb::ColumnFamilyHandle*> GetHandles(){ return handles_;};
117118
void GetRocksDBInfo(std::string &info, const char *prefix);
119+
void CheckBigKeyAndLog(const std::string& key, uint64_t size) {
120+
static const uint64_t kBigKeyThreshold = 10000;
121+
if (size > kBigKeyThreshold) {
122+
std::lock_guard<std::mutex> lock(big_key_access_mutex_);
123+
big_key_access_count_[key]++;
124+
std::cerr << "[BIGKEY DETECTED] Key: " << key
125+
<< ", Size: " << size
126+
<< ", Access Count: " << big_key_access_count_[key] << std::endl;
127+
}
128+
}
129+
130+
std::unordered_map<std::string, int> GetBigKeyStatistics() {
131+
std::lock_guard<std::mutex> lock(big_key_access_mutex_);
132+
return big_key_access_count_;
133+
}
118134

119135
protected:
120136
Storage* const storage_;
@@ -141,6 +157,8 @@ class Redis {
141157
Status UpdateSpecificKeyStatistics(const std::string& key, uint64_t count);
142158
Status UpdateSpecificKeyDuration(const std::string& key, uint64_t duration);
143159
Status AddCompactKeyTaskIfNeeded(const std::string& key, uint64_t count, uint64_t duration);
160+
std::unordered_map<std::string, int> big_key_access_count_;
161+
std::mutex big_key_access_mutex_;
144162
};
145163

146164
} // namespace storage

‎src/storage/src/redis_hashes.cc‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ Status RedisHashes::HDel(const Slice& key, const std::vector<std::string>& field
214214
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
215215
if (s.ok()) {
216216
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
217+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
217218
if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.count() == 0) {
218219
*ret = 0;
219220
return Status::OK();
@@ -266,6 +267,7 @@ Status RedisHashes::HGet(const Slice& key, const Slice& field, std::string* valu
266267
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
267268
if (s.ok()) {
268269
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
270+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
269271
if (parsed_hashes_meta_value.IsStale()) {
270272
return Status::NotFound("Stale");
271273
} else if (parsed_hashes_meta_value.count() == 0) {
@@ -290,6 +292,7 @@ Status RedisHashes::HGetall(const Slice& key, std::vector<FieldValue>* fvs) {
290292
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
291293
if (s.ok()) {
292294
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
295+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
293296
if (parsed_hashes_meta_value.IsStale()) {
294297
return Status::NotFound("Stale");
295298
} else if (parsed_hashes_meta_value.count() == 0) {
@@ -321,6 +324,7 @@ Status RedisHashes::HGetallWithTTL(const Slice& key, std::vector<FieldValue>* fv
321324
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
322325
if (s.ok()) {
323326
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
327+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
324328
if (parsed_hashes_meta_value.count() == 0) {
325329
return Status::NotFound();
326330
} else if (parsed_hashes_meta_value.IsStale()) {
@@ -368,6 +372,7 @@ Status RedisHashes::HIncrby(const Slice& key, const Slice& field, int64_t value,
368372
char meta_value_buf[4] = {0};
369373
if (s.ok()) {
370374
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
375+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
371376
if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.count() == 0) {
372377
version = parsed_hashes_meta_value.UpdateVersion();
373378
parsed_hashes_meta_value.set_count(1);
@@ -443,6 +448,7 @@ Status RedisHashes::HIncrbyfloat(const Slice& key, const Slice& field, const Sli
443448
char meta_value_buf[4] = {0};
444449
if (s.ok()) {
445450
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
451+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
446452
if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.count() == 0) {
447453
version = parsed_hashes_meta_value.UpdateVersion();
448454
parsed_hashes_meta_value.set_count(1);
@@ -509,6 +515,7 @@ Status RedisHashes::HKeys(const Slice& key, std::vector<std::string>* fields) {
509515
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
510516
if (s.ok()) {
511517
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
518+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
512519
if (parsed_hashes_meta_value.IsStale()) {
513520
return Status::NotFound("Stale");
514521
} else if (parsed_hashes_meta_value.count() == 0) {
@@ -535,6 +542,7 @@ Status RedisHashes::HLen(const Slice& key, int32_t* ret) {
535542
Status s = db_->Get(default_read_options_, handles_[0], key, &meta_value);
536543
if (s.ok()) {
537544
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
545+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
538546
if (parsed_hashes_meta_value.IsStale()) {
539547
*ret = 0;
540548
return Status::NotFound("Stale");
@@ -563,6 +571,7 @@ Status RedisHashes::HMGet(const Slice& key, const std::vector<std::string>& fiel
563571
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
564572
if (s.ok()) {
565573
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
574+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
566575
if ((is_stale = parsed_hashes_meta_value.IsStale()) || parsed_hashes_meta_value.count() == 0) {
567576
for (size_t idx = 0; idx < fields.size(); ++idx) {
568577
vss->push_back({std::string(), Status::NotFound()});
@@ -613,6 +622,7 @@ Status RedisHashes::HMSet(const Slice& key, const std::vector<FieldValue>& fvs)
613622
char meta_value_buf[4] = {0};
614623
if (s.ok()) {
615624
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
625+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
616626
if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.count() == 0) {
617627
version = parsed_hashes_meta_value.InitialMetaValue();
618628
if (!parsed_hashes_meta_value.check_set_count(static_cast<int32_t>(filtered_fvs.size()))) {
@@ -673,6 +683,7 @@ Status RedisHashes::HSet(const Slice& key, const Slice& field, const Slice& valu
673683
char meta_value_buf[4] = {0};
674684
if (s.ok()) {
675685
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
686+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
676687
if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.count() == 0) {
677688
version = parsed_hashes_meta_value.InitialMetaValue();
678689
parsed_hashes_meta_value.set_count(1);
@@ -731,6 +742,7 @@ Status RedisHashes::HSetnx(const Slice& key, const Slice& field, const Slice& va
731742
char meta_value_buf[4] = {0};
732743
if (s.ok()) {
733744
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
745+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
734746
if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.count() == 0) {
735747
version = parsed_hashes_meta_value.InitialMetaValue();
736748
parsed_hashes_meta_value.set_count(1);
@@ -782,6 +794,7 @@ Status RedisHashes::HVals(const Slice& key, std::vector<std::string>* values) {
782794
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
783795
if (s.ok()) {
784796
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
797+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
785798
if (parsed_hashes_meta_value.IsStale()) {
786799
return Status::NotFound("Stale");
787800
} else if (parsed_hashes_meta_value.count() == 0) {
@@ -832,6 +845,7 @@ Status RedisHashes::HScan(const Slice& key, int64_t cursor, const std::string& p
832845
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
833846
if (s.ok()) {
834847
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
848+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
835849
if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.count() == 0) {
836850
*next_cursor = 0;
837851
return Status::NotFound();
@@ -896,6 +910,7 @@ Status RedisHashes::HScanx(const Slice& key, const std::string& start_field, con
896910
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
897911
if (s.ok()) {
898912
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
913+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
899914
if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.count() == 0) {
900915
*next_field = "";
901916
return Status::NotFound();
@@ -954,6 +969,7 @@ Status RedisHashes::PKHScanRange(const Slice& key, const Slice& field_start, con
954969
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
955970
if (s.ok()) {
956971
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
972+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
957973
if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.count() == 0) {
958974
return Status::NotFound();
959975
} else {
@@ -1013,6 +1029,7 @@ Status RedisHashes::PKHRScanRange(const Slice& key, const Slice& field_start, co
10131029
Status s = db_->Get(read_options, handles_[0], key, &meta_value);
10141030
if (s.ok()) {
10151031
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
1032+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
10161033
if (parsed_hashes_meta_value.IsStale() || parsed_hashes_meta_value.count() == 0) {
10171034
return Status::NotFound();
10181035
} else {
@@ -1163,6 +1180,7 @@ Status RedisHashes::Expire(const Slice& key, int32_t ttl) {
11631180
Status s = db_->Get(default_read_options_, handles_[0], key, &meta_value);
11641181
if (s.ok()) {
11651182
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
1183+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
11661184
if (parsed_hashes_meta_value.IsStale()) {
11671185
return Status::NotFound("Stale");
11681186
} else if (parsed_hashes_meta_value.count() == 0) {
@@ -1186,6 +1204,7 @@ Status RedisHashes::Del(const Slice& key) {
11861204
Status s = db_->Get(default_read_options_, handles_[0], key, &meta_value);
11871205
if (s.ok()) {
11881206
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
1207+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
11891208
if (parsed_hashes_meta_value.IsStale()) {
11901209
return Status::NotFound("Stale");
11911210
} else if (parsed_hashes_meta_value.count() == 0) {
@@ -1281,6 +1300,7 @@ Status RedisHashes::Expireat(const Slice& key, int32_t timestamp) {
12811300
Status s = db_->Get(default_read_options_, handles_[0], key, &meta_value);
12821301
if (s.ok()) {
12831302
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
1303+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
12841304
if (parsed_hashes_meta_value.IsStale()) {
12851305
return Status::NotFound("Stale");
12861306
} else if (parsed_hashes_meta_value.count() == 0) {
@@ -1303,6 +1323,7 @@ Status RedisHashes::Persist(const Slice& key) {
13031323
Status s = db_->Get(default_read_options_, handles_[0], key, &meta_value);
13041324
if (s.ok()) {
13051325
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
1326+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
13061327
if (parsed_hashes_meta_value.IsStale()) {
13071328
return Status::NotFound("Stale");
13081329
} else if (parsed_hashes_meta_value.count() == 0) {
@@ -1325,6 +1346,7 @@ Status RedisHashes::TTL(const Slice& key, int64_t* timestamp) {
13251346
Status s = db_->Get(default_read_options_, handles_[0], key, &meta_value);
13261347
if (s.ok()) {
13271348
ParsedHashesMetaValue parsed_hashes_meta_value(&meta_value);
1349+
CheckBigKeyAndLog(key.ToString(), parsed_hashes_meta_value.count());
13281350
if (parsed_hashes_meta_value.IsStale()) {
13291351
*timestamp = -2;
13301352
return Status::NotFound("Stale");

0 commit comments

Comments
 (0)