From 0aa43fcb86748333639260f5e85a1626d25bdcc9 Mon Sep 17 00:00:00 2001 From: Mixficsol <838844609@qq.com> Date: Wed, 5 Mar 2025 14:26:31 +0800 Subject: [PATCH 01/15] cherry-pick 1 --- src/cache/include/cache.h | 2 +- src/cache/src/bit.cc | 3 +++ src/cache/src/hash.cc | 10 +++++++-- src/cache/src/list.cc | 7 ++++++- src/cache/src/set.cc | 3 +++ src/cache/src/string.cc | 3 +++ src/cache/src/zset.cc | 6 ++++++ src/pika_cache.cc | 28 ++++++-------------------- src/pika_cache_load_thread.cc | 9 +++++++++ src/pika_command.cc | 38 +++++++++++++++++------------------ src/pika_conf.cc | 2 +- src/pika_hash.cc | 3 ++- 12 files changed, 67 insertions(+), 47 deletions(-) diff --git a/src/cache/include/cache.h b/src/cache/include/cache.h index 20c93715f0..16fa131b6f 100644 --- a/src/cache/include/cache.h +++ b/src/cache/include/cache.h @@ -74,7 +74,7 @@ class RedisCache { // Hash Commands Status HDel(std::string& key, std::vector &fields); - Status HSet(std::string& key, std::string &field, std::string &value); + Status HSetIfKeyExist(std::string& key, std::string &field, std::string &value); Status HSetnx(std::string& key, std::string &field, std::string &value); Status HMSet(std::string& key, std::vector &fvs); Status HGet(std::string& key, std::string &field, std::string *value); diff --git a/src/cache/src/bit.cc b/src/cache/src/bit.cc index d8955875a2..d73974d676 100644 --- a/src/cache/src/bit.cc +++ b/src/cache/src/bit.cc @@ -15,6 +15,9 @@ Status RedisCache::SetBit(std::string& key, size_t offset, int64_t value) { return Status::Corruption("[error] Free memory faild !"); } + if (!Exists(key)) { + return Status::NotFound("key not exist"); + } // createObject is a function in redis, the init ref count of robj is 1 robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { diff --git a/src/cache/src/hash.cc b/src/cache/src/hash.cc index 19fc37092b..087754e156 100644 --- a/src/cache/src/hash.cc +++ b/src/cache/src/hash.cc @@ -31,12 +31,15 @@ Status RedisCache::HDel(std::string& key, std::vector &fields) { return Status::OK(); } -Status RedisCache::HSet(std::string& key, std::string &field, std::string &value) { +Status RedisCache::HSetIfKeyExist(std::string& key, std::string &field, std::string &value) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); } + if (!Exists(key)) { + return Status::NotFound("key not exist"); + } robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *fobj = createObject(OBJ_STRING, sdsnewlen(field.data(), field.size())); robj *vobj = createObject(OBJ_STRING, sdsnewlen(value.data(), value.size())); @@ -75,6 +78,9 @@ Status RedisCache::HMSet(std::string& key, std::vector &fvs return Status::Corruption("[error] Free memory faild !"); } + if (!Exists(key)) { + return Status::NotFound("key not exist"); + } robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); unsigned int items_size = fvs.size() * 2; robj **items = (robj **)zcallocate(sizeof(robj *) * items_size); @@ -141,7 +147,7 @@ Status RedisCache::HMGet(std::string& key, std::vector &fields, std if (C_OK == items[i].status) { vss->push_back({std::string(items[i].value, sdslen(items[i].value)), rocksdb::Status::OK()}); } else { - vss->push_back({std::string(), rocksdb::Status::NotFound()}); + return Status::NotFound("field not in cache"); } } diff --git a/src/cache/src/list.cc b/src/cache/src/list.cc index e063eed150..941f5c5313 100644 --- a/src/cache/src/list.cc +++ b/src/cache/src/list.cc @@ -99,6 +99,9 @@ Status RedisCache::LPush(std::string& key, std::vector &values) { return Status::Corruption("[error] Free memory faild !"); } + if (!Exists(key)) { + return Status::NotFound("key not exist"); + } robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj **vals = (robj **)zcallocate(sizeof(robj *) * values.size()); for (unsigned int i = 0; i < values.size(); ++i) { @@ -244,7 +247,9 @@ Status RedisCache::RPush(std::string& key, std::vector &values) { if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); } - + if (!Exists(key)) { + return Status::NotFound("key not exist"); + } robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj **vals = (robj **)zcallocate(sizeof(robj *) * values.size()); for (unsigned int i = 0; i < values.size(); ++i) { diff --git a/src/cache/src/set.cc b/src/cache/src/set.cc index 3c2fcd2598..ab4524d971 100644 --- a/src/cache/src/set.cc +++ b/src/cache/src/set.cc @@ -14,6 +14,9 @@ Status RedisCache::SAdd(std::string& key, std::vector &members) { return Status::Corruption("[error] Free memory faild !"); } + if (!Exists(key)) { + return Status::NotFound("key not exist"); + } robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj **vals = (robj **)zcallocate(sizeof(robj *) * members.size()); for (unsigned int i = 0; i < members.size(); ++i) { diff --git a/src/cache/src/string.cc b/src/cache/src/string.cc index 4195fe7b6e..af7323caf9 100644 --- a/src/cache/src/string.cc +++ b/src/cache/src/string.cc @@ -260,6 +260,9 @@ Status RedisCache::SetRange(std::string& key, int64_t start, std::string &value) return Status::Corruption("[error] Free memory faild !"); } + if (!Exists(key)) { + return Status::NotFound("key not exist"); + } uint64_t ret = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *vobj = createObject(OBJ_STRING, sdsnewlen(value.data(), value.size())); diff --git a/src/cache/src/zset.cc b/src/cache/src/zset.cc index 4e4de799fe..a06d999a5e 100644 --- a/src/cache/src/zset.cc +++ b/src/cache/src/zset.cc @@ -14,6 +14,9 @@ Status RedisCache::ZAdd(std::string& key, std::vector &sco return Status::Corruption("[error] Free memory faild !"); } + if (!Exists(key)) { + return Status::NotFound("key not exist"); + } robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); unsigned int items_size = score_members.size() * 2; robj **items = (robj **)zcallocate(sizeof(robj *) * items_size); @@ -73,6 +76,9 @@ Status RedisCache::ZIncrby(std::string& key, std::string& member, double increme return Status::Corruption("[error] Free memory faild !"); } + if (!Exists(key)) { + return Status::NotFound("key not exist"); + } robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj **items = (robj **)zcallocate(sizeof(robj *) * 2); items[0] = createStringObjectFromLongDouble(increment, 0); diff --git a/src/pika_cache.cc b/src/pika_cache.cc index 03badab177..4be4ee3ec6 100644 --- a/src/pika_cache.cc +++ b/src/pika_cache.cc @@ -373,10 +373,7 @@ Status PikaCache::GetRange(std::string& key, int64_t start, int64_t end, std::st Status PikaCache::SetRangexx(std::string& key, int64_t start, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); - if (caches_[cache_index]->Exists(key)) { - return caches_[cache_index]->SetRange(key, start, value); - } - return Status::NotFound("key not exist"); + return caches_[cache_index]->SetRange(key, start, value); } Status PikaCache::Strlen(std::string& key, int32_t *len) { @@ -397,16 +394,13 @@ Status PikaCache::HDel(std::string& key, std::vector &fields) { Status PikaCache::HSet(std::string& key, std::string &field, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); - return caches_[cache_index]->HSet(key, field, value); + return caches_[cache_index]->HSetIfKeyExist(key, field, value); } Status PikaCache::HSetIfKeyExist(std::string& key, std::string &field, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); - if (caches_[cache_index]->Exists(key)) { - return caches_[cache_index]->HSet(key, field, value); - } - return Status::NotFound("key not exist"); + return caches_[cache_index]->HSetIfKeyExist(key, field, value); } Status PikaCache::HSetIfKeyExistAndFieldNotExist(std::string& key, std::string &field, std::string &value) { @@ -450,11 +444,7 @@ Status PikaCache::HMSetnxWithoutTTL(std::string& key, std::vector &fvs) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); - if (caches_[cache_index]->Exists(key)) { - return caches_[cache_index]->HMSet(key, fvs); - } else { - return Status::NotFound("key not exist"); - } + return caches_[cache_index]->HMSet(key, fvs); } Status PikaCache::HGet(std::string& key, std::string &field, std::string *value) { @@ -641,10 +631,7 @@ Status PikaCache::SAdd(std::string& key, std::vector &members) { Status PikaCache::SAddIfKeyExist(std::string& key, std::vector &members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); - if (caches_[cache_index]->Exists(key)) { - return caches_[cache_index]->SAdd(key, members); - } - return Status::NotFound("key not exist"); + return caches_[cache_index]->SAdd(key, members); } Status PikaCache::SAddnx(std::string& key, std::vector &members, int64_t ttl) { @@ -1508,10 +1495,7 @@ Status PikaCache::SetBit(std::string& key, size_t offset, int64_t value) { Status PikaCache::SetBitIfKeyExist(std::string& key, size_t offset, int64_t value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); - if (caches_[cache_index]->Exists(key)) { - return caches_[cache_index]->SetBit(key, offset, value); - } - return Status::NotFound("key not exist"); + return caches_[cache_index]->SetBit(key, offset, value); } Status PikaCache::GetBit(std::string& key, size_t offset, int64_t *value) { diff --git a/src/pika_cache_load_thread.cc b/src/pika_cache_load_thread.cc index f9bb040a40..e815f760d0 100644 --- a/src/pika_cache_load_thread.cc +++ b/src/pika_cache_load_thread.cc @@ -68,6 +68,8 @@ bool PikaCacheLoadThread::LoadKV(std::string& key, const std::shared_ptr& db bool PikaCacheLoadThread::LoadHash(std::string& key, const std::shared_ptr& db) { int32_t len = 0; db->storage()->HLen(key, &len); + // If the Hash type contains more than 2048 data members, + // it will not be updated to RedisCache if (0 >= len || CACHE_VALUE_ITEM_MAX_SIZE < len) { return false; } @@ -86,6 +88,8 @@ bool PikaCacheLoadThread::LoadHash(std::string& key, const std::shared_ptr& bool PikaCacheLoadThread::LoadList(std::string& key, const std::shared_ptr& db) { uint64_t len = 0; db->storage()->LLen(key, &len); + // If the List type contains more than 2048 data members, + // it will not be updated to RedisCache if (len <= 0 || CACHE_VALUE_ITEM_MAX_SIZE < len) { LOG(WARNING) << "can not load key, because item size:" << len << " beyond max item size:" << CACHE_VALUE_ITEM_MAX_SIZE; @@ -106,6 +110,8 @@ bool PikaCacheLoadThread::LoadList(std::string& key, const std::shared_ptr& bool PikaCacheLoadThread::LoadSet(std::string& key, const std::shared_ptr& db) { int32_t len = 0; db->storage()->SCard(key, &len); + // If the Set type contains more than 2048 data members, + // it will not be updated to RedisCache if (0 >= len || CACHE_VALUE_ITEM_MAX_SIZE < len) { LOG(WARNING) << "can not load key, because item size:" << len << " beyond max item size:" << CACHE_VALUE_ITEM_MAX_SIZE; @@ -137,6 +143,9 @@ bool PikaCacheLoadThread::LoadZset(std::string& key, const std::shared_ptr& if (cache_len != 0) { return true; } + // Only 512 members will be cached (in the default configuration), + // and the first or last 512 elements will be cached depending on + // whether the zset-cache-start-direction is 0 or 1 if (zset_cache_start_direction_ == cache::CACHE_START_FROM_BEGIN) { if (zset_cache_field_num_per_key_ <= len) { stop_index = zset_cache_field_num_per_key_ - 1; diff --git a/src/pika_command.cc b/src/pika_command.cc index 7e2c9ba83c..7d8443001d 100644 --- a/src/pika_command.cc +++ b/src/pika_command.cc @@ -152,7 +152,7 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameDisableWal, std::move(disablewalptr))); std::unique_ptr cacheptr = std::make_unique(kCmdNameCache, -2, kCmdFlagsAdmin | kCmdFlagsRead); cmd_table->insert(std::pair>(kCmdNameCache, std::move(cacheptr))); - std::unique_ptr clearcacheptr = std::make_unique(kCmdNameClearCache, 1, kCmdFlagsAdmin | kCmdFlagsWrite); + std::unique_ptr clearcacheptr = std::make_unique(kCmdNameClearCache, 1, kCmdFlagsAdmin | kCmdFlagsRead); cmd_table->insert(std::pair>(kCmdNameClearCache, std::move(clearcacheptr))); std::unique_ptr lastsaveptr = std::make_unique(kCmdNameLastSave, 1, kCmdFlagsAdmin | kCmdFlagsRead | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameLastSave, std::move(lastsaveptr))); @@ -412,11 +412,11 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameHIncrbyfloat, std::move(hincrbyfloatptr))); ////HKeysCmd std::unique_ptr hkeysptr = - std::make_unique(kCmdNameHKeys, 2, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameHKeys, 2, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameHKeys, std::move(hkeysptr))); ////HLenCmd std::unique_ptr hlenptr = - std::make_unique(kCmdNameHLen, 2, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameHLen, 2, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameHLen, std::move(hlenptr))); ////HMgetCmd std::unique_ptr hmgetptr = @@ -432,7 +432,7 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameHSetnx, std::move(hsetnxptr))); ////HStrlenCmd std::unique_ptr hstrlenptr = - std::make_unique(kCmdNameHStrlen, 3, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameHStrlen, 3, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameHStrlen, std::move(hstrlenptr))); ////HValsCmd std::unique_ptr hvalsptr = @@ -457,14 +457,14 @@ void InitCmdTable(CmdTable* cmd_table) { // List std::unique_ptr lindexptr = - std::make_unique(kCmdNameLIndex, 3, kCmdFlagsRead | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameLIndex, 3, kCmdFlagsRead | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameLIndex, std::move(lindexptr))); std::unique_ptr linsertptr = std::make_unique(kCmdNameLInsert, 5, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameLInsert, std::move(linsertptr))); std::unique_ptr llenptr = - std::make_unique(kCmdNameLLen, 2, kCmdFlagsRead | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameLLen, 2, kCmdFlagsRead | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameLLen, std::move(llenptr))); std::unique_ptr blpopptr = std::make_unique( kCmdNameBLPop, -3, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsSlow); @@ -482,7 +482,7 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameLPushx, std::move(lpushxptr))); std::unique_ptr lrangeptr = std::make_unique( - kCmdNameLRange, 4, kCmdFlagsRead | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + kCmdNameLRange, 4, kCmdFlagsRead | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameLRange, std::move(lrangeptr))); std::unique_ptr lremptr = std::make_unique(kCmdNameLRem, 4, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); @@ -517,7 +517,7 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameZAdd, std::move(zaddptr))); ////ZCardCmd std::unique_ptr zcardptr = - std::make_unique(kCmdNameZCard, 2, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameZCard, 2, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameZCard, std::move(zcardptr))); ////ZScanCmd std::unique_ptr zscanptr = std::make_unique( @@ -529,11 +529,11 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameZIncrby, std::move(zincrbyptr))); ////ZRangeCmd std::unique_ptr zrangeptr = - std::make_unique(kCmdNameZRange, -4, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameZRange, -4, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameZRange, std::move(zrangeptr))); ////ZRevrangeCmd std::unique_ptr zrevrangeptr = - std::make_unique(kCmdNameZRevrange, -4, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameZRevrange, -4, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameZRevrange, std::move(zrevrangeptr))); ////ZRangebyscoreCmd std::unique_ptr zrangebyscoreptr = std::make_unique( @@ -546,7 +546,7 @@ void InitCmdTable(CmdTable* cmd_table) { std::pair>(kCmdNameZRevrangebyscore, std::move(zrevrangebyscoreptr))); ////ZCountCmd std::unique_ptr zcountptr = - std::make_unique(kCmdNameZCount, 4, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameZCount, 4, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameZCount, std::move(zcountptr))); ////ZRemCmd std::unique_ptr zremptr = @@ -562,15 +562,15 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameZInterstore, std::move(zinterstoreptr))); ////ZRankCmd std::unique_ptr zrankptr = - std::make_unique(kCmdNameZRank, 3, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameZRank, 3, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameZRank, std::move(zrankptr))); ////ZRevrankCmd std::unique_ptr zrevrankptr = - std::make_unique(kCmdNameZRevrank, 3, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameZRevrank, 3, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameZRevrank, std::move(zrevrankptr))); ////ZScoreCmd std::unique_ptr zscoreptr = - std::make_unique(kCmdNameZScore, 3, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameZScore, 3, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameZScore, std::move(zscoreptr))); ////ZRangebylexCmd std::unique_ptr zrangebylexptr = @@ -618,11 +618,11 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameSPop, std::move(spopptr))); ////SCardCmd std::unique_ptr scardptr = - std::make_unique(kCmdNameSCard, 2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameSCard, 2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameSCard, std::move(scardptr))); ////SMembersCmd std::unique_ptr smembersptr = - std::make_unique(kCmdNameSMembers, 2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameSMembers, 2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow | kCmdFlagsReadCache ); cmd_table->insert(std::pair>(kCmdNameSMembers, std::move(smembersptr))); ////SScanCmd std::unique_ptr sscanptr = @@ -650,7 +650,7 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameSInterstore, std::move(sinterstoreptr))); ////SIsmemberCmd std::unique_ptr sismemberptr = - std::make_unique(kCmdNameSIsmember, 3, kCmdFlagsRead | kCmdFlagsSet |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameSIsmember, 3, kCmdFlagsRead | kCmdFlagsSet |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameSIsmember, std::move(sismemberptr))); ////SDiffCmd std::unique_ptr sdiffptr = @@ -666,7 +666,7 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameSMove, std::move(smoveptr))); ////SRandmemberCmd std::unique_ptr srandmemberptr = - std::make_unique(kCmdNameSRandmember, -2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameSRandmember, -2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameSRandmember, std::move(srandmemberptr))); // BitMap @@ -680,7 +680,7 @@ void InitCmdTable(CmdTable* cmd_table) { cmd_table->insert(std::pair>(kCmdNameBitGet, std::move(bitgetptr))); ////bitcountCmd std::unique_ptr bitcountptr = - std::make_unique(kCmdNameBitCount, -2, kCmdFlagsRead | kCmdFlagsBit | kCmdFlagsSlow | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache); + std::make_unique(kCmdNameBitCount, -2, kCmdFlagsRead | kCmdFlagsBit | kCmdFlagsSlow | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsReadCache); cmd_table->insert(std::pair>(kCmdNameBitCount, std::move(bitcountptr))); ////bitposCmd std::unique_ptr bitposptr = diff --git a/src/pika_conf.cc b/src/pika_conf.cc index 9cbbdcadc4..6339d7077c 100644 --- a/src/pika_conf.cc +++ b/src/pika_conf.cc @@ -692,7 +692,7 @@ int PikaConf::Load() { int64_t cache_maxmemory = PIKA_CACHE_SIZE_DEFAULT; GetConfInt64("cache-maxmemory", &cache_maxmemory); - cache_maxmemory_ = (PIKA_CACHE_SIZE_MIN > cache_maxmemory) ? PIKA_CACHE_SIZE_DEFAULT : cache_maxmemory; + cache_maxmemory_ = (PIKA_CACHE_SIZE_MIN > cache_maxmemory) ? PIKA_CACHE_SIZE_MIN : cache_maxmemory; int cache_maxmemory_policy = 1; GetConfInt("cache-maxmemory-policy", &cache_maxmemory_policy); diff --git a/src/pika_hash.cc b/src/pika_hash.cc index d1b7498bcb..948b223071 100644 --- a/src/pika_hash.cc +++ b/src/pika_hash.cc @@ -560,7 +560,8 @@ void HSetnxCmd::DoThroughDB() { void HSetnxCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->HSetIfKeyExistAndFieldNotExist(key_, field_, value_); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + db_->cache()->HSetIfKeyExist(CachePrefixKeyH, field_, value_); } } From 2898a92a8045259d90cb69d3bd81981a7425c48a Mon Sep 17 00:00:00 2001 From: Mixficsol <838844609@qq.com> Date: Thu, 6 Mar 2025 11:30:00 +0800 Subject: [PATCH 02/15] cherry-pick 2 --- include/pika_cache.h | 6 ++++- src/cache/include/cache.h | 12 ++++++++-- src/cache/src/bit.cc | 19 +++++++++++++++ src/cache/src/hash.cc | 49 ++++++++++++++++++++++++++++++++++++-- src/cache/src/list.cc | 50 +++++++++++++++++++++++++++++++++++++-- src/cache/src/set.cc | 28 ++++++++++++++++++++-- src/cache/src/string.cc | 21 +++++++++++++++- src/cache/src/zset.cc | 30 +++++++++++++++++++++-- src/pika_cache.cc | 45 +++++++++++++++++++++++++---------- src/pika_hash.cc | 9 +++---- src/pika_kv.cc | 3 ++- src/pika_list.cc | 18 +++++++++----- 12 files changed, 255 insertions(+), 35 deletions(-) diff --git a/include/pika_cache.h b/include/pika_cache.h index 3485ecd3a4..bbac769a04 100644 --- a/include/pika_cache.h +++ b/include/pika_cache.h @@ -91,6 +91,7 @@ class PikaCache : public pstd::noncopyable, public std::enable_shared_from_this< rocksdb::Status Appendxx(std::string& key, std::string& value); rocksdb::Status GetRange(std::string& key, int64_t start, int64_t end, std::string* value); rocksdb::Status SetRangexx(std::string& key, int64_t start, std::string& value); + rocksdb::Status SetRangeIfKeyExist(std::string& key, int64_t start, std::string &value); rocksdb::Status Strlen(std::string& key, int32_t* len); // Hash Commands @@ -112,6 +113,7 @@ class PikaCache : public pstd::noncopyable, public std::enable_shared_from_this< rocksdb::Status HIncrbyfloatxx(std::string& key, std::string& field, long double value); rocksdb::Status HLen(std::string& key, uint64_t* len); rocksdb::Status HStrlen(std::string& key, std::string& field, uint64_t* len); + rocksdb::Status HMSetIfKeyExist(std::string& key, std::vector &fvs); // List Commands rocksdb::Status LIndex(std::string& key, int64_t index, std::string* element); @@ -126,10 +128,12 @@ class PikaCache : public pstd::noncopyable, public std::enable_shared_from_this< rocksdb::Status LTrim(std::string& key, int64_t start, int64_t stop); rocksdb::Status RPop(std::string& key, std::string* element); rocksdb::Status RPush(std::string& key, std::vector &values); + rocksdb::Status RPushIfKeyExist(std::string& key, std::vector &values); rocksdb::Status RPushx(std::string& key, std::vector &values); rocksdb::Status RPushnx(std::string& key, std::vector &values, int64_t ttl); rocksdb::Status RPushnxWithoutTTL(std::string& key, std::vector &values); - + rocksdb::Status LPushIfKeyExist(std::string& key, std::vector &values); + // Set Commands rocksdb::Status SAdd(std::string& key, std::vector& members); rocksdb::Status SAddIfKeyExist(std::string& key, std::vector& members); diff --git a/src/cache/include/cache.h b/src/cache/include/cache.h index 16fa131b6f..070504c4a7 100644 --- a/src/cache/include/cache.h +++ b/src/cache/include/cache.h @@ -70,13 +70,16 @@ class RedisCache { Status Append(std::string& key, std::string &value); Status GetRange(std::string& key, int64_t start, int64_t end, std::string *value); Status SetRange(std::string& key, int64_t start, std::string &value); + Status SetRangeIfKeyExist(std::string& key, int64_t start, std::string &value); Status Strlen(std::string& key, int32_t *len); // Hash Commands Status HDel(std::string& key, std::vector &fields); Status HSetIfKeyExist(std::string& key, std::string &field, std::string &value); Status HSetnx(std::string& key, std::string &field, std::string &value); + Status HSetnxIfKeyExist(std::string& key, std::string &field, std::string &value); Status HMSet(std::string& key, std::vector &fvs); + Status HMSetIfKeyExist(std::string& key, std::vector &fvs); Status HGet(std::string& key, std::string &field, std::string *value); Status HMGet(std::string& key, std::vector &fields, @@ -96,6 +99,7 @@ class RedisCache { std::string &pivot, std::string &value); Status LLen(const std::string& key, uint64_t *len); Status LPop(std::string& key, std::string *element); + Status LPushIfKeyExist(std::string& key, std::vector &values); Status LPush(std::string& key, std::vector &values); Status LPushx(std::string& key, std::vector &values); Status LRange(std::string& key, int64_t start, int64_t stop, std::vector *values); @@ -104,11 +108,13 @@ class RedisCache { Status LTrim(std::string& key, int64_t start, int64_t stop); Status RPop(std::string& key, std::string *element); Status RPush(std::string& key, std::vector &values); + Status RPushIfKeyExist(std::string& key, std::vector &values); Status RPushx(std::string& key, std::vector &values); // Set Commands Status SAdd(std::string& key, std::vector &members); - Status SCard(const std::string& key, uint64_t *len); + Status SAddIfKeyExist(std::string& key, std::vector &members); + Status SCard(std::string& key, uint64_t *len); Status SIsmember(std::string& key, std::string& member); Status SMembers(std::string& key, std::vector *members); Status SRem(std::string& key, std::vector &members); @@ -116,7 +122,8 @@ class RedisCache { // Zset Commands Status ZAdd(std::string& key, std::vector &score_members); - Status ZCard(const std::string& key, uint64_t *len); + Status ZAddIfKeyExist(std::string& key, std::vector &score_members); + Status ZCard(std::string& key, uint64_t *len); Status ZCount(std::string& key, std::string &min, std::string &max, uint64_t *len); Status ZIncrby(std::string& key, std::string& member, double increment); Status ZRange(std::string& key, @@ -151,6 +158,7 @@ class RedisCache { Status ZPopMax(std::string& key, int64_t count, std::vector* score_members); // Bit Commands + Status SetBitIfKeyExist(std::string& key, size_t offset, int64_t value); Status SetBit(std::string& key, size_t offset, int64_t value); Status GetBit(std::string& key, size_t offset, int64_t *value); Status BitCount(std::string& key, int64_t start, int64_t end, int64_t *value, bool have_offset); diff --git a/src/cache/src/bit.cc b/src/cache/src/bit.cc index d73974d676..576461b2ca 100644 --- a/src/cache/src/bit.cc +++ b/src/cache/src/bit.cc @@ -15,6 +15,25 @@ Status RedisCache::SetBit(std::string& key, size_t offset, int64_t value) { return Status::Corruption("[error] Free memory faild !"); } + // createObject is a function in redis, the init ref count of robj is 1 + robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); + DEFER { + DecrObjectsRefCount(kobj); + }; + int ret = RcSetBit(cache_, kobj, offset, value); + if (C_OK != ret) { + return Status::Corruption("RcSetBit failed"); + } + + return Status::OK(); +} + +Status RedisCache::SetBitIfKeyExist(std::string& key, size_t offset, int64_t value) { + int res = RcFreeMemoryIfNeeded(cache_); + if (C_OK != res) { + return Status::Corruption("[error] Free memory faild !"); + } + if (!Exists(key)) { return Status::NotFound("key not exist"); } diff --git a/src/cache/src/hash.cc b/src/cache/src/hash.cc index 087754e156..8974dcb723 100644 --- a/src/cache/src/hash.cc +++ b/src/cache/src/hash.cc @@ -54,6 +54,27 @@ Status RedisCache::HSetIfKeyExist(std::string& key, std::string &field, std::str return Status::OK(); } +Status RedisCache::HSetnxIfKeyExist(std::string& key, std::string &field, std::string &value) { + if (C_OK != RcFreeMemoryIfNeeded(cache_)) { + return Status::Corruption("[error] Free memory faild !"); + } + + if (!Exists(key)) { + return Status::NotFound("key not exist"); + } + robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); + robj *fobj = createObject(OBJ_STRING, sdsnewlen(field.data(), field.size())); + robj *vobj = createObject(OBJ_STRING, sdsnewlen(value.data(), value.size())); + DEFER { + DecrObjectsRefCount(kobj, fobj, vobj); + }; + if (C_OK != RcHSetnx(cache_, kobj, fobj, vobj)) { + return Status::Corruption("RcHSetnx failed"); + } + + return Status::OK(); +} + Status RedisCache::HSetnx(std::string& key, std::string &field, std::string &value) { if (C_OK != RcFreeMemoryIfNeeded(cache_)) { return Status::Corruption("[error] Free memory faild !"); @@ -72,11 +93,11 @@ Status RedisCache::HSetnx(std::string& key, std::string &field, std::string &val return Status::OK(); } -Status RedisCache::HMSet(std::string& key, std::vector &fvs) { +Status RedisCache::HMSetIfKeyExist(std::string& key, std::vector &fvs) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); - } + } if (!Exists(key)) { return Status::NotFound("key not exist"); @@ -99,6 +120,30 @@ Status RedisCache::HMSet(std::string& key, std::vector &fvs return Status::OK(); } +Status RedisCache::HMSet(std::string& key, std::vector &fvs) { + int res = RcFreeMemoryIfNeeded(cache_); + if (C_OK != res) { + return Status::Corruption("[error] Free memory faild !"); + } + + robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); + unsigned int items_size = fvs.size() * 2; + robj **items = (robj **)zcallocate(sizeof(robj *) * items_size); + for (unsigned int i = 0; i < fvs.size(); ++i) { + items[i * 2] = createObject(OBJ_STRING, sdsnewlen(fvs[i].field.data(), fvs[i].field.size())); + items[i * 2 + 1] = createObject(OBJ_STRING, sdsnewlen(fvs[i].value.data(), fvs[i].value.size())); + } + DEFER { + FreeObjectList(items, items_size); + DecrObjectsRefCount(kobj); + }; + int ret = RcHMSet(cache_, kobj, items, items_size); + if (C_OK != ret) { + return Status::Corruption("RcHMSet failed"); + } + return Status::OK(); +} + Status RedisCache::HGet(std::string& key, std::string &field, std::string *value) { sds val; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); diff --git a/src/cache/src/list.cc b/src/cache/src/list.cc index 941f5c5313..766062f94b 100644 --- a/src/cache/src/list.cc +++ b/src/cache/src/list.cc @@ -93,7 +93,7 @@ Status RedisCache::LPop(std::string& key, std::string *element) { return Status::OK(); } -Status RedisCache::LPush(std::string& key, std::vector &values) { +Status RedisCache::LPushIfKeyExist(std::string& key, std::vector &values) { int ret = RcFreeMemoryIfNeeded(cache_); if (C_OK != ret) { return Status::Corruption("[error] Free memory faild !"); @@ -119,6 +119,29 @@ Status RedisCache::LPush(std::string& key, std::vector &values) { return Status::OK(); } +Status RedisCache::LPush(std::string& key, std::vector &values) { + int ret = RcFreeMemoryIfNeeded(cache_); + if (C_OK != ret) { + return Status::Corruption("[error] Free memory faild !"); + } + + robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); + robj **vals = (robj **)zcallocate(sizeof(robj *) * values.size()); + for (unsigned int i = 0; i < values.size(); ++i) { + vals[i] = createObject(OBJ_STRING, sdsnewlen(values[i].data(), values[i].size())); + } + DEFER { + FreeObjectList(vals, values.size()); + DecrObjectsRefCount(kobj); + }; + int res = RcLPush(cache_, kobj, vals, values.size()); + if (C_OK != res) { + return Status::Corruption("RcLPush failed"); + } + + return Status::OK(); +} + Status RedisCache::LPushx(std::string& key, std::vector &values) { int ret = RcFreeMemoryIfNeeded(cache_); if (C_OK != ret) { @@ -242,7 +265,7 @@ Status RedisCache::RPop(std::string& key, std::string *element) { return Status::OK(); } -Status RedisCache::RPush(std::string& key, std::vector &values) { +Status RedisCache::RPushIfKeyExist(std::string& key, std::vector &values) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); @@ -267,6 +290,29 @@ Status RedisCache::RPush(std::string& key, std::vector &values) { return Status::OK(); } +Status RedisCache::RPush(std::string& key, std::vector &values) { + int res = RcFreeMemoryIfNeeded(cache_); + if (C_OK != res) { + return Status::Corruption("[error] Free memory faild !"); + } + + robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); + robj **vals = (robj **)zcallocate(sizeof(robj *) * values.size()); + for (unsigned int i = 0; i < values.size(); ++i) { + vals[i] = createObject(OBJ_STRING, sdsnewlen(values[i].data(), values[i].size())); + } + DEFER { + FreeObjectList(vals, values.size()); + DecrObjectsRefCount(kobj); + }; + int ret = RcRPush(cache_, kobj, vals, values.size()); + if (C_OK != ret) { + return Status::Corruption("RcRPush failed"); + } + + return Status::OK(); +} + Status RedisCache::RPushx(std::string& key, std::vector &values) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { diff --git a/src/cache/src/set.cc b/src/cache/src/set.cc index ab4524d971..1c0d0ff30e 100644 --- a/src/cache/src/set.cc +++ b/src/cache/src/set.cc @@ -8,7 +8,8 @@ namespace cache { -Status RedisCache::SAdd(std::string& key, std::vector &members) { + +Status RedisCache::SAddIfKeyExist(std::string& key, std::vector &members) { int ret = RcFreeMemoryIfNeeded(cache_); if (C_OK != ret) { return Status::Corruption("[error] Free memory faild !"); @@ -34,7 +35,30 @@ Status RedisCache::SAdd(std::string& key, std::vector &members) { return Status::OK(); } -Status RedisCache::SCard(const std::string& key, uint64_t *len) { +Status RedisCache::SAdd(std::string& key, std::vector &members) { + int ret = RcFreeMemoryIfNeeded(cache_); + if (C_OK != ret) { + return Status::Corruption("[error] Free memory faild !"); + } + + robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); + robj **vals = (robj **)zcallocate(sizeof(robj *) * members.size()); + for (unsigned int i = 0; i < members.size(); ++i) { + vals[i] = createObject(OBJ_STRING, sdsnewlen(members[i].data(), members[i].size())); + } + DEFER { + FreeObjectList(vals, members.size()); + DecrObjectsRefCount(kobj); + }; + int res = RcSAdd(cache_, kobj, vals, members.size()); + if (C_OK != res) { + return Status::Corruption("RcSAdd failed"); + } + + return Status::OK(); +} + +Status RedisCache::SCard(std::string& key, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); diff --git a/src/cache/src/string.cc b/src/cache/src/string.cc index af7323caf9..5015b1f86e 100644 --- a/src/cache/src/string.cc +++ b/src/cache/src/string.cc @@ -255,7 +255,7 @@ Status RedisCache::GetRange(std::string& key, int64_t start, int64_t end, std::s return Status::OK(); } -Status RedisCache::SetRange(std::string& key, int64_t start, std::string &value) { +Status RedisCache::SetRangeIfKeyExist(std::string& key, int64_t start, std::string &value) { if (C_OK != RcFreeMemoryIfNeeded(cache_)) { return Status::Corruption("[error] Free memory faild !"); } @@ -277,6 +277,25 @@ Status RedisCache::SetRange(std::string& key, int64_t start, std::string &value) return Status::OK(); } +Status RedisCache::SetRange(std::string& key, int64_t start, std::string &value) { + if (C_OK != RcFreeMemoryIfNeeded(cache_)) { + return Status::Corruption("[error] Free memory faild !"); + } + + uint64_t ret = 0; + robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); + robj *vobj = createObject(OBJ_STRING, sdsnewlen(value.data(), value.size())); + DEFER { + DecrObjectsRefCount(kobj, vobj); + }; + int res = RcSetRange(cache_, kobj, start, vobj, reinterpret_cast(&ret)); + if (C_OK != res) { + return Status::Corruption("SetRange failed!"); + } + + return Status::OK(); +} + Status RedisCache::Strlen(std::string& key, int32_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { diff --git a/src/cache/src/zset.cc b/src/cache/src/zset.cc index a06d999a5e..6fec5aac2f 100644 --- a/src/cache/src/zset.cc +++ b/src/cache/src/zset.cc @@ -8,7 +8,7 @@ namespace cache { -Status RedisCache::ZAdd(std::string& key, std::vector &score_members) { +Status RedisCache::ZAddIfKeyExist(std::string& key, std::vector &score_members) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); @@ -37,7 +37,33 @@ Status RedisCache::ZAdd(std::string& key, std::vector &sco return Status::OK(); } -Status RedisCache::ZCard(const std::string& key, uint64_t *len) { +Status RedisCache::ZAdd(std::string& key, std::vector &score_members) { + int res = RcFreeMemoryIfNeeded(cache_); + if (C_OK != res) { + return Status::Corruption("[error] Free memory faild !"); + } + + robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); + unsigned int items_size = score_members.size() * 2; + robj **items = (robj **)zcallocate(sizeof(robj *) * items_size); + for (unsigned int i = 0; i < score_members.size(); ++i) { + items[i * 2] = createStringObjectFromLongDouble(score_members[i].score, 0); + items[i * 2 + 1] = + createObject(OBJ_STRING, sdsnewlen(score_members[i].member.data(), score_members[i].member.size())); + } + DEFER { + FreeObjectList(items, items_size); + DecrObjectsRefCount(kobj); + }; + int ret = RcZAdd(cache_, kobj, items, items_size); + if (C_OK != ret) { + return Status::Corruption("RcZAdd failed"); + } + + return Status::OK(); +} + +Status RedisCache::ZCard(std::string& key, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); diff --git a/src/pika_cache.cc b/src/pika_cache.cc index 4be4ee3ec6..0345a7b880 100644 --- a/src/pika_cache.cc +++ b/src/pika_cache.cc @@ -370,6 +370,12 @@ Status PikaCache::GetRange(std::string& key, int64_t start, int64_t end, std::st return caches_[cache_index]->GetRange(key, start, end, value); } +Status PikaCache::SetRangeIfKeyExist(std::string& key, int64_t start, std::string &value) { + int cache_index = CacheIndex(key); + std::lock_guard lm(*cache_mutexs_[cache_index]); + return caches_[cache_index]->SetRangeIfKeyExist(key, start, value); +} + Status PikaCache::SetRangexx(std::string& key, int64_t start, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -406,10 +412,7 @@ Status PikaCache::HSetIfKeyExist(std::string& key, std::string &field, std::stri Status PikaCache::HSetIfKeyExistAndFieldNotExist(std::string& key, std::string &field, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); - if (caches_[cache_index]->Exists(key)) { - return caches_[cache_index]->HSetnx(key, field, value); - } - return Status::NotFound("key not exist"); + return caches_[cache_index]->HSetnxIfKeyExist(key, field, value); } Status PikaCache::HMSet(std::string& key, std::vector &fvs) { @@ -418,6 +421,12 @@ Status PikaCache::HMSet(std::string& key, std::vector &fvs) return caches_[cache_index]->HMSet(key, fvs); } +Status PikaCache::HMSetIfKeyExist(std::string& key, std::vector &fvs) { + int cache_index = CacheIndex(key); + std::lock_guard lm(*cache_mutexs_[cache_index]); + return caches_[cache_index]->HMSetIfKeyExist(key, fvs); +} + Status PikaCache::HMSetnx(std::string& key, std::vector &fvs, int64_t ttl) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -542,6 +551,12 @@ Status PikaCache::LPop(std::string& key, std::string *element) { return caches_[cache_index]->LPop(key, element); } +Status PikaCache::LPushIfKeyExist(std::string& key, std::vector &values) { + int cache_index = CacheIndex(key); + std::lock_guard lm(*cache_mutexs_[cache_index]); + return caches_[cache_index]->LPushIfKeyExist(key, values); +} + Status PikaCache::LPush(std::string& key, std::vector &values) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -590,6 +605,12 @@ Status PikaCache::RPush(std::string& key, std::vector &values) { return caches_[cache_index]->RPush(key, values); } +Status PikaCache::RPushIfKeyExist(std::string& key, std::vector &values) { + int cache_index = CacheIndex(key); + std::lock_guard lm(*cache_mutexs_[cache_index]); + return caches_[cache_index]->RPushIfKeyExist(key, values); +} + Status PikaCache::RPushx(std::string& key, std::vector &values) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -631,7 +652,7 @@ Status PikaCache::SAdd(std::string& key, std::vector &members) { Status PikaCache::SAddIfKeyExist(std::string& key, std::vector &members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); - return caches_[cache_index]->SAdd(key, members); + return caches_[cache_index]->SAddIfKeyExist(key, members); } Status PikaCache::SAddnx(std::string& key, std::vector &members, int64_t ttl) { @@ -764,7 +785,7 @@ Status PikaCache::ZAddIfKeyExist(std::string& key, std::vectorZAdd(key, new_score_members); + cache_obj->ZAddIfKeyExist(key, new_score_members); } else { std::vector score_members_can_add; std::vector members_need_remove; @@ -782,7 +803,7 @@ Status PikaCache::ZAddIfKeyExist(std::string& key, std::vectorZAdd(key, score_members_can_add); + cache_obj->ZAddIfKeyExist(key, score_members_can_add); std::string cache_max_score_str = left_close ? "" : "(" + std::to_string(cache_max_score); std::string max_str = "+inf"; cache_obj->ZRemrangebyscore(key, cache_max_score_str, max_str); @@ -793,7 +814,7 @@ Status PikaCache::ZAddIfKeyExist(std::string& key, std::vector cache_min_score) { - cache_obj->ZAdd(key, new_score_members); + cache_obj->ZAddIfKeyExist(key, new_score_members); } else { std::vector score_members_can_add; std::vector members_need_remove; @@ -811,7 +832,7 @@ Status PikaCache::ZAddIfKeyExist(std::string& key, std::vectorZAdd(key, score_members_can_add); + cache_obj->ZAddIfKeyExist(key, score_members_can_add); std::string cache_min_score_str = right_close ? "" : "(" + std::to_string(cache_min_score); std::string min_str = "-inf"; cache_obj->ZRemrangebyscore(key, min_str, cache_min_score_str); @@ -1055,14 +1076,14 @@ Status PikaCache::ZIncrbyIfKeyExist(std::string& key, std::string& member, doubl return RemCacheRangebyscoreAndCheck(cache_max_score); } else { std::vector score_member = {{cmd->Score(), member}}; - auto s = cache_obj->ZAdd(key, score_member); + auto s = cache_obj->ZAddIfKeyExist(key, score_member); CleanCacheKeyIfNeeded(cache_obj, key); return s; } } else if (zset_cache_start_direction_ == cache::CACHE_START_FROM_END) { if (cmd->Score() > cache_min_score) { std::vector score_member = {{cmd->Score(), member}}; - auto s = cache_obj->ZAdd(key, score_member); + auto s = cache_obj->ZAddIfKeyExist(key, score_member); CleanCacheKeyIfNeeded(cache_obj, key); return s; } else if (cmd->Score() == cache_min_score) { @@ -1495,7 +1516,7 @@ Status PikaCache::SetBit(std::string& key, size_t offset, int64_t value) { Status PikaCache::SetBitIfKeyExist(std::string& key, size_t offset, int64_t value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); - return caches_[cache_index]->SetBit(key, offset, value); + return caches_[cache_index]->SetBitIfKeyExist(key, offset, value); } Status PikaCache::GetBit(std::string& key, size_t offset, int64_t *value) { diff --git a/src/pika_hash.cc b/src/pika_hash.cc index 948b223071..8a190d3d8a 100644 --- a/src/pika_hash.cc +++ b/src/pika_hash.cc @@ -79,8 +79,8 @@ void HSetCmd::DoUpdateCache() { if (IsTooLargeKey(g_pika_conf->max_key_size_in_cache())) { return; } - if (s_.ok()) { - db_->cache()->HSetIfKeyExist(key_, field_, value_); + else if (argv_.size() > 4 && argv_.size() % 2 == 0) { + db_->cache()->HMSetIfKeyExist(CachePrefixKeyH, fields_values_); } } @@ -527,7 +527,8 @@ void HMsetCmd::DoThroughDB() { void HMsetCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->HMSetxx(key_, fvs_); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + db_->cache()->HMSetIfKeyExist(CachePrefixKeyH, fvs_); } } @@ -561,7 +562,7 @@ void HSetnxCmd::DoThroughDB() { void HSetnxCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - db_->cache()->HSetIfKeyExist(CachePrefixKeyH, field_, value_); + db_->cache()->HSetIfKeyExistAndFieldNotExist(CachePrefixKeyH, field_, value_); } } diff --git a/src/pika_kv.cc b/src/pika_kv.cc index ec7f5caaf1..92642560b7 100644 --- a/src/pika_kv.cc +++ b/src/pika_kv.cc @@ -1205,7 +1205,8 @@ void SetrangeCmd::DoThroughDB() { void SetrangeCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->SetRangexx(key_, offset_, value_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->SetRangeIfKeyExist(CachePrefixKeyK, offset_, value_); } } diff --git a/src/pika_list.cc b/src/pika_list.cc index fa57dde372..8a0cb16168 100644 --- a/src/pika_list.cc +++ b/src/pika_list.cc @@ -288,7 +288,8 @@ void LPushCmd::DoThroughDB() { void LPushCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->LPushx(key_, values_); + std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + db_->cache()->LPushIfKeyExist(CachePrefixKeyL, values_); } } @@ -482,7 +483,8 @@ void LPushxCmd::DoThroughDB() { void LPushxCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->LPushx(key_, values_); + std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + db_->cache()->LPushIfKeyExist(CachePrefixKeyL, values_); } } @@ -851,8 +853,10 @@ void RPopLPushCmd::DoUpdateCache() { if (s_.ok()) { std::vector value; value.resize(1); - db_->cache()->RPop(source_, &value[0]); - db_->cache()->LPushx(receiver_, value); + std::string CachePrefixKeyLR = PCacheKeyPrefixL + source_; + db_->cache()->RPop(CachePrefixKeyLR, &value[0]); + std::string CachePrefixKeyLL = PCacheKeyPrefixL + receiver_; + db_->cache()->LPushIfKeyExist(CachePrefixKeyLL, value); } } void RPopLPushCmd::DoThroughDB() { @@ -896,7 +900,8 @@ void RPushCmd::DoThroughDB() { void RPushCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->RPushx(key_, values_); + std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + db_->cache()->RPushIfKeyExist(CachePrefixKeyL, values_); } } @@ -931,6 +936,7 @@ void RPushxCmd::DoThroughDB() { void RPushxCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->RPushx(key_, values_); + std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + db_->cache()->RPushIfKeyExist(CachePrefixKeyL, values_); } } \ No newline at end of file From 2af64e5eff65719137f7d51e08e3b8ed96d129f5 Mon Sep 17 00:00:00 2001 From: wangshao1 <30471730+wangshao1@users.noreply.github.com> Date: Thu, 6 Mar 2025 14:05:44 +0800 Subject: [PATCH 03/15] cherry-pick 3 --- include/pika_client_conn.h | 2 +- include/pika_command.h | 17 +- include/pika_conf.h | 39 ++++ src/pika_admin.cc | 38 ++++ src/pika_client_conn.cc | 23 +- src/pika_command.cc | 34 ++- src/pika_hash.cc | 192 +++++++++++++---- src/pika_kv.cc | 372 +++++++++++++++++++++++++++------ src/pika_list.cc | 61 +++++- src/pika_set.cc | 70 +++++-- src/pika_zset.cc | 89 +++++++- src/pstd/include/stage_timer.h | 56 +++++ 12 files changed, 842 insertions(+), 151 deletions(-) create mode 100644 src/pstd/include/stage_timer.h diff --git a/include/pika_client_conn.h b/include/pika_client_conn.h index 3124d2036c..bc4c28db6a 100644 --- a/include/pika_client_conn.h +++ b/include/pika_client_conn.h @@ -130,7 +130,7 @@ class PikaClientConn : public net::RedisConn { std::shared_ptr DoCmd(const PikaCmdArgsType& argv, const std::string& opt, const std::shared_ptr& resp_ptr, bool cache_miss_in_rtc); - void ProcessSlowlog(const PikaCmdArgsType& argv, uint64_t do_duration); + void ProcessSlowlog(const PikaCmdArgsType& argv, std::shared_ptr c_ptr); void ProcessMonitor(const PikaCmdArgsType& argv); void ExecRedisCmd(const PikaCmdArgsType& argv, std::shared_ptr& resp_ptr, bool cache_miss_in_rtc); diff --git a/include/pika_command.h b/include/pika_command.h index c132eae9c5..77b0eca6b8 100644 --- a/include/pika_command.h +++ b/include/pika_command.h @@ -17,9 +17,16 @@ #include "net/include/net_conn.h" #include "net/include/redis_conn.h" #include "pstd/include/pstd_string.h" +#include "pstd/include/stage_timer.h" #include "net/src/dispatch_thread.h" +// Declare and set start time of the timer +#define STAGE_TIMER_GUARD(metric, enabled) \ + pstd::StageTimer stage_timer_##metric( \ + &metric, enabled); \ + stage_timer_##metric.Start(); + class SyncMasterDB; class SyncSlaveDB; class DB; @@ -552,7 +559,7 @@ class Cmd : public std::enable_shared_from_this { bool IsNeedReadCache() const; bool IsNeedCacheDo() const; bool HashtagIsConsistent(const std::string& lhs, const std::string& rhs) const; - uint64_t GetDoDuration() const { return do_duration_; }; + virtual std::string StagesDurationSummary(bool exclude_zero_value) const; std::shared_ptr GetDB() const { return db_; }; uint32_t AclCategory() const; void AddAclCategory(uint32_t aclCategory); @@ -607,7 +614,13 @@ class Cmd : public std::enable_shared_from_this { std::weak_ptr conn_; std::weak_ptr resp_; CmdStage stage_ = kNone; - uint64_t do_duration_ = 0; + + uint64_t acquire_lock_duration_ms = 0; + uint64_t command_duration_ms = 0; + uint64_t binlog_duration_ms = 0; + uint64_t storage_duration_ms = 0; + uint64_t cache_duration_ms = 0; + uint32_t cmdId_ = 0; uint32_t aclCategory_ = 0; bool cache_missed_in_rtc_{false}; diff --git a/include/pika_conf.h b/include/pika_conf.h index 6b64ba56b0..37a1f21d18 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -858,6 +858,30 @@ class PikaConf : public pstd::BaseConf { rsync_timeout_ms_.store(value); } + int RocksDBPerfLevel() const { + return rocksdb_perf_level_.load(); + } + + bool UpdateRocksDBPerfLevel(int perf_level) { + if (perf_level >= 6 || perf_level < 0) { + return false; + } + rocksdb_perf_level_.store(perf_level); + return true; + } + + int RocksDBPerfPercent() const { + return rocksdb_perf_percent_.load(); + } + + bool UpdateRocksDBPerfPercent(int percent) { + if (percent > 100 || percent < 0) { + return false; + } + rocksdb_perf_percent_.store(percent); + return true; + } + void SetAclPubsubDefault(const std::string& value) { std::lock_guard l(rwlock_); TryPushDiffCommands("acl-pubsub-default", value); @@ -1121,6 +1145,21 @@ class PikaConf : public pstd::BaseConf { int max_rsync_parallel_num_ = kMaxRsyncParallelNum; std::atomic_int64_t rsync_timeout_ms_ = 1000; + /* + kUninitialized = 0, // unknown setting + kDisable = 1, // disable perf stats + kEnableCount = 2, // enable only count stats + kEnableTimeExceptForMutex = 3, // Other than count stats, also enable time + // stats except for mutexes + // Other than time, also measure CPU time counters. Still don't measure + // time (neither wall time nor CPU time) for mutexes. + kEnableTimeAndCPUTimeExceptForMutex = 4, + kEnableTime = 5, // enable count and time stats + kOutOfBounds = 6 // N.B. Must always be the last value! + */ + std::atomic_int rocksdb_perf_level_ = 2; + std::atomic_int rocksdb_perf_percent_ = 10; + //Internal used metrics Persisted by pika.conf std::unordered_set internal_used_unfinished_full_sync_; diff --git a/src/pika_admin.cc b/src/pika_admin.cc index 90860173f0..02d2e2b5e0 100644 --- a/src/pika_admin.cc +++ b/src/pika_admin.cc @@ -2198,6 +2198,18 @@ void ConfigCmd::ConfigGet(std::string& ret) { EncodeNumber(&config_body, g_pika_conf->throttle_bytes_per_second()); } + if (pstd::stringmatch(pattern.data(), "rocksdb-perf-level", 1) != 0) { + elements += 2; + EncodeString(&config_body, "rocksdb-perf-level"); + EncodeNumber(&config_body, g_pika_conf->RocksDBPerfLevel()); + } + + if (pstd::stringmatch(pattern.data(), "rocksdb-perf-percent", 1) != 0) { + elements += 2; + EncodeString(&config_body, "rocksdb-perf-percent"); + EncodeNumber(&config_body, g_pika_conf->RocksDBPerfPercent()); + } + if (pstd::stringmatch(pattern.data(), "max-rsync-parallel-num", 1) != 0) { elements += 2; EncodeString(&config_body, "max-rsync-parallel-num"); @@ -2864,6 +2876,32 @@ void ConfigCmd::ConfigSet(std::shared_ptr db) { } g_pika_conf->SetArenaBlockSize(static_cast(ival)); res_.AppendStringRaw("+OK\r\n"); + } else if (set_item == "rocksdb-perf-level") { + if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival <= 0) { + res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'rocksdb-perf-level'\r\n"); + return; + } + bool success = g_pika_conf->UpdateRocksDBPerfLevel(int(ival)); + LOG(INFO) << "update rocksdb-perf-level to " << ival + << (success ? " success" : " failed"); + if (!success) { + res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'rocksdb-perf-level', should between 1 and 5\r\n"); + return; + } + res_.AppendStringRaw("+OK\r\n"); + } else if (set_item == "rocksdb-perf-percent") { + if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival <= 0) { + res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'rocksdb-perf-percent'\r\n"); + return; + } + bool success = g_pika_conf->UpdateRocksDBPerfPercent(int(ival)); + LOG(INFO) << "update rocksdb-perf-percent to " << ival + << (success ? " success" : " failed"); + if (!success) { + res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'rocksdb-perf-percent', should between 0 and 100\r\n"); + return; + } + res_.AppendStringRaw("+OK\r\n"); } else if (set_item == "throttle-bytes-per-second") { if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival <= 0) { res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'throttle-bytes-per-second'\r\n"); diff --git a/src/pika_client_conn.cc b/src/pika_client_conn.cc index 768cb6d5ad..35ad92b4ed 100644 --- a/src/pika_client_conn.cc +++ b/src/pika_client_conn.cc @@ -20,6 +20,10 @@ #include "net/src/worker_thread.h" #include "src/pstd/include/scope_record_lock.h" +#include "rocksdb/perf_context.h" +#include "rocksdb/iostats_context.h" +#include "util/random.h" + extern std::unique_ptr g_pika_conf; extern PikaServer* g_pika_server; extern std::unique_ptr g_pika_rm; @@ -46,6 +50,7 @@ std::shared_ptr PikaClientConn::DoCmd(const PikaCmdArgsType& argv, const st } return tmp_ptr; } + c_ptr->SetCacheMissedInRtc(cache_miss_in_rtc); c_ptr->SetConn(shared_from_this()); c_ptr->SetResp(resp_ptr); @@ -202,21 +207,32 @@ std::shared_ptr PikaClientConn::DoCmd(const PikaCmdArgsType& argv, const st } } + + // set rocksdb perflevel based on RocksDBPerfLevel and RocksDBPerfPercent + int rocksdb_perf_level = 2; + if (rocksdb::Random::GetTLSInstance()->PercentTrue(g_pika_conf->RocksDBPerfPercent())) { + rocksdb_perf_level = g_pika_conf->RocksDBPerfLevel(); + } + rocksdb::SetPerfLevel(rocksdb::PerfLevel(rocksdb_perf_level)); + + // Perform some operations + rocksdb::get_perf_context()->Reset(); // Process Command c_ptr->Execute(); + time_stat_->process_done_ts_ = pstd::NowMicros(); auto cmdstat_map = g_pika_cmd_table_manager->GetCommandStatMap(); (*cmdstat_map)[opt].cmd_count.fetch_add(1); (*cmdstat_map)[opt].cmd_time_consuming.fetch_add(time_stat_->total_time()); if (g_pika_conf->slowlog_slower_than() >= 0) { - ProcessSlowlog(argv, c_ptr->GetDoDuration()); + ProcessSlowlog(argv, c_ptr); } return c_ptr; } -void PikaClientConn::ProcessSlowlog(const PikaCmdArgsType& argv, uint64_t do_duration) { +void PikaClientConn::ProcessSlowlog(const PikaCmdArgsType& argv, std::shared_ptr c_ptr) { if (time_stat_->total_time() > g_pika_conf->slowlog_slower_than()) { g_pika_server->SlowlogPushEntry(argv, time_stat_->start_ts() / 1000000, time_stat_->total_time()); if (g_pika_conf->slowlog_write_errorlog()) { @@ -241,7 +257,8 @@ void PikaClientConn::ProcessSlowlog(const PikaCmdArgsType& argv, uint64_t do_dur << ", before_queue_time(ms): " << time_stat_->before_queue_time() / 1000 << ", queue_time(ms): " << time_stat_->queue_time() / 1000 << ", process_time(ms): " << time_stat_->process_time() / 1000 - << ", cmd_time(ms): " << do_duration / 1000; + << ", " << c_ptr->StagesDurationSummary(true /*skip zero counter*/) + << ", " << rocksdb::get_perf_context()->ToString(true); } } } diff --git a/src/pika_command.cc b/src/pika_command.cc index 7d8443001d..93455644ef 100644 --- a/src/pika_command.cc +++ b/src/pika_command.cc @@ -4,6 +4,7 @@ // of patent rights can be found in the PATENTS file in the same directory. #include +#include #include #include @@ -859,23 +860,22 @@ void Cmd::ProcessCommand(const HintKeys& hint_keys) { } void Cmd::InternalProcessCommand(const HintKeys& hint_keys) { + uint64_t start_us = pstd::NowMicros(); pstd::lock::MultiRecordLock record_lock(db_->LockMgr()); if (is_write()) { record_lock.Lock(current_key()); } - uint64_t start_us = 0; - if (g_pika_conf->slowlog_slower_than() >= 0) { - start_us = pstd::NowMicros(); - } if (!IsSuspend()) { db_->DBLockShared(); } + uint64_t before_do_command_us = pstd::NowMicros(); + this->acquire_lock_duration_ms = (before_do_command_us - start_us) / 1000; DoCommand(hint_keys); - if (g_pika_conf->slowlog_slower_than() >= 0) { - do_duration_ += pstd::NowMicros() - start_us; - } + + uint64_t before_do_binlog_us = pstd::NowMicros(); + this->command_duration_ms = (before_do_binlog_us - before_do_command_us) / 1000; DoBinlog(); if (!IsSuspend()) { @@ -884,6 +884,9 @@ void Cmd::InternalProcessCommand(const HintKeys& hint_keys) { if (is_write()) { record_lock.Unlock(current_key()); } + + uint64_t end_us = pstd::NowMicros(); + this->binlog_duration_ms = (end_us - before_do_binlog_us) / 1000; } void Cmd::DoCommand(const HintKeys& hint_keys) { @@ -968,6 +971,23 @@ void Cmd::DoBinlog() { } } +#define PIKA_STAGE_DURATION_OUTPUT(duration) \ + if (!exclude_zero_value || duration > 0) { \ + ss << #duration << " = " << duration << ", "; \ + } + +std::string Cmd::StagesDurationSummary(bool exclude_zero_value) const { + std::ostringstream ss; + PIKA_STAGE_DURATION_OUTPUT(acquire_lock_duration_ms); + PIKA_STAGE_DURATION_OUTPUT(command_duration_ms); + PIKA_STAGE_DURATION_OUTPUT(binlog_duration_ms); + PIKA_STAGE_DURATION_OUTPUT(storage_duration_ms); + PIKA_STAGE_DURATION_OUTPUT(cache_duration_ms); + std::string str = ss.str(); + str.erase(str.find_last_not_of(", ") + 1); + return str; +} + bool Cmd::hasFlag(uint32_t flag) const { return (flag_ & flag); } bool Cmd::is_read() const { return (flag_ & kCmdFlagsRead); } bool Cmd::is_write() const { return (flag_ & kCmdFlagsWrite); } diff --git a/src/pika_hash.cc b/src/pika_hash.cc index 8a190d3d8a..e5e615d342 100644 --- a/src/pika_hash.cc +++ b/src/pika_hash.cc @@ -26,6 +26,7 @@ void HDelCmd::DoInitial() { } void HDelCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HDel(key_, fields_, &deleted_); if (s_.ok() || s_.IsNotFound()) { @@ -43,7 +44,9 @@ void HDelCmd::DoThroughDB() { void HDelCmd::DoUpdateCache() { if (s_.ok() && deleted_ > 0) { - db_->cache()->HDel(key_, fields_); + STAGE_TIMER_GUARD(cache_duration_ms, true); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + db_->cache()->HDel(CachePrefixKeyH, fields_); } } @@ -58,15 +61,24 @@ void HSetCmd::DoInitial() { } void HSetCmd::Do() { - int32_t ret = 0; - s_ = db_->storage()->HSet(key_, field_, value_, &ret); - if (s_.ok()) { - res_.AppendContent(":" + std::to_string(ret)); - AddSlotKey("h", key_, db_); - } else if (s_.IsInvalidArgument()) { - res_.SetRes(CmdRes::kMultiKey); - } else { - res_.SetRes(CmdRes::kErrOther, s_.ToString()); + STAGE_TIMER_GUARD(storage_duration_ms, true); + if (argv_.size() == 4) { + int32_t count = 0; + s_ = db_->storage()->HSet(key_, field_, value_, &count); + if (s_.ok()) { + res_.AppendContent(":" + std::to_string(count)); + AddSlotKey("h", key_, db_); + } else { + res_.SetRes(CmdRes::kErrOther, s_.ToString()); + } + } else if (argv_.size() > 4 && argv_.size() % 2 == 0) { + s_ = db_->storage()->HMSet(key_, fields_values_); + if (s_.ok()) { + res_.AppendContent(":" + std::to_string(fields_values_.size())); + AddSlotKey("h", key_, db_); + } else { + res_.SetRes(CmdRes::kErrOther, s_.ToString()); + } } } @@ -75,9 +87,10 @@ void HSetCmd::DoThroughDB() { } void HSetCmd::DoUpdateCache() { - // HSetIfKeyExist() can void storing large key, but IsTooLargeKey() can speed up it - if (IsTooLargeKey(g_pika_conf->max_key_size_in_cache())) { - return; + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + if (argv_.size() == 4) { + db_->cache()->HSetIfKeyExist(CachePrefixKeyH, field_, value_); } else if (argv_.size() > 4 && argv_.size() % 2 == 0) { db_->cache()->HMSetIfKeyExist(CachePrefixKeyH, fields_values_); @@ -94,30 +107,74 @@ void HGetCmd::DoInitial() { } void HGetCmd::Do() { - std::string value; - s_ = db_->storage()->HGet(key_, field_, &value); - if (s_.ok()) { - res_.AppendStringLenUint64(value.size()); - res_.AppendContent(value); - } else if (s_.IsInvalidArgument()) { - res_.SetRes(CmdRes::kMultiKey); - } else if (s_.IsNotFound()) { - res_.AppendContent("$-1"); - } else { - res_.SetRes(CmdRes::kErrOther, s_.ToString()); + STAGE_TIMER_GUARD(storage_duration_ms, true); + if (argv_.size() == 3) { + std::string value; + s_ = db_->storage()->HGet(key_, field_, &value); + + if (s_.ok()) { + res_.AppendStringLenUint64(value.size()); + res_.AppendContent(value); + } else if (s_.IsNotFound()) { + res_.AppendContent("$-1"); + } else { + res_.SetRes(CmdRes::kErrOther, s_.ToString()); + } + } + else if (argv_.size() > 3) { + std::vector values; + s_ = db_->storage()->HMGet(key_, fields_, &values); + + if (s_.ok()) { + res_.AppendArrayLen(values.size()); + for (const auto& vs : values) { + if (vs.status.ok()) { + res_.AppendStringLenUint64(vs.value.size()); + res_.AppendContent(vs.value); + } else { + res_.AppendContent("$-1"); + } + } + } else { + res_.SetRes(CmdRes::kErrOther, s_.ToString()); + } } } void HGetCmd::ReadCache() { - std::string value; - auto s = db_->cache()->HGet(key_, field_, &value); - if (s.ok()) { - res_.AppendStringLen(value.size()); - res_.AppendContent(value); - } else if (s.IsNotFound()) { - res_.SetRes(CmdRes::kCacheMiss); - } else { - res_.SetRes(CmdRes::kErrOther, s.ToString()); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + if (argv_.size() == 3) { + std::string value; + auto s = db_->cache()->HGet(CachePrefixKeyH, field_, &value); + if (s.ok()) { + res_.AppendStringLen(value.size()); + res_.AppendContent(value); + } else if (s.IsNotFound()) { + res_.SetRes(CmdRes::kCacheMiss); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + } + else if (argv_.size() > 3) { + std::vector vss; + vss.clear(); + auto s = db_->cache()->HMGet(CachePrefixKeyH, fields_, &vss); + if (s.ok()) { + res_.AppendArrayLen(vss.size()); + for (const auto& vs : vss) { + if (vs.status.ok()) { + res_.AppendStringLen(vs.value.size()); + res_.AppendContent(vs.value); + } else { + res_.AppendContent("$-1"); + } + } + } else if (s.IsNotFound()) { + res_.SetRes(CmdRes::kCacheMiss); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } } } @@ -131,6 +188,8 @@ void HGetCmd::DoUpdateCache() { return; } if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -151,6 +210,7 @@ void HGetallCmd::Do() { std::string raw; std::vector fvs; + STAGE_TIMER_GUARD(storage_duration_ms, true); do { fvs.clear(); s_ = db_->storage()->HScan(key_, cursor, "*", PIKA_SCAN_STEP_LENGTH, &fvs, &next_cursor); @@ -184,7 +244,9 @@ void HGetallCmd::Do() { void HGetallCmd::ReadCache() { std::vector fvs; - auto s = db_->cache()->HGetall(key_, &fvs); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->HGetall(CachePrefixKeyH, &fvs); if (s.ok()) { res_.AppendArrayLen(fvs.size() * 2); for (const auto& fv : fvs) { @@ -207,6 +269,8 @@ void HGetallCmd::DoThroughDB() { void HGetallCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -221,6 +285,7 @@ void HExistsCmd::DoInitial() { } void HExistsCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HExists(key_, field_); if (s_.ok()) { res_.AppendContent(":1"); @@ -234,7 +299,9 @@ void HExistsCmd::Do() { } void HExistsCmd::ReadCache() { - auto s = db_->cache()->HExists(key_, field_); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->HExists(CachePrefixKeyH, field_); if (s.ok()) { res_.AppendContent(":1"); } else if (s.IsNotFound()) { @@ -251,6 +318,8 @@ void HExistsCmd::DoThroughDB() { void HExistsCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -270,6 +339,7 @@ void HIncrbyCmd::DoInitial() { void HIncrbyCmd::Do() { int64_t new_value = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HIncrby(key_, field_, by_, &new_value); if (s_.ok() || s_.IsNotFound()) { res_.AppendContent(":" + std::to_string(new_value)); @@ -291,7 +361,9 @@ void HIncrbyCmd::DoThroughDB() { void HIncrbyCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->HIncrbyxx(key_, field_, by_); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->HIncrbyxx(CachePrefixKeyH, field_, by_); } } @@ -307,6 +379,7 @@ void HIncrbyfloatCmd::DoInitial() { void HIncrbyfloatCmd::Do() { std::string new_value; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HIncrbyfloat(key_, field_, by_, &new_value); if (s_.ok()) { res_.AppendStringLenUint64(new_value.size()); @@ -331,7 +404,9 @@ void HIncrbyfloatCmd::DoUpdateCache() { if (s_.ok()) { long double long_double_by; if (storage::StrToLongDouble(by_.data(), by_.size(), &long_double_by) != -1) { - db_->cache()->HIncrbyfloatxx(key_, field_, long_double_by); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->HIncrbyfloatxx(CachePrefixKeyH, field_, long_double_by); } } } @@ -346,6 +421,7 @@ void HKeysCmd::DoInitial() { void HKeysCmd::Do() { std::vector fields; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HKeys(key_, &fields); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(fields.size()); @@ -361,7 +437,9 @@ void HKeysCmd::Do() { void HKeysCmd::ReadCache() { std::vector fields; - auto s = db_->cache()->HKeys(key_, &fields); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->HKeys(CachePrefixKeyH, &fields); if (s.ok()) { res_.AppendArrayLen(fields.size()); for (const auto& field : fields) { @@ -381,6 +459,8 @@ void HKeysCmd::DoThroughDB() { void HKeysCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -395,6 +475,7 @@ void HLenCmd::DoInitial() { void HLenCmd::Do() { int32_t len = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HLen(key_, &len); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(len); @@ -407,7 +488,9 @@ void HLenCmd::Do() { void HLenCmd::ReadCache() { uint64_t len = 0; - auto s = db_->cache()->HLen(key_, &len); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->HLen(CachePrefixKeyH, &len); if (s.ok()) { res_.AppendInteger(len); } else if (s.IsNotFound()) { @@ -424,6 +507,8 @@ void HLenCmd::DoThroughDB() { void HLenCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -442,6 +527,7 @@ void HMgetCmd::DoInitial() { void HMgetCmd::Do() { std::vector vss; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HMGet(key_, fields_, &vss); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(vss.size()); @@ -462,7 +548,9 @@ void HMgetCmd::Do() { void HMgetCmd::ReadCache() { std::vector vss; - auto s = db_->cache()->HMGet(key_, fields_, &vss); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->HMGet(CachePrefixKeyH, fields_, &vss); if (s.ok()) { res_.AppendArrayLen(vss.size()); for (const auto& vs : vss) { @@ -487,6 +575,8 @@ void HMgetCmd::DoThroughDB() { void HMgetCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -510,6 +600,7 @@ void HMsetCmd::DoInitial() { } void HMsetCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HMSet(key_, fvs_); if (s_.ok()) { res_.SetRes(CmdRes::kOk); @@ -528,6 +619,7 @@ void HMsetCmd::DoThroughDB() { void HMsetCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->HMSetIfKeyExist(CachePrefixKeyH, fvs_); } } @@ -544,6 +636,7 @@ void HSetnxCmd::DoInitial() { void HSetnxCmd::Do() { int32_t ret = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HSetnx(key_, field_, value_, &ret); if (s_.ok()) { res_.AppendContent(":" + std::to_string(ret)); @@ -562,6 +655,7 @@ void HSetnxCmd::DoThroughDB() { void HSetnxCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->HSetIfKeyExistAndFieldNotExist(CachePrefixKeyH, field_, value_); } } @@ -577,6 +671,7 @@ void HStrlenCmd::DoInitial() { void HStrlenCmd::Do() { int32_t len = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HStrlen(key_, field_, &len); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(len); @@ -589,7 +684,9 @@ void HStrlenCmd::Do() { void HStrlenCmd::ReadCache() { uint64_t len = 0; - auto s = db_->cache()->HStrlen(key_, field_, &len); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->HStrlen(CachePrefixKeyH, field_, &len); if (s.ok()) { res_.AppendInteger(len); } else if (s.IsNotFound()) { @@ -607,6 +704,8 @@ void HStrlenCmd::DoThroughDB() { void HStrlenCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -621,6 +720,7 @@ void HValsCmd::DoInitial() { void HValsCmd::Do() { std::vector values; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->HVals(key_, &values); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(values.size()); @@ -637,7 +737,9 @@ void HValsCmd::Do() { void HValsCmd::ReadCache() { std::vector values; - auto s = db_->cache()->HVals(key_, &values); + std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->HVals(CachePrefixKeyH, &values); if (s.ok()) { res_.AppendArrayLen(values.size()); for (const auto& value : values) { @@ -658,6 +760,8 @@ void HValsCmd::DoThroughDB() { void HValsCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -704,6 +808,7 @@ void HScanCmd::DoInitial() { void HScanCmd::Do() { int64_t next_cursor = 0; std::vector field_values; + STAGE_TIMER_GUARD(storage_duration_ms, true); auto s = db_->storage()->HScan(key_, cursor_, pattern_, count_, &field_values, &next_cursor); if (s.ok() || s.IsNotFound()) { @@ -764,6 +869,7 @@ void HScanxCmd::DoInitial() { void HScanxCmd::Do() { std::string next_field; std::vector field_values; + STAGE_TIMER_GUARD(storage_duration_ms, true); rocksdb::Status s = db_->storage()->HScanx(key_, start_field_, pattern_, count_, &field_values, &next_field); if (s.ok() || s.IsNotFound()) { @@ -819,6 +925,7 @@ void PKHScanRangeCmd::DoInitial() { void PKHScanRangeCmd::Do() { std::string next_field; std::vector field_values; + STAGE_TIMER_GUARD(storage_duration_ms, true); rocksdb::Status s = db_->storage()->PKHScanRange(key_, field_start_, field_end_, pattern_, static_cast(limit_), &field_values, &next_field); @@ -874,6 +981,7 @@ void PKHRScanRangeCmd::DoInitial() { void PKHRScanRangeCmd::Do() { std::string next_field; std::vector field_values; + STAGE_TIMER_GUARD(storage_duration_ms, true); rocksdb::Status s = db_->storage()->PKHRScanRange(key_, field_start_, field_end_, pattern_, static_cast(limit_), &field_values, &next_field); diff --git a/src/pika_kv.cc b/src/pika_kv.cc index 92642560b7..c9fe261554 100644 --- a/src/pika_kv.cc +++ b/src/pika_kv.cc @@ -65,6 +65,7 @@ void SetCmd::DoInitial() { void SetCmd::Do() { int32_t res = 1; + STAGE_TIMER_GUARD(storage_duration_ms, true); switch (condition_) { case SetCmd::kXX: s_ = db_->storage()->Setxx(key_, value_, &res, ttl_millsec); @@ -110,6 +111,7 @@ void SetCmd::DoUpdateCache() { return; } if (s_.ok()) { + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; if (has_ttl_) { db_->cache()->Setxx(key_, value_, ttl_millsec > 0 ? ttl_millsec / 1000 : ttl_millsec); } else { @@ -158,7 +160,7 @@ void GetCmd::DoInitial() { } void GetCmd::Do() { - s_ = db_->storage()->GetWithTTL(key_, &value_, &ttl_millsec_); + s_ = db_->storage()->GetWithTTL(key_, &value_, &sec_); if (s_.ok()) { res_.AppendStringLenUint64(value_.size()); res_.AppendContent(value_); @@ -172,7 +174,8 @@ void GetCmd::Do() { } void GetCmd::ReadCache() { - auto s = db_->cache()->Get(key_, &value_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + auto s = db_->cache()->Get(CachePrefixKeyK, &value_); if (s.ok()) { res_.AppendStringLen(value_.size()); res_.AppendContent(value_); @@ -191,7 +194,8 @@ void GetCmd::DoUpdateCache() { return; } if (s_.ok()) { - db_->cache()->WriteKVToCache(key_, value_, ttl_millsec_ > 0 ? ttl_millsec_ / 1000 : ttl_millsec_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); } } @@ -205,7 +209,10 @@ void DelCmd::DoInitial() { } void DelCmd::Do() { - int64_t count = db_->storage()->Del(keys_); + std::map type_status; + + int64_t count = db_->storage()->Del(keys_, &type_status); + if (count >= 0) { res_.AppendInteger(count); s_ = rocksdb::Status::OK(); @@ -225,13 +232,21 @@ void DelCmd::DoThroughDB() { void DelCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Del(keys_); + std::vector v; + for (auto key : keys_) { + v.emplace_back(PCacheKeyPrefixK + key); + v.emplace_back(PCacheKeyPrefixL + key); + v.emplace_back(PCacheKeyPrefixZ + key); + v.emplace_back(PCacheKeyPrefixS + key); + v.emplace_back(PCacheKeyPrefixH + key); + } + db_->cache()->Del(v); } } void DelCmd::Split(const HintKeys& hint_keys) { std::map type_status; - int64_t count = db_->storage()->Del(hint_keys.keys); + int64_t count = db_->storage()->Del(hint_keys.keys, &type_status); if (count >= 0) { split_res_ += count; } else { @@ -260,7 +275,7 @@ void IncrCmd::DoInitial() { } void IncrCmd::Do() { - s_ = db_->storage()->Incrby(key_, 1, &new_value_, &expired_timestamp_millsec_); + s_ = db_->storage()->Incrby(key_, 1, &new_value_, &expired_timestamp_sec_); if (s_.ok()) { res_.AppendContent(":" + std::to_string(new_value_)); AddSlotKey("k", key_, db_); @@ -281,7 +296,8 @@ void IncrCmd::DoThroughDB() { void IncrCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Incrxx(key_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->Incrxx(CachePrefixKeyK); } } @@ -324,7 +340,7 @@ void IncrbyCmd::DoInitial() { } void IncrbyCmd::Do() { - s_ = db_->storage()->Incrby(key_, by_, &new_value_, &expired_timestamp_millsec_); + s_ = db_->storage()->Incrby(key_, by_, &new_value_, &expired_timestamp_sec_); if (s_.ok()) { res_.AppendContent(":" + std::to_string(new_value_)); AddSlotKey("k", key_, db_); @@ -345,7 +361,8 @@ void IncrbyCmd::DoThroughDB() { void IncrbyCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->IncrByxx(key_, by_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->IncrByxx(CachePrefixKeyK, by_); } } @@ -389,7 +406,7 @@ void IncrbyfloatCmd::DoInitial() { } void IncrbyfloatCmd::Do() { - s_ = db_->storage()->Incrbyfloat(key_, value_, &new_value_, &expired_timestamp_millsec_); + s_ = db_->storage()->Incrbyfloat(key_, value_, &new_value_, &expired_timestamp_sec_); if (s_.ok()) { res_.AppendStringLenUint64(new_value_.size()); res_.AppendContent(new_value_); @@ -413,7 +430,8 @@ void IncrbyfloatCmd::DoUpdateCache() { if (s_.ok()) { long double long_double_by; if (storage::StrToLongDouble(value_.data(), value_.size(), &long_double_by) != -1) { - db_->cache()->Incrbyfloatxx(key_, long_double_by); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->Incrbyfloatxx(CachePrefixKeyK, long_double_by); } } } @@ -453,6 +471,7 @@ void DecrCmd::DoInitial() { } void DecrCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_= db_->storage()->Decrby(key_, 1, &new_value_); if (s_.ok()) { res_.AppendContent(":" + std::to_string(new_value_)); @@ -473,7 +492,8 @@ void DecrCmd::DoThroughDB() { void DecrCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Decrxx(key_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->Decrxx(CachePrefixKeyK); } } @@ -490,6 +510,7 @@ void DecrbyCmd::DoInitial() { } void DecrbyCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->Decrby(key_, by_, &new_value_); if (s_.ok()) { AddSlotKey("k", key_, db_); @@ -511,7 +532,8 @@ void DecrbyCmd::DoThroughDB() { void DecrbyCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->DecrByxx(key_, by_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->DecrByxx(CachePrefixKeyK, by_); } } @@ -526,6 +548,7 @@ void GetsetCmd::DoInitial() { void GetsetCmd::Do() { std::string old_value; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->GetSet(key_, new_value_, &old_value); if (s_.ok()) { if (old_value.empty()) { @@ -548,7 +571,8 @@ void GetsetCmd::DoThroughDB() { void GetsetCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->SetxxWithoutTTL(key_, new_value_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->SetxxWithoutTTL(CachePrefixKeyK, new_value_); } } @@ -563,7 +587,7 @@ void AppendCmd::DoInitial() { void AppendCmd::Do() { int32_t new_len = 0; - s_ = db_->storage()->Append(key_, value_, &new_len, &expired_timestamp_millsec_, new_value_); + s_ = db_->storage()->Append(key_, value_, &new_len, &expired_timestamp_sec_, new_value_); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(new_len); AddSlotKey("k", key_, db_); @@ -580,7 +604,8 @@ void AppendCmd::DoThroughDB() { void AppendCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Appendxx(key_, value_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->Appendxx(CachePrefixKeyK, value_); } } @@ -641,6 +666,7 @@ void MgetCmd::Do() { cache_miss_keys_ = keys_; } db_value_status_array_.clear(); + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->MGetWithTTL(cache_miss_keys_, &db_value_status_array_); if (!s_.ok()) { if (s_.IsInvalidArgument()) { @@ -657,6 +683,7 @@ void MgetCmd::Do() { void MgetCmd::Split(const HintKeys& hint_keys) { std::vector vss; const std::vector& keys = hint_keys.keys; + STAGE_TIMER_GUARD(storage_duration_ms, true); rocksdb::Status s = db_->storage()->MGet(keys, &vss); if (s.ok()) { if (hint_keys.hints.size() != vss.size()) { @@ -689,6 +716,7 @@ void MgetCmd::DoThroughDB() { } void MgetCmd::ReadCache() { + STAGE_TIMER_GUARD(cache_duration_ms, true); for (const auto key : keys_) { std::string value; auto s = db_->cache()->Get(const_cast(key), &value); @@ -707,6 +735,7 @@ void MgetCmd::ReadCache() { void MgetCmd::DoUpdateCache() { size_t db_index = 0; + STAGE_TIMER_GUARD(cache_duration_ms, true); for (const auto key : cache_miss_keys_) { if (db_index < db_value_status_array_.size() && db_value_status_array_[db_index].status.ok()) { int64_t ttl_millsec = db_value_status_array_[db_index].ttl_millsec; @@ -779,6 +808,7 @@ void KeysCmd::Do() { size_t raw_limit = g_pika_conf->max_client_response_size(); std::string raw; std::vector keys; + STAGE_TIMER_GUARD(storage_duration_ms, true); do { keys.clear(); cursor = db_->storage()->Scan(type_, cursor, pattern_, PIKA_SCAN_STEP_LENGTH, &keys); @@ -808,6 +838,7 @@ void SetnxCmd::DoInitial() { void SetnxCmd::Do() { success_ = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->Setnx(key_, value_, &success_); if (s_.ok()) { res_.AppendInteger(success_); @@ -852,7 +883,7 @@ void SetexCmd::DoInitial() { } void SetexCmd::Do() { - s_ = db_->storage()->Setex(key_, value_, ttl_sec_ * 1000); + s_ = db_->storage()->Setex(key_, value_, static_cast(sec_)); if (s_.ok()) { res_.SetRes(CmdRes::kOk); AddSlotKey("k", key_, db_); @@ -869,7 +900,8 @@ void SetexCmd::DoThroughDB() { void SetexCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Setxx(key_, value_, ttl_sec_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->Setxx(CachePrefixKeyK, value_, sec_); } } @@ -912,7 +944,7 @@ void PsetexCmd::DoInitial() { } void PsetexCmd::Do() { - s_ = db_->storage()->Setex(key_, value_, ttl_millsec); + s_ = db_->storage()->Setex(key_, value_, static_cast(usec_ / 1000)); if (s_.ok()) { res_.SetRes(CmdRes::kOk); } else if (s_.IsInvalidArgument()) { @@ -928,7 +960,8 @@ void PsetexCmd::DoThroughDB() { void PsetexCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Setxx(key_, value_, ttl_millsec / 1000); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->Setxx(CachePrefixKeyK, value_, static_cast(usec_ / 1000)); } } @@ -967,6 +1000,7 @@ void DelvxCmd::DoInitial() { } void DelvxCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); rocksdb::Status s = db_->storage()->Delvx(key_, value_, &success_); if (s.ok() || s.IsNotFound()) { res_.AppendInteger(success_); @@ -994,6 +1028,7 @@ void MsetCmd::DoInitial() { } void MsetCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->MSet(kvs_); if (s_.ok()) { res_.SetRes(CmdRes::kOk); @@ -1014,6 +1049,7 @@ void MsetCmd::DoThroughDB() { void MsetCmd::DoUpdateCache() { if (s_.ok()) { + std::string CachePrefixKeyK; for (auto key : kvs_) { db_->cache()->SetxxWithoutTTL(key.key, key.value); } @@ -1035,6 +1071,7 @@ void MsetCmd::Split(const HintKeys& hint_keys) { return; } } + STAGE_TIMER_GUARD(storage_duration_ms, true); storage::Status s = db_->storage()->MSet(kvs); if (s.ok()) { res_.SetRes(CmdRes::kOk); @@ -1079,6 +1116,7 @@ void MsetnxCmd::DoInitial() { void MsetnxCmd::Do() { success_ = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); rocksdb::Status s = db_->storage()->MSetnx(kvs_, &success_); if (s.ok()) { res_.AppendInteger(success_); @@ -1130,6 +1168,7 @@ void GetrangeCmd::DoInitial() { void GetrangeCmd::Do() { std::string substr; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_= db_->storage()->Getrange(key_, start_, end_, &substr); if (s_.ok() || s_.IsNotFound()) { res_.AppendStringLenUint64(substr.size()); @@ -1143,7 +1182,8 @@ void GetrangeCmd::Do() { void GetrangeCmd::ReadCache() { std::string substr; - auto s = db_->cache()->GetRange(key_, start_, end_, &substr); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + auto s = db_->cache()->GetRange(CachePrefixKeyK, start_, end_, &substr); if (s.ok()) { res_.AppendStringLen(substr.size()); res_.AppendContent(substr); @@ -1155,6 +1195,7 @@ void GetrangeCmd::ReadCache() { void GetrangeCmd::DoThroughDB() { res_.clear(); std::string substr; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->GetrangeWithValue(key_, start_, end_, &substr, &value_, &sec_); if (s_.ok()) { res_.AppendStringLen(substr.size()); @@ -1169,7 +1210,8 @@ void GetrangeCmd::DoThroughDB() { void GetrangeCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->WriteKVToCache(key_, value_, sec_); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); } } @@ -1188,6 +1230,7 @@ void SetrangeCmd::DoInitial() { void SetrangeCmd::Do() { int32_t new_len = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->Setrange(key_, offset_, value_, &new_len); if (s_.ok()) { res_.AppendInteger(new_len); @@ -1206,6 +1249,7 @@ void SetrangeCmd::DoThroughDB() { void SetrangeCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->SetRangeIfKeyExist(CachePrefixKeyK, offset_, value_); } } @@ -1220,6 +1264,7 @@ void StrlenCmd::DoInitial() { void StrlenCmd::Do() { int32_t len = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->Strlen(key_, &len); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(len); @@ -1232,7 +1277,8 @@ void StrlenCmd::Do() { void StrlenCmd::ReadCache() { int32_t len = 0; - auto s= db_->cache()->Strlen(key_, &len); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + auto s= db_->cache()->Strlen(CachePrefixKeyK, &len); if (s.ok()) { res_.AppendInteger(len); } else { @@ -1242,7 +1288,7 @@ void StrlenCmd::ReadCache() { void StrlenCmd::DoThroughDB() { res_.clear(); - s_ = db_->storage()->GetWithTTL(key_, &value_, &ttl_millsec); + s_ = db_->storage()->GetWithTTL(key_, &value_, &sec_); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(value_.size()); } else { @@ -1252,7 +1298,8 @@ void StrlenCmd::DoThroughDB() { void StrlenCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->WriteKVToCache(key_, value_, ttl_millsec > 0 ? ttl_millsec : ttl_millsec / 1000); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + db_->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); } } @@ -1266,7 +1313,8 @@ void ExistsCmd::DoInitial() { } void ExistsCmd::Do() { - int64_t res = db_->storage()->Exists(keys_); + std::map type_status; + int64_t res = db_->storage()->Exists(keys_, &type_status); if (res != -1) { res_.AppendInteger(res); } else { @@ -1275,7 +1323,8 @@ void ExistsCmd::Do() { } void ExistsCmd::Split(const HintKeys& hint_keys) { - int64_t res = db_->storage()->Exists(hint_keys.keys); + std::map type_status; + int64_t res = db_->storage()->Exists(hint_keys.keys, &type_status); if (res != -1) { split_res_ += res; } else { @@ -1290,9 +1339,21 @@ void ExistsCmd::ReadCache() { res_.SetRes(CmdRes::kCacheMiss); return; } - bool exist = db_->cache()->Exists(keys_[0]); - if (exist) { - res_.AppendInteger(1); + uint32_t nums = 0; + std::vector v; + v.emplace_back(PCacheKeyPrefixK + keys_[0]); + v.emplace_back(PCacheKeyPrefixL + keys_[0]); + v.emplace_back(PCacheKeyPrefixZ + keys_[0]); + v.emplace_back(PCacheKeyPrefixS + keys_[0]); + v.emplace_back(PCacheKeyPrefixH + keys_[0]); + for (auto key : v) { + bool exist = db_->cache()->Exists(key); + if (exist) { + nums++; + } + } + if (nums > 0) { + res_.AppendInteger(nums); } else { res_.SetRes(CmdRes::kCacheMiss); } @@ -1316,7 +1377,8 @@ void ExpireCmd::DoInitial() { } void ExpireCmd::Do() { - int32_t res = db_->storage()->Expire(key_, ttl_sec_ * 1000); + std::map type_status; + int64_t res = db_->storage()->Expire(key_, static_cast(sec_), &type_status); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1354,7 +1416,15 @@ void ExpireCmd::DoThroughDB() { void ExpireCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Expire(key_, ttl_sec_); + std::vector v; + v.emplace_back(PCacheKeyPrefixK + key_); + v.emplace_back(PCacheKeyPrefixL + key_); + v.emplace_back(PCacheKeyPrefixZ + key_); + v.emplace_back(PCacheKeyPrefixS + key_); + v.emplace_back(PCacheKeyPrefixH + key_); + for (auto key : v) { + db_->cache()->Expire(key, sec_); + } } } @@ -1371,7 +1441,8 @@ void PexpireCmd::DoInitial() { } void PexpireCmd::Do() { - int64_t res = db_->storage()->Expire(key_, ttl_millsec); + std::map type_status; + int64_t res = db_->storage()->Expire(key_, static_cast(msec_ / 1000), &type_status); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1409,7 +1480,15 @@ void PexpireCmd::DoThroughDB() { void PexpireCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Expire(key_, ttl_millsec); + std::vector v; + v.emplace_back(PCacheKeyPrefixK + key_); + v.emplace_back(PCacheKeyPrefixL + key_); + v.emplace_back(PCacheKeyPrefixZ + key_); + v.emplace_back(PCacheKeyPrefixS + key_); + v.emplace_back(PCacheKeyPrefixH + key_); + for (auto key : v){ + db_->cache()->Expire(key, msec_/1000); + } } } @@ -1426,7 +1505,8 @@ void ExpireatCmd::DoInitial() { } void ExpireatCmd::Do() { - int32_t res = db_->storage()->Expireat(key_, time_stamp_sec_ * 1000); + std::map type_status; + int32_t res = db_->storage()->Expireat(key_, static_cast(time_stamp_), &type_status); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1442,7 +1522,15 @@ void ExpireatCmd::DoThroughDB() { void ExpireatCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Expireat(key_, time_stamp_sec_); + std::vector v; + v.emplace_back(PCacheKeyPrefixK + key_); + v.emplace_back(PCacheKeyPrefixL + key_); + v.emplace_back(PCacheKeyPrefixZ + key_); + v.emplace_back(PCacheKeyPrefixS + key_); + v.emplace_back(PCacheKeyPrefixH + key_); + for (auto key : v) { + db_->cache()->Expireat(key, time_stamp_); + } } } @@ -1459,7 +1547,8 @@ void PexpireatCmd::DoInitial() { } void PexpireatCmd::Do() { - int32_t res = db_->storage()->Expireat(key_, static_cast(time_stamp_millsec_)); + std::map type_status; + int32_t res = db_->storage()->Expireat(key_, static_cast(time_stamp_ms_ / 1000), &type_status); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1475,7 +1564,15 @@ void PexpireatCmd::DoThroughDB() { void PexpireatCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Expireat(key_, time_stamp_millsec_ / 1000); + std::vector v; + v.emplace_back(PCacheKeyPrefixK + key_); + v.emplace_back(PCacheKeyPrefixL + key_); + v.emplace_back(PCacheKeyPrefixZ + key_); + v.emplace_back(PCacheKeyPrefixS + key_); + v.emplace_back(PCacheKeyPrefixH + key_); + for (auto key : v) { + db_->cache()->Expireat(key, time_stamp_ms_ / 1000); + } } } @@ -1488,20 +1585,53 @@ void TtlCmd::DoInitial() { } void TtlCmd::Do() { - int64_t ttl_sec_ = db_->storage()->TTL(key_); - if (ttl_sec_ == -3) { - res_.SetRes(CmdRes::kErrOther, "ttl internal error"); + std::map type_timestamp; + std::map type_status; + type_timestamp = db_->storage()->TTL(key_, &type_status); + for (const auto& item : type_timestamp) { + // mean operation exception errors happen in database + if (item.second == -3) { + res_.SetRes(CmdRes::kErrOther, "ttl internal error"); + return; + } + } + if (type_timestamp[storage::kStrings] != -2) { + res_.AppendInteger(type_timestamp[storage::kStrings]); + } else if (type_timestamp[storage::kHashes] != -2) { + res_.AppendInteger(type_timestamp[storage::kHashes]); + } else if (type_timestamp[storage::kLists] != -2) { + res_.AppendInteger(type_timestamp[storage::kLists]); + } else if (type_timestamp[storage::kZSets] != -2) { + res_.AppendInteger(type_timestamp[storage::kZSets]); + } else if (type_timestamp[storage::kSets] != -2) { + res_.AppendInteger(type_timestamp[storage::kSets]); } else { res_.AppendInteger(ttl_sec_); } } void TtlCmd::ReadCache() { - int64_t timestamp = db_->cache()->TTL(key_); - if (timestamp == -3) { - res_.SetRes(CmdRes::kErrOther, "ttl internal error"); - } else if (timestamp != -2) { - res_.AppendInteger(timestamp); + rocksdb::Status s; + std::map type_timestamp; + std::map type_status; + type_timestamp = db_->cache()->TTL(key_, &type_status); + for (const auto& item : type_timestamp) { + // mean operation exception errors happen in database + if (item.second == -3) { + res_.SetRes(CmdRes::kErrOther, "ttl internal error"); + return; + } + } + if (type_timestamp[storage::kStrings] != -2) { + res_.AppendInteger(type_timestamp[storage::kStrings]); + } else if (type_timestamp[storage::kHashes] != -2) { + res_.AppendInteger(type_timestamp[storage::kHashes]); + } else if (type_timestamp[storage::kLists] != -2) { + res_.AppendInteger(type_timestamp[storage::kLists]); + } else if (type_timestamp[storage::kZSets] != -2) { + res_.AppendInteger(type_timestamp[storage::kZSets]); + } else if (type_timestamp[storage::kSets] != -2) { + res_.AppendInteger(type_timestamp[storage::kSets]); } else { res_.SetRes(CmdRes::kCacheMiss); } @@ -1521,17 +1651,97 @@ void PttlCmd::DoInitial() { } void PttlCmd::Do() { - int64_t ttl_millsec = db_->storage()->PTTL(key_); - if (ttl_millsec == -3) { - res_.SetRes(CmdRes::kErrOther, "ttl internal error"); + std::map type_timestamp; + std::map type_status; + type_timestamp = db_->storage()->TTL(key_, &type_status); + for (const auto& item : type_timestamp) { + // mean operation exception errors happen in database + if (item.second == -3) { + res_.SetRes(CmdRes::kErrOther, "ttl internal error"); + return; + } + } + if (type_timestamp[storage::kStrings] != -2) { + if (type_timestamp[storage::kStrings] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[storage::kStrings] * 1000); + } + } else if (type_timestamp[storage::kHashes] != -2) { + if (type_timestamp[storage::kHashes] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[storage::kHashes] * 1000); + } + } else if (type_timestamp[storage::kLists] != -2) { + if (type_timestamp[storage::kLists] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[storage::kLists] * 1000); + } + } else if (type_timestamp[storage::kSets] != -2) { + if (type_timestamp[storage::kSets] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[storage::kSets] * 1000); + } + } else if (type_timestamp[storage::kZSets] != -2) { + if (type_timestamp[storage::kZSets] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[storage::kZSets] * 1000); + } } else { - res_.AppendInteger(ttl_millsec); + // mean this key not exist + res_.AppendInteger(-2); } } void PttlCmd::ReadCache() { - // redis cache don't support pttl cache, so read directly from db - DoThroughDB(); + std::map type_timestamp; + std::map type_status; + type_timestamp = db_->cache()->TTL(key_, &type_status); + for (const auto& item : type_timestamp) { + // mean operation exception errors happen in database + if (item.second == -3) { + res_.SetRes(CmdRes::kErrOther, "ttl internal error"); + return; + } + } + if (type_timestamp[storage::kStrings] != -2) { + if (type_timestamp[storage::kStrings] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[storage::kStrings] * 1000); + } + } else if (type_timestamp[storage::kHashes] != -2) { + if (type_timestamp[storage::kHashes] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[storage::kHashes] * 1000); + } + } else if (type_timestamp[storage::kLists] != -2) { + if (type_timestamp[storage::kLists] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[storage::kLists] * 1000); + } + } else if (type_timestamp[storage::kSets] != -2) { + if (type_timestamp[storage::kSets] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[storage::kSets] * 1000); + } + } else if (type_timestamp[storage::kZSets] != -2) { + if (type_timestamp[storage::kZSets] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[storage::kZSets] * 1000); + } + } else { + // mean this key not exist + res_.SetRes(CmdRes::kCacheMiss); + } } void PttlCmd::DoThroughDB() { @@ -1548,7 +1758,8 @@ void PersistCmd::DoInitial() { } void PersistCmd::Do() { - int32_t res = db_->storage()->Persist(key_); + std::map type_status; + int32_t res = db_->storage()->Persist(key_, &type_status); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1564,7 +1775,15 @@ void PersistCmd::DoThroughDB() { void PersistCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->Persist(key_); + std::vector v; + v.emplace_back(PCacheKeyPrefixK + key_); + v.emplace_back(PCacheKeyPrefixL + key_); + v.emplace_back(PCacheKeyPrefixZ + key_); + v.emplace_back(PCacheKeyPrefixS + key_); + v.emplace_back(PCacheKeyPrefixH + key_); + for (auto key : v) { + db_->cache()->Persist(key); + } } } @@ -1577,9 +1796,8 @@ void TypeCmd::DoInitial() { } void TypeCmd::Do() { - enum storage::DataType type = storage::DataType::kNones; - std::string key_type; - rocksdb::Status s = db_->storage()->GetType(key_, type); + std::vector types(1); + rocksdb::Status s = db_->storage()->GetType(key_, true, types); if (s.ok()) { res_.AppendContent("+" + std::string(DataTypeToString(type))); } else if (s_.IsInvalidArgument()) { @@ -1590,10 +1808,8 @@ void TypeCmd::Do() { } void TypeCmd::ReadCache() { - enum storage::DataType type = storage::DataType::kNones; - std::string key_type; - // TODO Cache GetType function - rocksdb::Status s = db_->storage()->GetType(key_, type); + std::vector types(1); + rocksdb::Status s = db_->storage()->GetType(key_, true, types); if (s.ok()) { res_.AppendContent("+" + std::string(DataTypeToString(type))); } else { @@ -1606,6 +1822,29 @@ void TypeCmd::DoThroughDB() { Do(); } +void PTypeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameType); + return; + } + key_ = argv_[1]; +} + +void PTypeCmd::Do() { + std::vector types(5); + rocksdb::Status s = db_->storage()->GetType(key_, false, types); + + if (s.ok()) { + res_.AppendArrayLenUint64(types.size()); + for (const auto& vs : types) { + res_.AppendStringLenUint64(vs.size()); + res_.AppendContent(vs); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + void ScanCmd::DoInitial() { if (!CheckArg(argv_.size())) { res_.SetRes(CmdRes::kWrongNum, kCmdNameScan); @@ -1664,6 +1903,7 @@ void ScanCmd::Do() { size_t raw_limit = g_pika_conf->max_client_response_size(); std::string raw; std::vector keys; + STAGE_TIMER_GUARD(storage_duration_ms, true); // To avoid memory overflow, we call the Scan method in batches do { keys.clear(); @@ -1740,6 +1980,7 @@ void ScanxCmd::DoInitial() { void ScanxCmd::Do() { std::string next_key; std::vector keys; + STAGE_TIMER_GUARD(storage_duration_ms, true); rocksdb::Status s = db_->storage()->Scanx(type_, start_key_, pattern_, count_, &keys, &next_key); if (s.ok()) { @@ -1771,7 +2012,7 @@ void PKSetexAtCmd::DoInitial() { } void PKSetexAtCmd::Do() { - s_ = db_->storage()->PKSetexAt(key_, value_, static_cast(time_stamp_sec_ * 1000)); + s_ = db_->storage()->PKSetexAt(key_, value_, static_cast(time_stamp_)); if (s_.ok()) { res_.SetRes(CmdRes::kOk); } else if (s_.IsInvalidArgument()) { @@ -1787,7 +2028,8 @@ void PKSetexAtCmd::DoThroughDB() { void PKSetexAtCmd::DoUpdateCache() { if (s_.ok()) { - auto expire = time_stamp_sec_ - static_cast(std::time(nullptr)); + auto expire = time_stamp_ - static_cast(std::time(nullptr)); + std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; if (expire <= 0) [[unlikely]] { db_->cache()->Del({key_}); return; @@ -1854,6 +2096,7 @@ void PKScanRangeCmd::Do() { std::string next_key; std::vector keys; std::vector kvs; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->PKScanRange(type_, key_start_, key_end_, pattern_, static_cast(limit_), &keys, &kvs, &next_key); if (s_.ok()) { @@ -1939,6 +2182,7 @@ void PKRScanRangeCmd::Do() { std::string next_key; std::vector keys; std::vector kvs; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->PKRScanRange(type_, key_start_, key_end_, pattern_, static_cast(limit_), &keys, &kvs, &next_key); diff --git a/src/pika_list.cc b/src/pika_list.cc index 8a0cb16168..c4fe88583e 100644 --- a/src/pika_list.cc +++ b/src/pika_list.cc @@ -30,6 +30,7 @@ void LIndexCmd::DoInitial() { void LIndexCmd::Do() { std::string value; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->LIndex(key_, index_, &value); if (s_.ok()) { res_.AppendString(value); @@ -44,7 +45,8 @@ void LIndexCmd::Do() { void LIndexCmd::ReadCache() { std::string value; - auto s = db_->cache()->LIndex(key_, index_, &value); + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->LIndex(CachePrefixKeyL, index_, &value); if (s.ok()) { res_.AppendString(value); } else if (s.IsNotFound()) { @@ -61,6 +63,8 @@ void LIndexCmd::DoThroughDB() { void LIndexCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_LIST, key_, db_); } } @@ -86,6 +90,7 @@ void LInsertCmd::DoInitial() { void LInsertCmd::Do() { int64_t llen = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->LInsert(key_, dir_, pivot_, value_, &llen); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(llen); @@ -103,7 +108,9 @@ void LInsertCmd::DoThroughDB() { void LInsertCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->LInsert(key_, dir_, pivot_, value_); + std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->LInsert(CachePrefixKeyL, dir_, pivot_, value_); } } @@ -117,6 +124,7 @@ void LLenCmd::DoInitial() { void LLenCmd::Do() { uint64_t llen = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->LLen(key_, &llen); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(static_cast(llen)); @@ -129,8 +137,9 @@ void LLenCmd::Do() { void LLenCmd::ReadCache() { uint64_t llen = 0; - auto s = db_->cache()->LLen(key_, &llen); - if (s.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->LLen(CachePrefixKeyL, &llen); + if (s.ok()){ res_.AppendInteger(llen); } else if (s.IsNotFound()) { res_.SetRes(CmdRes::kCacheMiss); @@ -146,6 +155,8 @@ void LLenCmd::DoThroughDB() { void LLenCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_LIST, key_, db_); } } @@ -265,6 +276,7 @@ void LPushCmd::DoInitial() { void LPushCmd::Do() { uint64_t llen = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->LPush(key_, values_, &llen); if (s_.ok()) { res_.AppendInteger(static_cast(llen)); @@ -289,6 +301,7 @@ void LPushCmd::DoThroughDB() { void LPushCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->LPushIfKeyExist(CachePrefixKeyL, values_); } } @@ -357,6 +370,7 @@ void BLPopCmd::DoInitial() { } void BLPopCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); for (auto& this_key : keys_) { std::vector values; rocksdb::Status s = db_->storage()->LPop(this_key, 1, &values); @@ -423,6 +437,7 @@ void LPopCmd::DoInitial() { void LPopCmd::Do() { std::vector elements; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->LPop(key_, count_, &elements); if (s_.ok()) { @@ -448,7 +463,8 @@ void LPopCmd::DoThroughDB() { void LPopCmd::DoUpdateCache() { if (s_.ok()) { std::string value; - db_->cache()->LPop(key_, &value); + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->LPop(CachePrefixKeyL, &value); } } @@ -466,6 +482,7 @@ void LPushxCmd::DoInitial() { void LPushxCmd::Do() { uint64_t llen = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->LPushx(key_, values_, &llen); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(static_cast(llen)); @@ -484,6 +501,7 @@ void LPushxCmd::DoThroughDB() { void LPushxCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->LPushIfKeyExist(CachePrefixKeyL, values_); } } @@ -507,6 +525,7 @@ void LRangeCmd::DoInitial() { void LRangeCmd::Do() { std::vector values; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->LRange(key_, left_, right_, &values); if (s_.ok()) { res_.AppendArrayLenUint64(values.size()); @@ -524,7 +543,9 @@ void LRangeCmd::Do() { void LRangeCmd::ReadCache() { std::vector values; - auto s = db_->cache()->LRange(key_, left_, right_, &values); + std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->LRange(CachePrefixKeyL, left_, right_, &values); if (s.ok()) { res_.AppendArrayLen(values.size()); for (const auto& value : values) { @@ -544,6 +565,8 @@ void LRangeCmd::DoThroughDB() { void LRangeCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_LIST, key_, db_); } } @@ -564,6 +587,7 @@ void LRemCmd::DoInitial() { void LRemCmd::Do() { uint64_t res = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->LRem(key_, count_, value_, &res); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(static_cast(res)); @@ -580,7 +604,9 @@ void LRemCmd::DoThroughDB() { void LRemCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->LRem(key_, count_, value_); + std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->LRem(CachePrefixKeyL, count_, value_); } } @@ -599,6 +625,7 @@ void LSetCmd::DoInitial() { } void LSetCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->LSet(key_, index_, value_); if (s_.ok()) { res_.SetRes(CmdRes::kOk); @@ -621,7 +648,9 @@ void LSetCmd::DoThroughDB() { void LSetCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->LSet(key_, index_, value_); + std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->LSet(CachePrefixKeyL, index_, value_); } } @@ -643,6 +672,7 @@ void LTrimCmd::DoInitial() { } void LTrimCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->LTrim(key_, start_, stop_); if (s_.ok() || s_.IsNotFound()) { res_.SetRes(CmdRes::kOk); @@ -659,11 +689,14 @@ void LTrimCmd::DoThroughDB() { void LTrimCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->LTrim(key_, start_, stop_); + std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->LTrim(CachePrefixKeyL, start_, stop_); } } void BRPopCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); for (auto& this_key : keys_) { std::vector values; s_ = db_->storage()->RPop(this_key, 1, &values); @@ -759,6 +792,7 @@ void RPopCmd::DoInitial() { void RPopCmd::Do() { std::vector elements; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->RPop(key_, count_, &elements); if (s_.ok()) { if (elements.size() > 1) { @@ -783,7 +817,8 @@ void RPopCmd::DoThroughDB() { void RPopCmd::DoUpdateCache() { if (s_.ok()) { std::string value; - db_->cache()->RPop(key_, &value); + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->RPop(CachePrefixKeyL, &value); } } @@ -801,6 +836,7 @@ void RPopLPushCmd::DoInitial() { void RPopLPushCmd::Do() { std::string value; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->RPoplpush(source_, receiver_, &value); if (s_.ok()) { AddSlotKey("k", receiver_, db_); @@ -851,6 +887,7 @@ void RPopLPushCmd::DoBinlog() { } void RPopLPushCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); std::vector value; value.resize(1); std::string CachePrefixKeyLR = PCacheKeyPrefixL + source_; @@ -877,6 +914,7 @@ void RPushCmd::DoInitial() { void RPushCmd::Do() { uint64_t llen = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->RPush(key_, values_, &llen); if (s_.ok()) { res_.AppendInteger(static_cast(llen)); @@ -901,6 +939,7 @@ void RPushCmd::DoThroughDB() { void RPushCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->RPushIfKeyExist(CachePrefixKeyL, values_); } } @@ -919,6 +958,7 @@ void RPushxCmd::DoInitial() { void RPushxCmd::Do() { uint64_t llen = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->RPushx(key_, values_, &llen); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(static_cast(llen)); @@ -937,6 +977,7 @@ void RPushxCmd::DoThroughDB() { void RPushxCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->RPushIfKeyExist(CachePrefixKeyL, values_); } } \ No newline at end of file diff --git a/src/pika_set.cc b/src/pika_set.cc index 66ca7f168e..963858c6ae 100644 --- a/src/pika_set.cc +++ b/src/pika_set.cc @@ -23,6 +23,7 @@ void SAddCmd::DoInitial() { void SAddCmd::Do() { int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SAdd(key_, members_, &count); if (s_.IsInvalidArgument()) { res_.SetRes(CmdRes::kMultiKey); @@ -41,7 +42,9 @@ void SAddCmd::DoThroughDB() { void SAddCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->SAddIfKeyExist(key_, members_); + std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->SAddIfKeyExist(CachePrefixKeyS, members_); } } @@ -68,6 +71,7 @@ void SPopCmd::DoInitial() { } void SPopCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SPop(key_, &members_, count_); if (s_.ok()) { res_.AppendArrayLenUint64(members_.size()); @@ -91,7 +95,9 @@ void SPopCmd::DoThroughDB() { void SPopCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->SRem(key_, members_); + std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->SRem(CachePrefixKeyS, members_); } } @@ -106,7 +112,7 @@ void SPopCmd::DoBinlog() { for (auto m = members_.begin(); m != members_.end(); ++m) { srem_args.emplace_back(*m); } - + srem_cmd_->Initial(srem_args, db_name_); srem_cmd_->SetConn(GetConn()); srem_cmd_->SetResp(resp_.lock()); @@ -123,6 +129,7 @@ void SCardCmd::DoInitial() { void SCardCmd::Do() { int32_t card = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SCard(key_, &card); if (s_.ok()) { res_.AppendInteger(card); @@ -138,7 +145,9 @@ void SCardCmd::Do() { void SCardCmd::ReadCache() { uint64_t card = 0; - auto s = db_->cache()->SCard(key_, &card); + std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->SCard(CachePrefixKeyS, &card); if (s.ok()) { res_.AppendInteger(card); } else if (s.IsNotFound()) { @@ -155,6 +164,8 @@ void SCardCmd::DoThroughDB() { void SCardCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, db_); } } @@ -169,6 +180,7 @@ void SMembersCmd::DoInitial() { void SMembersCmd::Do() { std::vector members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SMembers(key_, &members); if (s_.ok()) { res_.AppendArrayLenUint64(members.size()); @@ -188,7 +200,9 @@ void SMembersCmd::Do() { void SMembersCmd::ReadCache() { std::vector members; - auto s = db_->cache()->SMembers(key_, &members); + std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->SMembers(CachePrefixKeyS, &members); if (s.ok()) { res_.AppendArrayLen(members.size()); for (const auto& member : members) { @@ -209,6 +223,8 @@ void SMembersCmd::DoThroughDB() { void SMembersCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, db_); } } @@ -254,6 +270,7 @@ void SScanCmd::DoInitial() { void SScanCmd::Do() { int64_t next_cursor = 0; std::vector members; + STAGE_TIMER_GUARD(storage_duration_ms, true); rocksdb::Status s = db_->storage()->SScan(key_, cursor_, pattern_, count_, &members, &next_cursor); if (s.ok()) { @@ -295,6 +312,7 @@ void SRemCmd::DoInitial() { } void SRemCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SRem(key_, members_, &deleted_); if (s_.ok()) { res_.AppendInteger(deleted_); @@ -314,10 +332,13 @@ void SRemCmd::DoThroughDB() { void SRemCmd::DoUpdateCache() { if (s_.ok() && deleted_ > 0) { - db_->cache()->SRem(key_, members_); + std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->SRem(CachePrefixKeyS, members_); } } + void SUnionCmd::DoInitial() { if (!CheckArg(argv_.size())) { res_.SetRes(CmdRes::kWrongNum, kCmdNameSUnion); @@ -329,6 +350,7 @@ void SUnionCmd::DoInitial() { void SUnionCmd::Do() { std::vector members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SUnion(keys_, &members); if (s_.ok()) { res_.AppendArrayLenUint64(members.size()); @@ -356,6 +378,7 @@ void SUnionstoreCmd::DoInitial() { void SUnionstoreCmd::Do() { int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SUnionstore(dest_key_, keys_, value_to_dest_, &count); if (s_.ok()) { res_.AppendInteger(count); @@ -373,7 +396,8 @@ void SUnionstoreCmd::DoThroughDB() { void SUnionstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(dest_key_); + v.emplace_back(PCacheKeyPrefixS + dest_key_); + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->Del(v); } } @@ -429,6 +453,7 @@ void SInterCmd::DoInitial() { void SInterCmd::Do() { std::vector members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SInter(keys_, &members); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(members.size()); @@ -456,6 +481,7 @@ void SInterstoreCmd::DoInitial() { void SInterstoreCmd::Do() { int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SInterstore(dest_key_, keys_, value_to_dest_, &count); if (s_.ok()) { res_.AppendInteger(count); @@ -473,7 +499,8 @@ void SInterstoreCmd::DoThroughDB() { void SInterstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(dest_key_); + v.emplace_back(PCacheKeyPrefixS + dest_key_); + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->Del(v); } } @@ -489,6 +516,7 @@ void SIsmemberCmd::DoInitial() { void SIsmemberCmd::Do() { int32_t is_member = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SIsmember(key_, member_, &is_member); if (is_member != 0) { res_.AppendContent(":1"); @@ -503,7 +531,9 @@ void SIsmemberCmd::Do() { } void SIsmemberCmd::ReadCache() { - auto s = db_->cache()->SIsmember(key_, member_); + std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->SIsmember(CachePrefixKeyS, member_); if (s.ok()) { res_.AppendContent(":1"); } else if (s.IsNotFound()) { @@ -521,6 +551,8 @@ void SIsmemberCmd::DoThroughDB() { void SIsmemberCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, db_); } } @@ -536,6 +568,7 @@ void SDiffCmd::DoInitial() { void SDiffCmd::Do() { std::vector members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SDiff(keys_, &members); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(members.size()); @@ -563,6 +596,7 @@ void SDiffstoreCmd::DoInitial() { void SDiffstoreCmd::Do() { int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SDiffstore(dest_key_, keys_, value_to_dest_, &count); if (s_.ok()) { res_.AppendInteger(count); @@ -580,7 +614,8 @@ void SDiffstoreCmd::DoThroughDB() { void SDiffstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(dest_key_); + v.emplace_back(PCacheKeyPrefixS + dest_key_); + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->Del(v); } } @@ -597,6 +632,7 @@ void SMoveCmd::DoInitial() { void SMoveCmd::Do() { int32_t res = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SMove(src_key_, dest_key_, member_, &res); if (s_.ok()) { res_.AppendInteger(res); @@ -620,8 +656,11 @@ void SMoveCmd::DoUpdateCache() { if (s_.ok()) { std::vector members; members.emplace_back(member_); - db_->cache()->SRem(src_key_, members); - db_->cache()->SAddIfKeyExist(dest_key_, members); + std::string CachePrefixKeyS = PCacheKeyPrefixS + src_key_; + std::string CachePrefixKeyD = PCacheKeyPrefixS + dest_key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->SRem(CachePrefixKeyS, members); + db_->cache()->SAddIfKeyExist(CachePrefixKeyD, members); } } @@ -673,6 +712,7 @@ void SRandmemberCmd::DoInitial() { void SRandmemberCmd::Do() { std::vector members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->SRandmember(key_, static_cast(count_), &members); if (s_.ok()) { if (!reply_arr && (static_cast(!members.empty()) != 0U)) { @@ -697,7 +737,9 @@ void SRandmemberCmd::Do() { void SRandmemberCmd::ReadCache() { std::vector members; - auto s = db_->cache()->SRandmember(key_, count_, &members); + std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->SRandmember(CachePrefixKeyS, count_, &members); if (s.ok()) { if (!reply_arr && members.size()) { res_.AppendStringLen(members[0].size()); @@ -723,6 +765,8 @@ void SRandmemberCmd::DoThroughDB() { void SRandmemberCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, db_); } } diff --git a/src/pika_zset.cc b/src/pika_zset.cc index 87b45723f0..8f06e885c3 100644 --- a/src/pika_zset.cc +++ b/src/pika_zset.cc @@ -36,6 +36,7 @@ void ZAddCmd::DoInitial() { void ZAddCmd::Do() { int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZAdd(key_, score_members, &count); if (s_.ok()) { res_.AppendInteger(count); @@ -53,7 +54,9 @@ void ZAddCmd::DoThroughDB() { void ZAddCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->ZAddIfKeyExist(key_, score_members); + std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->ZAddIfKeyExist(CachePrefixKeyZ, score_members); } } @@ -67,6 +70,7 @@ void ZCardCmd::DoInitial() { void ZCardCmd::Do() { int32_t card = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZCard(key_, &card); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(card); @@ -131,6 +135,7 @@ void ZScanCmd::DoInitial() { void ZScanCmd::Do() { int64_t next_cursor = 0; std::vector score_members; + STAGE_TIMER_GUARD(storage_duration_ms, true); rocksdb::Status s = db_->storage()->ZScan(key_, cursor_, pattern_, count_, &score_members, &next_cursor); if (s.ok() || s.IsNotFound()) { res_.AppendContent("*2"); @@ -169,6 +174,7 @@ void ZIncrbyCmd::DoInitial() { void ZIncrbyCmd::Do() { double score = 0.0; + STAGE_TIMER_GUARD(storage_duration_ms, true); rocksdb::Status s = db_->storage()->ZIncrby(key_, member_, by_, &score); if (s.ok()) { score_ = score; @@ -190,7 +196,9 @@ void ZIncrbyCmd::DoThroughDB() { void ZIncrbyCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->ZIncrbyIfKeyExist(key_, member_, by_, this, db_); + std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->ZIncrbyIfKeyExist(CachePrefixKeyZ, member_, by_, this, db_); } } @@ -222,6 +230,7 @@ void ZRangeCmd::DoInitial() { void ZRangeCmd::Do() { std::vector score_members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRange(key_, static_cast(start_), static_cast(stop_), &score_members); if (s_.ok() || s_.IsNotFound()) { if (is_ws_) { @@ -251,6 +260,7 @@ void ZRangeCmd::Do() { void ZRangeCmd::ReadCache() { std::vector score_members; + STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->ZRange(key_, start_, stop_, &score_members, db_); if (s.ok()) { if (is_ws_) { @@ -286,6 +296,8 @@ void ZRangeCmd::DoThroughDB() { void ZRangeCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -300,6 +312,7 @@ void ZRevrangeCmd::DoInitial() { void ZRevrangeCmd::Do() { std::vector score_members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRevrange(key_, static_cast(start_), static_cast(stop_), &score_members); if (s_.ok() || s_.IsNotFound()) { if (is_ws_) { @@ -329,6 +342,7 @@ void ZRevrangeCmd::Do() { void ZRevrangeCmd::ReadCache() { std::vector score_members; + STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->ZRevrange(key_, start_, stop_, &score_members, db_); if (s.ok()) { @@ -365,6 +379,8 @@ void ZRevrangeCmd::DoThroughDB() { void ZRevrangeCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -457,6 +473,7 @@ void ZRangebyscoreCmd::Do() { return; } std::vector score_members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &score_members); if (s_.IsInvalidArgument()) { res_.SetRes(CmdRes::kMultiKey); @@ -497,6 +514,7 @@ void ZRangebyscoreCmd::ReadCache() { std::vector score_members; min_ = std::to_string(min_score_); max_ = std::to_string(max_score_); + STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->ZRangebyscore(key_, min_, max_, &score_members, this); if (s.ok()) { auto sm_count = score_members.size(); @@ -532,6 +550,8 @@ void ZRangebyscoreCmd::DoThroughDB() { void ZRangebyscoreCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -559,6 +579,7 @@ void ZRevrangebyscoreCmd::Do() { return; } std::vector score_members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRevrangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &score_members); if (s_.IsInvalidArgument()) { res_.SetRes(CmdRes::kMultiKey); @@ -597,6 +618,7 @@ void ZRevrangebyscoreCmd::ReadCache() { return; } std::vector score_members; + STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->ZRevrangebyscore(key_, min_, max_, &score_members, this, db_); if (s.ok()) { auto sm_count = score_members.size(); @@ -632,6 +654,8 @@ void ZRevrangebyscoreCmd::DoThroughDB() { void ZRevrangebyscoreCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -658,6 +682,7 @@ void ZCountCmd::Do() { } int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZCount(key_, min_score_, max_score_, left_close_, right_close_, &count); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(count); @@ -674,6 +699,7 @@ void ZCountCmd::ReadCache() { return; } uint64_t count = 0; + STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->ZCount(key_, min_, max_, &count, this); if (s.ok()) { res_.AppendInteger(count); @@ -691,6 +717,8 @@ void ZCountCmd::DoThroughDB() { void ZCountCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -706,6 +734,7 @@ void ZRemCmd::DoInitial() { } void ZRemCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRem(key_, members_, &deleted_); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(deleted_); @@ -722,7 +751,9 @@ void ZRemCmd::DoThroughDB() { void ZRemCmd::DoUpdateCache() { if (s_.ok() && deleted_ > 0) { - db_->cache()->ZRem(key_, members_, db_); + std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->ZRem(CachePrefixKeyZ, members_, db_); } } @@ -794,6 +825,7 @@ void ZUnionstoreCmd::DoInitial() { void ZUnionstoreCmd::Do() { int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZUnionstore(dest_key_, keys_, weights_, aggregate_, value_to_dest_, &count); if (s_.ok()) { res_.AppendInteger(count); @@ -812,7 +844,8 @@ void ZUnionstoreCmd::DoThroughDB() { void ZUnionstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(dest_key_); + v.emplace_back(PCacheKeyPrefixZ + dest_key_); + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->Del(v); } } @@ -875,6 +908,7 @@ void ZInterstoreCmd::DoInitial() { void ZInterstoreCmd::Do() { int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZInterstore(dest_key_, keys_, weights_, aggregate_, value_to_dest_, &count); if (s_.ok()) { res_.AppendInteger(count); @@ -892,7 +926,8 @@ void ZInterstoreCmd::DoThroughDB() { void ZInterstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(dest_key_); + v.emplace_back(PCacheKeyPrefixZ + dest_key_); + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->Del(v); } } @@ -958,6 +993,7 @@ void ZRankCmd::DoInitial() { void ZRankCmd::Do() { int32_t rank = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRank(key_, member_, &rank); if (s_.ok()) { res_.AppendInteger(rank); @@ -972,6 +1008,7 @@ void ZRankCmd::Do() { void ZRankCmd::ReadCache() { int64_t rank = 0; + STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->ZRank(key_, member_, &rank, db_); if (s.ok()) { res_.AppendInteger(rank); @@ -989,6 +1026,8 @@ void ZRankCmd::DoThroughDB() { void ZRankCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -1003,6 +1042,7 @@ void ZRevrankCmd::DoInitial() { void ZRevrankCmd::Do() { int32_t revrank = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRevrank(key_, member_, &revrank); if (s_.ok()) { res_.AppendInteger(revrank); @@ -1017,6 +1057,7 @@ void ZRevrankCmd::Do() { void ZRevrankCmd::ReadCache() { int64_t revrank = 0; + STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->ZRevrank(key_, member_, &revrank, db_); if (s.ok()) { res_.AppendInteger(revrank); @@ -1034,6 +1075,8 @@ void ZRevrankCmd::DoThroughDB() { void ZRevrankCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -1049,6 +1092,7 @@ void ZScoreCmd::DoInitial() { void ZScoreCmd::Do() { double score = 0.0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZScore(key_, member_, &score); if (s_.ok()) { char buf[32]; @@ -1066,7 +1110,9 @@ void ZScoreCmd::Do() { void ZScoreCmd::ReadCache() { double score = 0.0; - auto s = db_->cache()->ZScore(key_, member_, &score, db_); + std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + auto s = db_->cache()->ZScore(CachePrefixKeyZ, member_, &score, db_); if (s.ok()) { char buf[32]; int64_t len = pstd::d2string(buf, sizeof(buf), score); @@ -1162,6 +1208,7 @@ void ZRangebylexCmd::Do() { return; } std::vector members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRangebylex(key_, min_member_, max_member_, left_close_, right_close_, &members); if (s_.IsInvalidArgument()) { res_.SetRes(CmdRes::kMultiKey); @@ -1187,6 +1234,7 @@ void ZRangebylexCmd::ReadCache() { return; } std::vector members; + STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->ZRangebylex(key_, min_, max_, &members, db_); if (s.ok()) { FitLimit(count_, offset_, members.size()); @@ -1212,6 +1260,8 @@ void ZRangebylexCmd::DoThroughDB() { void ZRangebylexCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -1240,6 +1290,7 @@ void ZRevrangebylexCmd::Do() { return; } std::vector members; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRangebylex(key_, min_member_, max_member_, left_close_, right_close_, &members); if (s_.IsInvalidArgument()) { res_.SetRes(CmdRes::kMultiKey); @@ -1266,6 +1317,7 @@ void ZRevrangebylexCmd::ReadCache() { return; } std::vector members; + STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->ZRevrangebylex(key_, min_, max_, &members, db_); if (s.ok()) { auto size = count_ < members.size() ? count_ : members.size(); @@ -1287,6 +1339,8 @@ void ZRevrangebylexCmd::DoThroughDB() { void ZRevrangebylexCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -1312,6 +1366,7 @@ void ZLexcountCmd::Do() { return; } int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZLexcount(key_, min_member_, max_member_, left_close_, right_close_, &count); if (s_.IsInvalidArgument()) { res_.SetRes(CmdRes::kMultiKey); @@ -1329,6 +1384,7 @@ void ZLexcountCmd::ReadCache() { return; } uint64_t count = 0; + STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->ZLexcount(key_, min_, max_, &count, db_); if (s.ok()) { res_.AppendInteger(count); @@ -1346,6 +1402,8 @@ void ZLexcountCmd::DoThroughDB() { void ZLexcountCmd::DoUpdateCache() { if (s_.ok()) { + STAGE_TIMER_GUARD(cache_duration_ms, true); + // record time cost in push key to queue db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -1370,6 +1428,7 @@ void ZRemrangebyrankCmd::DoInitial() { void ZRemrangebyrankCmd::Do() { int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRemrangebyrank(key_, static_cast(start_rank_), static_cast(stop_rank_), &count); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(count); @@ -1386,7 +1445,9 @@ void ZRemrangebyrankCmd::DoThroughDB() { void ZRemrangebyrankCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->ZRemrangebyrank(key_, min_, max_, ele_deleted_, db_); + std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->ZRemrangebyrank(CachePrefixKeyZ, min_, max_, ele_deleted_, db_); } } @@ -1409,6 +1470,7 @@ void ZRemrangebyscoreCmd::Do() { return; } int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRemrangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &count); if (s_.IsInvalidArgument()) { res_.SetRes(CmdRes::kMultiKey); @@ -1426,7 +1488,9 @@ void ZRemrangebyscoreCmd::DoThroughDB() { void ZRemrangebyscoreCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->ZRemrangebyscore(key_, min_, max_, db_); + std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->ZRemrangebyscore(CachePrefixKeyZ, min_, max_, db_); } } @@ -1450,6 +1514,7 @@ void ZRemrangebylexCmd::Do() { } int32_t count = 0; + STAGE_TIMER_GUARD(storage_duration_ms, true); s_ = db_->storage()->ZRemrangebylex(key_, min_member_, max_member_, left_close_, right_close_, &count); if (s_.IsInvalidArgument()) { res_.SetRes(CmdRes::kMultiKey); @@ -1467,7 +1532,9 @@ void ZRemrangebylexCmd::DoThroughDB() { void ZRemrangebylexCmd::DoUpdateCache() { if (s_.ok()) { - db_->cache()->ZRemrangebylex(key_, min_, max_, db_); + std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); + db_->cache()->ZRemrangebylex(CachePrefixKeyZ, min_, max_, db_); } } @@ -1488,6 +1555,7 @@ void ZPopmaxCmd::DoInitial() { } void ZPopmaxCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); std::vector score_members; rocksdb::Status s = db_->storage()->ZPopMax(key_, count_, &score_members); if (s.ok() || s.IsNotFound()) { @@ -1514,6 +1582,7 @@ void ZPopmaxCmd::DoThroughDB(){ void ZPopmaxCmd::DoUpdateCache(){ std::vector score_members; if(s_.ok() || s_.IsNotFound()){ + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->ZPopMax(key_, count_, &score_members, db_); } } @@ -1541,11 +1610,13 @@ void ZPopminCmd::DoThroughDB(){ void ZPopminCmd::DoUpdateCache(){ std::vector score_members; if(s_.ok() || s_.IsNotFound()){ + STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->ZPopMin(key_, count_, &score_members, db_); } } void ZPopminCmd::Do() { + STAGE_TIMER_GUARD(storage_duration_ms, true); std::vector score_members; rocksdb::Status s = db_->storage()->ZPopMin(key_, count_, &score_members); if (s.ok() || s.IsNotFound()) { diff --git a/src/pstd/include/stage_timer.h b/src/pstd/include/stage_timer.h new file mode 100644 index 0000000000..a6b491f1bd --- /dev/null +++ b/src/pstd/include/stage_timer.h @@ -0,0 +1,56 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#ifndef STAGE_TIMER_H_ +#define STAGE_TIMER_H_ + +#include "pstd/include/env.h" + +namespace pstd { +class StageTimer { + public: + explicit StageTimer (uint64_t* metric_ms, bool enabled) + : perf_counter_enabled_(enabled), + start_(0), + metric_ms_(metric_ms) {} + + ~StageTimer() { Stop(); } + + void Start() { + if (perf_counter_enabled_) { + start_ = time_now(); + } + } + + void Measure() { + if (start_) { + uint64_t now = time_now(); + *metric_ms_ += (now - start_) / 1000; + start_ = now; + } + } + + void Stop() { + if (start_) { + uint64_t duration = (time_now() - start_) / 1000; + if (perf_counter_enabled_) { + *metric_ms_ += duration; + } + start_ = 0; + } + } + + private: + uint64_t time_now() { + return NowMicros(); + } + + const bool perf_counter_enabled_; + uint64_t start_; + uint64_t* metric_ms_; +}; +} // namespace pstd +#endif From 5d0b5877c175e6190774fa9bbe15e314a8cb4faf Mon Sep 17 00:00:00 2001 From: Mixficsol <838844609@qq.com> Date: Fri, 14 Mar 2025 11:02:01 +0800 Subject: [PATCH 04/15] fix: The number of element updates in RedisCache is configurable (#3043) * The number of element updates in RedisCache is configurable Co-authored-by: wuxianrong --- conf/pika.conf | 6 ++++++ include/pika_command.h | 2 +- include/pika_conf.h | 4 +++- include/pika_hash.h | 2 +- include/pika_kv.h | 4 ++-- src/cache/include/config.h | 8 ++++++++ src/pika_cache_load_thread.cc | 11 ++++++----- src/pika_conf.cc | 9 +++++++++ 8 files changed, 36 insertions(+), 10 deletions(-) diff --git a/conf/pika.conf b/conf/pika.conf index 6fb55c97f1..ba03390037 100644 --- a/conf/pika.conf +++ b/conf/pika.conf @@ -570,6 +570,12 @@ cache-model : 1 # cache-type: string, set, zset, list, hash, bit cache-type: string, set, zset, list, hash, bit +# Set the maximum number of elements in the cache of the Set, list, Zset data types +cache-value-item-max-size: 1024 + +# Sets the maximum number of bytes for Key when the String data type is updated in the cache +max-key-size-in-cache: 1048576 + # Maximum number of keys in the zset redis cache # On the disk DB, a zset field may have many fields. In the memory cache, we limit the maximum # number of keys that can exist in a zset, which is zset-zset-cache-field-num-per-key, with a diff --git a/include/pika_command.h b/include/pika_command.h index 77b0eca6b8..99ca05f087 100644 --- a/include/pika_command.h +++ b/include/pika_command.h @@ -539,7 +539,7 @@ class Cmd : public std::enable_shared_from_this { // used for execute multikey command into different slots virtual void Split(const HintKeys& hint_keys) = 0; virtual void Merge() = 0; - virtual bool IsTooLargeKey(const int &max_sz) { return false; } + virtual bool IsTooLargeKey(const size_t &max_sz) { return false; } int8_t SubCmdIndex(const std::string& cmdName); // if the command no subCommand，return -1； diff --git a/include/pika_conf.h b/include/pika_conf.h index 37a1f21d18..ecb08e9680 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -959,6 +959,7 @@ class PikaConf : public pstd::BaseConf { int zset_cache_start_direction() { return zset_cache_start_direction_; } int zset_cache_field_num_per_key() { return zset_cache_field_num_per_key_; } int max_key_size_in_cache() { return max_key_size_in_cache_; } + int value_item_max_size_in_cache() { return cache_value_item_max_size_; } int cache_maxmemory_policy() { return cache_maxmemory_policy_; } int cache_maxmemory_samples() { return cache_maxmemory_samples_; } int cache_lfu_decay_time() { return cache_lfu_decay_time_; } @@ -1120,7 +1121,8 @@ class PikaConf : public pstd::BaseConf { std::atomic_int cache_bit_ = 1; std::atomic_int zset_cache_start_direction_ = 0; std::atomic_int zset_cache_field_num_per_key_ = 512; - std::atomic_int max_key_size_in_cache_ = 512; + std::atomic_int cache_value_item_max_size_ = 1024; + std::atomic_int max_key_size_in_cache_ = 1024 * 1024; std::atomic_int cache_maxmemory_policy_ = 1; std::atomic_int cache_maxmemory_samples_ = 5; std::atomic_int cache_lfu_decay_time_ = 1; diff --git a/include/pika_hash.h b/include/pika_hash.h index 1362040682..a7c4385d72 100644 --- a/include/pika_hash.h +++ b/include/pika_hash.h @@ -54,7 +54,7 @@ class HGetCmd : public Cmd { void DoUpdateCache() override; void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; - bool IsTooLargeKey(const int &max_sz) override { return key_.size() > static_cast(max_sz); } + bool IsTooLargeKey(const size_t &max_sz) override { return key_.size() > max_sz; } Cmd* Clone() override { return new HGetCmd(*this); } private: diff --git a/include/pika_kv.h b/include/pika_kv.h index 7da694705b..8d8da95e04 100644 --- a/include/pika_kv.h +++ b/include/pika_kv.h @@ -29,7 +29,7 @@ class SetCmd : public Cmd { void DoThroughDB() override; void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; - bool IsTooLargeKey(const int& max_sz) override { return key_.size() > static_cast(max_sz); } + bool IsTooLargeKey(const size_t &max_sz) override { return key_.size() > max_sz; } Cmd* Clone() override { return new SetCmd(*this); } private: @@ -65,7 +65,7 @@ class GetCmd : public Cmd { void ReadCache() override; void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; - bool IsTooLargeKey(const int &max_sz) override { return key_.size() > static_cast(max_sz); } + bool IsTooLargeKey(const size_t &max_sz) override { return key_.size() > max_sz; } Cmd* Clone() override { return new GetCmd(*this); } private: diff --git a/src/cache/include/config.h b/src/cache/include/config.h index e0a261542c..6b98709a19 100644 --- a/src/cache/include/config.h +++ b/src/cache/include/config.h @@ -40,6 +40,12 @@ constexpr int CACHE_START_FROM_END = -1; #define DEFAULT_CACHE_ITEMS_PER_KEY 512 #define DEFAULT_CACHE_MAX_KEY_SIZE 512 +/* + * cache value item default size + */ +#define DEFAULT_CACHE_ITEMS_SIZE 1024 +#define MAX_CACHE_ITEMS_SIZE 2048 + struct CacheConfig { uint64_t maxmemory; /* Can used max memory */ int32_t maxmemory_policy; /* Policy for key eviction */ @@ -47,6 +53,8 @@ struct CacheConfig { int32_t lfu_decay_time; /* LFU counter decay factor. */ int32_t zset_cache_start_direction; int32_t zset_cache_field_num_per_key; + int32_t cache_value_item_max_size; + CacheConfig() : maxmemory(CACHE_DEFAULT_MAXMEMORY) diff --git a/src/pika_cache_load_thread.cc b/src/pika_cache_load_thread.cc index e815f760d0..19892d712b 100644 --- a/src/pika_cache_load_thread.cc +++ b/src/pika_cache_load_thread.cc @@ -10,6 +10,7 @@ #include "pstd/include/scope_record_lock.h" extern PikaServer* g_pika_server; +extern std::unique_ptr g_pika_conf; PikaCacheLoadThread::PikaCacheLoadThread(int zset_cache_start_direction, int zset_cache_field_num_per_key) : should_exit_(false) @@ -70,7 +71,7 @@ bool PikaCacheLoadThread::LoadHash(std::string& key, const std::shared_ptr& db->storage()->HLen(key, &len); // If the Hash type contains more than 2048 data members, // it will not be updated to RedisCache - if (0 >= len || CACHE_VALUE_ITEM_MAX_SIZE < len) { + if (0 >= len || g_pika_conf->value_item_max_size_in_cache() < len) { return false; } @@ -90,9 +91,9 @@ bool PikaCacheLoadThread::LoadList(std::string& key, const std::shared_ptr& db->storage()->LLen(key, &len); // If the List type contains more than 2048 data members, // it will not be updated to RedisCache - if (len <= 0 || CACHE_VALUE_ITEM_MAX_SIZE < len) { + if (len <= 0 || g_pika_conf->value_item_max_size_in_cache() < len) { LOG(WARNING) << "can not load key, because item size:" << len - << " beyond max item size:" << CACHE_VALUE_ITEM_MAX_SIZE; + << " beyond max item size:" << g_pika_conf->value_item_max_size_in_cache(); return false; } @@ -112,9 +113,9 @@ bool PikaCacheLoadThread::LoadSet(std::string& key, const std::shared_ptr& d db->storage()->SCard(key, &len); // If the Set type contains more than 2048 data members, // it will not be updated to RedisCache - if (0 >= len || CACHE_VALUE_ITEM_MAX_SIZE < len) { + if (0 >= len || g_pika_conf->value_item_max_size_in_cache() < len) { LOG(WARNING) << "can not load key, because item size:" << len - << " beyond max item size:" << CACHE_VALUE_ITEM_MAX_SIZE; + << " beyond max item size:" << g_pika_conf->value_item_max_size_in_cache(); return false; } diff --git a/src/pika_conf.cc b/src/pika_conf.cc index 6339d7077c..6f4aecd78e 100644 --- a/src/pika_conf.cc +++ b/src/pika_conf.cc @@ -683,6 +683,15 @@ int PikaConf::Load() { } zset_cache_field_num_per_key_ = zset_cache_field_num_per_key; + int cache_value_item_max_size = DEFAULT_CACHE_ITEMS_SIZE; + GetConfInt("cache-value-item-max-size", &cache_value_item_max_size); + if (cache_value_item_max_size <= 0) { + cache_value_item_max_size = DEFAULT_CACHE_ITEMS_SIZE; + } else if (cache_value_item_max_size > MAX_CACHE_ITEMS_SIZE) { + cache_value_item_max_size = MAX_CACHE_ITEMS_SIZE; + } + cache_value_item_max_size_ = cache_value_item_max_size; + int max_key_size_in_cache = DEFAULT_CACHE_MAX_KEY_SIZE; GetConfInt("max-key-size-in-cache", &max_key_size_in_cache); if (max_key_size_in_cache <= 0) { From 1be9d3f46c759ef733de4681de177fecfd563af3 Mon Sep 17 00:00:00 2001 From: wangshao1 <30471730+wangshao1@users.noreply.github.com> Date: Fri, 14 Mar 2025 14:53:40 +0800 Subject: [PATCH 05/15] cherry-pick 4 --- codis/pkg/topom/topom_group.go | 13 +++++-------- codis/pkg/topom/topom_sentinel.go | 12 ++++++++++++ codis/pkg/utils/redis/sentinel.go | 4 ++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/codis/pkg/topom/topom_group.go b/codis/pkg/topom/topom_group.go index b1ed76836b..a08eec70cc 100644 --- a/codis/pkg/topom/topom_group.go +++ b/codis/pkg/topom/topom_group.go @@ -381,14 +381,11 @@ func (s *Topom) tryFixReplicationRelationship(group *models.Group, groupServer * } } } else { - // skip if it has right replication relationship - if state.Replication.GetMasterAddr() == curMasterAddr { - return nil - } - - // current server is slave, execute the command `slaveof [new master ip] [new master port] force` - if err = updateMasterToNewOneForcefully(groupServer.Addr, curMasterAddr, s.config.ProductAuth); err != nil { - return err + if state.Replication.GetMasterAddr() != curMasterAddr { + // current server is slave, execute the command `slaveof [new master ip] [new master port]` + if err = updateMasterToNewOne(groupServer.Addr, curMasterAddr, s.config.ProductAuth); err != nil { + return err + } } } diff --git a/codis/pkg/topom/topom_sentinel.go b/codis/pkg/topom/topom_sentinel.go index f1fc67fff6..f43a53fce6 100644 --- a/codis/pkg/topom/topom_sentinel.go +++ b/codis/pkg/topom/topom_sentinel.go @@ -85,6 +85,18 @@ func (s *Topom) checkAndUpdateGroupServerState(conf *Config, group *models.Group *recoveredGroupServers = append(*recoveredGroupServers, state) // update GroupServer to GroupServerStateNormal state later } else { + // This may contains any of following condition: + // 1. groupServer.State is Normal + // 2. groupServer.State is GroupServerStateSubjectiveOffline and is Master + // 3. groupServer.State is GroupServerStateSubjectiveOffline and is Slave + // for condition 3, if current server's previous state is SubjectiveOffline + // and has been added to slaveofflinegroups before, + // should also resync mappings to proxy to enable replicationgroup + if groupServer.State == models.GroupServerStateSubjectiveOffline && + !isGroupMaster(state, group) && + group.OutOfSync { + *recoveredGroupServers = append(*recoveredGroupServers, state) + } // Update the offset information of the state and role nodes groupServer.State = models.GroupServerStateNormal groupServer.ReCallTimes = 0 diff --git a/codis/pkg/utils/redis/sentinel.go b/codis/pkg/utils/redis/sentinel.go index 0415ede6f1..5bdf132661 100644 --- a/codis/pkg/utils/redis/sentinel.go +++ b/codis/pkg/utils/redis/sentinel.go @@ -106,8 +106,8 @@ func (i *InfoReplication) UnmarshalJSON(b []byte) error { } i.Role = kvmap["role"] - i.MasterPort = kvmap["master_host"] - i.MasterHost = kvmap["master_port"] + i.MasterPort = kvmap["master_port"] + i.MasterHost = kvmap["master_host"] i.MasterLinkStatus = kvmap["master_link_status"] i.IsEligibleForMasterElection = kvmap["is_eligible_for_master_election"] == "true" From fc14508da6d0b0a0a8b637c8d7a7bb1bb565b850 Mon Sep 17 00:00:00 2001 From: Mixficsol <838844609@qq.com> Date: Fri, 14 Mar 2025 15:00:34 +0800 Subject: [PATCH 06/15] cherry-pick 5 --- src/storage/src/redis_strings.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storage/src/redis_strings.cc b/src/storage/src/redis_strings.cc index 36e4014390..04bf17e41d 100644 --- a/src/storage/src/redis_strings.cc +++ b/src/storage/src/redis_strings.cc @@ -99,9 +99,9 @@ Status Redis::Append(const Slice& key, const Slice& value, int32_t* ret, int64_t } } else if (s.IsNotFound()) { *ret = static_cast(value.size()); + out_new_value = value.ToString(); StringsValue strings_value(value); - *expired_timestamp_millsec = 0; - return db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); + return db_->Put(default_write_options_, key, strings_value.Encode()); } return s; } From a47a7fcfeb6dc1cb40804730f8c60d947dfde17d Mon Sep 17 00:00:00 2001 From: Mixficsol <838844609@qq.com> Date: Mon, 17 Mar 2025 16:46:18 +0800 Subject: [PATCH 07/15] Example Modify the policy for updating big keys except String to Redis-Cache (#3047) Co-authored-by: wuxianrong --- include/pika_conf.h | 27 ++++++++++++++++++++++++- src/cache/include/config.h | 3 ++- src/pika_admin.cc | 38 +++++++++++++++++++++++++++++++++++ src/pika_cache_load_thread.cc | 10 ++++----- src/pika_conf.cc | 2 ++ 5 files changed, 73 insertions(+), 7 deletions(-) diff --git a/include/pika_conf.h b/include/pika_conf.h index ecb08e9680..e25b3549ce 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -18,6 +18,7 @@ #include "pstd/include/pstd_string.h" #include "acl.h" +#include "cache/include/config.h" #include "include/pika_define.h" #include "rocksdb/compression_type.h" @@ -862,6 +863,30 @@ class PikaConf : public pstd::BaseConf { return rocksdb_perf_level_.load(); } + int CacheValueItemMaxSize() const { + return cache_value_item_max_size_.load(); + } + + bool UpdateCacheValueItemMaxSize(int size) { + if (size >= MAX_CACHE_ITEMS_SIZE || size <= 0) { + return false; + } + cache_value_item_max_size_.store(size); + return true; + } + + size_t MaxKeySizeInCache() const { + return max_key_size_in_cache_.load(); + } + + bool UpdateMaxKeySizeInCache(size_t size) { + if (size >= MAX_CACHE_MAX_KEY_SIZE || size <= 0) { + return false; + } + max_key_size_in_cache_.store(size); + return true; + } + bool UpdateRocksDBPerfLevel(int perf_level) { if (perf_level >= 6 || perf_level < 0) { return false; @@ -1122,7 +1147,7 @@ class PikaConf : public pstd::BaseConf { std::atomic_int zset_cache_start_direction_ = 0; std::atomic_int zset_cache_field_num_per_key_ = 512; std::atomic_int cache_value_item_max_size_ = 1024; - std::atomic_int max_key_size_in_cache_ = 1024 * 1024; + std::atomic_size_t max_key_size_in_cache_ = 1024 * 1024; std::atomic_int cache_maxmemory_policy_ = 1; std::atomic_int cache_maxmemory_samples_ = 5; std::atomic_int cache_lfu_decay_time_ = 1; diff --git a/src/cache/include/config.h b/src/cache/include/config.h index 6b98709a19..5eee3bdebf 100644 --- a/src/cache/include/config.h +++ b/src/cache/include/config.h @@ -38,7 +38,8 @@ constexpr int CACHE_START_FROM_END = -1; * cache items per key */ #define DEFAULT_CACHE_ITEMS_PER_KEY 512 -#define DEFAULT_CACHE_MAX_KEY_SIZE 512 +#define DEFAULT_CACHE_MAX_KEY_SIZE 1048576 // 1M +#define MAX_CACHE_MAX_KEY_SIZE 2097152 // 2M /* * cache value item default size diff --git a/src/pika_admin.cc b/src/pika_admin.cc index 02d2e2b5e0..3275e6cb7b 100644 --- a/src/pika_admin.cc +++ b/src/pika_admin.cc @@ -2132,6 +2132,18 @@ void ConfigCmd::ConfigGet(std::string& ret) { EncodeNumber(&config_body, g_pika_conf->blob_file_size()); } + if (pstd::stringmatch(pattern.data(), "cache-value-item-max-size", 1) != 0) { + elements += 2; + EncodeString(&config_body, "cache-value-item-max-size"); + EncodeNumber(&config_body, g_pika_conf->CacheValueItemMaxSize()); + } + + if (pstd::stringmatch(pattern.data(), "max-key-size-in-cache", 1) != 0) { + elements += 2; + EncodeString(&config_body, "max-key-size-in-cache"); + EncodeNumber(&config_body, g_pika_conf->MaxKeySizeInCache()); + } + if (pstd::stringmatch(pattern.data(), "blob-garbage-collection-age-cutoff", 1) != 0) { elements += 2; EncodeString(&config_body, "blob-garbage-collection-age-cutoff"); @@ -2902,6 +2914,32 @@ void ConfigCmd::ConfigSet(std::shared_ptr db) { return; } res_.AppendStringRaw("+OK\r\n"); + } else if (set_item == "cache-value-item-max-size") { + if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival <= 0) { + res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'cache-value-item-max-size'\r\n"); + return; + } + bool success = g_pika_conf->UpdateCacheValueItemMaxSize(int(ival)); + LOG(INFO) << "update cache-value-item-max-size to " << ival + << (success ? " success" : " failed"); + if (!success) { + res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'cache-value-item-max-size', should between 1 and 2048\r\n"); + return; + } + res_.AppendStringRaw("+OK\r\n"); + } else if (set_item == "max-key-size-in-cache") { + if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival <= 0) { + res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'max-key-size-in-cache'\r\n"); + return; + } + bool success = g_pika_conf->UpdateMaxKeySizeInCache(size_t(ival)); + LOG(INFO) << "update max-key-size-in-cache to " << ival + << (success ? " success" : " failed"); + if (!success) { + res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'max-key-size-in-cache', should between 1 and 2097152 \r\n"); + return; + } + res_.AppendStringRaw("+OK\r\n"); } else if (set_item == "throttle-bytes-per-second") { if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival <= 0) { res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'throttle-bytes-per-second'\r\n"); diff --git a/src/pika_cache_load_thread.cc b/src/pika_cache_load_thread.cc index 19892d712b..d24b7b975a 100644 --- a/src/pika_cache_load_thread.cc +++ b/src/pika_cache_load_thread.cc @@ -58,7 +58,7 @@ bool PikaCacheLoadThread::LoadKV(std::string& key, const std::shared_ptr& db std::string value; int64_t ttl = -1; rocksdb::Status s = db->storage()->GetWithTTL(key, &value, &ttl); - if (!s.ok()) { + if (!s.ok() || key.size() > g_pika_conf->max_key_size_in_cache()) { LOG(WARNING) << "load kv failed, key=" << key; return false; } @@ -71,7 +71,7 @@ bool PikaCacheLoadThread::LoadHash(std::string& key, const std::shared_ptr& db->storage()->HLen(key, &len); // If the Hash type contains more than 2048 data members, // it will not be updated to RedisCache - if (0 >= len || g_pika_conf->value_item_max_size_in_cache() < len) { + if (0 >= len || g_pika_conf->value_item_max_size_in_cache() < len || key.size() > g_pika_conf->max_key_size_in_cache()) { return false; } @@ -91,7 +91,7 @@ bool PikaCacheLoadThread::LoadList(std::string& key, const std::shared_ptr& db->storage()->LLen(key, &len); // If the List type contains more than 2048 data members, // it will not be updated to RedisCache - if (len <= 0 || g_pika_conf->value_item_max_size_in_cache() < len) { + if (len <= 0 || g_pika_conf->value_item_max_size_in_cache() < len || key.size() > g_pika_conf->max_key_size_in_cache()) { LOG(WARNING) << "can not load key, because item size:" << len << " beyond max item size:" << g_pika_conf->value_item_max_size_in_cache(); return false; @@ -113,7 +113,7 @@ bool PikaCacheLoadThread::LoadSet(std::string& key, const std::shared_ptr& d db->storage()->SCard(key, &len); // If the Set type contains more than 2048 data members, // it will not be updated to RedisCache - if (0 >= len || g_pika_conf->value_item_max_size_in_cache() < len) { + if (0 >= len || g_pika_conf->value_item_max_size_in_cache() < len || key.size() > g_pika_conf->max_key_size_in_cache()) { LOG(WARNING) << "can not load key, because item size:" << len << " beyond max item size:" << g_pika_conf->value_item_max_size_in_cache(); return false; @@ -135,7 +135,7 @@ bool PikaCacheLoadThread::LoadZset(std::string& key, const std::shared_ptr& int start_index = 0; int stop_index = -1; db->storage()->ZCard(key, &len); - if (0 >= len) { + if (0 >= len || key.size() > g_pika_conf->max_key_size_in_cache()) { return false; } diff --git a/src/pika_conf.cc b/src/pika_conf.cc index 6f4aecd78e..cb004f2d2a 100644 --- a/src/pika_conf.cc +++ b/src/pika_conf.cc @@ -696,6 +696,8 @@ int PikaConf::Load() { GetConfInt("max-key-size-in-cache", &max_key_size_in_cache); if (max_key_size_in_cache <= 0) { max_key_size_in_cache = DEFAULT_CACHE_MAX_KEY_SIZE; + } else if (max_key_size_in_cache > MAX_CACHE_MAX_KEY_SIZE) { + max_key_size_in_cache = MAX_CACHE_MAX_KEY_SIZE; } max_key_size_in_cache_ = max_key_size_in_cache; From 899b3479563ff1da135e4b26bc22b827a5e7d159 Mon Sep 17 00:00:00 2001 From: Mixficsol <838844609@qq.com> Date: Wed, 19 Mar 2025 14:17:26 +0800 Subject: [PATCH 08/15] add auth command to admin-thread-pool (#3048) Co-authored-by: wuxianrong --- include/pika_conf.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/pika_conf.h b/include/pika_conf.h index e25b3549ce..556cb72ee1 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -1002,7 +1002,10 @@ class PikaConf : public pstd::BaseConf { int slow_cmd_thread_pool_size_ = 0; int admin_thread_pool_size_ = 0; std::unordered_set slow_cmd_set_; - std::unordered_set admin_cmd_set_ = {"info", "ping", "monitor"}; + // Because the exporter of Pika_exporter implements Auth authentication + // with the Exporter of Pika, and the Exporter authenticates the Auth when + // users connect to Pika, the Auth is added to the management command thread pool + std::unordered_set admin_cmd_set_ = {"info", "ping", "monitor", "auth"}; int sync_thread_num_ = 0; int sync_binlog_thread_num_ = 0; int expire_dump_days_ = 3; From 013ac005eb633b7a7ca4e93cef6cf2db05f060b4 Mon Sep 17 00:00:00 2001 From: Mixficsol <838844609@qq.com> Date: Wed, 19 Mar 2025 14:17:55 +0800 Subject: [PATCH 09/15] fix: Example Repair the subjective and objective offline logic of noeds (#3049) * Example Repair the subjective and objective offline logic of primary and secondary nodes * fix comment --------- Co-authored-by: wuxianrong --- codis/pkg/topom/topom_sentinel.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/codis/pkg/topom/topom_sentinel.go b/codis/pkg/topom/topom_sentinel.go index f43a53fce6..b190b93988 100644 --- a/codis/pkg/topom/topom_sentinel.go +++ b/codis/pkg/topom/topom_sentinel.go @@ -74,10 +74,17 @@ func (s *Topom) checkAndUpdateGroupServerState(conf *Config, group *models.Group } // Start the election master node - if groupServer.State == models.GroupServerStateOffline && isGroupMaster(state, group) { - *masterOfflineGroups = append(*masterOfflineGroups, group) - } else { - *slaveOfflineGroups = append(*slaveOfflineGroups, group) + // Currently, both primary and secondary nodes have subjective and objective logics. + // If it is subjective, we will not perform any operation. If more than 10 probe counts + // fail, it is defined as objective logics, If it is an objective offline, we will add the + // node to masterOfflineGroups or slaveOfflineGroups respectively, and then notify the Proxy + // to change the meta information + if groupServer.State == models.GroupServerStateOffline { + if isGroupMaster(state, group) { + *masterOfflineGroups = append(*masterOfflineGroups, group) + } else { + *slaveOfflineGroups = append(*slaveOfflineGroups, group) + } } } } else { From e780a9fe679daef2b8efc2475627e3906e165965 Mon Sep 17 00:00:00 2001 From: Mixficsol <838844609@qq.com> Date: Thu, 20 Mar 2025 21:12:39 +0800 Subject: [PATCH 10/15] fix: Fixed version information for Pika, Pika_exporter, Codis (#3054) * Fixed version information for Pika, Pika_exporter, Codis * Delete redundant logic --------- Co-authored-by: wuxianrong --- CMakeLists.txt | 2 +- codis/Makefile | 7 +++++-- tools/pika_exporter/Makefile | 6 +----- tools/pika_exporter/main.go | 5 ++++- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 81bf712133..38e9991ad4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -739,7 +739,7 @@ aux_source_directory(src DIR_SRCS) # # generate version string(TIMESTAMP TS "%Y-%m-%d %H:%M:%S" UTC) -set(PIKA_BUILD_DATE "${TS}" CACHE STRING "the time we first built pika") +set(PIKA_BUILD_DATE "${TS}") find_package(Git) diff --git a/codis/Makefile b/codis/Makefile index c3dd8851d9..7226dfbd8f 100644 --- a/codis/Makefile +++ b/codis/Makefile @@ -2,14 +2,13 @@ export GO111MODULE=on -build-all: codis-dashboard codis-proxy codis-admin codis-ha codis-fe +build-all: codis-dashboard codis-proxy codis-admin codis-ha codis-fe generate-version PRJ_ROOT=${CURDIR} codis-deps: @mkdir -p ./bin && go version - codis-dashboard: codis-deps $(info build codis-dashboard) @cd ${PRJ_ROOT}/cmd/dashboard && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-dashboard . @@ -33,6 +32,10 @@ codis-fe: codis-deps @cd ${PRJ_ROOT}/cmd/fe && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-fe . @rm -rf ${PRJ_ROOT}/bin/assets && cp -rf ${PRJ_ROOT}/cmd/fe/assets ./bin/ +generate-version: + $(info generate version) + @./version + clean: $(info ...Clean Start!) @rm -rf bin diff --git a/tools/pika_exporter/Makefile b/tools/pika_exporter/Makefile index b5178df6e0..7e47d19a8e 100644 --- a/tools/pika_exporter/Makefile +++ b/tools/pika_exporter/Makefile @@ -4,11 +4,7 @@ # export PATH := $(PATH):$(GOPATH)/bin -# for mac -BRANCH := $(shell git branch | sed 's/* \(.*\)/\1/p') -# for Linux -# BRANCH := $(shell git branch | sed --quiet 's/* \(.*\)/\1/p') -GITREV := $(shell git rev-parse --short HEAD) +GITREV := $(shell git rev-parse HEAD) BUILDTIME := $(shell date '+%F %T %Z') COMPILERVERSION := $(subst go version ,,$(shell go version)) PROJNAME := pika_exporter diff --git a/tools/pika_exporter/main.go b/tools/pika_exporter/main.go index 07b4cf5f40..6af84bc4b4 100644 --- a/tools/pika_exporter/main.go +++ b/tools/pika_exporter/main.go @@ -55,7 +55,10 @@ func getEnvInt(key string, defaultVal int) int { func main() { flag.Parse() - log.Println("Pika Metrics Exporter ", BuildVersion, "build date:", BuildDate, "sha:", BuildCommitSha, "go version:", GoVersion) + log.Println("Pika Metrics Exporter") + log.Println("Build Date: ", BuildDate) + log.Println("Commit SHA: ", BuildCommitSha) + log.Println("Go Version: ", GoVersion) if *showVersion { return } From 08f1a858dab2d01d808dab2ef4ce87ed0a1ea7d7 Mon Sep 17 00:00:00 2001 From: wuxianrong Date: Wed, 26 Mar 2025 11:09:48 +0800 Subject: [PATCH 11/15] Resolving code conflicts --- CMakeLists.txt | 2 +- codis/Makefile | 11 +- codis/cmd/dashboard/main.go | 7 +- codis/cmd/fe/main.go | 7 +- codis/cmd/ha/main.go | 7 +- codis/cmd/proxy/main.go | 7 +- codis/version | 22 ++- include/pika_cache.h | 4 +- src/cache/include/cache.h | 4 +- src/cache/src/set.cc | 2 +- src/cache/src/zset.cc | 2 +- src/pika_cache.cc | 4 +- src/pika_hash.cc | 152 +++++---------- src/pika_kv.cc | 345 +++++++---------------------------- src/pika_list.cc | 42 ++--- src/pika_set.cc | 35 ++-- src/pika_zset.cc | 27 ++- tools/pika_exporter/Makefile | 6 +- 18 files changed, 202 insertions(+), 484 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 38e9991ad4..ad05bccae1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -738,7 +738,7 @@ endif() aux_source_directory(src DIR_SRCS) # # generate version -string(TIMESTAMP TS "%Y-%m-%d %H:%M:%S" UTC) +string(TIMESTAMP TS "%Y-%m-%d %H:%M:%S") set(PIKA_BUILD_DATE "${TS}") find_package(Git) diff --git a/codis/Makefile b/codis/Makefile index 7226dfbd8f..112006685e 100644 --- a/codis/Makefile +++ b/codis/Makefile @@ -2,13 +2,16 @@ export GO111MODULE=on -build-all: codis-dashboard codis-proxy codis-admin codis-ha codis-fe generate-version +build-all: generate-version codis-dashboard codis-proxy codis-admin codis-ha codis-fe PRJ_ROOT=${CURDIR} +generate-version: + $(info generate version) + @mkdir -p ./bin && ./version codis-deps: - @mkdir -p ./bin && go version + @go version codis-dashboard: codis-deps $(info build codis-dashboard) @cd ${PRJ_ROOT}/cmd/dashboard && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-dashboard . @@ -32,10 +35,6 @@ codis-fe: codis-deps @cd ${PRJ_ROOT}/cmd/fe && go mod tidy && go build -buildvcs=false -o ${PRJ_ROOT}/bin/codis-fe . @rm -rf ${PRJ_ROOT}/bin/assets && cp -rf ${PRJ_ROOT}/cmd/fe/assets ./bin/ -generate-version: - $(info generate version) - @./version - clean: $(info ...Clean Start!) @rm -rf bin diff --git a/codis/cmd/dashboard/main.go b/codis/cmd/dashboard/main.go index a4ef83a18a..a67e39a282 100644 --- a/codis/cmd/dashboard/main.go +++ b/codis/cmd/dashboard/main.go @@ -48,8 +48,11 @@ Options: return case d["--version"].(bool): - fmt.Println("version:", utils.Version) - fmt.Println("compile:", utils.Compile) + fmt.Printf("-----------Codis Dashboard----------\n") + fmt.Println("codis_version:", utils.Version) + fmt.Println("codis_git_sha:", utils.Gitsha) + fmt.Println("codis_build_compile_date:", utils.Compile) + fmt.Println("go version:", utils.GoVersion) return } diff --git a/codis/cmd/fe/main.go b/codis/cmd/fe/main.go index 663cc47086..1e7cdad9d9 100644 --- a/codis/cmd/fe/main.go +++ b/codis/cmd/fe/main.go @@ -72,8 +72,11 @@ Options: } if d["--version"].(bool) { - fmt.Println("version:", utils.Version) - fmt.Println("compile:", utils.Compile) + fmt.Printf("-----------Codis FE----------\n") + fmt.Println("codis_version:", utils.Version) + fmt.Println("codis_git_sha:", utils.Gitsha) + fmt.Println("codis_build_compile_date:", utils.Compile) + fmt.Println("go version:", utils.GoVersion) return } diff --git a/codis/cmd/ha/main.go b/codis/cmd/ha/main.go index e00d26fbe2..f30c60516d 100644 --- a/codis/cmd/ha/main.go +++ b/codis/cmd/ha/main.go @@ -33,8 +33,11 @@ Options: } if d["--version"].(bool) { - fmt.Println("version:", utils.Version) - fmt.Println("compile:", utils.Compile) + fmt.Printf("-----------Codis HA----------\n") + fmt.Println("codis_version:", utils.Version) + fmt.Println("codis_git_sha:", utils.Gitsha) + fmt.Println("codis_build_compile_date:", utils.Compile) + fmt.Println("go version:", utils.GoVersion) return } diff --git a/codis/cmd/proxy/main.go b/codis/cmd/proxy/main.go index 26cfb37197..fa1f6ddfe2 100644 --- a/codis/cmd/proxy/main.go +++ b/codis/cmd/proxy/main.go @@ -55,8 +55,11 @@ Options: return case d["--version"].(bool): - fmt.Println("version:", utils.Version) - fmt.Println("compile:", utils.Compile) + fmt.Printf("-----------Codis Proxy----------\n") + fmt.Println("codis_version:", utils.Version) + fmt.Println("codis_git_sha:", utils.Gitsha) + fmt.Println("codis_build_compile_date:", utils.Compile) + fmt.Println("go version:", utils.GoVersion) return } diff --git a/codis/version b/codis/version index 9ac6d5d40f..038f50cfbf 100755 --- a/codis/version +++ b/codis/version @@ -1,30 +1,36 @@ #!/bin/bash -version=`git log --date=iso --pretty=format:"%cd @%H" -1` +CODIS_MAJOR=3 +CODIS_MINOR=5 +CODIS_PATCH=4 + +gitsha=`git log --pretty=format:"%H" -1` if [ $? -ne 0 ]; then - version="unknown version" + gitsha ="unknown version gitsha" fi -compile=`date +"%F %T %z"`" by "`go version` +compile=`date +"%F %T %Z"` if [ $? -ne 0 ]; then compile="unknown datetime" fi -describe=`git describe --tags 2>/dev/null` -if [ $? -eq 0 ]; then - version="${version} @${describe}" +goversion=$(go version | sed 's/go version //') +if [ $? -ne 0 ]; then + compile="unknown go version" fi cat << EOF | gofmt > pkg/utils/version.go package utils const ( - Version = "$version" + Version = "$CODIS_MAJOR.$CODIS_MINOR.$CODIS_PATCH" + Gitsha = "$gitsha" Compile = "$compile" + GoVersion = "$goversion" ) EOF cat << EOF > bin/version -version = $version +gitsha = $gitsha compile = $compile EOF diff --git a/include/pika_cache.h b/include/pika_cache.h index bbac769a04..41f71ba1c0 100644 --- a/include/pika_cache.h +++ b/include/pika_cache.h @@ -139,7 +139,7 @@ class PikaCache : public pstd::noncopyable, public std::enable_shared_from_this< rocksdb::Status SAddIfKeyExist(std::string& key, std::vector& members); rocksdb::Status SAddnx(std::string& key, std::vector& members, int64_t ttl); rocksdb::Status SAddnxWithoutTTL(std::string& key, std::vector& members); - rocksdb::Status SCard(std::string& key, uint64_t* len); + rocksdb::Status SCard(const std::string& key, uint64_t* len); rocksdb::Status SIsmember(std::string& key, std::string& member); rocksdb::Status SMembers(std::string& key, std::vector* members); rocksdb::Status SRem(std::string& key, std::vector& members); @@ -150,7 +150,7 @@ class PikaCache : public pstd::noncopyable, public std::enable_shared_from_this< rocksdb::Status ZAddIfKeyExist(std::string& key, std::vector& score_members); rocksdb::Status ZAddnx(std::string& key, std::vector& score_members, int64_t ttl); rocksdb::Status ZAddnxWithoutTTL(std::string& key, std::vector& score_members); - rocksdb::Status ZCard(std::string& key, uint32_t* len, const std::shared_ptr& db); + rocksdb::Status ZCard(const std::string& key, uint32_t* len, const std::shared_ptr& db); rocksdb::Status ZCount(std::string& key, std::string& min, std::string& max, uint64_t* len, ZCountCmd* cmd); rocksdb::Status ZIncrby(std::string& key, std::string& member, double increment); rocksdb::Status ZIncrbyIfKeyExist(std::string& key, std::string& member, double increment, ZIncrbyCmd* cmd, const std::shared_ptr& db); diff --git a/src/cache/include/cache.h b/src/cache/include/cache.h index 070504c4a7..68a23b9338 100644 --- a/src/cache/include/cache.h +++ b/src/cache/include/cache.h @@ -114,7 +114,7 @@ class RedisCache { // Set Commands Status SAdd(std::string& key, std::vector &members); Status SAddIfKeyExist(std::string& key, std::vector &members); - Status SCard(std::string& key, uint64_t *len); + Status SCard(const std::string& key, uint64_t *len); Status SIsmember(std::string& key, std::string& member); Status SMembers(std::string& key, std::vector *members); Status SRem(std::string& key, std::vector &members); @@ -123,7 +123,7 @@ class RedisCache { // Zset Commands Status ZAdd(std::string& key, std::vector &score_members); Status ZAddIfKeyExist(std::string& key, std::vector &score_members); - Status ZCard(std::string& key, uint64_t *len); + Status ZCard(const std::string& key, uint64_t *len); Status ZCount(std::string& key, std::string &min, std::string &max, uint64_t *len); Status ZIncrby(std::string& key, std::string& member, double increment); Status ZRange(std::string& key, diff --git a/src/cache/src/set.cc b/src/cache/src/set.cc index 1c0d0ff30e..8d0406df38 100644 --- a/src/cache/src/set.cc +++ b/src/cache/src/set.cc @@ -58,7 +58,7 @@ Status RedisCache::SAdd(std::string& key, std::vector &members) { return Status::OK(); } -Status RedisCache::SCard(std::string& key, uint64_t *len) { +Status RedisCache::SCard(const std::string& key, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); diff --git a/src/cache/src/zset.cc b/src/cache/src/zset.cc index 6fec5aac2f..655e7a817c 100644 --- a/src/cache/src/zset.cc +++ b/src/cache/src/zset.cc @@ -63,7 +63,7 @@ Status RedisCache::ZAdd(std::string& key, std::vector &sco return Status::OK(); } -Status RedisCache::ZCard(std::string& key, uint64_t *len) { +Status RedisCache::ZCard(const std::string& key, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); diff --git a/src/pika_cache.cc b/src/pika_cache.cc index 0345a7b880..da1fa714e3 100644 --- a/src/pika_cache.cc +++ b/src/pika_cache.cc @@ -678,7 +678,7 @@ Status PikaCache::SAddnxWithoutTTL(std::string& key, std::vector &m } } -Status PikaCache::SCard(std::string& key, uint64_t *len) { +Status PikaCache::SCard(const std::string& key, uint64_t *len) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->SCard(key, len); @@ -892,7 +892,7 @@ Status PikaCache::ZAddnxWithoutTTL(std::string& key, std::vector& db) { +Status PikaCache::ZCard(const std::string& key, uint32_t *len, const std::shared_ptr& db) { int32_t db_len = 0; db->storage()->ZCard(key, &db_len); *len = db_len; diff --git a/src/pika_hash.cc b/src/pika_hash.cc index e5e615d342..e9c333d8fd 100644 --- a/src/pika_hash.cc +++ b/src/pika_hash.cc @@ -45,8 +45,7 @@ void HDelCmd::DoThroughDB() { void HDelCmd::DoUpdateCache() { if (s_.ok() && deleted_ > 0) { STAGE_TIMER_GUARD(cache_duration_ms, true); - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - db_->cache()->HDel(CachePrefixKeyH, fields_); + db_->cache()->HDel(key_, fields_); } } @@ -62,23 +61,15 @@ void HSetCmd::DoInitial() { void HSetCmd::Do() { STAGE_TIMER_GUARD(storage_duration_ms, true); - if (argv_.size() == 4) { - int32_t count = 0; - s_ = db_->storage()->HSet(key_, field_, value_, &count); - if (s_.ok()) { - res_.AppendContent(":" + std::to_string(count)); - AddSlotKey("h", key_, db_); - } else { - res_.SetRes(CmdRes::kErrOther, s_.ToString()); - } - } else if (argv_.size() > 4 && argv_.size() % 2 == 0) { - s_ = db_->storage()->HMSet(key_, fields_values_); - if (s_.ok()) { - res_.AppendContent(":" + std::to_string(fields_values_.size())); - AddSlotKey("h", key_, db_); - } else { - res_.SetRes(CmdRes::kErrOther, s_.ToString()); - } + int32_t ret = 0; + s_ = db_->storage()->HSet(key_, field_, value_, &ret); + if (s_.ok()) { + res_.AppendContent(":" + std::to_string(ret)); + AddSlotKey("h", key_, db_); + } else if (s_.IsInvalidArgument()) { + res_.SetRes(CmdRes::kMultiKey); + } else { + res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } @@ -87,13 +78,9 @@ void HSetCmd::DoThroughDB() { } void HSetCmd::DoUpdateCache() { - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - if (argv_.size() == 4) { - db_->cache()->HSetIfKeyExist(CachePrefixKeyH, field_, value_); - } - else if (argv_.size() > 4 && argv_.size() % 2 == 0) { - db_->cache()->HMSetIfKeyExist(CachePrefixKeyH, fields_values_); + if (s_.ok()) { + db_->cache()->HSetIfKeyExist(key_, field_, value_); } } @@ -108,73 +95,31 @@ void HGetCmd::DoInitial() { void HGetCmd::Do() { STAGE_TIMER_GUARD(storage_duration_ms, true); - if (argv_.size() == 3) { - std::string value; - s_ = db_->storage()->HGet(key_, field_, &value); - - if (s_.ok()) { - res_.AppendStringLenUint64(value.size()); - res_.AppendContent(value); - } else if (s_.IsNotFound()) { - res_.AppendContent("$-1"); - } else { - res_.SetRes(CmdRes::kErrOther, s_.ToString()); - } - } - else if (argv_.size() > 3) { - std::vector values; - s_ = db_->storage()->HMGet(key_, fields_, &values); - - if (s_.ok()) { - res_.AppendArrayLen(values.size()); - for (const auto& vs : values) { - if (vs.status.ok()) { - res_.AppendStringLenUint64(vs.value.size()); - res_.AppendContent(vs.value); - } else { - res_.AppendContent("$-1"); - } - } - } else { - res_.SetRes(CmdRes::kErrOther, s_.ToString()); - } + std::string value; + s_ = db_->storage()->HGet(key_, field_, &value); + if (s_.ok()) { + res_.AppendStringLenUint64(value.size()); + res_.AppendContent(value); + } else if (s_.IsInvalidArgument()) { + res_.SetRes(CmdRes::kMultiKey); + } else if (s_.IsNotFound()) { + res_.AppendContent("$-1"); + } else { + res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } void HGetCmd::ReadCache() { - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - if (argv_.size() == 3) { - std::string value; - auto s = db_->cache()->HGet(CachePrefixKeyH, field_, &value); - if (s.ok()) { - res_.AppendStringLen(value.size()); - res_.AppendContent(value); - } else if (s.IsNotFound()) { - res_.SetRes(CmdRes::kCacheMiss); - } else { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - } - } - else if (argv_.size() > 3) { - std::vector vss; - vss.clear(); - auto s = db_->cache()->HMGet(CachePrefixKeyH, fields_, &vss); - if (s.ok()) { - res_.AppendArrayLen(vss.size()); - for (const auto& vs : vss) { - if (vs.status.ok()) { - res_.AppendStringLen(vs.value.size()); - res_.AppendContent(vs.value); - } else { - res_.AppendContent("$-1"); - } - } - } else if (s.IsNotFound()) { - res_.SetRes(CmdRes::kCacheMiss); - } else { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - } + std::string value; + auto s = db_->cache()->HGet(key_, field_, &value); + if (s.ok()) { + res_.AppendStringLen(value.size()); + res_.AppendContent(value); + } else if (s.IsNotFound()) { + res_.SetRes(CmdRes::kCacheMiss); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); } } @@ -244,9 +189,8 @@ void HGetallCmd::Do() { void HGetallCmd::ReadCache() { std::vector fvs; - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->HGetall(CachePrefixKeyH, &fvs); + auto s = db_->cache()->HGetall(key_, &fvs); if (s.ok()) { res_.AppendArrayLen(fvs.size() * 2); for (const auto& fv : fvs) { @@ -299,9 +243,8 @@ void HExistsCmd::Do() { } void HExistsCmd::ReadCache() { - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->HExists(CachePrefixKeyH, field_); + auto s = db_->cache()->HExists(key_, field_); if (s.ok()) { res_.AppendContent(":1"); } else if (s.IsNotFound()) { @@ -361,9 +304,8 @@ void HIncrbyCmd::DoThroughDB() { void HIncrbyCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->HIncrbyxx(CachePrefixKeyH, field_, by_); + db_->cache()->HIncrbyxx(key_, field_, by_); } } @@ -404,9 +346,8 @@ void HIncrbyfloatCmd::DoUpdateCache() { if (s_.ok()) { long double long_double_by; if (storage::StrToLongDouble(by_.data(), by_.size(), &long_double_by) != -1) { - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->HIncrbyfloatxx(CachePrefixKeyH, field_, long_double_by); + db_->cache()->HIncrbyfloatxx(key_, field_, long_double_by); } } } @@ -437,9 +378,8 @@ void HKeysCmd::Do() { void HKeysCmd::ReadCache() { std::vector fields; - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->HKeys(CachePrefixKeyH, &fields); + auto s = db_->cache()->HKeys(key_, &fields); if (s.ok()) { res_.AppendArrayLen(fields.size()); for (const auto& field : fields) { @@ -488,9 +428,8 @@ void HLenCmd::Do() { void HLenCmd::ReadCache() { uint64_t len = 0; - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->HLen(CachePrefixKeyH, &len); + auto s = db_->cache()->HLen(key_, &len); if (s.ok()) { res_.AppendInteger(len); } else if (s.IsNotFound()) { @@ -548,9 +487,8 @@ void HMgetCmd::Do() { void HMgetCmd::ReadCache() { std::vector vss; - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->HMGet(CachePrefixKeyH, fields_, &vss); + auto s = db_->cache()->HMGet(key_, fields_, &vss); if (s.ok()) { res_.AppendArrayLen(vss.size()); for (const auto& vs : vss) { @@ -618,9 +556,8 @@ void HMsetCmd::DoThroughDB() { void HMsetCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->HMSetIfKeyExist(CachePrefixKeyH, fvs_); + db_->cache()->HMSetIfKeyExist(key_, fvs_); } } @@ -654,9 +591,8 @@ void HSetnxCmd::DoThroughDB() { void HSetnxCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->HSetIfKeyExistAndFieldNotExist(CachePrefixKeyH, field_, value_); + db_->cache()->HSetIfKeyExistAndFieldNotExist(key_, field_, value_); } } @@ -684,9 +620,8 @@ void HStrlenCmd::Do() { void HStrlenCmd::ReadCache() { uint64_t len = 0; - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->HStrlen(CachePrefixKeyH, field_, &len); + auto s = db_->cache()->HStrlen(key_, field_, &len); if (s.ok()) { res_.AppendInteger(len); } else if (s.IsNotFound()) { @@ -737,9 +672,8 @@ void HValsCmd::Do() { void HValsCmd::ReadCache() { std::vector values; - std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->HVals(CachePrefixKeyH, &values); + auto s = db_->cache()->HVals(key_, &values); if (s.ok()) { res_.AppendArrayLen(values.size()); for (const auto& value : values) { diff --git a/src/pika_kv.cc b/src/pika_kv.cc index c9fe261554..9f59aea2ee 100644 --- a/src/pika_kv.cc +++ b/src/pika_kv.cc @@ -111,7 +111,6 @@ void SetCmd::DoUpdateCache() { return; } if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; if (has_ttl_) { db_->cache()->Setxx(key_, value_, ttl_millsec > 0 ? ttl_millsec / 1000 : ttl_millsec); } else { @@ -160,7 +159,7 @@ void GetCmd::DoInitial() { } void GetCmd::Do() { - s_ = db_->storage()->GetWithTTL(key_, &value_, &sec_); + s_ = db_->storage()->GetWithTTL(key_, &value_, &ttl_millsec_); if (s_.ok()) { res_.AppendStringLenUint64(value_.size()); res_.AppendContent(value_); @@ -174,8 +173,7 @@ void GetCmd::Do() { } void GetCmd::ReadCache() { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - auto s = db_->cache()->Get(CachePrefixKeyK, &value_); + auto s = db_->cache()->Get(key_, &value_); if (s.ok()) { res_.AppendStringLen(value_.size()); res_.AppendContent(value_); @@ -194,8 +192,7 @@ void GetCmd::DoUpdateCache() { return; } if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); + db_->cache()->WriteKVToCache(key_, value_, ttl_millsec_ > 0 ? ttl_millsec_ / 1000 : ttl_millsec_); } } @@ -209,9 +206,8 @@ void DelCmd::DoInitial() { } void DelCmd::Do() { - std::map type_status; - int64_t count = db_->storage()->Del(keys_, &type_status); + int64_t count = db_->storage()->Del(keys_); if (count >= 0) { res_.AppendInteger(count); @@ -234,11 +230,7 @@ void DelCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; for (auto key : keys_) { - v.emplace_back(PCacheKeyPrefixK + key); - v.emplace_back(PCacheKeyPrefixL + key); - v.emplace_back(PCacheKeyPrefixZ + key); - v.emplace_back(PCacheKeyPrefixS + key); - v.emplace_back(PCacheKeyPrefixH + key); + v.emplace_back(key); } db_->cache()->Del(v); } @@ -246,7 +238,7 @@ void DelCmd::DoUpdateCache() { void DelCmd::Split(const HintKeys& hint_keys) { std::map type_status; - int64_t count = db_->storage()->Del(hint_keys.keys, &type_status); + int64_t count = db_->storage()->Del(hint_keys.keys); if (count >= 0) { split_res_ += count; } else { @@ -275,7 +267,7 @@ void IncrCmd::DoInitial() { } void IncrCmd::Do() { - s_ = db_->storage()->Incrby(key_, 1, &new_value_, &expired_timestamp_sec_); + s_ = db_->storage()->Incrby(key_, 1, &new_value_, &expired_timestamp_millsec_); if (s_.ok()) { res_.AppendContent(":" + std::to_string(new_value_)); AddSlotKey("k", key_, db_); @@ -296,8 +288,7 @@ void IncrCmd::DoThroughDB() { void IncrCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->Incrxx(CachePrefixKeyK); + db_->cache()->Incrxx(key_); } } @@ -340,7 +331,7 @@ void IncrbyCmd::DoInitial() { } void IncrbyCmd::Do() { - s_ = db_->storage()->Incrby(key_, by_, &new_value_, &expired_timestamp_sec_); + s_ = db_->storage()->Incrby(key_, by_, &new_value_, &expired_timestamp_millsec_); if (s_.ok()) { res_.AppendContent(":" + std::to_string(new_value_)); AddSlotKey("k", key_, db_); @@ -361,8 +352,7 @@ void IncrbyCmd::DoThroughDB() { void IncrbyCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->IncrByxx(CachePrefixKeyK, by_); + db_->cache()->IncrByxx(key_, by_); } } @@ -406,7 +396,7 @@ void IncrbyfloatCmd::DoInitial() { } void IncrbyfloatCmd::Do() { - s_ = db_->storage()->Incrbyfloat(key_, value_, &new_value_, &expired_timestamp_sec_); + s_ = db_->storage()->Incrbyfloat(key_, value_, &new_value_, &expired_timestamp_millsec_); if (s_.ok()) { res_.AppendStringLenUint64(new_value_.size()); res_.AppendContent(new_value_); @@ -430,8 +420,7 @@ void IncrbyfloatCmd::DoUpdateCache() { if (s_.ok()) { long double long_double_by; if (storage::StrToLongDouble(value_.data(), value_.size(), &long_double_by) != -1) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->Incrbyfloatxx(CachePrefixKeyK, long_double_by); + db_->cache()->Incrbyfloatxx(key_, long_double_by); } } } @@ -492,8 +481,7 @@ void DecrCmd::DoThroughDB() { void DecrCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->Decrxx(CachePrefixKeyK); + db_->cache()->Decrxx(key_); } } @@ -532,8 +520,7 @@ void DecrbyCmd::DoThroughDB() { void DecrbyCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->DecrByxx(CachePrefixKeyK, by_); + db_->cache()->DecrByxx(key_, by_); } } @@ -571,8 +558,7 @@ void GetsetCmd::DoThroughDB() { void GetsetCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->SetxxWithoutTTL(CachePrefixKeyK, new_value_); + db_->cache()->SetxxWithoutTTL(key_, new_value_); } } @@ -587,7 +573,7 @@ void AppendCmd::DoInitial() { void AppendCmd::Do() { int32_t new_len = 0; - s_ = db_->storage()->Append(key_, value_, &new_len, &expired_timestamp_sec_, new_value_); + s_ = db_->storage()->Append(key_, value_, &new_len, &expired_timestamp_millsec_, new_value_); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(new_len); AddSlotKey("k", key_, db_); @@ -604,8 +590,7 @@ void AppendCmd::DoThroughDB() { void AppendCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->Appendxx(CachePrefixKeyK, value_); + db_->cache()->Appendxx(key_, value_); } } @@ -883,7 +868,7 @@ void SetexCmd::DoInitial() { } void SetexCmd::Do() { - s_ = db_->storage()->Setex(key_, value_, static_cast(sec_)); + s_ = db_->storage()->Setex(key_, value_, ttl_sec_ * 1000); if (s_.ok()) { res_.SetRes(CmdRes::kOk); AddSlotKey("k", key_, db_); @@ -900,8 +885,7 @@ void SetexCmd::DoThroughDB() { void SetexCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->Setxx(CachePrefixKeyK, value_, sec_); + db_->cache()->Setxx(key_, value_, ttl_sec_); } } @@ -944,7 +928,7 @@ void PsetexCmd::DoInitial() { } void PsetexCmd::Do() { - s_ = db_->storage()->Setex(key_, value_, static_cast(usec_ / 1000)); + s_ = db_->storage()->Setex(key_, value_, ttl_millsec); if (s_.ok()) { res_.SetRes(CmdRes::kOk); } else if (s_.IsInvalidArgument()) { @@ -960,8 +944,7 @@ void PsetexCmd::DoThroughDB() { void PsetexCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->Setxx(CachePrefixKeyK, value_, static_cast(usec_ / 1000)); + db_->cache()->Setxx(key_, value_, ttl_millsec / 1000); } } @@ -1049,7 +1032,6 @@ void MsetCmd::DoThroughDB() { void MsetCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK; for (auto key : kvs_) { db_->cache()->SetxxWithoutTTL(key.key, key.value); } @@ -1182,8 +1164,7 @@ void GetrangeCmd::Do() { void GetrangeCmd::ReadCache() { std::string substr; - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - auto s = db_->cache()->GetRange(CachePrefixKeyK, start_, end_, &substr); + auto s = db_->cache()->GetRange(key_, start_, end_, &substr); if (s.ok()) { res_.AppendStringLen(substr.size()); res_.AppendContent(substr); @@ -1210,8 +1191,7 @@ void GetrangeCmd::DoThroughDB() { void GetrangeCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); + db_->cache()->WriteKVToCache(key_, value_, sec_); } } @@ -1248,9 +1228,9 @@ void SetrangeCmd::DoThroughDB() { void SetrangeCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->SetRangeIfKeyExist(CachePrefixKeyK, offset_, value_); + db_->cache()->SetRangeIfKeyExist(key_, offset_, value_); } } @@ -1277,8 +1257,7 @@ void StrlenCmd::Do() { void StrlenCmd::ReadCache() { int32_t len = 0; - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - auto s= db_->cache()->Strlen(CachePrefixKeyK, &len); + auto s= db_->cache()->Strlen(key_, &len); if (s.ok()) { res_.AppendInteger(len); } else { @@ -1288,7 +1267,7 @@ void StrlenCmd::ReadCache() { void StrlenCmd::DoThroughDB() { res_.clear(); - s_ = db_->storage()->GetWithTTL(key_, &value_, &sec_); + s_ = db_->storage()->GetWithTTL(key_, &value_, &ttl_millsec); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(value_.size()); } else { @@ -1298,8 +1277,7 @@ void StrlenCmd::DoThroughDB() { void StrlenCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - db_->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); + db_->cache()->WriteKVToCache(key_, value_, ttl_millsec > 0 ? ttl_millsec : ttl_millsec / 1000); } } @@ -1313,8 +1291,7 @@ void ExistsCmd::DoInitial() { } void ExistsCmd::Do() { - std::map type_status; - int64_t res = db_->storage()->Exists(keys_, &type_status); + int64_t res = db_->storage()->Exists(keys_); if (res != -1) { res_.AppendInteger(res); } else { @@ -1323,8 +1300,7 @@ void ExistsCmd::Do() { } void ExistsCmd::Split(const HintKeys& hint_keys) { - std::map type_status; - int64_t res = db_->storage()->Exists(hint_keys.keys, &type_status); + int64_t res = db_->storage()->Exists(hint_keys.keys); if (res != -1) { split_res_ += res; } else { @@ -1339,21 +1315,9 @@ void ExistsCmd::ReadCache() { res_.SetRes(CmdRes::kCacheMiss); return; } - uint32_t nums = 0; - std::vector v; - v.emplace_back(PCacheKeyPrefixK + keys_[0]); - v.emplace_back(PCacheKeyPrefixL + keys_[0]); - v.emplace_back(PCacheKeyPrefixZ + keys_[0]); - v.emplace_back(PCacheKeyPrefixS + keys_[0]); - v.emplace_back(PCacheKeyPrefixH + keys_[0]); - for (auto key : v) { - bool exist = db_->cache()->Exists(key); - if (exist) { - nums++; - } - } - if (nums > 0) { - res_.AppendInteger(nums); + bool exist = db_->cache()->Exists(keys_[0]); + if (exist) { + res_.AppendInteger(1); } else { res_.SetRes(CmdRes::kCacheMiss); } @@ -1377,8 +1341,7 @@ void ExpireCmd::DoInitial() { } void ExpireCmd::Do() { - std::map type_status; - int64_t res = db_->storage()->Expire(key_, static_cast(sec_), &type_status); + int64_t res = db_->storage()->Expire(key_, ttl_sec_ * 1000); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1416,15 +1379,7 @@ void ExpireCmd::DoThroughDB() { void ExpireCmd::DoUpdateCache() { if (s_.ok()) { - std::vector v; - v.emplace_back(PCacheKeyPrefixK + key_); - v.emplace_back(PCacheKeyPrefixL + key_); - v.emplace_back(PCacheKeyPrefixZ + key_); - v.emplace_back(PCacheKeyPrefixS + key_); - v.emplace_back(PCacheKeyPrefixH + key_); - for (auto key : v) { - db_->cache()->Expire(key, sec_); - } + db_->cache()->Expire(key_, ttl_sec_); } } @@ -1441,8 +1396,8 @@ void PexpireCmd::DoInitial() { } void PexpireCmd::Do() { - std::map type_status; - int64_t res = db_->storage()->Expire(key_, static_cast(msec_ / 1000), &type_status); + + int64_t res = db_->storage()->Expire(key_, ttl_millsec); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1480,15 +1435,7 @@ void PexpireCmd::DoThroughDB() { void PexpireCmd::DoUpdateCache() { if (s_.ok()) { - std::vector v; - v.emplace_back(PCacheKeyPrefixK + key_); - v.emplace_back(PCacheKeyPrefixL + key_); - v.emplace_back(PCacheKeyPrefixZ + key_); - v.emplace_back(PCacheKeyPrefixS + key_); - v.emplace_back(PCacheKeyPrefixH + key_); - for (auto key : v){ - db_->cache()->Expire(key, msec_/1000); - } + db_->cache()->Expire(key_, ttl_millsec); } } @@ -1505,8 +1452,7 @@ void ExpireatCmd::DoInitial() { } void ExpireatCmd::Do() { - std::map type_status; - int32_t res = db_->storage()->Expireat(key_, static_cast(time_stamp_), &type_status); + int32_t res = db_->storage()->Expireat(key_, time_stamp_sec_ * 100); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1522,15 +1468,7 @@ void ExpireatCmd::DoThroughDB() { void ExpireatCmd::DoUpdateCache() { if (s_.ok()) { - std::vector v; - v.emplace_back(PCacheKeyPrefixK + key_); - v.emplace_back(PCacheKeyPrefixL + key_); - v.emplace_back(PCacheKeyPrefixZ + key_); - v.emplace_back(PCacheKeyPrefixS + key_); - v.emplace_back(PCacheKeyPrefixH + key_); - for (auto key : v) { - db_->cache()->Expireat(key, time_stamp_); - } + db_->cache()->Expireat(key_, time_stamp_sec_); } } @@ -1547,8 +1485,7 @@ void PexpireatCmd::DoInitial() { } void PexpireatCmd::Do() { - std::map type_status; - int32_t res = db_->storage()->Expireat(key_, static_cast(time_stamp_ms_ / 1000), &type_status); + int32_t res = db_->storage()->Expireat(key_, static_cast(time_stamp_millsec_)); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1564,15 +1501,7 @@ void PexpireatCmd::DoThroughDB() { void PexpireatCmd::DoUpdateCache() { if (s_.ok()) { - std::vector v; - v.emplace_back(PCacheKeyPrefixK + key_); - v.emplace_back(PCacheKeyPrefixL + key_); - v.emplace_back(PCacheKeyPrefixZ + key_); - v.emplace_back(PCacheKeyPrefixS + key_); - v.emplace_back(PCacheKeyPrefixH + key_); - for (auto key : v) { - db_->cache()->Expireat(key, time_stamp_ms_ / 1000); - } + db_->cache()->Expireat(key_, time_stamp_millsec_ / 1000); } } @@ -1585,53 +1514,20 @@ void TtlCmd::DoInitial() { } void TtlCmd::Do() { - std::map type_timestamp; - std::map type_status; - type_timestamp = db_->storage()->TTL(key_, &type_status); - for (const auto& item : type_timestamp) { - // mean operation exception errors happen in database - if (item.second == -3) { - res_.SetRes(CmdRes::kErrOther, "ttl internal error"); - return; - } - } - if (type_timestamp[storage::kStrings] != -2) { - res_.AppendInteger(type_timestamp[storage::kStrings]); - } else if (type_timestamp[storage::kHashes] != -2) { - res_.AppendInteger(type_timestamp[storage::kHashes]); - } else if (type_timestamp[storage::kLists] != -2) { - res_.AppendInteger(type_timestamp[storage::kLists]); - } else if (type_timestamp[storage::kZSets] != -2) { - res_.AppendInteger(type_timestamp[storage::kZSets]); - } else if (type_timestamp[storage::kSets] != -2) { - res_.AppendInteger(type_timestamp[storage::kSets]); + int64_t ttl_sec_ = db_->storage()->TTL(key_); + if (ttl_sec_ == -3) { + res_.SetRes(CmdRes::kErrOther, "ttl internal error"); } else { res_.AppendInteger(ttl_sec_); } } void TtlCmd::ReadCache() { - rocksdb::Status s; - std::map type_timestamp; - std::map type_status; - type_timestamp = db_->cache()->TTL(key_, &type_status); - for (const auto& item : type_timestamp) { - // mean operation exception errors happen in database - if (item.second == -3) { - res_.SetRes(CmdRes::kErrOther, "ttl internal error"); - return; - } - } - if (type_timestamp[storage::kStrings] != -2) { - res_.AppendInteger(type_timestamp[storage::kStrings]); - } else if (type_timestamp[storage::kHashes] != -2) { - res_.AppendInteger(type_timestamp[storage::kHashes]); - } else if (type_timestamp[storage::kLists] != -2) { - res_.AppendInteger(type_timestamp[storage::kLists]); - } else if (type_timestamp[storage::kZSets] != -2) { - res_.AppendInteger(type_timestamp[storage::kZSets]); - } else if (type_timestamp[storage::kSets] != -2) { - res_.AppendInteger(type_timestamp[storage::kSets]); + int64_t timestamp = db_->cache()->TTL(key_); + if (timestamp == -3) { + res_.SetRes(CmdRes::kErrOther, "ttl internal error"); + } else if (timestamp != -2) { + res_.AppendInteger(timestamp); } else { res_.SetRes(CmdRes::kCacheMiss); } @@ -1651,46 +1547,9 @@ void PttlCmd::DoInitial() { } void PttlCmd::Do() { - std::map type_timestamp; - std::map type_status; - type_timestamp = db_->storage()->TTL(key_, &type_status); - for (const auto& item : type_timestamp) { - // mean operation exception errors happen in database - if (item.second == -3) { - res_.SetRes(CmdRes::kErrOther, "ttl internal error"); - return; - } - } - if (type_timestamp[storage::kStrings] != -2) { - if (type_timestamp[storage::kStrings] == -1) { - res_.AppendInteger(-1); - } else { - res_.AppendInteger(type_timestamp[storage::kStrings] * 1000); - } - } else if (type_timestamp[storage::kHashes] != -2) { - if (type_timestamp[storage::kHashes] == -1) { - res_.AppendInteger(-1); - } else { - res_.AppendInteger(type_timestamp[storage::kHashes] * 1000); - } - } else if (type_timestamp[storage::kLists] != -2) { - if (type_timestamp[storage::kLists] == -1) { - res_.AppendInteger(-1); - } else { - res_.AppendInteger(type_timestamp[storage::kLists] * 1000); - } - } else if (type_timestamp[storage::kSets] != -2) { - if (type_timestamp[storage::kSets] == -1) { - res_.AppendInteger(-1); - } else { - res_.AppendInteger(type_timestamp[storage::kSets] * 1000); - } - } else if (type_timestamp[storage::kZSets] != -2) { - if (type_timestamp[storage::kZSets] == -1) { - res_.AppendInteger(-1); - } else { - res_.AppendInteger(type_timestamp[storage::kZSets] * 1000); - } + int64_t ttl_millsec = db_->storage()->PTTL(key_); + if (ttl_millsec == -3) { + res_.SetRes(CmdRes::kErrOther, "ttl internal error"); } else { // mean this key not exist res_.AppendInteger(-2); @@ -1698,50 +1557,7 @@ void PttlCmd::Do() { } void PttlCmd::ReadCache() { - std::map type_timestamp; - std::map type_status; - type_timestamp = db_->cache()->TTL(key_, &type_status); - for (const auto& item : type_timestamp) { - // mean operation exception errors happen in database - if (item.second == -3) { - res_.SetRes(CmdRes::kErrOther, "ttl internal error"); - return; - } - } - if (type_timestamp[storage::kStrings] != -2) { - if (type_timestamp[storage::kStrings] == -1) { - res_.AppendInteger(-1); - } else { - res_.AppendInteger(type_timestamp[storage::kStrings] * 1000); - } - } else if (type_timestamp[storage::kHashes] != -2) { - if (type_timestamp[storage::kHashes] == -1) { - res_.AppendInteger(-1); - } else { - res_.AppendInteger(type_timestamp[storage::kHashes] * 1000); - } - } else if (type_timestamp[storage::kLists] != -2) { - if (type_timestamp[storage::kLists] == -1) { - res_.AppendInteger(-1); - } else { - res_.AppendInteger(type_timestamp[storage::kLists] * 1000); - } - } else if (type_timestamp[storage::kSets] != -2) { - if (type_timestamp[storage::kSets] == -1) { - res_.AppendInteger(-1); - } else { - res_.AppendInteger(type_timestamp[storage::kSets] * 1000); - } - } else if (type_timestamp[storage::kZSets] != -2) { - if (type_timestamp[storage::kZSets] == -1) { - res_.AppendInteger(-1); - } else { - res_.AppendInteger(type_timestamp[storage::kZSets] * 1000); - } - } else { - // mean this key not exist - res_.SetRes(CmdRes::kCacheMiss); - } + DoThroughDB(); } void PttlCmd::DoThroughDB() { @@ -1758,8 +1574,7 @@ void PersistCmd::DoInitial() { } void PersistCmd::Do() { - std::map type_status; - int32_t res = db_->storage()->Persist(key_, &type_status); + int32_t res = db_->storage()->Persist(key_); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1775,15 +1590,7 @@ void PersistCmd::DoThroughDB() { void PersistCmd::DoUpdateCache() { if (s_.ok()) { - std::vector v; - v.emplace_back(PCacheKeyPrefixK + key_); - v.emplace_back(PCacheKeyPrefixL + key_); - v.emplace_back(PCacheKeyPrefixZ + key_); - v.emplace_back(PCacheKeyPrefixS + key_); - v.emplace_back(PCacheKeyPrefixH + key_); - for (auto key : v) { - db_->cache()->Persist(key); - } + db_->cache()->Persist(key_); } } @@ -1795,9 +1602,9 @@ void TypeCmd::DoInitial() { key_ = argv_[1]; } -void TypeCmd::Do() { - std::vector types(1); - rocksdb::Status s = db_->storage()->GetType(key_, true, types); +void TypeCmd::Do() { enum storage::DataType type = storage::DataType::kNones; + std::string key_type; + rocksdb::Status s = db_->storage()->GetType(key_, type); if (s.ok()) { res_.AppendContent("+" + std::string(DataTypeToString(type))); } else if (s_.IsInvalidArgument()) { @@ -1808,8 +1615,10 @@ void TypeCmd::Do() { } void TypeCmd::ReadCache() { - std::vector types(1); - rocksdb::Status s = db_->storage()->GetType(key_, true, types); + enum storage::DataType type = storage::DataType::kNones; + std::string key_type; + // TODO Cache GetType function + rocksdb::Status s = db_->storage()->GetType(key_, type); if (s.ok()) { res_.AppendContent("+" + std::string(DataTypeToString(type))); } else { @@ -1822,29 +1631,6 @@ void TypeCmd::DoThroughDB() { Do(); } -void PTypeCmd::DoInitial() { - if (!CheckArg(argv_.size())) { - res_.SetRes(CmdRes::kWrongNum, kCmdNameType); - return; - } - key_ = argv_[1]; -} - -void PTypeCmd::Do() { - std::vector types(5); - rocksdb::Status s = db_->storage()->GetType(key_, false, types); - - if (s.ok()) { - res_.AppendArrayLenUint64(types.size()); - for (const auto& vs : types) { - res_.AppendStringLenUint64(vs.size()); - res_.AppendContent(vs); - } - } else { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - } -} - void ScanCmd::DoInitial() { if (!CheckArg(argv_.size())) { res_.SetRes(CmdRes::kWrongNum, kCmdNameScan); @@ -2012,7 +1798,7 @@ void PKSetexAtCmd::DoInitial() { } void PKSetexAtCmd::Do() { - s_ = db_->storage()->PKSetexAt(key_, value_, static_cast(time_stamp_)); + s_ = db_->storage()->PKSetexAt(key_, value_, static_cast(time_stamp_sec_ * 1000)); if (s_.ok()) { res_.SetRes(CmdRes::kOk); } else if (s_.IsInvalidArgument()) { @@ -2028,8 +1814,7 @@ void PKSetexAtCmd::DoThroughDB() { void PKSetexAtCmd::DoUpdateCache() { if (s_.ok()) { - auto expire = time_stamp_ - static_cast(std::time(nullptr)); - std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; + auto expire = time_stamp_sec_ - static_cast(std::time(nullptr)); if (expire <= 0) [[unlikely]] { db_->cache()->Del({key_}); return; diff --git a/src/pika_list.cc b/src/pika_list.cc index c4fe88583e..9660fd9628 100644 --- a/src/pika_list.cc +++ b/src/pika_list.cc @@ -46,7 +46,7 @@ void LIndexCmd::Do() { void LIndexCmd::ReadCache() { std::string value; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->LIndex(CachePrefixKeyL, index_, &value); + auto s = db_->cache()->LIndex(key_, index_, &value); if (s.ok()) { res_.AppendString(value); } else if (s.IsNotFound()) { @@ -108,9 +108,8 @@ void LInsertCmd::DoThroughDB() { void LInsertCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->LInsert(CachePrefixKeyL, dir_, pivot_, value_); + db_->cache()->LInsert(key_, dir_, pivot_, value_); } } @@ -138,7 +137,7 @@ void LLenCmd::Do() { void LLenCmd::ReadCache() { uint64_t llen = 0; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->LLen(CachePrefixKeyL, &llen); + auto s = db_->cache()->LLen(key_, &llen); if (s.ok()){ res_.AppendInteger(llen); } else if (s.IsNotFound()) { @@ -300,9 +299,8 @@ void LPushCmd::DoThroughDB() { void LPushCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->LPushIfKeyExist(CachePrefixKeyL, values_); + db_->cache()->LPushIfKeyExist(key_, values_); } } @@ -464,7 +462,7 @@ void LPopCmd::DoUpdateCache() { if (s_.ok()) { std::string value; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->LPop(CachePrefixKeyL, &value); + db_->cache()->LPop(key_, &value); } } @@ -500,9 +498,8 @@ void LPushxCmd::DoThroughDB() { void LPushxCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->LPushIfKeyExist(CachePrefixKeyL, values_); + db_->cache()->LPushIfKeyExist(key_, values_); } } @@ -543,9 +540,8 @@ void LRangeCmd::Do() { void LRangeCmd::ReadCache() { std::vector values; - std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->LRange(CachePrefixKeyL, left_, right_, &values); + auto s = db_->cache()->LRange(key_, left_, right_, &values); if (s.ok()) { res_.AppendArrayLen(values.size()); for (const auto& value : values) { @@ -604,9 +600,8 @@ void LRemCmd::DoThroughDB() { void LRemCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->LRem(CachePrefixKeyL, count_, value_); + db_->cache()->LRem(key_, count_, value_); } } @@ -648,9 +643,8 @@ void LSetCmd::DoThroughDB() { void LSetCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->LSet(CachePrefixKeyL, index_, value_); + db_->cache()->LSet(key_, index_, value_); } } @@ -689,9 +683,8 @@ void LTrimCmd::DoThroughDB() { void LTrimCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->LTrim(CachePrefixKeyL, start_, stop_); + db_->cache()->LTrim(key_, start_, stop_); } } @@ -818,7 +811,7 @@ void RPopCmd::DoUpdateCache() { if (s_.ok()) { std::string value; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->RPop(CachePrefixKeyL, &value); + db_->cache()->RPop(key_, &value); } } @@ -890,10 +883,9 @@ void RPopLPushCmd::DoUpdateCache() { STAGE_TIMER_GUARD(cache_duration_ms, true); std::vector value; value.resize(1); - std::string CachePrefixKeyLR = PCacheKeyPrefixL + source_; - db_->cache()->RPop(CachePrefixKeyLR, &value[0]); - std::string CachePrefixKeyLL = PCacheKeyPrefixL + receiver_; - db_->cache()->LPushIfKeyExist(CachePrefixKeyLL, value); + db_->cache()->RPop(source_, &value[0]); + + db_->cache()->LPushIfKeyExist(receiver_, value); } } void RPopLPushCmd::DoThroughDB() { @@ -938,9 +930,8 @@ void RPushCmd::DoThroughDB() { void RPushCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->RPushIfKeyExist(CachePrefixKeyL, values_); + db_->cache()->RPushIfKeyExist(key_, values_); } } @@ -976,8 +967,7 @@ void RPushxCmd::DoThroughDB() { void RPushxCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->RPushIfKeyExist(CachePrefixKeyL, values_); + db_->cache()->RPushIfKeyExist(key_, values_); } } \ No newline at end of file diff --git a/src/pika_set.cc b/src/pika_set.cc index 963858c6ae..1745375e59 100644 --- a/src/pika_set.cc +++ b/src/pika_set.cc @@ -42,9 +42,8 @@ void SAddCmd::DoThroughDB() { void SAddCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->SAddIfKeyExist(CachePrefixKeyS, members_); + db_->cache()->SAddIfKeyExist(key_, members_); } } @@ -95,9 +94,8 @@ void SPopCmd::DoThroughDB() { void SPopCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->SRem(CachePrefixKeyS, members_); + db_->cache()->SRem(key_, members_); } } @@ -145,9 +143,8 @@ void SCardCmd::Do() { void SCardCmd::ReadCache() { uint64_t card = 0; - std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->SCard(CachePrefixKeyS, &card); + auto s = db_->cache()->SCard(key_, &card); if (s.ok()) { res_.AppendInteger(card); } else if (s.IsNotFound()) { @@ -200,9 +197,8 @@ void SMembersCmd::Do() { void SMembersCmd::ReadCache() { std::vector members; - std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->SMembers(CachePrefixKeyS, &members); + auto s = db_->cache()->SMembers(key_, &members); if (s.ok()) { res_.AppendArrayLen(members.size()); for (const auto& member : members) { @@ -332,9 +328,8 @@ void SRemCmd::DoThroughDB() { void SRemCmd::DoUpdateCache() { if (s_.ok() && deleted_ > 0) { - std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->SRem(CachePrefixKeyS, members_); + db_->cache()->SRem(key_, members_); } } @@ -396,7 +391,7 @@ void SUnionstoreCmd::DoThroughDB() { void SUnionstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(PCacheKeyPrefixS + dest_key_); + v.emplace_back(dest_key_); STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->Del(v); } @@ -499,7 +494,7 @@ void SInterstoreCmd::DoThroughDB() { void SInterstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(PCacheKeyPrefixS + dest_key_); + v.emplace_back(dest_key_); STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->Del(v); } @@ -531,9 +526,9 @@ void SIsmemberCmd::Do() { } void SIsmemberCmd::ReadCache() { - std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->SIsmember(CachePrefixKeyS, member_); + auto s = db_->cache()->SIsmember(key_, member_); if (s.ok()) { res_.AppendContent(":1"); } else if (s.IsNotFound()) { @@ -614,7 +609,7 @@ void SDiffstoreCmd::DoThroughDB() { void SDiffstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(PCacheKeyPrefixS + dest_key_); + v.emplace_back(dest_key_); STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->Del(v); } @@ -656,11 +651,9 @@ void SMoveCmd::DoUpdateCache() { if (s_.ok()) { std::vector members; members.emplace_back(member_); - std::string CachePrefixKeyS = PCacheKeyPrefixS + src_key_; - std::string CachePrefixKeyD = PCacheKeyPrefixS + dest_key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->SRem(CachePrefixKeyS, members); - db_->cache()->SAddIfKeyExist(CachePrefixKeyD, members); + db_->cache()->SRem(src_key_, members); + db_->cache()->SAddIfKeyExist(dest_key_, members); } } @@ -737,9 +730,9 @@ void SRandmemberCmd::Do() { void SRandmemberCmd::ReadCache() { std::vector members; - std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->SRandmember(CachePrefixKeyS, count_, &members); + auto s = db_->cache()->SRandmember(key_, count_, &members); if (s.ok()) { if (!reply_arr && members.size()) { res_.AppendStringLen(members[0].size()); diff --git a/src/pika_zset.cc b/src/pika_zset.cc index 8f06e885c3..54865cf8cb 100644 --- a/src/pika_zset.cc +++ b/src/pika_zset.cc @@ -54,9 +54,8 @@ void ZAddCmd::DoThroughDB() { void ZAddCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->ZAddIfKeyExist(CachePrefixKeyZ, score_members); + db_->cache()->ZAddIfKeyExist(key_, score_members); } } @@ -196,9 +195,8 @@ void ZIncrbyCmd::DoThroughDB() { void ZIncrbyCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->ZIncrbyIfKeyExist(CachePrefixKeyZ, member_, by_, this, db_); + db_->cache()->ZIncrbyIfKeyExist(key_, member_, by_, this, db_); } } @@ -751,9 +749,8 @@ void ZRemCmd::DoThroughDB() { void ZRemCmd::DoUpdateCache() { if (s_.ok() && deleted_ > 0) { - std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->ZRem(CachePrefixKeyZ, members_, db_); + db_->cache()->ZRem(key_, members_, db_); } } @@ -844,7 +841,7 @@ void ZUnionstoreCmd::DoThroughDB() { void ZUnionstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(PCacheKeyPrefixZ + dest_key_); + v.emplace_back(dest_key_); STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->Del(v); } @@ -926,7 +923,7 @@ void ZInterstoreCmd::DoThroughDB() { void ZInterstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(PCacheKeyPrefixZ + dest_key_); + v.emplace_back(dest_key_); STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->Del(v); } @@ -1110,9 +1107,9 @@ void ZScoreCmd::Do() { void ZScoreCmd::ReadCache() { double score = 0.0; - std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); - auto s = db_->cache()->ZScore(CachePrefixKeyZ, member_, &score, db_); + auto s = db_->cache()->ZScore(key_, member_, &score, db_); if (s.ok()) { char buf[32]; int64_t len = pstd::d2string(buf, sizeof(buf), score); @@ -1445,9 +1442,9 @@ void ZRemrangebyrankCmd::DoThroughDB() { void ZRemrangebyrankCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; + STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->ZRemrangebyrank(CachePrefixKeyZ, min_, max_, ele_deleted_, db_); + db_->cache()->ZRemrangebyrank(key_, min_, max_, ele_deleted_, db_); } } @@ -1488,9 +1485,8 @@ void ZRemrangebyscoreCmd::DoThroughDB() { void ZRemrangebyscoreCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->ZRemrangebyscore(CachePrefixKeyZ, min_, max_, db_); + db_->cache()->ZRemrangebyscore(key_, min_, max_, db_); } } @@ -1532,9 +1528,8 @@ void ZRemrangebylexCmd::DoThroughDB() { void ZRemrangebylexCmd::DoUpdateCache() { if (s_.ok()) { - std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; STAGE_TIMER_GUARD(cache_duration_ms, true); - db_->cache()->ZRemrangebylex(CachePrefixKeyZ, min_, max_, db_); + db_->cache()->ZRemrangebylex(key_, min_, max_, db_); } } diff --git a/tools/pika_exporter/Makefile b/tools/pika_exporter/Makefile index 7e47d19a8e..64620c181e 100644 --- a/tools/pika_exporter/Makefile +++ b/tools/pika_exporter/Makefile @@ -3,17 +3,21 @@ # endif # export PATH := $(PATH):$(GOPATH)/bin - +BRANCH := $(shell git branch | sed -n 's/* \(.*\)/\1/p') GITREV := $(shell git rev-parse HEAD) BUILDTIME := $(shell date '+%F %T %Z') COMPILERVERSION := $(subst go version ,,$(shell go version)) PROJNAME := pika_exporter +PIKA_EXPORTER_MAJOR := 3 +PIKA_EXPORTER_MINOR := 5 +PIKA_EXPORTER_PATCH := 5 define GENERATE_VERSION_CODE cat << EOF | gofmt > version.go package main const ( + BuildVersion = "$(BRANCH)" BuildCommitSha = "$(GITREV)" BuildDate = "$(BUILDTIME)" From 5c5d89d5478138d64bd071840beab30a64105075 Mon Sep 17 00:00:00 2001 From: wuxianrong Date: Wed, 26 Mar 2025 11:23:04 +0800 Subject: [PATCH 12/15] code format --- src/pika_client_conn.cc | 2 -- src/pika_kv.cc | 23 ++++++++--------------- src/pika_list.cc | 2 +- src/pika_set.cc | 3 --- src/pika_zset.cc | 3 +-- tools/pika_exporter/Makefile | 6 +++--- 6 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/pika_client_conn.cc b/src/pika_client_conn.cc index 35ad92b4ed..f3c34af159 100644 --- a/src/pika_client_conn.cc +++ b/src/pika_client_conn.cc @@ -50,7 +50,6 @@ std::shared_ptr PikaClientConn::DoCmd(const PikaCmdArgsType& argv, const st } return tmp_ptr; } - c_ptr->SetCacheMissedInRtc(cache_miss_in_rtc); c_ptr->SetConn(shared_from_this()); c_ptr->SetResp(resp_ptr); @@ -219,7 +218,6 @@ std::shared_ptr PikaClientConn::DoCmd(const PikaCmdArgsType& argv, const st rocksdb::get_perf_context()->Reset(); // Process Command c_ptr->Execute(); - time_stat_->process_done_ts_ = pstd::NowMicros(); auto cmdstat_map = g_pika_cmd_table_manager->GetCommandStatMap(); (*cmdstat_map)[opt].cmd_count.fetch_add(1); diff --git a/src/pika_kv.cc b/src/pika_kv.cc index 9f59aea2ee..71a3a5d70e 100644 --- a/src/pika_kv.cc +++ b/src/pika_kv.cc @@ -206,9 +206,7 @@ void DelCmd::DoInitial() { } void DelCmd::Do() { - int64_t count = db_->storage()->Del(keys_); - if (count >= 0) { res_.AppendInteger(count); s_ = rocksdb::Status::OK(); @@ -228,11 +226,7 @@ void DelCmd::DoThroughDB() { void DelCmd::DoUpdateCache() { if (s_.ok()) { - std::vector v; - for (auto key : keys_) { - v.emplace_back(key); - } - db_->cache()->Del(v); + db_->cache()->Del(keys_); } } @@ -1228,7 +1222,6 @@ void SetrangeCmd::DoThroughDB() { void SetrangeCmd::DoUpdateCache() { if (s_.ok()) { - STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->SetRangeIfKeyExist(key_, offset_, value_); } @@ -1341,7 +1334,7 @@ void ExpireCmd::DoInitial() { } void ExpireCmd::Do() { - int64_t res = db_->storage()->Expire(key_, ttl_sec_ * 1000); + int32_t res = db_->storage()->Expire(key_, ttl_sec_ * 1000); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1396,7 +1389,6 @@ void PexpireCmd::DoInitial() { } void PexpireCmd::Do() { - int64_t res = db_->storage()->Expire(key_, ttl_millsec); if (res != -1) { res_.AppendInteger(res); @@ -1452,7 +1444,7 @@ void ExpireatCmd::DoInitial() { } void ExpireatCmd::Do() { - int32_t res = db_->storage()->Expireat(key_, time_stamp_sec_ * 100); + int32_t res = db_->storage()->Expireat(key_, time_stamp_sec_ * 1000); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1527,7 +1519,7 @@ void TtlCmd::ReadCache() { if (timestamp == -3) { res_.SetRes(CmdRes::kErrOther, "ttl internal error"); } else if (timestamp != -2) { - res_.AppendInteger(timestamp); + res_.AppendInteger(timestamp); } else { res_.SetRes(CmdRes::kCacheMiss); } @@ -1551,12 +1543,12 @@ void PttlCmd::Do() { if (ttl_millsec == -3) { res_.SetRes(CmdRes::kErrOther, "ttl internal error"); } else { - // mean this key not exist - res_.AppendInteger(-2); + res_.AppendInteger(ttl_millsec); } } void PttlCmd::ReadCache() { + // redis cache don't support pttl cache, so read directly from db DoThroughDB(); } @@ -1602,7 +1594,8 @@ void TypeCmd::DoInitial() { key_ = argv_[1]; } -void TypeCmd::Do() { enum storage::DataType type = storage::DataType::kNones; +void TypeCmd::Do() { + enum storage::DataType type = storage::DataType::kNones; std::string key_type; rocksdb::Status s = db_->storage()->GetType(key_, type); if (s.ok()) { diff --git a/src/pika_list.cc b/src/pika_list.cc index 9660fd9628..f03d692904 100644 --- a/src/pika_list.cc +++ b/src/pika_list.cc @@ -138,7 +138,7 @@ void LLenCmd::ReadCache() { uint64_t llen = 0; STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->LLen(key_, &llen); - if (s.ok()){ + if (s.ok()) { res_.AppendInteger(llen); } else if (s.IsNotFound()) { res_.SetRes(CmdRes::kCacheMiss); diff --git a/src/pika_set.cc b/src/pika_set.cc index 1745375e59..0643ab4836 100644 --- a/src/pika_set.cc +++ b/src/pika_set.cc @@ -333,7 +333,6 @@ void SRemCmd::DoUpdateCache() { } } - void SUnionCmd::DoInitial() { if (!CheckArg(argv_.size())) { res_.SetRes(CmdRes::kWrongNum, kCmdNameSUnion); @@ -526,7 +525,6 @@ void SIsmemberCmd::Do() { } void SIsmemberCmd::ReadCache() { - STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->SIsmember(key_, member_); if (s.ok()) { @@ -730,7 +728,6 @@ void SRandmemberCmd::Do() { void SRandmemberCmd::ReadCache() { std::vector members; - STAGE_TIMER_GUARD(cache_duration_ms, true); auto s = db_->cache()->SRandmember(key_, count_, &members); if (s.ok()) { diff --git a/src/pika_zset.cc b/src/pika_zset.cc index 54865cf8cb..7ee5291626 100644 --- a/src/pika_zset.cc +++ b/src/pika_zset.cc @@ -1441,8 +1441,7 @@ void ZRemrangebyrankCmd::DoThroughDB() { } void ZRemrangebyrankCmd::DoUpdateCache() { - if (s_.ok()) { - + if (s_.ok()) { STAGE_TIMER_GUARD(cache_duration_ms, true); db_->cache()->ZRemrangebyrank(key_, min_, max_, ele_deleted_, db_); } diff --git a/tools/pika_exporter/Makefile b/tools/pika_exporter/Makefile index 64620c181e..bc1d9460a5 100644 --- a/tools/pika_exporter/Makefile +++ b/tools/pika_exporter/Makefile @@ -8,9 +8,9 @@ GITREV := $(shell git rev-parse HEAD) BUILDTIME := $(shell date '+%F %T %Z') COMPILERVERSION := $(subst go version ,,$(shell go version)) PROJNAME := pika_exporter -PIKA_EXPORTER_MAJOR := 3 -PIKA_EXPORTER_MINOR := 5 -PIKA_EXPORTER_PATCH := 5 +PIKA_EXPORTER_MAJOR := 4 +PIKA_EXPORTER_MINOR := 0 +PIKA_EXPORTER_PATCH := 2 define GENERATE_VERSION_CODE cat << EOF | gofmt > version.go From 057720a76dc13ade9f91ffe60dfda11c4745c581 Mon Sep 17 00:00:00 2001 From: wuxianrong Date: Wed, 26 Mar 2025 11:35:30 +0800 Subject: [PATCH 13/15] code format --- codis/pkg/utils/version.go | 6 ++++-- tools/pika_exporter/Makefile | 2 +- tools/pika_exporter/main.go | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/codis/pkg/utils/version.go b/codis/pkg/utils/version.go index 424aeb7498..abea224a4d 100644 --- a/codis/pkg/utils/version.go +++ b/codis/pkg/utils/version.go @@ -1,6 +1,8 @@ package utils const ( - Version = "2018-11-04 16:22:35 +0800 @de1ad026e329561c22e2a3035fbfe89dc7fef764 @3.2.2-12-gde1ad026" - Compile = "2023-02-23 11:25:09 +0800 by go version go1.19.6 linux/amd64" + Version = "3.5.4" + Gitsha = "5c5d89d5478138d64bd071840beab30a64105075" + Compile = "2025-03-26 11:30:14 CST" + GoVersion = "go1.23.4 darwin/amd64" ) diff --git a/tools/pika_exporter/Makefile b/tools/pika_exporter/Makefile index bc1d9460a5..94b7ff9348 100644 --- a/tools/pika_exporter/Makefile +++ b/tools/pika_exporter/Makefile @@ -17,7 +17,7 @@ cat << EOF | gofmt > version.go package main const ( - + PikaExporterVersion = "$(PIKA_EXPORTER_MAJOR).$(PIKA_EXPORTER_MINOR).$(PIKA_EXPORTER_PATCH)" BuildVersion = "$(BRANCH)" BuildCommitSha = "$(GITREV)" BuildDate = "$(BUILDTIME)" diff --git a/tools/pika_exporter/main.go b/tools/pika_exporter/main.go index 6af84bc4b4..7e728d9a2e 100644 --- a/tools/pika_exporter/main.go +++ b/tools/pika_exporter/main.go @@ -56,6 +56,7 @@ func main() { flag.Parse() log.Println("Pika Metrics Exporter") + log.Println("Pika Exporter Version: ", PikaExporterVersion) log.Println("Build Date: ", BuildDate) log.Println("Commit SHA: ", BuildCommitSha) log.Println("Go Version: ", GoVersion) From e23ec9d6c8782c733cd9622e0feaf12e966c37fb Mon Sep 17 00:00:00 2001 From: wuxianrong Date: Wed, 26 Mar 2025 11:41:21 +0800 Subject: [PATCH 14/15] code format --- src/pika_kv.cc | 2 +- src/pika_list.cc | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pika_kv.cc b/src/pika_kv.cc index 71a3a5d70e..2adcb357e9 100644 --- a/src/pika_kv.cc +++ b/src/pika_kv.cc @@ -1594,7 +1594,7 @@ void TypeCmd::DoInitial() { key_ = argv_[1]; } -void TypeCmd::Do() { +void TypeCmd::Do() { enum storage::DataType type = storage::DataType::kNones; std::string key_type; rocksdb::Status s = db_->storage()->GetType(key_, type); diff --git a/src/pika_list.cc b/src/pika_list.cc index f03d692904..4832f42047 100644 --- a/src/pika_list.cc +++ b/src/pika_list.cc @@ -884,7 +884,6 @@ void RPopLPushCmd::DoUpdateCache() { std::vector value; value.resize(1); db_->cache()->RPop(source_, &value[0]); - db_->cache()->LPushIfKeyExist(receiver_, value); } } From 605e7a07497886e16dc1f2c70b294af566ec6e61 Mon Sep 17 00:00:00 2001 From: wuxianrong Date: Wed, 26 Mar 2025 14:04:12 +0800 Subject: [PATCH 15/15] fix string test --- src/storage/src/redis_strings.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/src/redis_strings.cc b/src/storage/src/redis_strings.cc index 04bf17e41d..0a209d7f30 100644 --- a/src/storage/src/redis_strings.cc +++ b/src/storage/src/redis_strings.cc @@ -101,7 +101,7 @@ Status Redis::Append(const Slice& key, const Slice& value, int32_t* ret, int64_t *ret = static_cast(value.size()); out_new_value = value.ToString(); StringsValue strings_value(value); - return db_->Put(default_write_options_, key, strings_value.Encode()); + return db_->Put(default_write_options_, base_key.Encode(), strings_value.Encode()); } return s; }