Example Modify the policy for updating big keys except String to Redis-Cache (OpenAtomFoundation#3047)

Co-authored-by: wuxianrong <[email protected]>

Author: Mixficsol

include/pika_conf.h

@@ -16,6 +16,7 @@
 #include "pstd/include/pstd_string.h"
 
 #include "acl.h"
+#include "cache/include/config.h"
 #include "include/pika_define.h"
 #include "include/pika_meta.h"
 #include "rocksdb/compression_type.h"
@@ -748,6 +749,30 @@ class PikaConf : public pstd::BaseConf {
     return rocksdb_perf_level_.load();
   }
 
+  int CacheValueItemMaxSize() const {
+    return cache_value_item_max_size_.load();
+  } 
+
+  bool UpdateCacheValueItemMaxSize(int size) {
+    if (size >= MAX_CACHE_ITEMS_SIZE || size <= 0) {
+      return false;
+    }
+    cache_value_item_max_size_.store(size);
+    return true;
+  }
+
+  size_t MaxKeySizeInCache() const {
+    return max_key_size_in_cache_.load();
+  } 
+
+  bool UpdateMaxKeySizeInCache(size_t size) {
+    if (size >= MAX_CACHE_MAX_KEY_SIZE || size <= 0) {
+      return false;
+    }
+    max_key_size_in_cache_.store(size);
+    return true;
+  }
+
   bool UpdateRocksDBPerfLevel(int perf_level) {
     if (perf_level >= 6 || perf_level < 0) {
       return false;
@@ -984,7 +1009,7 @@ class PikaConf : public pstd::BaseConf {
   std::atomic_int zset_cache_start_direction_ = 0;
   std::atomic_int zset_cache_field_num_per_key_ = 512;
   std::atomic_int cache_value_item_max_size_ = 1024;
-  std::atomic_int max_key_size_in_cache_ = 1024 * 1024;
+  std::atomic_size_t max_key_size_in_cache_ = 1024 * 1024;
   std::atomic_int cache_maxmemory_policy_ = 1;
   std::atomic_int cache_maxmemory_samples_ = 5;
   std::atomic_int cache_lfu_decay_time_ = 1;

src/cache/include/config.h

@@ -38,7 +38,8 @@ constexpr int CACHE_START_FROM_END = -1;
  * cache items per key
  */
 #define DEFAULT_CACHE_ITEMS_PER_KEY 512
-#define DEFAULT_CACHE_MAX_KEY_SIZE 512
+#define DEFAULT_CACHE_MAX_KEY_SIZE 1048576  // 1M
+#define MAX_CACHE_MAX_KEY_SIZE 2097152    // 2M
 
 /*
  * cache value item default size

src/pika_admin.cc

@@ -2213,6 +2213,18 @@ void ConfigCmd::ConfigGet(std::string& ret) {
     EncodeNumber(&config_body, g_pika_conf->blob_file_size());
   }
 
+  if (pstd::stringmatch(pattern.data(), "cache-value-item-max-size", 1) != 0) {
+    elements += 2;
+    EncodeString(&config_body, "cache-value-item-max-size");
+    EncodeNumber(&config_body, g_pika_conf->CacheValueItemMaxSize());
+  }
+
+  if (pstd::stringmatch(pattern.data(), "max-key-size-in-cache", 1) != 0) {
+    elements += 2;
+    EncodeString(&config_body, "max-key-size-in-cache");
+    EncodeNumber(&config_body, g_pika_conf->MaxKeySizeInCache());
+  }
+
   if (pstd::stringmatch(pattern.data(), "blob-garbage-collection-age-cutoff", 1) != 0) {
     elements += 2;
     EncodeString(&config_body, "blob-garbage-collection-age-cutoff");
@@ -2861,6 +2873,32 @@ void ConfigCmd::ConfigSet(std::shared_ptr<DB> db) {
       return;
     }
     res_.AppendStringRaw("+OK\r\n");
+  } else if (set_item == "cache-value-item-max-size") {
+    if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival <= 0) {
+      res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'cache-value-item-max-size'\r\n");
+      return;
+    }
+    bool success = g_pika_conf->UpdateCacheValueItemMaxSize(int(ival));
+    LOG(INFO) << "update cache-value-item-max-size to " << ival
+              << (success ? " success" : " failed");
+    if (!success) {
+      res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'cache-value-item-max-size', should between 1 and 2048\r\n");
+      return;
+    }
+    res_.AppendStringRaw("+OK\r\n");
+  } else if (set_item == "max-key-size-in-cache") {
+    if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival <= 0) {
+      res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'max-key-size-in-cache'\r\n");
+      return;
+    }
+    bool success = g_pika_conf->UpdateMaxKeySizeInCache(size_t(ival));
+    LOG(INFO) << "update max-key-size-in-cache to " << ival
+              << (success ? " success" : " failed");
+    if (!success) {
+      res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'max-key-size-in-cache', should between 1 and 2097152 \r\n");
+      return;
+    }
+    res_.AppendStringRaw("+OK\r\n");
   } else if (set_item == "throttle-bytes-per-second") {
     if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival <= 0) {
       res_.AppendStringRaw("-ERR Invalid argument \'" + value + "\' for CONFIG SET 'throttle-bytes-per-second'\r\n");

src/pika_cache_load_thread.cc

@@ -58,7 +58,7 @@ bool PikaCacheLoadThread::LoadKV(std::string& key, const std::shared_ptr<DB>& db
   std::string value;
   int64_t ttl = -1;
   rocksdb::Status s = db->storage()->GetWithTTL(key, &value, &ttl);
-  if (!s.ok()) {
+  if (!s.ok() || key.size() > g_pika_conf->max_key_size_in_cache()) {
     LOG(WARNING) << "load kv failed, key=" << key;
     return false;
   }
@@ -72,7 +72,7 @@ bool PikaCacheLoadThread::LoadHash(std::string& key, const std::shared_ptr<DB>&
   db->storage()->HLen(key, &len);
   // If the Hash type contains more than 2048 data members, 
   // it will not be updated to RedisCache
-  if (0 >= len || g_pika_conf->value_item_max_size_in_cache() < len) {
+  if (0 >= len || g_pika_conf->value_item_max_size_in_cache() < len || key.size() > g_pika_conf->max_key_size_in_cache()) {
     return false;
   }
 
@@ -93,7 +93,7 @@ bool PikaCacheLoadThread::LoadList(std::string& key, const std::shared_ptr<DB>&
   db->storage()->LLen(key, &len);
   // If the List type contains more than 2048 data members, 
   // it will not be updated to RedisCache
-  if (len <= 0 || g_pika_conf->value_item_max_size_in_cache() < len) {
+  if (len <= 0 || g_pika_conf->value_item_max_size_in_cache() < len || key.size() > g_pika_conf->max_key_size_in_cache()) {
     LOG(WARNING) << "can not load key, because item size:" << len
                  << " beyond max item size:" << g_pika_conf->value_item_max_size_in_cache();
     return false;
@@ -116,7 +116,7 @@ bool PikaCacheLoadThread::LoadSet(std::string& key, const std::shared_ptr<DB>& d
   db->storage()->SCard(key, &len);
   // If the Set type contains more than 2048 data members, 
   // it will not be updated to RedisCache
-  if (0 >= len || g_pika_conf->value_item_max_size_in_cache() < len) {
+  if (0 >= len || g_pika_conf->value_item_max_size_in_cache() < len || key.size() > g_pika_conf->max_key_size_in_cache()) {
     LOG(WARNING) << "can not load key, because item size:" << len
                  << " beyond max item size:" << g_pika_conf->value_item_max_size_in_cache();
     return false;
@@ -139,7 +139,7 @@ bool PikaCacheLoadThread::LoadZset(std::string& key, const std::shared_ptr<DB>&
   int start_index = 0;
   int stop_index = -1;
   db->storage()->ZCard(key, &len);
-  if (0 >= len) {
+  if (0 >= len || key.size() > g_pika_conf->max_key_size_in_cache()) {
     return false;
   }
 

src/pika_conf.cc

@@ -622,6 +622,8 @@ int PikaConf::Load() {
   GetConfInt("max-key-size-in-cache", &max_key_size_in_cache);
   if (max_key_size_in_cache <= 0) {
     max_key_size_in_cache = DEFAULT_CACHE_MAX_KEY_SIZE;
+  } else if (max_key_size_in_cache > MAX_CACHE_MAX_KEY_SIZE) {
+    max_key_size_in_cache = MAX_CACHE_MAX_KEY_SIZE; 
   }
   max_key_size_in_cache_ = max_key_size_in_cache;