Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 0 additions & 20 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -630,24 +630,6 @@ ExternalProject_Add(rocksdb
make -j${CPU_CORE}
)

ExternalProject_Add(prometheus_cpp
URL
https://github.com/jupp0r/prometheus-cpp/releases/download/v1.2.4/prometheus-cpp-with-submodules.tar.gz
CMAKE_ARGS
-DBUILD_SHARED_LIBS=OFF
-DENABLE_PUSH=OFF
-DENABLE_COMPRESSION=OFF
-DCMAKE_INSTALL_LIBDIR=${INSTALL_LIBDIR}
-DCMAKE_INSTALL_INCLUDEDIR=${INSTALL_INCLUDEDIR}
BUILD_ALWAYS
1
BUILD_COMMAND
make -j${CPU_CORE}
)

set(PROMETHEUS_CPP_CORE_LIB ${INSTALL_LIBDIR}/libprometheus-cpp-core.a)
set(PROMETHEUS_CPP_PULL_LIB ${INSTALL_LIBDIR}/libprometheus-cpp-pull.a)

ExternalProject_Add(rediscache
URL
https://github.com/pikiwidb/rediscache/archive/refs/tags/v1.0.7.tar.gz
Expand Down Expand Up @@ -840,8 +822,6 @@ target_link_libraries(${PROJECT_NAME}
${LIB_PROTOBUF}
${LIB_GFLAGS}
${LIB_FMT}
${PROMETHEUS_CPP_PULL_LIB}
${PROMETHEUS_CPP_CORE_LIB}
libsnappy.a
libzstd.a
liblz4.a
Expand Down
6 changes: 0 additions & 6 deletions include/pika_admin.h
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,6 @@ class InfoCmd : public Cmd {
kInfoAll,
kInfoDebug,
kInfoCommandStats,
kInfoSlowCommand,
kInfoCommandP99,
kInfoCache
};
InfoCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {}
Expand Down Expand Up @@ -296,8 +294,6 @@ class InfoCmd : public Cmd {
const static std::string kRocksDBSection;
const static std::string kDebugSection;
const static std::string kCommandStatsSection;
const static std::string kCommandP99Section;
const static std::string kSlowCommandSection;
const static std::string kCacheSection;

void DoInitial() override;
Expand All @@ -318,8 +314,6 @@ class InfoCmd : public Cmd {
void InfoRocksDB(std::string& info);
void InfoDebug(std::string& info);
void InfoCommandStats(std::string& info);
void InfoCommandP99(std::string& info);
void InfoSlowCommand(std::string& info);
void InfoCache(std::string& info, std::shared_ptr<DB> db);

std::string CacheStatusToString(int status);
Expand Down
2 changes: 1 addition & 1 deletion include/pika_client_conn.h
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ class PikaClientConn : public net::RedisConn {
std::shared_ptr<Cmd> DoCmd(const PikaCmdArgsType& argv, const std::string& opt,
const std::shared_ptr<std::string>& resp_ptr, bool cache_miss_in_rtc);

void ProcessSlowlog(const PikaCmdArgsType& argv, std::shared_ptr<Cmd> c_ptr, const std::string& opt);
void ProcessSlowlog(const PikaCmdArgsType& argv, std::shared_ptr<Cmd> c_ptr);
void ProcessMonitor(const PikaCmdArgsType& argv);

void ExecRedisCmd(const PikaCmdArgsType& argv, std::shared_ptr<std::string>& resp_ptr, bool cache_miss_in_rtc);
Expand Down
20 changes: 0 additions & 20 deletions include/pika_cmd_table_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,11 @@

#include <shared_mutex>
#include <thread>
#include <prometheus/exposer.h>
#include <prometheus/registry.h>
#include <prometheus/counter.h>
#include <prometheus/histogram.h>

#include "include/acl.h"
#include "include/pika_command.h"
#include "include/pika_data_distribution.h"

using namespace prometheus;

struct CommandStatistics {
CommandStatistics() = default;
CommandStatistics(const CommandStatistics& other) {
Expand All @@ -36,7 +30,6 @@ class PikaCmdTableManager {
PikaCmdTableManager();
virtual ~PikaCmdTableManager() = default;
void InitCmdTable(void);
void InitHistograms();
void RenameCommand(const std::string before, const std::string after);
std::shared_ptr<Cmd> GetCmd(const std::string& opt);
bool CmdExist(const std::string& cmd) const;
Expand All @@ -49,11 +42,6 @@ class PikaCmdTableManager {
* Info Commandstats used
*/
std::unordered_map<std::string, CommandStatistics>* GetCommandStatMap();
std::unordered_map<std::string, CommandStatistics> GetSlowCommandCount();
void UpdateSlowCommandCount(const std::string& opt);
void ResetCommandCount();
prometheus::Histogram& GetHistogram(const std::string& opt);
prometheus::Family<prometheus::Histogram>* GetHistograms();

private:
std::shared_ptr<Cmd> NewCommand(const std::string& opt);
Expand All @@ -72,13 +60,5 @@ class PikaCmdTableManager {
* Info Commandstats used
*/
std::unordered_map<std::string, CommandStatistics> cmdstat_map_;
std::unordered_map<std::string, CommandStatistics> slow_command_count_;
std::thread reset_thread_;
std::mutex command_mutex_;
std::shared_mutex histograms_mutex_;
std::shared_mutex slow_command_mutex_;
std::shared_ptr<prometheus::Registry> prometheus_registry_;
prometheus::Family<prometheus::Histogram>* histogram_family_;
std::unordered_map<std::string, prometheus::Histogram*> histograms_;
};
#endif
83 changes: 0 additions & 83 deletions src/pika_admin.cc
Original file line number Diff line number Diff line change
Expand Up @@ -882,8 +882,6 @@ const std::string InfoCmd::kKeyspaceSection = "keyspace";
const std::string InfoCmd::kDataSection = "data";
const std::string InfoCmd::kRocksDBSection = "rocksdb";
const std::string InfoCmd::kDebugSection = "debug";
const std::string InfoCmd::kCommandP99Section = "commandp99";
const std::string InfoCmd::kSlowCommandSection = "slowcommand";
const std::string InfoCmd::kCommandStatsSection = "commandstats";
const std::string InfoCmd::kCacheSection = "cache";

Expand Down Expand Up @@ -969,10 +967,6 @@ void InfoCmd::DoInitial() {
info_section_ = kInfoDebug;
} else if (strcasecmp(argv_[1].data(), kCommandStatsSection.data()) == 0) {
info_section_ = kInfoCommandStats;
} else if (strcasecmp(argv_[1].data(), kCommandP99Section.data()) == 0) {
info_section_ = kInfoCommandP99;
} else if (strcasecmp(argv_[1].data(), kSlowCommandSection.data()) == 0) {
info_section_ = kInfoSlowCommand;
} else if (strcasecmp(argv_[1].data(), kCacheSection.data()) == 0) {
info_section_ = kInfoCache;
} else {
Expand Down Expand Up @@ -1014,10 +1008,6 @@ void InfoCmd::Do() {
info.append("\r\n");
InfoCommandStats(info);
info.append("\r\n");
InfoCommandP99(info);
info.append("\r\n");
InfoSlowCommand(info);
info.append("\r\n");
InfoCache(info, db_);
info.append("\r\n");
InfoCPU(info);
Expand Down Expand Up @@ -1061,12 +1051,6 @@ void InfoCmd::Do() {
case kInfoCommandStats:
InfoCommandStats(info);
break;
case kInfoCommandP99:
InfoCommandP99(info);
break;
case kInfoSlowCommand:
InfoSlowCommand(info);
break;
case kInfoCache:
InfoCache(info, db_);
break;
Expand Down Expand Up @@ -1497,73 +1481,6 @@ void InfoCmd::InfoDebug(std::string& info) {
g_pika_server->ServerStatus(&info);
}

void InfoCmd::InfoCommandP99(std::string& info) {
std::stringstream tmp_stream;
tmp_stream.precision(2);
tmp_stream.setf(std::ios::fixed);
tmp_stream << "# Commands P99" << "\r\n";
auto histogram_family = g_pika_cmd_table_manager->GetHistograms();

for (const auto& metric_family : histogram_family->Collect()) {
for (const auto& metric : metric_family.metric) {
std::string command_name;

for (const auto& label : metric.label) {
if (label.name == "command") {
command_name = label.value;
break;
}
}

double total_count = metric.histogram.sample_count;

if (command_name.empty()) {
tmp_stream << "Command: UNKNOWN\r\n";
} else {
tmp_stream << "Command: " << command_name << "\r\n";
}

double tp99_threshold = total_count * 0.99;
double tp999_threshold = total_count * 0.999;
double tp9999_threshold = total_count * 0.9999;
double tp99 = 0, tp999 = 0, tp9999 = 0;

for (const auto& bucket : metric.histogram.bucket) {
if (bucket.cumulative_count >= tp99_threshold && tp99 == 0) {
tp99 = bucket.upper_bound;
}
if (bucket.cumulative_count >= tp999_threshold && tp999 == 0) {
tp999 = bucket.upper_bound;
}
if (bucket.cumulative_count >= tp9999_threshold && tp9999 == 0) {
tp9999 = bucket.upper_bound;
break;
}
}
tmp_stream << "TP99 ms: " << tp99 << "\r\n";
tmp_stream << "TP999 ms: " << tp999 << "\r\n";
tmp_stream << "TP9999 ms: " << tp9999 << "\r\n";
tmp_stream << "----------------------\r\n";
}
}

info.append(tmp_stream.str());
}

void InfoCmd::InfoSlowCommand(std::string& info) {
std::stringstream tmp_stream;
tmp_stream.precision(2);
tmp_stream.setf(std::ios::fixed);
auto stats = g_pika_cmd_table_manager->GetSlowCommandCount();
tmp_stream << "# SlowCommand Count" << "\r\n";
for (auto iter : stats) {
if (iter.second.cmd_count != 0) {
tmp_stream << iter.first << ":slow_count=" << iter.second.cmd_count << "\r\n";
}
}
info.append(tmp_stream.str());
}

void InfoCmd::InfoCommandStats(std::string& info) {
std::stringstream tmp_stream;
tmp_stream.precision(2);
Expand Down
10 changes: 2 additions & 8 deletions src/pika_client_conn.cc
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,6 @@
#include <glog/logging.h>
#include <utility>
#include <vector>
#include <prometheus/exposer.h>
#include <prometheus/registry.h>
#include <prometheus/counter.h>
#include <prometheus/histogram.h>

#include "include/pika_admin.h"
#include "include/pika_client_conn.h"
Expand Down Expand Up @@ -225,21 +221,19 @@ std::shared_ptr<Cmd> PikaClientConn::DoCmd(const PikaCmdArgsType& argv, const st
c_ptr->Execute();

time_stat_->process_done_ts_ = pstd::NowMicros();
g_pika_cmd_table_manager->GetHistogram(opt).Observe(time_stat_->total_time() / 1000);
auto cmdstat_map = g_pika_cmd_table_manager->GetCommandStatMap();
(*cmdstat_map)[opt].cmd_count.fetch_add(1);
(*cmdstat_map)[opt].cmd_time_consuming.fetch_add(time_stat_->total_time());

if (g_pika_conf->slowlog_slower_than() >= 0) {
ProcessSlowlog(argv, c_ptr, opt);
ProcessSlowlog(argv, c_ptr);
}

return c_ptr;
}

void PikaClientConn::ProcessSlowlog(const PikaCmdArgsType& argv, std::shared_ptr<Cmd> c_ptr, const std::string& opt) {
void PikaClientConn::ProcessSlowlog(const PikaCmdArgsType& argv, std::shared_ptr<Cmd> c_ptr) {
if (time_stat_->total_time() > g_pika_conf->slowlog_slower_than()) {
g_pika_cmd_table_manager->UpdateSlowCommandCount(opt);
g_pika_server->SlowlogPushEntry(argv, time_stat_->start_ts() / 1000000, time_stat_->total_time());
if (g_pika_conf->slowlog_write_errorlog()) {
bool trim = false;
Expand Down
65 changes: 0 additions & 65 deletions src/pika_cmd_table_manager.cc
Original file line number Diff line number Diff line change
Expand Up @@ -14,31 +14,9 @@

extern std::unique_ptr<PikaConf> g_pika_conf;

void PikaCmdTableManager::ResetCommandCount() {
while (true) {
std::this_thread::sleep_for(std::chrono::minutes(1));
{
std::lock_guard<std::mutex> lock(command_mutex_);
slow_command_count_.clear();
InitHistograms();
}
}
}

void PikaCmdTableManager::InitHistograms() {
histograms_.clear();
prometheus_registry_ = std::make_shared<prometheus::Registry>();
histogram_family_ = &prometheus::BuildHistogram()
.Name("pika_command_duration_seconds")
.Help("Execution time of Pika commands in seconds")
.Register(*prometheus_registry_);
}

PikaCmdTableManager::PikaCmdTableManager() {
cmds_ = std::make_unique<CmdTable>();
cmds_->reserve(300);
InitHistograms();
reset_thread_ = std::thread(&PikaCmdTableManager::ResetCommandCount, this);
}

void PikaCmdTableManager::InitCmdTable(void) {
Expand Down Expand Up @@ -85,49 +63,6 @@ void PikaCmdTableManager::RenameCommand(const std::string before, const std::str
}
}

prometheus::Histogram& PikaCmdTableManager::GetHistogram(const std::string& opt) {
{
std::shared_lock<std::shared_mutex> read_lock(histograms_mutex_);
auto it = histograms_.find(opt);
if (it != histograms_.end()) {
return *(it->second);
}
}
std::unique_lock<std::shared_mutex> write_lock(histograms_mutex_);
auto& new_histogram = histogram_family_->Add(
{{"command", opt}},
prometheus::Histogram::BucketBoundaries{0.5, 1, 2, 3, 5, 7, 10, 15, 20, 30, 40, 50, 65, 75, 85, 100, 125, 140, 150, 160, 175, 185, 200, 300, 400, 500, 750, 1000, 2000, 5000, 10000}
);
histograms_[opt] = &new_histogram;
return new_histogram;
}

prometheus::Family<prometheus::Histogram>* PikaCmdTableManager::GetHistograms() {
return histogram_family_;
}

void PikaCmdTableManager::UpdateSlowCommandCount(const std::string& opt) {
{
std::shared_lock<std::shared_mutex> read_lock(slow_command_mutex_);
if (slow_command_count_.find(opt) != slow_command_count_.end()) {
slow_command_count_[opt].cmd_count.fetch_add(1);
return;
}
}

{
std::unique_lock<std::shared_mutex> write_lock(slow_command_mutex_);
slow_command_count_[opt];
}

slow_command_count_[opt].cmd_count.fetch_add(1);
}

std::unordered_map<std::string, CommandStatistics> PikaCmdTableManager::GetSlowCommandCount() {
std::shared_lock<std::shared_mutex> lock(slow_command_mutex_);
return slow_command_count_;
}

std::unordered_map<std::string, CommandStatistics>* PikaCmdTableManager::GetCommandStatMap() {
return &cmdstat_map_;
}
Expand Down
5 changes: 2 additions & 3 deletions tools/pika_exporter/config/info.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,8 @@ stats = true
cpu = true
replication = true
keyspace = true
cache = false
commandp99 = true
slowcommand = true
cache = true

execcount = false
commandstats = false
rocksdb = false
4 changes: 0 additions & 4 deletions tools/pika_exporter/exporter/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,6 @@ func (c *client) InfoNoneCommandList() (string, error) {
"COMMAND_EXEC_COUNT": InfoConf.Execcount,
"COMMANDSTATS": InfoConf.Commandstats,
"ROCKSDB": InfoConf.Rocksdb,
"CommandP99": InfoConf.CommandP99,
"SlowCommand": InfoConf.SlowCommand,
}
for section, flag := range sectionsMap {
if flag {
Expand Down Expand Up @@ -157,8 +155,6 @@ func (c *client) InfoAllCommandList() (string, error) {
"COMMAND_EXEC_COUNT": InfoConf.Execcount,
"COMMANDSTATS": InfoConf.Commandstats,
"ROCKSDB": InfoConf.Rocksdb,
"CommandP99": InfoConf.CommandP99,
"SlowCommand": InfoConf.SlowCommand,
}
for section, flag := range sectionsMap {
if flag {
Expand Down
2 changes: 0 additions & 2 deletions tools/pika_exporter/exporter/conf.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@ type InfoConfig struct {
Keyspace bool `toml:"keyspace"`
Execcount bool `toml:"execcount"`
Commandstats bool `toml:"commandstats"`
CommandP99 bool `toml:"commandp99"`
SlowCommand bool `toml:"slowcommand"`
Rocksdb bool `toml:"rocksdb"`
Cache bool `toml:"cache"`

Expand Down
Loading
Loading