Skip to content

Commit 5ed38e4

Browse files
orangesaporluozhuojun
orangesapor
and
luozhuojun
authored
feat:expoter (#3237)
* fix: 在 Makefile 中添加 tidy 目标以自动运行 go mod tidy * feat: 更新 pika_exporter 组件代码 --------- Co-authored-by: luozhuojun <[email protected]>
1 parent 4b65ebb commit 5ed38e4

30 files changed

Lines changed: 7913 additions & 4796 deletions

‎tools/pika_exporter/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ cat << EOF | gofmt > version.go
1717
package main
1818

1919
const (
20-
PikaExporterVersion = "$(PIKA_EXPORTER_MAJOR).$(PIKA_EXPORTER_MINOR).$(PIKA_EXPORTER_PATCH)"
20+
PikaExporterVersion = "$(PIKA_EXPORTER_MAJOR).$(PIKA_EXPORTER_MINOR).$(PIKA_EXPORTER_PATCH)"
2121
BuildVersion = "$(BRANCH)"
2222
BuildCommitSha = "$(GITREV)"
2323
BuildDate = "$(BUILDTIME)"

‎tools/pika_exporter/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ prometheus --config.file=./grafana/prometheus.yml
5858
| check.scan-count | PIKA_EXPORTER_CHECK_SCAN_COUNT | 100 | When check keys and executing SCAN command, scan-count assigned to COUNT. | --check.scan-count 200 |
5959
| web.listen-address | PIKA_EXPORTER_WEB_LISTEN_ADDRESS | :9121 | Address to listen on for web interface and telemetry. | --web.listen-address ":9121" |
6060
| web.telemetry-path | PIKA_EXPORTER_WEB_TELEMETRY_PATH | /metrics | Path under which to expose metrics. | --web.telemetry-path "/metrics" |
61-
| log.level | PIKA_EXPORTER_LOG_LEVEL | info | Log level, valid options:`panic` `fatal` `error` `warn` `warning` `info` `debug`. | --log.level "debug" |
61+
| log.level | PIKA_EXPORTER_LOG_LEVEL | error | Log level, valid options:`panic` `fatal` `error` `warn` `warning` `info` `debug`. | --log.level "debug" |
6262
| log.format | PIKA_EXPORTER_LOG_FORMAT | json | Log format, valid options:`txt` `json`. | --log.format "json" |
6363
| version | | false | Show version information and exit. | --version |
6464

‎tools/pika_exporter/config/info.toml‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# Pika Exporter Configuration
12
server = true
23
data = true
34
clients = true
@@ -7,6 +8,53 @@ replication = true
78
keyspace = true
89
cache = true
910

10-
execcount = false
11-
commandstats = false
12-
rocksdb = false
11+
execcount = true
12+
commandstats = true
13+
rocksdb = true
14+
15+
16+
# The address of the pika instance to monitor
17+
# If not set, the exporter will use the discovery mechanism
18+
# pika_addr = "127.0.0.1:9379"
19+
20+
# The password for the pika instance
21+
# pika_password = ""
22+
23+
# The alias for the pika instance
24+
# pika_alias = ""
25+
26+
# The address of the codis topom
27+
# codis_addr = "http://127.0.0.1:18087"
28+
29+
# The namespace for the metrics
30+
# namespace = "pika"
31+
32+
# The path to the metrics definition file
33+
# metrics_file = ""
34+
35+
# The port to listen on for the web interface
36+
# web_listen_address = ":9121"
37+
38+
# The path under which to expose metrics
39+
# web_telemetry_path = "/metrics"
40+
41+
# The log level
42+
# log_level = "info"
43+
44+
# The log format
45+
# log_format = "text"
46+
47+
# The path to the config file
48+
# config = "config/info.toml"
49+
50+
# The key space stats clock
51+
# keyspace_stats_clock = -1
52+
53+
# The key patterns to check
54+
# check_key_patterns = ""
55+
56+
# The keys to check
57+
# check_keys = ""
58+
59+
# The scan count for checking keys
60+
# check_scan_count = 100

‎tools/pika_exporter/discovery/codis_dashboard.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,21 @@ type CmdInfo struct {
5050
Usecs_percall int64 `json:"usecs_percall"`
5151
Fails int64 `json:"fails"`
5252
MaxDelay int64 `json:"max_delay"`
53+
AVG int64 `json:"avg"`
54+
TP90 int64 `json:"tp90"`
55+
TP99 int64 `json:"tp99"`
56+
TP999 int64 `json:"tp999"`
57+
TP9999 int64 `json:"tp9999"`
58+
TP100 int64 `json:"tp100"`
59+
60+
Delay50ms int64 `json:"delay50ms"`
61+
Delay100ms int64 `json:"delay100ms"`
62+
Delay200ms int64 `json:"delay200ms"`
63+
Delay300ms int64 `json:"delay300ms"`
64+
Delay500ms int64 `json:"delay500ms"`
65+
Delay1s int64 `json:"delay1s"`
66+
Delay2s int64 `json:"delay2s"`
67+
Delay3s int64 `json:"delay3s"`
5368
}
5469

5570
type ProxyOpsInfo struct {

‎tools/pika_exporter/exporter/client.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ import (
77
"strings"
88
"time"
99

10-
"github.com/garyburd/redigo/redis"
10+
"github.com/gomodule/redigo/redis"
11+
log "github.com/sirupsen/logrus"
1112
)
1213

1314
const (
@@ -132,7 +133,8 @@ func (c *client) InfoNoneCommandList() (string, error) {
132133
if flag {
133134
info, err := c.InfoCommand(section)
134135
if err != nil {
135-
return "", err
136+
log.Warnf("Failed to get INFO %s: %v", section, err)
137+
continue // Skip this section but continue with others
136138
}
137139
rst = append(rst, info)
138140
}
@@ -160,7 +162,8 @@ func (c *client) InfoAllCommandList() (string, error) {
160162
if flag {
161163
info, err := c.InfoCommand(section)
162164
if err != nil {
163-
return "", err
165+
log.Warnf("Failed to get INFO %s: %v", section, err)
166+
continue // Skip this section but continue with others
164167
}
165168
rst = append(rst, info)
166169
}

‎tools/pika_exporter/exporter/conf.go‎

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,29 @@ type InfoConfig struct {
2929
}
3030

3131
func LoadConfig() error {
32-
log.Println("Update configuration")
33-
err := readConfig(InfoConfigPath)
34-
if err != nil {
35-
return err
32+
log.Debugln("Update configuration")
33+
34+
// Initialize default configuration
35+
InfoConf = &InfoConfig{
36+
Server: true,
37+
Data: true,
38+
Clients: true,
39+
Stats: true,
40+
CPU: true,
41+
Replication: true,
42+
Keyspace: true,
43+
Execcount: true,
44+
Commandstats: true,
45+
Rocksdb: false,
46+
Cache: true,
47+
}
48+
49+
// Try to load config file if path is provided
50+
if InfoConfigPath != "" {
51+
err := readConfig(InfoConfigPath)
52+
if err != nil {
53+
log.Warnf("Failed to load config file %s: %s, using default configuration", InfoConfigPath, err)
54+
}
3655
}
3756

3857
InfoConf.CheckInfo()
@@ -80,10 +99,18 @@ func (c *InfoConfig) CheckInfo() {
8099
c.InfoAll = false
81100
c.Info = false
82101

83-
if c.Server && c.Data && c.Clients && c.Stats && c.CPU && c.Replication && c.Keyspace {
102+
// For Pika versions, we need to enable Info if any of the core modules are enabled
103+
// This ensures basic metrics are collected
104+
if c.Server || c.Data || c.Clients || c.Stats || c.CPU || c.Replication || c.Keyspace {
84105
c.Info = true
85-
if c.Execcount && c.Commandstats && c.Rocksdb && c.Cache {
86-
c.InfoAll = true
87-
}
106+
}
107+
108+
// InfoAll should only be enabled if all modules are enabled
109+
// For Pika 3.2.x versions, we should NOT use InfoAll because INFO ALL command
110+
// has different output format compared to newer versions
111+
// The version detection will be handled in the exporter, but here we ensure
112+
// that Info is enabled when needed
113+
if c.Info && c.Execcount && c.Commandstats && c.Rocksdb && c.Cache {
114+
c.InfoAll = true
88115
}
89116
}

‎tools/pika_exporter/exporter/metrics/cache.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,16 @@ var collectCacheMetrics = map[string]MetricConfig{
3434
ValueName: "cache_db_num",
3535
},
3636
},
37+
"cache_keys": {
38+
Parser: &normalParser{},
39+
MetricMeta: &MetaData{
40+
Name: "cache_keys",
41+
Help: "pika serve instance cache keys count",
42+
Type: metricTypeGauge,
43+
Labels: []string{LabelNameAddr, LabelNameAlias},
44+
ValueName: "cache_keys",
45+
},
46+
},
3747
"cache_memory": {
3848
Parser: &normalParser{},
3949
MetricMeta: &MetaData{
@@ -44,6 +54,26 @@ var collectCacheMetrics = map[string]MetricConfig{
4454
ValueName: "cache_memory",
4555
},
4656
},
57+
"hits": {
58+
Parser: &normalParser{},
59+
MetricMeta: &MetaData{
60+
Name: "cache_hits",
61+
Help: "pika serve instance cache hit count",
62+
Type: metricTypeCounter,
63+
Labels: []string{LabelNameAddr, LabelNameAlias},
64+
ValueName: "hits",
65+
},
66+
},
67+
"all_cmds": {
68+
Parser: &normalParser{},
69+
MetricMeta: &MetaData{
70+
Name: "cache_all_cmds",
71+
Help: "pika serve instance cache all commands count",
72+
Type: metricTypeCounter,
73+
Labels: []string{LabelNameAddr, LabelNameAlias},
74+
ValueName: "all_cmds",
75+
},
76+
},
4777
"hits_per_sec": {
4878
Parser: &normalParser{},
4979
MetricMeta: &MetaData{
@@ -54,6 +84,16 @@ var collectCacheMetrics = map[string]MetricConfig{
5484
ValueName: "hits_per_sec",
5585
},
5686
},
87+
"read_cmd_per_sec": {
88+
Parser: &normalParser{},
89+
MetricMeta: &MetaData{
90+
Name: "read_cmd_per_sec",
91+
Help: "pika serve instance cache read command count per second",
92+
Type: metricTypeGauge,
93+
Labels: []string{LabelNameAddr, LabelNameAlias},
94+
ValueName: "read_cmd_per_sec",
95+
},
96+
},
5797
"hitratio_per_second": {
5898
Parser: &regexParser{
5999
name: "hitratio_per_sec",
@@ -84,4 +124,24 @@ var collectCacheMetrics = map[string]MetricConfig{
84124
ValueName: "hitratio_all",
85125
},
86126
},
127+
"load_keys_per_sec": {
128+
Parser: &normalParser{},
129+
MetricMeta: &MetaData{
130+
Name: "load_keys_per_sec",
131+
Help: "pika serve instance cache load keys count per second",
132+
Type: metricTypeGauge,
133+
Labels: []string{LabelNameAddr, LabelNameAlias},
134+
ValueName: "load_keys_per_sec",
135+
},
136+
},
137+
"waitting_load_keys_num": {
138+
Parser: &normalParser{},
139+
MetricMeta: &MetaData{
140+
Name: "waitting_load_keys_num",
141+
Help: "pika serve instance cache waiting load keys number",
142+
Type: metricTypeGauge,
143+
Labels: []string{LabelNameAddr, LabelNameAlias},
144+
ValueName: "waitting_load_keys_num",
145+
},
146+
},
87147
}

‎tools/pika_exporter/exporter/metrics/command_exec_count.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ var collectCommandExecCountMetrics = map[string]MetricConfig{
1616
Parser: &regexParser{
1717
name: "command_exec_count_command",
1818
source: "commands_count",
19-
reg: regexp.MustCompile(`(\r|\n)*(?P<command>[^:]+):(?P<count>[\d]*)`),
19+
reg: regexp.MustCompile(`[\r\n]+(?P<command>[^:\r\n]+):(?P<count>[\d]+)`),
2020
Parser: &normalParser{},
2121
},
2222
},

‎tools/pika_exporter/exporter/metrics/keyspace.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ var collectKeySpaceMetrics = map[string]MetricConfig{
4040
Parser: &regexParser{
4141
name: "keyspace_info_3.1.0-3.3.2",
4242
reg: regexp.MustCompile(`(?P<db>db[\d]+)\s*(?P<data_type>[^_]+)\w*keys=(?P<keys>[\d]+)[,\s]*` +
43-
`expires=(?P<expire_keys>[\d]+)[,\s]*invaild_keys=(?P<invalid_keys>[\d]+)`),
43+
`expires=(?P<expire_keys>[\d]+)[,\s]*(?:invaild_keys|invalid_keys)=(?P<invalid_keys>[\d]+)`),
4444
Parser: &normalParser{},
4545
},
4646
},
@@ -49,7 +49,7 @@ var collectKeySpaceMetrics = map[string]MetricConfig{
4949
Parser: &regexParser{
5050
name: "keyspace_info_>=3.1.0",
5151
reg: regexp.MustCompile(`(?P<db>db[\d]+)\s*(?P<data_type>[^_]+)\w*keys=(?P<keys>[\d]+)[,\s]*` +
52-
`expires=(?P<expire_keys>[\d]+)[,\s]*invalid_keys=(?P<invalid_keys>[\d]+)`),
52+
`expires=(?P<expire_keys>[\d]+)[,\s]*(?:invaild_keys|invalid_keys)=(?P<invalid_keys>[\d]+)`),
5353
Parser: &normalParser{},
5454
},
5555
},
@@ -88,7 +88,7 @@ var collectKeySpaceMetrics = map[string]MetricConfig{
8888
Parser: &regexParser{
8989
name: "keyspace_info_all_~3.0.5",
9090
reg: regexp.MustCompile(`(?P<data_type>\w*):\s*keys=(?P<keys>[\d]+)[,\s]*` +
91-
`expires=(?P<expire_keys>[\d]+)[,\s]*invaild_keys=(?P<invalid_keys>[\d]+)`),
91+
`expires=(?P<expire_keys>[\d]+)[,\s]*(?:invaild_keys|invalid_keys)=(?P<invalid_keys>[\d]+)`),
9292
Parser: &normalParser{},
9393
},
9494
},
@@ -97,7 +97,7 @@ var collectKeySpaceMetrics = map[string]MetricConfig{
9797
Parser: &regexParser{
9898
name: "keyspace_info_all_~3.1.0",
9999
reg: regexp.MustCompile(`(?P<db>db[\d]+)_\s*(?P<data_type>[^:]+):\s*keys=(?P<keys>[\d]+)[,\s]*` +
100-
`expires=(?P<expire_keys>[\d]+)[,\s]*invaild_keys=(?P<invalid_keys>[\d]+)`),
100+
`expires=(?P<expire_keys>[\d]+)[,\s]*(?:invaild_keys|invalid_keys)=(?P<invalid_keys>[\d]+)`),
101101
Parser: &normalParser{},
102102
},
103103
},
@@ -106,7 +106,7 @@ var collectKeySpaceMetrics = map[string]MetricConfig{
106106
Parser: &regexParser{
107107
name: "keyspace_info_all_3.1.0-3.3.2",
108108
reg: regexp.MustCompile(`(?P<db>db[\d]+)\s*(?P<data_type>[^_]+)\w*keys=(?P<keys>[\d]+)[,\s]*` +
109-
`expires=(?P<expire_keys>[\d]+)[,\s]*invaild_keys=(?P<invalid_keys>[\d]+)`),
109+
`expires=(?P<expire_keys>[\d]+)[,\s]*(?:invaild_keys|invalid_keys)=(?P<invalid_keys>[\d]+)`),
110110
Parser: &normalParser{},
111111
},
112112
},
@@ -115,7 +115,7 @@ var collectKeySpaceMetrics = map[string]MetricConfig{
115115
Parser: &regexParser{
116116
name: "keyspace_info_all_>=3.3.3",
117117
reg: regexp.MustCompile(`(?P<db>db[\d]+)\s*(?P<data_type>[^_]+)\w*keys=(?P<keys>[\d]+)[,\s]*` +
118-
`expires=(?P<expire_keys>[\d]+)[,\s]*invalid_keys=(?P<invalid_keys>[\d]+)`),
118+
`expires=(?P<expire_keys>[\d]+)[,\s]*(?:invaild_keys|invalid_keys)=(?P<invalid_keys>[\d]+)`),
119119
Parser: &normalParser{},
120120
},
121121
},

0 commit comments

Comments
 (0)