Skip to content

Commit 00ede3a

Browse files
0xgouda0xgouda
authored
[+] add graceful fallback for local log parsing (#1109)
* Check privileges before invoking `ParseLogsLocal()` I didn't consider testing all possible directory and file privileges, assuming the default setup so if a user (postgres or superuser) has read privilege on the logs directory then he should be able to read all the files. * Update log parsing docs We don't neccesarily require superuser privileges in local mode, e.g, `postgres` user will be enough as he can access the log dir and its content. * Move `... on the same host ...` log message to the first condition
1 parent 50cead2 commit 00ede3a

3 files changed

Lines changed: 35 additions & 25 deletions

File tree

‎docs/reference/advanced_features.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ Only event counts are stored - no error messages, usernames, or other details. E
4646

4747
pgwatch automatically selects between two parsing modes:
4848

49-
1. **Local mode** - Used when pgwatch runs on the same host as the database server and can access log files directly. Requires superuser privileges and `pg_read_all_settings` role.
49+
1. **Local mode** - Used when pgwatch runs on the same host as the database server and can access log files directly. Requires OS user with read privileges on the logs directory and its files and `pg_read_all_settings` role.
5050

5151
2. **Remote mode** - Used when pgwatch runs on a different host. Requires `pg_monitor` role and execute privilege on `pg_read_file()`.
5252

‎internal/reaper/logparser.go‎

Lines changed: 33 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package reaper
22

33
import (
44
"context"
5+
"os"
56
"path"
67
"path/filepath"
78
"regexp"
@@ -15,6 +16,26 @@ import (
1516
"github.com/jackc/pgx/v5"
1617
)
1718

19+
// Constants and types
20+
var pgSeverities = [...]string{"DEBUG", "INFO", "NOTICE", "WARNING", "ERROR", "LOG", "FATAL", "PANIC"}
21+
var pgSeveritiesLocale = map[string]map[string]string{
22+
"C.": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTICE": "NOTICE", "WARNING": "WARNING", "ERROR": "ERROR", "FATAL": "FATAL", "PANIC": "PANIC"},
23+
"de": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "HINWEIS": "NOTICE", "WARNUNG": "WARNING", "FEHLER": "ERROR", "FATAL": "FATAL", "PANIK": "PANIC"},
24+
"fr": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTICE": "NOTICE", "ATTENTION": "WARNING", "ERREUR": "ERROR", "FATAL": "FATAL", "PANIK": "PANIC"},
25+
"it": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTIFICA": "NOTICE", "ATTENZIONE": "WARNING", "ERRORE": "ERROR", "FATALE": "FATAL", "PANICO": "PANIC"},
26+
"ko": {"디버그": "DEBUG", "로그": "LOG", "정보": "INFO", "알림": "NOTICE", "경고": "WARNING", "오류": "ERROR", "치명적오류": "FATAL", "손상": "PANIC"},
27+
"pl": {"DEBUG": "DEBUG", "DZIENNIK": "LOG", "INFORMACJA": "INFO", "UWAGA": "NOTICE", "OSTRZEŻENIE": "WARNING", "BŁĄD": "ERROR", "KATASTROFALNY": "FATAL", "PANIKA": "PANIC"},
28+
"ru": {"ОТЛАДКА": "DEBUG", "СООБЩЕНИЕ": "LOG", "ИНФОРМАЦИЯ": "INFO", "ЗАМЕЧАНИЕ": "NOTICE", "ПРЕДУПРЕЖДЕНИЕ": "WARNING", "ОШИБКА": "ERROR", "ВАЖНО": "FATAL", "ПАНИКА": "PANIC"},
29+
"sv": {"DEBUG": "DEBUG", "LOGG": "LOG", "INFO": "INFO", "NOTIS": "NOTICE", "VARNING": "WARNING", "FEL": "ERROR", "FATALT": "FATAL", "PANIK": "PANIC"},
30+
"tr": {"DEBUG": "DEBUG", "LOG": "LOG", "BİLGİ": "INFO", "NOT": "NOTICE", "UYARI": "WARNING", "HATA": "ERROR", "ÖLÜMCÜL (FATAL)": "FATAL", "KRİTİK": "PANIC"},
31+
"zh": {"调试": "DEBUG", "日志": "LOG", "信息": "INFO", "注意": "NOTICE", "警告": "WARNING", "错误": "ERROR", "致命错误": "FATAL", "比致命错误还过分的错误": "PANIC"},
32+
}
33+
34+
const csvLogDefaultRegEx = `^^(?P<log_time>.*?),"?(?P<user_name>.*?)"?,"?(?P<database_name>.*?)"?,(?P<process_id>\d+),"?(?P<connection_from>.*?)"?,(?P<session_id>.*?),(?P<session_line_num>\d+),"?(?P<command_tag>.*?)"?,(?P<session_start_time>.*?),(?P<virtual_transaction_id>.*?),(?P<transaction_id>.*?),(?P<error_severity>\w+),`
35+
const csvLogDefaultGlobSuffix = "*.csv"
36+
37+
const maxChunkSize int32 = 10 * 1024 * 1024 // 10 MB
38+
1839
type LogParser struct {
1940
ctx context.Context
2041
LogsMatchRegex *regexp.Regexp
@@ -74,12 +95,14 @@ func (lp *LogParser) ParseLogs() error {
7495
l := log.GetLogger(lp.ctx)
7596
if ok, err := db.IsClientOnSameHost(lp.SourceConn.Conn); ok && err == nil {
7697
l.Info("DB is on the same host. parsing logs locally")
77-
// TODO: check privileges before invoking local parsing
78-
return lp.parseLogsLocal()
98+
if err = checkHasLocalPrivileges(lp.LogFolder); err == nil {
99+
return lp.parseLogsLocal()
100+
}
101+
l.WithError(err).Error("Could't parse logs locally. lacking required privileges")
79102
}
80103

81104
l.Info("DB is not detected to be on the same host. parsing logs remotely")
82-
if err := checkHasPrivileges(lp.ctx, lp.SourceConn, lp.LogFolder); err != nil {
105+
if err := checkHasRemotePrivileges(lp.ctx, lp.SourceConn, lp.LogFolder); err != nil {
83106
l.WithError(err).Error("Could't parse logs remotely. lacking required privileges")
84107
return err
85108
}
@@ -113,7 +136,7 @@ func tryDetermineLogMessagesLanguage(ctx context.Context, conn db.PgxIface) (str
113136
return lc, nil
114137
}
115138

116-
func checkHasPrivileges(ctx context.Context, mdb *sources.SourceConn, logsDirPath string) error {
139+
func checkHasRemotePrivileges(ctx context.Context, mdb *sources.SourceConn, logsDirPath string) error {
117140
var logFile string
118141
err := mdb.Conn.QueryRow(ctx, "select name from pg_ls_logdir() limit 1").Scan(&logFile)
119142
if err != nil && err != pgx.ErrNoRows {
@@ -125,27 +148,14 @@ func checkHasPrivileges(ctx context.Context, mdb *sources.SourceConn, logsDirPat
125148
return err
126149
}
127150

128-
// Constants and types
129-
130-
var pgSeverities = [...]string{"DEBUG", "INFO", "NOTICE", "WARNING", "ERROR", "LOG", "FATAL", "PANIC"}
131-
var pgSeveritiesLocale = map[string]map[string]string{
132-
"C.": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTICE": "NOTICE", "WARNING": "WARNING", "ERROR": "ERROR", "FATAL": "FATAL", "PANIC": "PANIC"},
133-
"de": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "HINWEIS": "NOTICE", "WARNUNG": "WARNING", "FEHLER": "ERROR", "FATAL": "FATAL", "PANIK": "PANIC"},
134-
"fr": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTICE": "NOTICE", "ATTENTION": "WARNING", "ERREUR": "ERROR", "FATAL": "FATAL", "PANIK": "PANIC"},
135-
"it": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTIFICA": "NOTICE", "ATTENZIONE": "WARNING", "ERRORE": "ERROR", "FATALE": "FATAL", "PANICO": "PANIC"},
136-
"ko": {"디버그": "DEBUG", "로그": "LOG", "정보": "INFO", "알림": "NOTICE", "경고": "WARNING", "오류": "ERROR", "치명적오류": "FATAL", "손상": "PANIC"},
137-
"pl": {"DEBUG": "DEBUG", "DZIENNIK": "LOG", "INFORMACJA": "INFO", "UWAGA": "NOTICE", "OSTRZEŻENIE": "WARNING", "BŁĄD": "ERROR", "KATASTROFALNY": "FATAL", "PANIKA": "PANIC"},
138-
"ru": {"ОТЛАДКА": "DEBUG", "СООБЩЕНИЕ": "LOG", "ИНФОРМАЦИЯ": "INFO", "ЗАМЕЧАНИЕ": "NOTICE", "ПРЕДУПРЕЖДЕНИЕ": "WARNING", "ОШИБКА": "ERROR", "ВАЖНО": "FATAL", "ПАНИКА": "PANIC"},
139-
"sv": {"DEBUG": "DEBUG", "LOGG": "LOG", "INFO": "INFO", "NOTIS": "NOTICE", "VARNING": "WARNING", "FEL": "ERROR", "FATALT": "FATAL", "PANIK": "PANIC"},
140-
"tr": {"DEBUG": "DEBUG", "LOG": "LOG", "BİLGİ": "INFO", "NOT": "NOTICE", "UYARI": "WARNING", "HATA": "ERROR", "ÖLÜMCÜL (FATAL)": "FATAL", "KRİTİK": "PANIC"},
141-
"zh": {"调试": "DEBUG", "日志": "LOG", "信息": "INFO", "注意": "NOTICE", "警告": "WARNING", "错误": "ERROR", "致命错误": "FATAL", "比致命错误还过分的错误": "PANIC"},
151+
func checkHasLocalPrivileges(logsDirPath string) error {
152+
_, err := os.ReadDir(logsDirPath)
153+
if err != nil {
154+
return err
155+
}
156+
return nil
142157
}
143158

144-
const csvLogDefaultRegEx = `^^(?P<log_time>.*?),"?(?P<user_name>.*?)"?,"?(?P<database_name>.*?)"?,(?P<process_id>\d+),"?(?P<connection_from>.*?)"?,(?P<session_id>.*?),(?P<session_line_num>\d+),"?(?P<command_tag>.*?)"?,(?P<session_start_time>.*?),(?P<virtual_transaction_id>.*?),(?P<transaction_id>.*?),(?P<error_severity>\w+),`
145-
const csvLogDefaultGlobSuffix = "*.csv"
146-
147-
const maxChunkSize int32 = 10 * 1024 * 1024 // 10 MB
148-
149159
func severityToEnglish(serverLang, errorSeverity string) string {
150160
if serverLang == "en" {
151161
return errorSeverity

‎internal/reaper/logparser_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ func TestCheckHasPrivileges(t *testing.T) {
206206

207207
names := [2]string{"pg_ls_logdir() fails", "pg_read_file() permission denied"}
208208
for _, name := range names {
209-
t.Run("checkHasPrivileges fails - "+name, func(t *testing.T) {
209+
t.Run("checkHasRemotePrivileges fails - "+name, func(t *testing.T) {
210210
mock, err := pgxmock.NewPool()
211211
require.NoError(t, err)
212212
defer mock.Close()

0 commit comments

Comments
 (0)