Skip to content

Commit e18425b

Browse files
MoonBow-1MoonBow-1
authored
Pgsqlserverlogs support (#73)
* Initial object for PGSQLServerLogsType - Requires unit tests * Unit tests and fixes for PGSQLServerLogsType * Remove unnecessary import in PGSQLServerLogsTypeTest * Add PGSQLServerLogsType to NLFPlugin.syslogMessage and test it * Throw Exceptions early in PGSQLServerLogsType.appName and add more tests for that
1 parent 3a62835 commit e18425b

8 files changed

Lines changed: 607 additions & 0 deletions

File tree

‎src/main/java/com/teragrep/nlf_01/NLFPlugin.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,9 @@ else if (jsonObject.getString("Type").equals("FunctionAppLogs")) {
124124
else if (jsonObject.getString("Type").equals("LogicAppWorkflowRuntime")) {
125125
eventTypes.add(new LogicAppWorkflowRuntimeType(parsedEvent, realHostname));
126126
}
127+
else if (jsonObject.getString("Type").equals("PGSQLServerLogs")) {
128+
eventTypes.add(new PGSQLServerLogsType(parsedEvent, realHostname));
129+
}
127130
else if (jsonObject.getString("Type").equals("PowerAutomateActivity")) {
128131
eventTypes.add(new PowerAutomateActivityType(parsedEvent, realHostname));
129132
}

‎src/main/java/com/teragrep/nlf_01/types/PGSQLServerLogsType.java‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
/*
2+
* Teragrep Neon log format plugin for AKV_01
3+
* Copyright (C) 2025 Suomen Kanuuna Oy
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU Affero General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU Affero General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU Affero General Public License
16+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
17+
*
18+
*
19+
* Additional permission under GNU Affero General Public License version 3
20+
* section 7
21+
*
22+
* If you modify this Program, or any covered work, by linking or combining it
23+
* with other code, such other code is not for that reason alone subject to any
24+
* of the requirements of the GNU Affero GPL version 3 as long as this Program
25+
* is the same Program as licensed from Suomen Kanuuna Oy without any additional
26+
* modifications.
27+
*
28+
* Supplemented terms under GNU Affero General Public License version 3
29+
* section 7
30+
*
31+
* Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified
32+
* versions must be marked as "Modified version of" The Program.
33+
*
34+
* Names of the licensors and authors may not be used for publicity purposes.
35+
*
36+
* No rights are granted for use of trade names, trademarks, or service marks
37+
* which are in The Program if any.
38+
*
39+
* Licensee must indemnify licensors and authors for any liability that these
40+
* contractual assumptions impose on licensors and authors.
41+
*
42+
* To the extent this program is licensed as part of the Commercial versions of
43+
* Teragrep, the applicable Commercial License may apply to this file if you as
44+
* a licensee so wish it.
45+
*/
46+
package com.teragrep.nlf_01.types;
47+
48+
import com.teragrep.akv_01.event.ParsedEvent;
49+
import com.teragrep.akv_01.plugin.PluginException;
50+
import com.teragrep.nlf_01.PropertiesJson;
51+
import com.teragrep.nlf_01.util.ASCIIString;
52+
import com.teragrep.nlf_01.util.MD5Hash;
53+
import com.teragrep.nlf_01.util.ResourceId;
54+
import com.teragrep.nlf_01.util.ValidKey;
55+
import com.teragrep.nlf_01.util.ValidRFC5424AppName;
56+
import com.teragrep.nlf_01.util.ValidRFC5424Hostname;
57+
import com.teragrep.nlf_01.util.ValidRFC5424Timestamp;
58+
import com.teragrep.nlf_01.util.ValidStringKey;
59+
import com.teragrep.rlo_14.Facility;
60+
import com.teragrep.rlo_14.SDElement;
61+
import com.teragrep.rlo_14.Severity;
62+
import jakarta.json.JsonObject;
63+
import java.time.Instant;
64+
import java.util.HashSet;
65+
import java.util.Set;
66+
import java.util.UUID;
67+
import java.util.regex.Matcher;
68+
import java.util.regex.Pattern;
69+
70+
public final class PGSQLServerLogsType implements EventType {
71+
72+
private final ParsedEvent parsedEvent;
73+
private final String realHostname;
74+
private final Pattern appNamePattern;
75+
76+
public PGSQLServerLogsType(final ParsedEvent parsedEvent, final String realHostname) {
77+
this(parsedEvent, realHostname, Pattern.compile("^.*?db=(?<dbName>.*?),"));
78+
}
79+
80+
private PGSQLServerLogsType(
81+
final ParsedEvent parsedEvent,
82+
final String realHostname,
83+
final Pattern appNamePattern
84+
) {
85+
this.parsedEvent = parsedEvent;
86+
this.realHostname = realHostname;
87+
this.appNamePattern = appNamePattern;
88+
}
89+
90+
@Override
91+
public Severity severity() throws PluginException {
92+
return Severity.NOTICE;
93+
}
94+
95+
@Override
96+
public Facility facility() throws PluginException {
97+
return Facility.AUDIT;
98+
}
99+
100+
@Override
101+
public String hostname() throws PluginException {
102+
final JsonObject record = parsedEvent.asJsonStructure().asJsonObject();
103+
104+
final ValidKey<String> validKey = new ValidStringKey(record, "_Internal_WorkspaceResourceId");
105+
106+
return new ValidRFC5424Hostname(
107+
"md5-".concat(new MD5Hash(validKey.value()).md5().concat("-").concat(new ASCIIString(new ResourceId(validKey.value()).resourceName()).withNonAsciiCharsRemoved()))
108+
).hostnameWithInvalidCharsRemoved();
109+
}
110+
111+
@Override
112+
public String appName() throws PluginException {
113+
final JsonObject record = parsedEvent.asJsonStructure().asJsonObject();
114+
115+
final ValidKey<String> messageValidKey = new ValidStringKey(record, "Message");
116+
final String message = messageValidKey.value();
117+
118+
final Matcher matcher = appNamePattern.matcher(message);
119+
if (!matcher.find()) {
120+
throw new PluginException("Could not parse dbName from Message");
121+
}
122+
else {
123+
final String dbName = matcher.group("dbName");
124+
if (dbName == null || dbName.isEmpty()) {
125+
throw new PluginException("Capture group 'dbName' was not found");
126+
}
127+
return new ValidRFC5424AppName(new ASCIIString(dbName).withNonAsciiCharsRemoved()).appName();
128+
}
129+
}
130+
131+
@Override
132+
public long timestamp() throws PluginException {
133+
final JsonObject record = parsedEvent.asJsonStructure().asJsonObject();
134+
135+
return new ValidRFC5424Timestamp(new ValidStringKey(record, "TimeGenerated").value()).validTimestamp();
136+
}
137+
138+
@Override
139+
public Set<SDElement> sdElements() throws PluginException {
140+
final Set<SDElement> elems = new HashSet<>();
141+
final String time;
142+
if (!parsedEvent.enqueuedTimeUtc().isStub()) {
143+
time = parsedEvent.enqueuedTimeUtc().zonedDateTime().toString();
144+
}
145+
else {
146+
time = "";
147+
}
148+
149+
final String fullyQualifiedNamespace;
150+
final String eventHubName;
151+
final String partitionId;
152+
final String consumerGroup;
153+
if (!parsedEvent.partitionCtx().isStub()) {
154+
fullyQualifiedNamespace = String
155+
.valueOf(parsedEvent.partitionCtx().asMap().getOrDefault("FullyQualifiedNamespace", ""));
156+
eventHubName = String.valueOf(parsedEvent.partitionCtx().asMap().getOrDefault("EventHubName", ""));
157+
partitionId = String.valueOf(parsedEvent.partitionCtx().asMap().getOrDefault("PartitionId", ""));
158+
consumerGroup = String.valueOf(parsedEvent.partitionCtx().asMap().getOrDefault("ConsumerGroup", ""));
159+
}
160+
else {
161+
fullyQualifiedNamespace = "";
162+
eventHubName = "";
163+
partitionId = "";
164+
consumerGroup = "";
165+
}
166+
167+
elems
168+
.add(new SDElement("aer_02_partition@48577").addSDParam("fully_qualified_namespace", fullyQualifiedNamespace).addSDParam("eventhub_name", eventHubName).addSDParam("partition_id", partitionId).addSDParam("consumer_group", consumerGroup));
169+
170+
elems
171+
.add(new SDElement("event_id@48577").addSDParam("uuid", UUID.randomUUID().toString()).addSDParam("hostname", realHostname).addSDParam("unixtime", Instant.now().toString()).addSDParam("id_source", "aer_02"));
172+
173+
final String partitionKey;
174+
if (!parsedEvent.systemProperties().isStub()) {
175+
partitionKey = String.valueOf(parsedEvent.systemProperties().asMap().getOrDefault("PartitionKey", ""));
176+
}
177+
else {
178+
partitionKey = "";
179+
}
180+
181+
final String offset;
182+
if (!parsedEvent.offset().isStub()) {
183+
offset = parsedEvent.offset().value();
184+
}
185+
else {
186+
offset = "";
187+
}
188+
189+
elems
190+
.add(new SDElement("aer_02_event@48577").addSDParam("offset", offset).addSDParam("enqueued_time", time).addSDParam("partition_key", partitionKey).addSDParam("properties", new PropertiesJson(parsedEvent.properties()).toJsonObject().toString()));
191+
192+
elems
193+
.add(new SDElement("aer_02@48577").addSDParam("timestamp_source", time.isEmpty() ? "generated" : "timeEnqueued"));
194+
195+
final JsonObject mainObject = parsedEvent.asJsonStructure().asJsonObject();
196+
197+
final ValidKey<String> resourceIdValidKey = new ValidStringKey(mainObject, "_ResourceId");
198+
final String resourceId = resourceIdValidKey.value();
199+
200+
elems.add(new SDElement("origin@48577").addSDParam("_ResourceId", resourceId));
201+
elems.add(new SDElement("nlf_01@48577").addSDParam("eventType", this.getClass().getSimpleName()));
202+
203+
return elems;
204+
}
205+
206+
@Override
207+
public String msgId() throws PluginException {
208+
final String sequenceNumber;
209+
if (!parsedEvent.systemProperties().isStub()) {
210+
sequenceNumber = String.valueOf(parsedEvent.systemProperties().asMap().getOrDefault("SequenceNumber", ""));
211+
}
212+
else {
213+
sequenceNumber = "";
214+
}
215+
return sequenceNumber;
216+
}
217+
218+
@Override
219+
public String msg() throws PluginException {
220+
return parsedEvent.asString();
221+
}
222+
}

‎src/test/java/com/teragrep/nlf_01/NLFPluginTest.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -740,6 +740,38 @@ void functionAppLogsType() {
740740
Assertions.assertTrue(sdElementMap.get("aer_02_event@48577").containsKey("properties"));
741741
}
742742

743+
@Test
744+
void pgsqlServerLogsTypeTest() {
745+
final String json = Assertions
746+
.assertDoesNotThrow(() -> Files.readString(Paths.get("src/test/resources/pgsqlserverlogs.json")));
747+
final ParsedEvent parsedEvent = new ParsedEventFactory(
748+
new UnparsedEventImpl(json, new EventPartitionContextImpl(new HashMap<>()), new EventPropertiesImpl(new HashMap<>()), new EventSystemPropertiesImpl(new HashMap<>()), new EnqueuedTimeImpl("2020-01-01T00:00:00"), new EventOffsetImpl("0"))
749+
).parsedEvent();
750+
751+
final NLFPlugin plugin = new NLFPlugin(new FakeSourceable());
752+
final List<SyslogMessage> syslogMessages = Assertions
753+
.assertDoesNotThrow(() -> plugin.syslogMessage(parsedEvent));
754+
Assertions.assertEquals(1, syslogMessages.size());
755+
756+
final SyslogMessage syslogMessage = syslogMessages.get(0);
757+
Assertions.assertEquals("md5-0ded52ef915af563e25778bf26b0f129-resourceName", syslogMessage.getHostname());
758+
Assertions.assertEquals("dbase_maintenance", syslogMessage.getAppName());
759+
Assertions.assertEquals("2020-10-01T11:59:26.256Z", syslogMessage.getTimestamp());
760+
Assertions.assertEquals(json, syslogMessage.getMsg());
761+
final Map<String, Map<String, String>> sdElementMap = syslogMessage
762+
.getSDElements()
763+
.stream()
764+
.collect(Collectors.toMap((SDElement::getSdID), (sdElem) -> sdElem.getSdParams().stream().collect(Collectors.toMap(SDParam::getParamName, SDParam::getParamValue))));
765+
766+
Assertions
767+
.assertEquals(
768+
"/SUBSCRIPTIONS/uuid/RESOURCEGROUPS/ab-cd-efgh-ijklmn-xx-DEV-01/PROVIDERS/postgres-db/FLEXIBLESERVERS/efgh-ijklmn-xx-DEV-01",
769+
sdElementMap.get("origin@48577").get("_ResourceId")
770+
);
771+
772+
Assertions.assertTrue(sdElementMap.get("aer_02_event@48577").containsKey("properties"));
773+
}
774+
743775
@Test
744776
void powerAutomateActivityType() {
745777
final String json = Assertions

0 commit comments

Comments
 (0)