#435 fix: +1sec to latest timeQualifier if sub-seconds are used (#454)

Includes commits:

change DefaultTimeFormat, DPLTimeFormat and EpochTimestamp(now InstantTimestamp) to return Instant. Add RoundedUpTimestamp decorator, which is called when LATEST timeQualifier is used, otherwise InstantTimestamp is used directly.

cleanup DefaultTimeFormat as suggested in PR review

add invalidTimeformat test

added timezones to test data (earliestLatestTest_data/earliestLatestTest_epoch_data.jsonl); fixed failing tests due to that

#435 initial commit: add +1sec to latest time if sub-seconds used

Author: eemhu

src/main/java/com/teragrep/pth10/ast/DPLParserConfig.java

@@ -111,7 +111,7 @@ public void setEarliest(String earliest) {
         // Try to check if it is relative and catch exception
         try {
             RelativeTimestamp rtTimestamp = rtParser.parse(earliest); // can throw error if not relative timestamp
-            earliestEpoch = rtTimestamp.calculate(now);
+            earliestEpoch = rtTimestamp.calculate(now).getEpochSecond();
         }
         catch (NumberFormatException ne) {
             // absolute time
@@ -145,7 +145,7 @@ public void setLatest(String latest) {
         // Try to check if it is relative and catch exception
         try {
             RelativeTimestamp rtTimestamp = rtParser.parse(latest); // can throw exception if not relative timestamp
-            latestEpoch = rtTimestamp.calculate(now);
+            latestEpoch = rtTimestamp.calculate(now).getEpochSecond();
         }
         catch (NumberFormatException ne) {
             // absolute time

src/main/java/com/teragrep/pth10/ast/DPLTimeFormat.java

@@ -47,6 +47,7 @@
 
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.time.Instant;
 
 /**
  * For using the DPL custom timeformat like Java's SimpleDateFormat. Get a Date object with parse -function for example.
@@ -78,8 +79,8 @@ public SimpleDateFormat createSimpleDateFormat() {
      * @return Unix Epoch
      * @throws ParseException when dplTime doesn't have the correct format
      */
-    public long getEpoch(String dplTime) throws ParseException {
-        return createSimpleDateFormat().parse(dplTime).getTime() / 1000L;
+    public Instant instantOf(String dplTime) throws ParseException {
+        return createSimpleDateFormat().parse(dplTime).toInstant();
     }
 
     // replace dpl time units with java-compatible time units

src/main/java/com/teragrep/pth10/ast/DefaultTimeFormat.java

@@ -56,6 +56,22 @@
  */
 public class DefaultTimeFormat {
 
+    private final String[] formats;
+
+    public DefaultTimeFormat() {
+        this(new String[] {
+                "MM/dd/yyyy:HH:mm:ss",
+                "yyyy-MM-dd'T'HH:mm:ss.SSSXXX",
+                "yyyy-MM-dd'T'HH:mm:ss.SSS",
+                "yyyy-MM-dd'T'HH:mm:ssXXX",
+                "yyyy-MM-dd'T'HH:mm:ss"
+        });
+    }
+
+    public DefaultTimeFormat(String[] formats) {
+        this.formats = formats;
+    }
+
     /**
      * Calculate the epoch from given string.
      * 
@@ -73,40 +89,15 @@ public long getEpoch(String time) {
      * @return Date parsed from the given string
      */
     public Date parse(String time) {
-        // Try parsing all the three default timeformats
-        Date date;
-
-        int attempt = 0;
-        while (true) {
+        // Try parsing all provided time formats in order
+        for (final String format : formats) {
             try {
-                if (attempt == 0) {
-                    // Use default format (MM/dd/yyyy:HH:mm:ss)
-                    // Use system default timezone
-                    date = this.parseDate(time, "MM/dd/yyyy:HH:mm:ss");
-                }
-                else if (attempt == 1) {
-                    // On first fail, try ISO 8601 with timezone offset, e.g. '2011-12-03T10:15:30+01:00'
-                    date = this.parseDate(time, "yyyy-MM-dd'T'HH:mm:ssXXX");
-                }
-                else {
-                    // On second fail, try ISO 8601 without offset, e.g. '2011-12-03T10:15:30'
-                    // Use system default timezone
-                    date = this.parseDate(time, "yyyy-MM-dd'T'HH:mm:ss");
-                }
-                break;
-
+                return parseDate(time, format);
             }
-            catch (ParseException e) {
-                if (attempt > 1) {
-                    throw new RuntimeException("TimeQualifier conversion error: <" + time + "> can't be parsed.");
-                }
-            }
-            finally {
-                attempt++;
+            catch (ParseException ignored) {
             }
         }
-
-        return date;
+        throw new RuntimeException("TimeQualifier conversion error: <" + time + "> can't be parsed.");
     }
 
     private Date parseDate(String time, String timeFormat) throws ParseException {

src/main/java/com/teragrep/pth10/ast/commands/evalstatement/UDFs/Mvrange.java

@@ -106,7 +106,7 @@ else if (stepObj instanceof String) {
             RelativeTimestamp rtTimestamp = rtParser.parse("+" + stepStr);
             // Go until incremented past end
             while (time < end) {
-                time = rtTimestamp.calculate(new Timestamp(time * 1000L));
+                time = rtTimestamp.calculate(new Timestamp(time * 1000L)).getEpochSecond();
 
                 // If time went past end, stop incrementing and don't add to mv field
                 if (time > end) {

src/main/java/com/teragrep/pth10/ast/commands/evalstatement/UDFs/Relative_time.java

@@ -65,7 +65,7 @@ public class Relative_time implements UDF2<Long, String, Long>, Serializable {
     public Long call(Long unixtime, String modifier) throws Exception {
         RelativeTimeParser rtParser = new RelativeTimeParser();
         RelativeTimestamp rtTimestamp = rtParser.parse(modifier);
-        return rtTimestamp.calculate(new Timestamp(unixtime * 1000L));
+        return rtTimestamp.calculate(new Timestamp(unixtime * 1000L)).getEpochSecond();
     }
 
 }

src/main/java/com/teragrep/pth10/ast/commands/transformstatement/convert/Mktime.java

@@ -61,7 +61,7 @@ public class Mktime implements UDF2<String, String, String> {
     @Override
     public String call(String hrt, String tf) throws Exception {
         DPLTimeFormat format = new DPLTimeFormat(tf);
-        Long rv = format.getEpoch(hrt);
+        Long rv = format.instantOf(hrt).getEpochSecond();
 
         return rv.toString();
     }

src/main/java/com/teragrep/pth10/ast/time/DPLTimestamp.java

@@ -0,0 +1,53 @@
+/*
+ * Teragrep Data Processing Language (DPL) translator for Apache Spark (pth_10)
+ * Copyright (C) 2019-2024 Suomen Kanuuna Oy
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ *
+ * Additional permission under GNU Affero General Public License version 3
+ * section 7
+ *
+ * If you modify this Program, or any covered work, by linking or combining it
+ * with other code, such other code is not for that reason alone subject to any
+ * of the requirements of the GNU Affero GPL version 3 as long as this Program
+ * is the same Program as licensed from Suomen Kanuuna Oy without any additional
+ * modifications.
+ *
+ * Supplemented terms under GNU Affero General Public License version 3
+ * section 7
+ *
+ * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified
+ * versions must be marked as "Modified version of" The Program.
+ *
+ * Names of the licensors and authors may not be used for publicity purposes.
+ *
+ * No rights are granted for use of trade names, trademarks, or service marks
+ * which are in The Program if any.
+ *
+ * Licensee must indemnify licensors and authors for any liability that these
+ * contractual assumptions impose on licensors and authors.
+ *
+ * To the extent this program is licensed as part of the Commercial versions of
+ * Teragrep, the applicable Commercial License may apply to this file if you as
+ * a licensee so wish it.
+ */
+package com.teragrep.pth10.ast.time;
+
+import java.time.Instant;
+
+public interface DPLTimestamp {
+
+    public abstract Instant instant();
+}

…agrep/pth10/ast/time/EpochTimestamp.java …rep/pth10/ast/time/InstantTimestamp.javasrc/main/java/com/teragrep/pth10/ast/time/EpochTimestamp.java renamed to src/main/java/com/teragrep/pth10/ast/time/InstantTimestamp.java src/main/java/com/teragrep/pth10/ast/time/EpochTimestamp.java renamed to src/main/java/com/teragrep/pth10/ast/time/InstantTimestamp.java

@@ -52,46 +52,47 @@
 
 import java.sql.Timestamp;
 import java.text.ParseException;
+import java.time.Instant;
 import java.util.Objects;
 
-public final class EpochTimestamp {
+public final class InstantTimestamp implements DPLTimestamp {
 
     private final String value;
     private final String timeformat;
 
-    public EpochTimestamp(final String value, final String timeformat) {
+    public InstantTimestamp(final String value, final String timeformat) {
         this.value = value;
         this.timeformat = timeformat;
     }
 
-    public long epoch() {
-        long rv;
+    public Instant instant() {
+        Instant rv;
         try {
             RelativeTimestamp relativeTimestamp = new RelativeTimeParser().parse(value);
             rv = relativeTimestamp.calculate(new Timestamp(System.currentTimeMillis()));
         }
         catch (NumberFormatException ne) {
-            rv = epochFromString(value, timeformat);
+            rv = instantFromString(value, timeformat);
         }
 
         return rv;
     }
 
     // Uses defaultTimeFormat if timeformat is null and DPLTimeFormat if timeformat isn't null (which means that the
     // timeformat= option was used).
-    private long epochFromString(final String value, final String timeFormatString) {
+    private Instant instantFromString(final String value, final String timeFormatString) {
         final String unquotedValue = new UnquotedText(new TextString(value)).read(); // erase the possible outer quotes
-        final long timevalue;
+        final Instant timevalue;
         if (timeFormatString == null || timeFormatString.isEmpty()) {
-            timevalue = new DefaultTimeFormat().getEpoch(unquotedValue);
+            timevalue = new DefaultTimeFormat().parse(unquotedValue).toInstant();
         }
         else {
             // TODO: should be included in DPLTimeFormat
             if (timeFormatString.equals("%s")) {
-                return Long.parseLong(unquotedValue);
+                return Instant.ofEpochSecond(Long.parseLong(unquotedValue));
             }
             try {
-                timevalue = new DPLTimeFormat(timeFormatString).getEpoch(unquotedValue);
+                timevalue = new DPLTimeFormat(timeFormatString).instantOf(unquotedValue);
             }
             catch (ParseException e) {
                 throw new RuntimeException("TimeQualifier conversion error: <" + unquotedValue + "> can't be parsed.");
@@ -106,7 +107,7 @@ public boolean equals(Object o) {
             return true;
         if (o == null || getClass() != o.getClass())
             return false;
-        EpochTimestamp that = (EpochTimestamp) o;
+        InstantTimestamp that = (InstantTimestamp) o;
         return Objects.equals(value, that.value) && Objects.equals(timeformat, that.timeformat);
     }
 

src/main/java/com/teragrep/pth10/ast/time/RelativeTimestamp.java

@@ -64,17 +64,22 @@ public RelativeTimestamp(RelativeOffset offset, SnapToTime snapToTime) {
      * @param timestamp A moment in time, usually the current time
      * @return Calculated time as epoch milliseconds
      */
-    public long calculate(Timestamp timestamp) {
+    public Instant calculate(Timestamp timestamp) {
         Instant time = timestamp.toInstant();
 
         // if both are null, "now" option is left. Therefore, returns current time.
-        if (offset == null && snapToTime == null)
+        if (offset == null && snapToTime == null) {
             time = new Timestamp(System.currentTimeMillis()).toInstant();
+        }
 
-        if (offset != null)
+        if (offset != null) {
             time = offset.addOffset(time);
-        if (snapToTime != null)
+        }
+
+        if (snapToTime != null) {
             time = snapToTime.snap(time);
-        return time.getEpochSecond();
+        }
+
+        return time;
     }
 }

src/main/java/com/teragrep/pth10/ast/time/RoundedUpTimestamp.java

@@ -0,0 +1,67 @@
+/*
+ * Teragrep Data Processing Language (DPL) translator for Apache Spark (pth_10)
+ * Copyright (C) 2019-2024 Suomen Kanuuna Oy
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ *
+ * Additional permission under GNU Affero General Public License version 3
+ * section 7
+ *
+ * If you modify this Program, or any covered work, by linking or combining it
+ * with other code, such other code is not for that reason alone subject to any
+ * of the requirements of the GNU Affero GPL version 3 as long as this Program
+ * is the same Program as licensed from Suomen Kanuuna Oy without any additional
+ * modifications.
+ *
+ * Supplemented terms under GNU Affero General Public License version 3
+ * section 7
+ *
+ * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified
+ * versions must be marked as "Modified version of" The Program.
+ *
+ * Names of the licensors and authors may not be used for publicity purposes.
+ *
+ * No rights are granted for use of trade names, trademarks, or service marks
+ * which are in The Program if any.
+ *
+ * Licensee must indemnify licensors and authors for any liability that these
+ * contractual assumptions impose on licensors and authors.
+ *
+ * To the extent this program is licensed as part of the Commercial versions of
+ * Teragrep, the applicable Commercial License may apply to this file if you as
+ * a licensee so wish it.
+ */
+package com.teragrep.pth10.ast.time;
+
+import java.time.Instant;
+
+public final class RoundedUpTimestamp implements DPLTimestamp {
+
+    private final DPLTimestamp dplTimestamp;
+
+    public RoundedUpTimestamp(final DPLTimestamp dplTimestamp) {
+        this.dplTimestamp = dplTimestamp;
+    }
+
+    public Instant instant() {
+        Instant origin = dplTimestamp.instant();
+        // If date is for latest timeQualifier and has fractions-of-second, add 1 second to capture events
+        // that are on the same second
+        if (origin.getNano() > 0) {
+            origin = origin.plusSeconds(1);
+        }
+        return origin;
+    }
+}