Skip to content

Commit e4f55e2

Browse files
elliVMelliVM
authored
better gson exception messages when parsing filter fields from gson (#586)
* gson parse JSON filter configuration to a java object or throw an understandable exception if parse fails * fix exception type, comparable logic and testing * add missing cases to bloom filter configuration comparable test * use Long instead of primitive in all BloomFilterConfiguration methods
1 parent f1a9275 commit e4f55e2

4 files changed

Lines changed: 281 additions & 21 deletions

File tree

‎src/main/java/com/teragrep/pth10/steps/teragrep/bloomfilter/BloomFilterConfiguration.java‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* Teragrep Data Processing Language (DPL) translator for Apache Spark (pth_10)
3+
* Copyright (C) 2019-2024 Suomen Kanuuna Oy
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU Affero General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU Affero General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU Affero General Public License
16+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
17+
*
18+
*
19+
* Additional permission under GNU Affero General Public License version 3
20+
* section 7
21+
*
22+
* If you modify this Program, or any covered work, by linking or combining it
23+
* with other code, such other code is not for that reason alone subject to any
24+
* of the requirements of the GNU Affero GPL version 3 as long as this Program
25+
* is the same Program as licensed from Suomen Kanuuna Oy without any additional
26+
* modifications.
27+
*
28+
* Supplemented terms under GNU Affero General Public License version 3
29+
* section 7
30+
*
31+
* Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified
32+
* versions must be marked as "Modified version of" The Program.
33+
*
34+
* Names of the licensors and authors may not be used for publicity purposes.
35+
*
36+
* No rights are granted for use of trade names, trademarks, or service marks
37+
* which are in The Program if any.
38+
*
39+
* Licensee must indemnify licensors and authors for any liability that these
40+
* contractual assumptions impose on licensors and authors.
41+
*
42+
* To the extent this program is licensed as part of the Commercial versions of
43+
* Teragrep, the applicable Commercial License may apply to this file if you as
44+
* a licensee so wish it.
45+
*/
46+
package com.teragrep.pth10.steps.teragrep.bloomfilter;
47+
48+
import java.util.Objects;
49+
50+
public final class BloomFilterConfiguration implements Comparable<BloomFilterConfiguration> {
51+
52+
// member field names must be exactly these so Gson can parse them from JSON
53+
private final Long expected;
54+
private final Double fpp;
55+
56+
public BloomFilterConfiguration(final Long expected, final Double fpp) {
57+
this.expected = expected;
58+
this.fpp = fpp;
59+
}
60+
61+
public Long expectedNumOfItems() {
62+
return expected;
63+
}
64+
65+
public Double falsePositiveProbability() {
66+
return fpp;
67+
}
68+
69+
@Override
70+
public boolean equals(final Object object) {
71+
if (this == object)
72+
return true;
73+
if (object == null)
74+
return false;
75+
if (object.getClass() != this.getClass())
76+
return false;
77+
final BloomFilterConfiguration cast = (BloomFilterConfiguration) object;
78+
return expected.equals(cast.expected) && fpp.equals(cast.fpp);
79+
}
80+
81+
@Override
82+
public int hashCode() {
83+
return Objects.hash(expected, fpp);
84+
}
85+
86+
@Override
87+
public int compareTo(final BloomFilterConfiguration other) {
88+
if (equals(other)) {
89+
return 0;
90+
}
91+
if (expected.equals(other.expected)) {
92+
// larger fpp results in a smaller filter bit size
93+
if (fpp < other.fpp) {
94+
return 1;
95+
}
96+
else {
97+
return -1;
98+
}
99+
}
100+
else if (expected > other.expected) {
101+
return 1;
102+
}
103+
else {
104+
return -1;
105+
}
106+
}
107+
}

‎src/main/java/com/teragrep/pth10/steps/teragrep/bloomfilter/FilterTypes.java‎

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@
4646
package com.teragrep.pth10.steps.teragrep.bloomfilter;
4747

4848
import com.google.gson.Gson;
49-
import com.google.gson.JsonObject;
49+
import com.google.gson.JsonIOException;
50+
import com.google.gson.JsonSyntaxException;
5051
import com.typesafe.config.Config;
5152
import org.apache.spark.util.sketch.BloomFilter;
5253
import org.slf4j.Logger;
@@ -78,22 +79,30 @@ public FilterTypes(Config config) {
7879
*/
7980
public SortedMap<Long, Double> sortedMap() {
8081
final SortedMap<Long, Double> sizesMapFromJson = new TreeMap<>();
81-
final Gson gson = new Gson();
82-
final List<JsonObject> jsonArray = gson.fromJson(sizesJsonString(), new TypeToken<List<JsonObject>>() {
83-
}.getType());
84-
for (final JsonObject object : jsonArray) {
85-
if (object.has("expected") && object.has("fpp")) {
86-
final Long expectedNumOfItems = Long.parseLong(object.get("expected").toString());
87-
final Double fpp = Double.parseDouble(object.get("fpp").toString());
88-
if (sizesMapFromJson.containsKey(expectedNumOfItems)) {
89-
LOGGER.error("Duplicate value of expected number of items value: <[{}]>", expectedNumOfItems);
90-
throw new RuntimeException("Duplicate entry expected num of items");
91-
}
92-
sizesMapFromJson.put(expectedNumOfItems, fpp);
93-
}
94-
else {
95-
throw new RuntimeException("JSON did not have expected values of 'expected' or 'fpp'");
96-
}
82+
final List<BloomFilterConfiguration> filterConfigurationList;
83+
84+
try {
85+
final Gson gson = new Gson();
86+
filterConfigurationList = gson.fromJson(sizesJsonString(), new TypeToken<List<BloomFilterConfiguration>>() {
87+
}.getType());
88+
}
89+
catch (final JsonIOException | JsonSyntaxException e) {
90+
throw new IllegalArgumentException(
91+
"Error parsing 'dpl.pth_06.bloom.db.fields' option to JSON. ensure that filter size options are formated as an JSON array and that there are no duplicate values. "
92+
+ "example '[{expected: 1000, fpp: 0.01},{expected: 2000, fpp: 0.01}]'. message: "
93+
+ e.getMessage()
94+
);
95+
}
96+
97+
for (final BloomFilterConfiguration configuration : filterConfigurationList) {
98+
final Long expectedNumOfItems = configuration.expectedNumOfItems();
99+
final Double falsePositiveProbability = configuration.falsePositiveProbability();
100+
sizesMapFromJson.put(expectedNumOfItems, falsePositiveProbability);
101+
}
102+
103+
final boolean hasDuplicates = new HashSet<>(filterConfigurationList).size() != filterConfigurationList.size();
104+
if (hasDuplicates) {
105+
throw new IllegalArgumentException("Found duplicate values in 'dpl.pth_06.bloom.db.fields'");
97106
}
98107
return sizesMapFromJson;
99108
}

‎src/test/java/com/teragrep/pth10/steps/teragrep/bloomfilter/BloomFilterConfigurationTest.java‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/*
2+
* Teragrep Data Processing Language (DPL) translator for Apache Spark (pth_10)
3+
* Copyright (C) 2019-2024 Suomen Kanuuna Oy
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU Affero General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU Affero General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU Affero General Public License
16+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
17+
*
18+
*
19+
* Additional permission under GNU Affero General Public License version 3
20+
* section 7
21+
*
22+
* If you modify this Program, or any covered work, by linking or combining it
23+
* with other code, such other code is not for that reason alone subject to any
24+
* of the requirements of the GNU Affero GPL version 3 as long as this Program
25+
* is the same Program as licensed from Suomen Kanuuna Oy without any additional
26+
* modifications.
27+
*
28+
* Supplemented terms under GNU Affero General Public License version 3
29+
* section 7
30+
*
31+
* Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified
32+
* versions must be marked as "Modified version of" The Program.
33+
*
34+
* Names of the licensors and authors may not be used for publicity purposes.
35+
*
36+
* No rights are granted for use of trade names, trademarks, or service marks
37+
* which are in The Program if any.
38+
*
39+
* Licensee must indemnify licensors and authors for any liability that these
40+
* contractual assumptions impose on licensors and authors.
41+
*
42+
* To the extent this program is licensed as part of the Commercial versions of
43+
* Teragrep, the applicable Commercial License may apply to this file if you as
44+
* a licensee so wish it.
45+
*/
46+
package com.teragrep.pth10.steps.teragrep.bloomfilter;
47+
48+
import com.google.gson.Gson;
49+
import nl.jqno.equalsverifier.EqualsVerifier;
50+
import org.junit.jupiter.api.Assertions;
51+
import org.junit.jupiter.api.Test;
52+
import org.sparkproject.guava.reflect.TypeToken;
53+
54+
public final class BloomFilterConfigurationTest {
55+
56+
@Test
57+
public void testOptions() {
58+
final BloomFilterConfiguration configuration = new BloomFilterConfiguration(1000L, 0.01);
59+
Assertions.assertEquals(1000L, configuration.expectedNumOfItems());
60+
Assertions.assertEquals(0.01, configuration.falsePositiveProbability());
61+
}
62+
63+
@Test
64+
public void testGsonParseable() {
65+
final String json = "{expected:1000,fpp:0.01}";
66+
final Gson gson = new Gson();
67+
final BloomFilterConfiguration parsedObject = Assertions
68+
.assertDoesNotThrow(() -> gson.fromJson(json, new TypeToken<BloomFilterConfiguration>() {
69+
}.getType()));
70+
Assertions.assertEquals(1000L, parsedObject.expectedNumOfItems());
71+
Assertions.assertEquals(0.01, parsedObject.falsePositiveProbability());
72+
}
73+
74+
@Test
75+
public void testContract() {
76+
EqualsVerifier.forClass(BloomFilterConfiguration.class).withNonnullFields("expected", "fpp").verify();
77+
}
78+
79+
@Test
80+
public void testEquality() {
81+
final BloomFilterConfiguration base = new BloomFilterConfiguration(1000L, 0.01);
82+
final BloomFilterConfiguration equals = new BloomFilterConfiguration(1000L, 0.01);
83+
Assertions.assertEquals(base, equals);
84+
}
85+
86+
@Test
87+
public void testNonEquality() {
88+
final BloomFilterConfiguration base = new BloomFilterConfiguration(1000L, 0.01);
89+
final BloomFilterConfiguration nonEquals = new BloomFilterConfiguration(2000L, 0.01);
90+
final BloomFilterConfiguration nonEquals2 = new BloomFilterConfiguration(1000L, 0.02);
91+
Assertions.assertNotEquals(base, nonEquals);
92+
Assertions.assertNotEquals(base, nonEquals2);
93+
}
94+
95+
@Test
96+
public void testComparable() {
97+
final BloomFilterConfiguration base = new BloomFilterConfiguration(1000L, 0.02);
98+
final BloomFilterConfiguration equals = new BloomFilterConfiguration(1000L, 0.02);
99+
final BloomFilterConfiguration smaller = new BloomFilterConfiguration(500L, 0.02);
100+
final BloomFilterConfiguration larger = new BloomFilterConfiguration(2000L, 0.02);
101+
final BloomFilterConfiguration smallerFpp = new BloomFilterConfiguration(1000L, 0.01);
102+
final BloomFilterConfiguration largerFpp = new BloomFilterConfiguration(1000L, 0.03);
103+
final BloomFilterConfiguration bothLarger = new BloomFilterConfiguration(2000L, 0.03);
104+
final BloomFilterConfiguration bothSmaller = new BloomFilterConfiguration(500L, 0.01);
105+
final BloomFilterConfiguration smallerExpectedLargerFpp = new BloomFilterConfiguration(500L, 0.03);
106+
final BloomFilterConfiguration largerExpectedSmallerFpp = new BloomFilterConfiguration(2000L, 0.01);
107+
Assertions.assertEquals(0, base.compareTo(equals));
108+
Assertions.assertEquals(1, base.compareTo(smaller));
109+
Assertions.assertEquals(-1, base.compareTo(larger));
110+
Assertions.assertEquals(-1, base.compareTo(smallerFpp));
111+
Assertions.assertEquals(1, base.compareTo(largerFpp));
112+
Assertions.assertEquals(-1, base.compareTo(bothLarger));
113+
Assertions.assertEquals(1, base.compareTo(bothSmaller));
114+
Assertions.assertEquals(1, base.compareTo(smallerExpectedLargerFpp));
115+
Assertions.assertEquals(-1, base.compareTo(largerExpectedSmallerFpp));
116+
}
117+
}

‎src/test/java/com/teragrep/pth10/steps/teragrep/bloomfilter/FilterTypesTest.java‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565

6666
import static org.junit.jupiter.api.Assertions.*;
6767

68-
class FilterTypesTest {
68+
public final class FilterTypesTest {
6969

7070
private final String username = "sa";
7171
private final String password = "";
@@ -78,9 +78,6 @@ public void setup() {
7878
Assertions.assertDoesNotThrow(() -> {
7979
conn.prepareStatement("DROP ALL OBJECTS").execute(); // h2 clear database
8080
});
81-
Assertions.assertDoesNotThrow(() -> {
82-
Class.forName("org.h2.Driver");
83-
});
8481
String createFilterType = "CREATE TABLE `filtertype` ("
8582
+ "`id` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,"
8683
+ "`expectedElements` bigint(20) NOT NULL," + "`targetFpp` DOUBLE UNSIGNED NOT NULL,"
@@ -146,6 +143,36 @@ public void testWriteFilterTypesToDatabase() {
146143
});
147144
}
148145

146+
@Test
147+
public void testMalformattedFieldsOption() {
148+
final Properties properties = new Properties();
149+
properties
150+
.put(
151+
"dpl.pth_06.bloom.db.fields",
152+
"{expected: 1000, fpp: 0.01},{expected: 2000, fpp: 0.01},{expected: 3000, fpp: 0.01}"
153+
);
154+
final Config config = ConfigFactory.parseProperties(properties);
155+
final FilterTypes filterTypes = new FilterTypes(config);
156+
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, filterTypes::sortedMap);
157+
final String expectedMessage = "Error parsing 'dpl.pth_06.bloom.db.fields' option to JSON. ensure that filter size options are formated as an JSON array and that there are no duplicate values. example '[{expected: 1000, fpp: 0.01},{expected: 2000, fpp: 0.01}]'. message:";
158+
Assertions.assertTrue(exception.getMessage().startsWith(expectedMessage));
159+
}
160+
161+
@Test
162+
public void testDuplicateValues() {
163+
final Properties properties = new Properties();
164+
properties
165+
.put(
166+
"dpl.pth_06.bloom.db.fields",
167+
"[{expected: 1000, fpp: 0.01},{expected: 1000, fpp: 0.01},{expected: 3000, fpp: 0.01}]"
168+
);
169+
final Config config = ConfigFactory.parseProperties(properties);
170+
final FilterTypes filterTypes = new FilterTypes(config);
171+
final IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, filterTypes::sortedMap);
172+
final String expectedMessage = "Found duplicate values in 'dpl.pth_06.bloom.db.fields'";
173+
Assertions.assertEquals(expectedMessage, exception.getMessage());
174+
}
175+
149176
@Test
150177
public void testEquals() {
151178
Config config = ConfigFactory.parseProperties(defaultProperties());

0 commit comments

Comments
 (0)