From c119f1d00714aafa3c2f6a58636c4839fc4d38c8 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Fri, 16 Jan 2026 16:06:14 +0000 Subject: [PATCH 1/4] experimental coefficient validator --- src/main/java/simpaths/data/Parameters.java | 27 ++++++++++++++++ .../java/simpaths/data/ParametersTest.java | 32 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 src/test/java/simpaths/data/ParametersTest.java diff --git a/src/main/java/simpaths/data/Parameters.java b/src/main/java/simpaths/data/Parameters.java index ea8b52e66..9651ed2c3 100644 --- a/src/main/java/simpaths/data/Parameters.java +++ b/src/main/java/simpaths/data/Parameters.java @@ -10,6 +10,7 @@ import org.apache.commons.io.FileUtils; import simpaths.data.startingpop.DataParser; import simpaths.model.AnnuityRates; +import simpaths.model.Person; import simpaths.model.enums.*; import org.apache.commons.collections4.keyvalue.MultiKey; import org.apache.commons.collections4.map.LinkedMap; @@ -1138,6 +1139,8 @@ public static void loadParameters(Country country, int maxAgeModel, boolean enab coeffCovarianceHM2CaseMales = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_health_mental.xlsx", countryString + "_HM2_Males_C", 1); coeffCovarianceHM2CaseFemales = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_health_mental.xlsx", countryString + "_HM2_Females_C", 1); + validateRegressors(coeffCovarianceHM2CaseMales, "HM2_Males_C"); + //Health coeffCovarianceDHE_MCS1 = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_health_wellbeing.xlsx", countryString + "_DHE_MCS1", 1); coeffCovarianceDHE_MCS2Males = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_health_wellbeing.xlsx", countryString + "_DHE_MCS2_Males", 1); @@ -3230,6 +3233,30 @@ public static void setEuromodOutputDirectory(String euromodOutputDirectory) { EUROMOD_TRAINING_DIRECTORY = EUROMOD_OUTPUT_DIRECTORY + "training" + File.separator; } + public static void validateRegressors(MultiKeyCoefficientMap map, String mapName) { + if (map == null) return; + + // Get the values read from the REGRESSOR column by ExcelAssistant (excludes 'Constant') + Set regressorNames = map.keySet(); + + // Check across all + for (Object regressor : regressorNames) { + if (regressor instanceof MultiKey mk) { + String keyName = mk.getKey(0).toString(); + + // Test if a Person Enum + try { + Person.DoublesVariables.valueOf(keyName); + } catch (IllegalArgumentException e) { + // This fires if the string isn't in the Enum + throw new RuntimeException("Validation failed for " + mapName + + ": Regressor '" + keyName + "' not found in Person.DoublesVariables. " + + "Check for typos in Excel or missing Enums in Person.java."); + } + } + } + } + public static String getInputDirectory() { return INPUT_DIRECTORY; } diff --git a/src/test/java/simpaths/data/ParametersTest.java b/src/test/java/simpaths/data/ParametersTest.java new file mode 100644 index 000000000..83cc2d2ae --- /dev/null +++ b/src/test/java/simpaths/data/ParametersTest.java @@ -0,0 +1,32 @@ +package simpaths.data; + +import microsim.data.MultiKeyCoefficientMap; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.*; + +class ParametersTest { + + /** + * Tests regressor validation logic using valid/invalid maps + */ + @Test + void validateRegressors() { + + String[] badValueVector = new String[] {"Dag", "Not_a_valid_value"}; + String[] goodValueVector = new String[] {"Dag", "D_Home_owner", "PovertyToNonPoverty"}; + String[] keyVector = new String[] {"REGRESSOR"}; + + MultiKeyCoefficientMap badMap = new MultiKeyCoefficientMap(keyVector, badValueVector); + for (String badValue : badValueVector) {badMap.putValue(badValue, 0);} + + MultiKeyCoefficientMap goodMap = new MultiKeyCoefficientMap(keyVector, goodValueVector); + for (String goodValue : goodValueVector) {goodMap.putValue(goodValue, 0);} + + assertThrows(RuntimeException.class, () -> Parameters.validateRegressors(badMap, "A map designed to contain invalid values")); + assertDoesNotThrow(() -> Parameters.validateRegressors(goodMap, "A map designed to contain valid values")); + + } +} \ No newline at end of file From fe4a1e7095a48ea6195c176fc8cf2a43a82b3679 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Fri, 16 Jan 2026 18:13:03 +0000 Subject: [PATCH 2/4] experimental safe read excel --- src/main/java/simpaths/data/Parameters.java | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/simpaths/data/Parameters.java b/src/main/java/simpaths/data/Parameters.java index 9651ed2c3..06a73e0fb 100644 --- a/src/main/java/simpaths/data/Parameters.java +++ b/src/main/java/simpaths/data/Parameters.java @@ -3257,6 +3257,26 @@ public static void validateRegressors(MultiKeyCoefficientMap map, String mapName } } + public static MultiKeyCoefficientMap safeReadExcel(String excelFileName, String sheetName, int keyColumns) { + + MultiKeyCoefficientMap map = ExcelAssistant.loadCoefficientMap(excelFileName, sheetName, keyColumns); + + validateRegressors(map, sheetName); + + return map; + + } + + public static MultiKeyCoefficientMap safeReadExcel(String excelFileName, String sheetName, int keyColumns, int valueColumns) { + + MultiKeyCoefficientMap map = ExcelAssistant.loadCoefficientMap(excelFileName, sheetName, keyColumns, valueColumns); + + validateRegressors(map, sheetName); + + return map; + + } + public static String getInputDirectory() { return INPUT_DIRECTORY; } From e7969dfe7d5270dcc1b20fbcbb426f602bf75e41 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Mon, 19 Jan 2026 15:13:20 +0000 Subject: [PATCH 3/4] added sheet name and BenefitUnit handling to safe excel read --- src/main/java/simpaths/data/Parameters.java | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/simpaths/data/Parameters.java b/src/main/java/simpaths/data/Parameters.java index 06a73e0fb..0110c4c8a 100644 --- a/src/main/java/simpaths/data/Parameters.java +++ b/src/main/java/simpaths/data/Parameters.java @@ -10,6 +10,7 @@ import org.apache.commons.io.FileUtils; import simpaths.data.startingpop.DataParser; import simpaths.model.AnnuityRates; +import simpaths.model.BenefitUnit; import simpaths.model.Person; import simpaths.model.enums.*; import org.apache.commons.collections4.keyvalue.MultiKey; @@ -3233,7 +3234,7 @@ public static void setEuromodOutputDirectory(String euromodOutputDirectory) { EUROMOD_TRAINING_DIRECTORY = EUROMOD_OUTPUT_DIRECTORY + "training" + File.separator; } - public static void validateRegressors(MultiKeyCoefficientMap map, String mapName) { + public static void validatePersonRegressors(MultiKeyCoefficientMap map, String excelFileName, String sheetName) { if (map == null) return; // Get the values read from the REGRESSOR column by ExcelAssistant (excludes 'Constant') @@ -3248,10 +3249,15 @@ public static void validateRegressors(MultiKeyCoefficientMap map, String mapName try { Person.DoublesVariables.valueOf(keyName); } catch (IllegalArgumentException e) { - // This fires if the string isn't in the Enum - throw new RuntimeException("Validation failed for " + mapName + - ": Regressor '" + keyName + "' not found in Person.DoublesVariables. " + - "Check for typos in Excel or missing Enums in Person.java."); + try { + BenefitUnit.Regressors.valueOf(keyName); + } catch (IllegalArgumentException e2) { + + // This fires if the string isn't in the Enum + throw new RuntimeException("Validation failed for " + excelFileName + " in " + sheetName + + ": Regressor '" + keyName + "' not found in Person.DoublesVariables. " + + "Check for typos in Excel or missing Enums in Person.java."); + } } } } @@ -3261,7 +3267,7 @@ public static MultiKeyCoefficientMap safeReadExcel(String excelFileName, String MultiKeyCoefficientMap map = ExcelAssistant.loadCoefficientMap(excelFileName, sheetName, keyColumns); - validateRegressors(map, sheetName); + validatePersonRegressors(map, excelFileName, sheetName); return map; @@ -3271,7 +3277,7 @@ public static MultiKeyCoefficientMap safeReadExcel(String excelFileName, String MultiKeyCoefficientMap map = ExcelAssistant.loadCoefficientMap(excelFileName, sheetName, keyColumns, valueColumns); - validateRegressors(map, sheetName); + validatePersonRegressors(map, excelFileName, sheetName); return map; From c624bc214470044808a5ce6074156587125d6b0c Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Tue, 27 Jan 2026 11:10:17 +0000 Subject: [PATCH 4/4] fixing validator naming and testing --- src/main/java/simpaths/data/Parameters.java | 8 ++++---- src/test/java/simpaths/data/ParametersTest.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/simpaths/data/Parameters.java b/src/main/java/simpaths/data/Parameters.java index 0110c4c8a..1b1d18075 100644 --- a/src/main/java/simpaths/data/Parameters.java +++ b/src/main/java/simpaths/data/Parameters.java @@ -1140,7 +1140,7 @@ public static void loadParameters(Country country, int maxAgeModel, boolean enab coeffCovarianceHM2CaseMales = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_health_mental.xlsx", countryString + "_HM2_Males_C", 1); coeffCovarianceHM2CaseFemales = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_health_mental.xlsx", countryString + "_HM2_Females_C", 1); - validateRegressors(coeffCovarianceHM2CaseMales, "HM2_Males_C"); + validateRegressors(coeffCovarianceHM2CaseMales, "reg_health_mental.xlsx", "HM2_Males_C"); //Health coeffCovarianceDHE_MCS1 = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_health_wellbeing.xlsx", countryString + "_DHE_MCS1", 1); @@ -3234,7 +3234,7 @@ public static void setEuromodOutputDirectory(String euromodOutputDirectory) { EUROMOD_TRAINING_DIRECTORY = EUROMOD_OUTPUT_DIRECTORY + "training" + File.separator; } - public static void validatePersonRegressors(MultiKeyCoefficientMap map, String excelFileName, String sheetName) { + public static void validateRegressors(MultiKeyCoefficientMap map, String excelFileName, String sheetName) { if (map == null) return; // Get the values read from the REGRESSOR column by ExcelAssistant (excludes 'Constant') @@ -3267,7 +3267,7 @@ public static MultiKeyCoefficientMap safeReadExcel(String excelFileName, String MultiKeyCoefficientMap map = ExcelAssistant.loadCoefficientMap(excelFileName, sheetName, keyColumns); - validatePersonRegressors(map, excelFileName, sheetName); + validateRegressors(map, excelFileName, sheetName); return map; @@ -3277,7 +3277,7 @@ public static MultiKeyCoefficientMap safeReadExcel(String excelFileName, String MultiKeyCoefficientMap map = ExcelAssistant.loadCoefficientMap(excelFileName, sheetName, keyColumns, valueColumns); - validatePersonRegressors(map, excelFileName, sheetName); + validateRegressors(map, excelFileName, sheetName); return map; diff --git a/src/test/java/simpaths/data/ParametersTest.java b/src/test/java/simpaths/data/ParametersTest.java index 83cc2d2ae..b0ead97d4 100644 --- a/src/test/java/simpaths/data/ParametersTest.java +++ b/src/test/java/simpaths/data/ParametersTest.java @@ -25,8 +25,8 @@ void validateRegressors() { MultiKeyCoefficientMap goodMap = new MultiKeyCoefficientMap(keyVector, goodValueVector); for (String goodValue : goodValueVector) {goodMap.putValue(goodValue, 0);} - assertThrows(RuntimeException.class, () -> Parameters.validateRegressors(badMap, "A map designed to contain invalid values")); - assertDoesNotThrow(() -> Parameters.validateRegressors(goodMap, "A map designed to contain valid values")); + assertThrows(RuntimeException.class, () -> Parameters.validateRegressors(badMap, "A map designed to contain invalid values", "Sheet1")); + assertDoesNotThrow(() -> Parameters.validateRegressors(goodMap, "A map designed to contain valid values", "Sheet1")); } } \ No newline at end of file