Provide warnings for unknown fields for cross sample type import (#7370)

cnathe · web-flow · commit 9f6cca0ac543 · 2026-02-05T14:25:18.000-06:00
- update @labkey/components package version
- InferDomainAction update to take optional param for a list of columns to get distinct values for based on the data file contents
- test fixes for error message single quote to bold
diff --git a/api/src/org/labkey/api/data/ConvertHelper.java b/api/src/org/labkey/api/data/ConvertHelper.java
@@ -1342,7 +1342,7 @@ public static String getStandardConversionErrorMessage(Object value, String fiel
 
         // Issue 50768: Need a better error message if date value is not in the expected format.
         if (fieldType.equalsIgnoreCase("date") || fieldType.equalsIgnoreCase("datetime") || fieldType.equalsIgnoreCase("timestamp"))
-            return "'" + value + "' is not a valid " + fieldType + " for " + fieldName + " using " + LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getDateParsingMode().getDisplayString();
+            return "'" + value + "' is not a valid " + fieldType + " for '" + fieldName + "' using " + LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getDateParsingMode().getDisplayString();
 
         return "Could not convert value '" + value + "' (" + value.getClass().getSimpleName() + ") for " + fieldType + (null==fieldName ? "" : " field '" + fieldName + "'");
     }
diff --git a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java
@@ -172,7 +172,7 @@ else if (attachmentValue instanceof File file)
 
     private ValidationException propertyValidationException(DomainProperty property, Object value)
     {
-        return rowValidationException(String.format("Can't upload '%s' to field %s with type %s.", value, property.getName(), property.getType().getLabel()));
+        return rowValidationException(String.format("Cannot upload '%s' to %s type field '%s'.", value, property.getType().getLabel(), property.getName()));
     }
 
     private ValidationException rowValidationException(String message)
diff --git a/api/src/org/labkey/api/exp/api/SampleTypeService.java b/api/src/org/labkey/api/exp/api/SampleTypeService.java
@@ -55,9 +55,9 @@
 
 public interface SampleTypeService
 {
-    String MISSING_AMOUNT_ERROR_MESSAGE = "An Amount value must be provided when Units are provided.";
-    String MISSING_UNITS_ERROR_MESSAGE = "A Units value must be provided when Amounts are provided.";
-    String UNPROVIDED_VALUE_ERROR_MESSAGE_PATTERN = "No %s value provided for %s %s.";
+    String MISSING_AMOUNT_ERROR_MESSAGE = "An 'Amount' value must be provided when 'Units' are provided.";
+    String MISSING_UNITS_ERROR_MESSAGE = "A 'Units' value must be provided when 'Amounts' are provided.";
+    String UNPROVIDED_VALUE_ERROR_MESSAGE_PATTERN = "No '%s' value provided for %s '%s'.";
     String NEW_SAMPLE_TYPE_ALIAS_VALUE = "{{this_sample_set}}";
     String MATERIAL_INPUTS_PREFIX = "MaterialInputs/";
     String MODULE_NAME = "Experiment";
diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java
@@ -214,7 +214,7 @@ public Map<Integer, Map<String, Object>> getExistingRows(User user, Container co
                     if (StringUtils.isEmpty(dataContainer))
                         dataContainer = (String) row.get("folder");
                     if (!container.getId().equals(dataContainer))
-                        throw new InvalidKeyException("Data doesn't belong to folder '" + container.getName() + "': " + key.getValue().values());
+                        throw new InvalidKeyException("Data does not belong to folder '" + container.getName() + "': " + key.getValue().values());
                 }
             }
             else if (verifyExisting)
diff --git a/assay/package-lock.json b/assay/package-lock.json
diff --git a/assay/package.json b/assay/package.json
@@ -12,7 +12,7 @@
     "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen"
   },
   "dependencies": {
-    "@labkey/components": "7.15.0"
+    "@labkey/components": "7.16.0"
   },
   "devDependencies": {
     "@labkey/build": "8.8.0",
diff --git a/core/package-lock.json b/core/package-lock.json
diff --git a/core/package.json b/core/package.json
@@ -53,7 +53,7 @@
     }
   },
   "dependencies": {
-    "@labkey/components": "7.15.0",
+    "@labkey/components": "7.16.0",
     "@labkey/themes": "1.6.0"
   },
   "devDependencies": {
diff --git a/experiment/package-lock.json b/experiment/package-lock.json
diff --git a/experiment/package.json b/experiment/package.json
@@ -13,7 +13,7 @@
     "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js"
   },
   "dependencies": {
-    "@labkey/components": "7.15.0"
+    "@labkey/components": "7.16.0"
   },
   "devDependencies": {
     "@labkey/build": "8.8.0",
diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java
@@ -1461,7 +1461,7 @@ protected Map<String, Object> _update(User user, Container c, Map<String, Object
                     else if (value != null && !StringUtils.isEmpty(String.valueOf(value)))
                     {
                         // Issue 53498: string value for attachment field is not allowed
-                        throw new ValidationException("Can't upload '" + value + "' to field " + name + " with type Attachment.");
+                        throw new ValidationException("Cannot upload '" + value + "' to Attachment type field '" + name + "'.");
                     }
                     else
                         rowStripped.put(name, value); // if null or empty, remove attachment
diff --git a/experiment/src/org/labkey/experiment/controllers/property/PropertyController.java b/experiment/src/org/labkey/experiment/controllers/property/PropertyController.java
@@ -126,6 +126,7 @@
 import java.io.PrintWriter;
 import java.io.Writer;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -1215,17 +1216,21 @@ else if (fileMap.size() == 1)
                     throw new IllegalArgumentException("Unable to find a posted file or the file for the posted id/path.");
                 }
 
-                return getInferDomainResponse(loader, form.getNumLinesToInclude(), form.isCheckCommentLineCount(), form.getDomainKindName());
+                return getInferDomainResponse(loader, form);
             }
             finally
             {
                 IOUtils.closeQuietly(loader);
             }
         }
 
-        private ApiSimpleResponse getInferDomainResponse(DataLoader loader, Integer numLinesToInclude, boolean checkCommentLineCount, String domainKindName) throws IOException
+        private ApiSimpleResponse getInferDomainResponse(DataLoader loader, InferDomainForm form) throws IOException
         {
             ApiSimpleResponse response = new ApiSimpleResponse();
+            Integer numLinesToInclude = form.getNumLinesToInclude();
+            boolean checkCommentLineCount = form.isCheckCommentLineCount();
+            String domainKindName = form.getDomainKindName();
+            Collection<String> distinctValueColumns = form.getDistinctValueColumns();
 
             List<GWTPropertyDescriptor> fields = new ArrayList<>();
             List<GWTPropertyDescriptor> reservedFields = new ArrayList<>();
@@ -1258,12 +1263,45 @@ private ApiSimpleResponse getInferDomainResponse(DataLoader loader, Integer numL
                 {
                     response.put("commentLineCount", loader.getCommentLineCount());
                 }
+                if (distinctValueColumns != null && !distinctValueColumns.isEmpty())
+                {
+                    response.put("distinctValues", getDistinctValues(loader, distinctValueColumns));
+                }
             }
 
             response.put("fields", fields);
             response.put("reservedFields", reservedFields);
             return response;
         }
+
+        /**
+         * Iterate through the data and get the distinct values for the specified columns.
+         * Response is a map of column name to set of distinct string values, for those columns
+         * that have at least one non-null value.
+         */
+        private Map<String, Set<String>> getDistinctValues(DataLoader loader, Collection<String> colKeys) throws IOException
+        {
+            Map<String, Set<String>> distinctValuesMap = new HashMap<>();
+            try (var iter = loader.iterator())
+            {
+                while (iter.hasNext())
+                {
+                    Map<String, Object> values = iter.next();
+                    if (values == null)
+                        continue;
+
+                    for (String key : colKeys)
+                    {
+                        if (values.containsKey(key))
+                        {
+                            Object val = values.get(key);
+                            distinctValuesMap.computeIfAbsent(key, k -> new CaseInsensitiveHashSet()).add(val == null ? null : val.toString());
+                        }
+                    }
+                }
+            }
+            return distinctValuesMap;
+        }
     }
 
     public static class InferDomainForm
@@ -1274,6 +1312,7 @@ public static class InferDomainForm
         private String _domainKindName;
         private boolean _guessFormatAsTSV;
         private boolean _checkCommentLineCount;
+        private @Nullable Collection<String> _distinctValueColumns;
 
         public Integer getNumLinesToInclude()
         {
@@ -1324,6 +1363,16 @@ public void setCheckCommentLineCount(boolean checkCommentLineCount)
         {
             _checkCommentLineCount = checkCommentLineCount;
         }
+
+        public Collection<String> getDistinctValueColumns()
+        {
+            return _distinctValueColumns;
+        }
+
+        public void setDistinctValueColumns(Collection<String> distinctValueColumns)
+        {
+            _distinctValueColumns = distinctValueColumns;
+        }
     }
 
     /**
diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java
@@ -371,7 +371,7 @@ protected Map<String, Object> updateRow(User user, Container container, Map<Stri
                     }
                     else if (r.getValue() != null && !StringUtils.isEmpty(String.valueOf(r.getValue())))
                     {
-                        throw new ValidationException("Can't upload '" + r.getValue() + "' to field " + r.getKey() + " with type Attachment.");
+                        throw new ValidationException("Cannot upload '" + r.getValue() + "' to Attachment type field '" + r.getKey() + "'.");
                     }
                 }
             }
diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json
diff --git a/pipeline/package.json b/pipeline/package.json
@@ -14,7 +14,7 @@
     "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile"
   },
   "dependencies": {
-    "@labkey/components": "7.15.0"
+    "@labkey/components": "7.16.0"
   },
   "devDependencies": {
     "@labkey/build": "8.8.0",
diff --git a/study/test/src/org/labkey/test/tests/study/AssayTest.java b/study/test/src/org/labkey/test/tests/study/AssayTest.java
@@ -393,7 +393,7 @@ private void uploadRuns(String folder, String asUser)
         assertTextPresent(
                 "Could not convert value 'Bad Test' (String) for Double field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1) + "'.",
                 "Could not convert value 'Bad Test' (String) for Integer field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2) + "'.",
-                "'Bad Test' is not a valid Date for " + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3) + " using U.S. date parsing (MDY).");
+                "'Bad Test' is not a valid Date for '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3) + "' using U.S. date parsing (MDY).");
         setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1)), TEST_ASSAY_SET_PROPERTIES[1]);
         setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2)), TEST_ASSAY_SET_PROPERTIES[2]);
         setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3)), TEST_ASSAY_SET_PROPERTIES[3]);
diff --git a/study/test/src/org/labkey/test/tests/study/LinkAssayToStudyTest.java b/study/test/src/org/labkey/test/tests/study/LinkAssayToStudyTest.java
@@ -158,7 +158,7 @@ private void uploadRuns(String folder, String asUser)
         assertTextPresent(
                 "Could not convert value 'Bad Test' (String) for Double field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1) + "'.",
                 "Could not convert value 'Bad Test' (String) for Integer field '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2) + "'.",
-                "'Bad Test' is not a valid Date for " + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3) + " using U.S. date parsing (MDY).");
+                "'Bad Test' is not a valid Date for '" + TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3) + "' using U.S. date parsing (MDY).");
         setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 1)), TEST_ASSAY_SET_PROPERTIES[1]);
         setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 2)), TEST_ASSAY_SET_PROPERTIES[2]);
         setFormElement(Locator.name(TEST_ASSAY_SET_PROP_NAME + (TEST_ASSAY_SET_PREDEFINED_PROP_COUNT + 3)), TEST_ASSAY_SET_PROPERTIES[3]);

Original file line number	Diff line number	Diff line change
`@@ -1342,7 +1342,7 @@ public static String getStandardConversionErrorMessage(Object value, String fiel`
`1342`	`1342`
`1343`	`1343`	`// Issue 50768: Need a better error message if date value is not in the expected format.`
`1344`	`1344`	`if (fieldType.equalsIgnoreCase("date") \|\| fieldType.equalsIgnoreCase("datetime") \|\| fieldType.equalsIgnoreCase("timestamp"))`
`1345`		`- return "'" + value + "' is not a valid " + fieldType + " for " + fieldName + " using " + LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getDateParsingMode().getDisplayString();`
	`1345`	`+ return "'" + value + "' is not a valid " + fieldType + " for '" + fieldName + "' using " + LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getDateParsingMode().getDisplayString();`
`1346`	`1346`
`1347`	`1347`	`return "Could not convert value '" + value + "' (" + value.getClass().getSimpleName() + ") for " + fieldType + (null==fieldName ? "" : " field '" + fieldName + "'");`
`1348`	`1348`	`}`
Original file line number	Diff line number	Diff line change
`@@ -172,7 +172,7 @@ else if (attachmentValue instanceof File file)`
`172`	`172`
`173`	`173`	`private ValidationException propertyValidationException(DomainProperty property, Object value)`
`174`	`174`	`{`
`175`		`- return rowValidationException(String.format("Can't upload '%s' to field %s with type %s.", value, property.getName(), property.getType().getLabel()));`
	`175`	`+ return rowValidationException(String.format("Cannot upload '%s' to %s type field '%s'.", value, property.getType().getLabel(), property.getName()));`
`176`	`176`	`}`
`177`	`177`
`178`	`178`	`private ValidationException rowValidationException(String message)`
Original file line number	Diff line number	Diff line change
`@@ -214,7 +214,7 @@ public Map<Integer, Map<String, Object>> getExistingRows(User user, Container co`
`214`	`214`	`if (StringUtils.isEmpty(dataContainer))`
`215`	`215`	`dataContainer = (String) row.get("folder");`
`216`	`216`	`if (!container.getId().equals(dataContainer))`
`217`		`- throw new InvalidKeyException("Data doesn't belong to folder '" + container.getName() + "': " + key.getValue().values());`
	`217`	`+ throw new InvalidKeyException("Data does not belong to folder '" + container.getName() + "': " + key.getValue().values());`
`218`	`218`	`}`
`219`	`219`	`}`
`220`	`220`	`else if (verifyExisting)`
Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@`
`53`	`53`	`}`
`54`	`54`	`},`
`55`	`55`	`"dependencies": {`
`56`		`- "@labkey/components": "7.15.0",`
	`56`	`+ "@labkey/components": "7.16.0",`
`57`	`57`	`"@labkey/themes": "1.6.0"`
`58`	`58`	`},`
`59`	`59`	`"devDependencies": {`