Merge remote-tracking branch 'origin/develop' into fb_mvtc_convert

Author: XingY

api/src/org/labkey/api/exp/OntologyManager.java

@@ -1338,6 +1338,7 @@ public static void deleteAllObjects(Container c, User user) throws ValidationExc
             String deleteObjPropSql = "DELETE FROM " + getTinfoObjectProperty() + " WHERE  ObjectId IN (SELECT ObjectId FROM " + getTinfoObject() + " WHERE Container = ?)";
             executor.execute(deleteObjPropSql, c);
             String deleteObjSql = "DELETE FROM " + getTinfoObject() + " WHERE Container = ?";
+            _log.info("Deleting from exp.object in container {}", c);
             executor.execute(deleteObjSql, c);
 
             // delete property validator references on property descriptors

api/src/org/labkey/api/exp/api/ExperimentService.java

@@ -61,6 +61,7 @@
 import org.labkey.api.exp.query.ExpRunTable;
 import org.labkey.api.exp.query.ExpSampleTypeTable;
 import org.labkey.api.exp.query.ExpSchema;
+import org.labkey.api.exp.query.ExpUnreferencedSampleFilesTable;
 import org.labkey.api.exp.query.SampleStatusTable;
 import org.labkey.api.gwt.client.AuditBehaviorType;
 import org.labkey.api.gwt.client.model.GWTDomain;
@@ -661,6 +662,8 @@ static void validateParentAlias(Map<String, String> aliasMap, Set<String> reserv
 
     SampleStatusTable createSampleStatusTable(ExpSchema expSchema, ContainerFilter cf);
 
+    ExpUnreferencedSampleFilesTable createUnreferencedSampleFilesTable(ExpSchema expSchema, ContainerFilter cf);
+
     FilteredTable<ExpSchema> createFieldsTable(ExpSchema expSchema, ContainerFilter cf);
 
     FilteredTable<ExpSchema> createPhiFieldsTable(ExpSchema expSchema, ContainerFilter cf);

api/src/org/labkey/api/exp/query/ExpDataTable.java

@@ -36,6 +36,7 @@ enum Column
         SourceProtocolApplication,
         SourceApplicationInput,
         DataFileUrl,
+        ReferenceCount,
         Run,
         RunApplication,
         RunApplicationOutput,

api/src/org/labkey/api/exp/query/ExpMaterialTable.java

@@ -43,6 +43,7 @@ enum Column
         Inputs,
         IsAliquot,
         IsPlated,
+        LastIndexed,
         LSID,
         MaterialExpDate,
         MaterialSourceId,

api/src/org/labkey/api/exp/query/ExpSchema.java

@@ -48,6 +48,7 @@
 import org.labkey.api.security.User;
 import org.labkey.api.security.permissions.InsertPermission;
 import org.labkey.api.security.permissions.ReadPermission;
+import org.labkey.api.settings.AppProps;
 import org.labkey.api.util.StringExpression;
 import org.labkey.api.view.ActionURL;
 import org.labkey.api.view.ViewContext;
@@ -71,6 +72,7 @@ public class ExpSchema extends AbstractExpSchema
     public static final String SAMPLE_STATE_TYPE_TABLE = "SampleStateType";
     public static final String SAMPLE_TYPE_CATEGORY_TABLE = "SampleTypeCategoryType";
     public static final String MEASUREMENT_UNITS_TABLE = "MeasurementUnits";
+    public static final String SAMPLE_FILES_TABLE = "UnreferencedSampleFiles";
 
     public static final SchemaKey SCHEMA_EXP = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME);
     public static final SchemaKey SCHEMA_EXP_DATA = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.data.name());
@@ -221,6 +223,20 @@ public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFil
                 return expSchema.setupTable(result);
             }
         },
+        UnreferencedSampleFiles
+        {
+             @Override
+             public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf)
+             {
+                 return ExperimentService.get().createUnreferencedSampleFilesTable(expSchema, cf);
+             }
+
+            @Override
+            public boolean includeTable()
+            {
+                return AppProps.getInstance().isOptionalFeatureEnabled(SAMPLE_FILES_TABLE);
+            }
+        },
         SampleStatus
         {
             @Override

api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java

@@ -0,0 +1,8 @@
+package org.labkey.api.exp.query;
+
+import org.labkey.api.data.ContainerFilterable;
+import org.labkey.api.data.TableInfo;
+
+public interface ExpUnreferencedSampleFilesTable extends ContainerFilterable, TableInfo
+{
+}

api/src/org/labkey/api/files/FileContentService.java

@@ -326,6 +326,8 @@ default void fireFileDeletedEvent(@NotNull Path deleted, @Nullable User user, @N
      */
     SQLFragment listFilesQuery(@NotNull User currentUser);
 
+    SQLFragment listSampleFilesQuery(@NotNull User currentUser);
+
     void setWebfilesEnabled(boolean enabled, User user);
 
     /**
@@ -349,6 +351,14 @@ enum PathType { full, serverRelative, folderRelative }
      */
     void ensureFileData(@NotNull ExpDataTable table);
 
+    /**
+     * Fix the container column in the exp.data table for files that were moved as part of a sample move operation
+     * but did not have their containers updated
+     * @param admin The user doing the repair
+     * @return Number of duplicate rows removed from exp.data table
+     */
+    int fixContainerForExpDataFiles(User admin);
+
     /**
      * Allows a module to register a directory pattern to be checked in the files webpart in order to zip the matching directory before uploading.
      * @param directoryPattern DirectoryPattern

api/src/org/labkey/api/files/FileListener.java

@@ -94,4 +94,9 @@ default void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable C
      * </ul>
      */
     SQLFragment listFilesQuery();
+
+    @Nullable default SQLFragment listSampleFilesQuery()
+    {
+        return null;
+    }
 }

api/src/org/labkey/api/files/TableUpdaterFileListener.java

@@ -16,7 +16,6 @@
 package org.labkey.api.files;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -34,6 +33,7 @@
 import org.labkey.api.data.dialect.SqlDialect;
 import org.labkey.api.security.User;
 import org.labkey.api.util.FileUtil;
+import org.labkey.api.util.logging.LogHelper;
 
 import java.io.File;
 import java.nio.file.Files;
@@ -45,13 +45,13 @@
 
 /**
  * FileListener implementation that can update tables that store file paths in various flavors (URI, standard OS
- * paths, etc).
+ * paths, etc.).
  * User: jeckels
  * Date: 11/7/12
  */
 public class TableUpdaterFileListener implements FileListener
 {
-    private static final Logger LOG = LogManager.getLogger(TableUpdaterFileListener.class);
+    protected static final Logger LOG = LogHelper.getLogger(TableUpdaterFileListener.class, "File listener activity");
 
     public static final String TABLE_ALIAS = "x";
 
@@ -273,7 +273,7 @@ public int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user,
         singleEntrySQL.append(")");
 
         int rows = schema.getScope().executeWithRetry(tx -> new SqlExecutor(schema).execute(singleEntrySQL));
-        LOG.info("Updated " + rows + " row in " + _table + " for move from " + src + " to " + dest);
+        LOG.info("Updated {} row in {} for move from {} to {}", rows, _table, src, dest);
 
         // Handle updating child paths, unless we know that the entry is a file. If it's not (either it's a
         // directory or it doesn't exist), then try to fix up child records
@@ -305,7 +305,7 @@ public int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user,
             childPathsSQL.append(whereClause);
             childRowsUpdated += new SqlExecutor(schema).execute(childPathsSQL);
 
-            LOG.info("Updated " + childRowsUpdated + " child paths in " + _table + " rows for move from " + src + " to " + dest);
+            LOG.info("Updated {} child paths in {} rows for move from {} to {}", childRowsUpdated, _table, src, dest);
             return childRowsUpdated;
         }
         return 0;
@@ -353,10 +353,10 @@ public Collection<File> listFiles(@Nullable Container container)
     @Override
     public SQLFragment listFilesQuery()
     {
-        return listFilesQuery(false, null);
+        return listFilesQuery(false, null, false);
     }
 
-    public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath)
+    public SQLFragment listFilesQuery(boolean skipCreatedModified, CharSequence filePath, boolean extractName)
     {
         SQLFragment selectFrag = new SQLFragment();
         selectFrag.append("SELECT\n");
@@ -395,6 +395,16 @@ else if (_table.getColumn("Folder") != null)
 
         selectFrag.append("  ").appendIdentifier(_pathColumn.getSelectIdentifier()).append(" AS FilePath,\n");
 
+        if (extractName)
+        {
+            SqlDialect dialect = _table.getSchema().getSqlDialect();
+            SQLFragment fileNameFrag = new SQLFragment();
+            fileNameFrag.append("regexp_replace(").appendIdentifier(_pathColumn.getSelectIdentifier()).append(", ");
+            fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral(".*/")).append(", ");
+            fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral("")).append(")");
+            selectFrag.append("  ").append(fileNameFrag).append(" AS FilePathShort,\n");
+        }
+
         if (_keyColumn != null)
             selectFrag.append("  ").appendIdentifier(_keyColumn.getSelectIdentifier()).append(" AS SourceKey,\n");
         else
@@ -408,6 +418,8 @@ else if (_table.getColumn("Folder") != null)
 
         if (StringUtils.isEmpty(filePath))
             selectFrag.append(" IS NOT NULL\n");
+        else if (filePath instanceof SQLFragment)
+            selectFrag.append(" = ").append(filePath).append("\n");
         else
             selectFrag.append(" = ").appendStringLiteral(filePath, _table.getSchema().getSqlDialect()).append("\n");
 

api/webapp/clientapi/dom/DataRegion.js

@@ -493,6 +493,7 @@ if (!LABKEY.DataRegions) {
          * Non-configurable Options
          */
         this.selectionModified = false;
+        this.selectionLoading = false;
 
         if (this.panelConfigurations === undefined) {
             this.panelConfigurations = {};
@@ -1171,19 +1172,11 @@ if (!LABKEY.DataRegions) {
         config.selectionKey = this.selectionKey;
         config.scope = config.scope || me;
 
-        config = _chainSelectionCountCallback(this, config);
+        // set loading flag for DataRegion, will be cleared in callback
+        this.selectionLoading = true;
+        _updateSelectedCountMessage(this);
 
-        var failure = LABKEY.Utils.getOnFailure(config);
-        if ($.isFunction(failure)) {
-            config.failure = failure;
-        }
-        else {
-            config.failure = function(error) {
-                let msg = 'Error setting selection';
-                if (error && error.exception) msg += ': ' + error.exception;
-                me.addMessage(msg, 'selection');
-            };
-        }
+        config = _chainSelectionCountCallback(this, config);
 
         if (config.selectionKey) {
             LABKEY.DataRegion.setSelected(config);
@@ -3058,12 +3051,39 @@ if (!LABKEY.DataRegions) {
 
     var _chainSelectionCountCallback = function(region, config) {
 
-        var success = LABKEY.Utils.getOnSuccess(config);
+        const failure = LABKEY.Utils.getOnFailure(config);
+        config.failure = function(error) {
+            region.selectionLoading = false;
+
+            let msg = 'Error setting selection';
+            if (error && error.exception) msg += ': ' + error.exception;
+            config.scope.addMessage(msg, 'selection');
+
+            if ($.isFunction(failure)) {
+                failure.call(config.scope, error);
+            }
+        }
 
         // On success, update the current selectedCount on this DataRegion and fire the 'selectchange' event
-        config.success = function(data) {
+        const success = LABKEY.Utils.getOnSuccess(config);
+        config.success = function(data, response) {
+
+            // Workaround for GitHub Issue 778 where the response payload is JSON but the response has been
+            // configured by the server as non-JSON.
+            if (!data && response?.responseText) {
+                try {
+                    data = JSON.parse(response.responseText);
+                } catch (e) {
+                    const msg = 'failed to parse response';
+                    console.error(msg, e, response);
+                    config.failure.call(config.scope, { exception: msg });
+                    return;
+                }
+            }
+
             region.removeMessage('selection');
             region.selectionModified = true;
+            region.selectionLoading = false;
             region.selectedCount = data.count;
             _onSelectionChange(region);
 
@@ -3363,7 +3383,10 @@ if (!LABKEY.DataRegions) {
     var _buttonSelectionBind = function(region, cls, fn) {
         var partEl = region.msgbox.getParent().find('div[data-msgpart="selection"]');
         partEl.find('.labkey-button' + cls).off('click').on('click', $.proxy(function() {
-            fn.call(this);
+            // if one of the buttons is clicked while another action is loading selections, skip the click (the button should also be disabled)
+            if (!region.selectionLoading) {
+                fn.call(this);
+            }
         }, region));
     };
 
@@ -3592,18 +3615,20 @@ if (!LABKEY.DataRegions) {
 
     var _showSelectMessage = function(region, msg) {
         if (region.showRecordSelectors) {
+            const cls = 'labkey-button ' + (region.selectionLoading ? 'disabled ' : '');
+
             if (_isShowSelectAll(region)) {
-                msg += "&nbsp;<span class='labkey-button select-all'>" + _getSelectAllText(region) + "</span>";
+                msg += "&nbsp;<span class='" + cls + " select-all'>" + _getSelectAllText(region) + "</span>";
             }
 
-            msg += "&nbsp;" + "<span class='labkey-button select-none'>Select None</span>";
+            msg += "&nbsp;" + "<span class='" + cls + " select-none'>Select None</span>";
             var showOpts = [];
             if (region.showRows !== 'all' && !_isMaxRowsAllRows(region))
-                showOpts.push("<span class='labkey-button show-all'>Show All</span>");
+                showOpts.push("<span class='" + cls + " show-all'>Show All</span>");
             if (region.showRows !== 'selected')
-                showOpts.push("<span class='labkey-button show-selected'>Show Selected</span>");
+                showOpts.push("<span class='" + cls + " show-selected'>Show Selected</span>");
             if (region.showRows !== 'unselected')
-                showOpts.push("<span class='labkey-button show-unselected'>Show Unselected</span>");
+                showOpts.push("<span class='" + cls + " show-unselected'>Show Unselected</span>");
             msg += "&nbsp;&nbsp;" + showOpts.join(" ");
         }
 
@@ -4081,6 +4106,21 @@ if (!LABKEY.DataRegions) {
         _setParameters(region, params, [OFFSET_PREFIX].concat(skipPrefixes));
     };
 
+    var _updateSelectedCountMessage = function(region) {
+        // If not all rows are visible and some rows are selected, show selection message
+        if (region.totalRows && 0 !== region.selectedCount && !region.complete) {
+            var msg;
+            if (region.selectedCount === region.totalRows) {
+                msg = 'All <span class="labkey-strong">' + region.totalRows.toLocaleString() + '</span> rows selected.';
+            } else if (region.selectionLoading) {
+                msg = 'Selected <i class="fa fa-spinner fa-pulse" ></i> of ' + region.totalRows.toLocaleString() + ' rows.';
+            } else {
+                msg = 'Selected <span class="labkey-strong">' + region.selectedCount.toLocaleString() + '</span> of ' + region.totalRows.toLocaleString() + ' rows.';
+            }
+            _showSelectMessage(region, msg);
+        }
+    }
+
     var _updateRequiresSelectionButtons = function(region, selectedCount) {
 
         // update the 'select all on page' checkbox state
@@ -4100,13 +4140,7 @@ if (!LABKEY.DataRegions) {
             }
         });
 
-        // If not all rows are visible and some rows are selected, show selection message
-        if (region.totalRows && 0 !== region.selectedCount && !region.complete) {
-            var msg = (region.selectedCount === region.totalRows) ?
-                        'All <span class="labkey-strong">' + region.totalRows.toLocaleString() + '</span> rows selected.' :
-                        'Selected <span class="labkey-strong">' + region.selectedCount.toLocaleString() + '</span> of ' + region.totalRows.toLocaleString() + ' rows.';
-            _showSelectMessage(region, msg);
-        }
+        _updateSelectedCountMessage(region);
 
         // Issue 10566: for javascript perf on IE stash the requires selection buttons
         if (!region._requiresSelectionButtons) {
@@ -4446,6 +4480,11 @@ if (!LABKEY.DataRegions) {
     };
 
     LABKEY.DataRegion.selectAll = function(config) {
+        // GitHub Issue 778: Track selection loading state on DataRegion so we can disable buttons and prevent another click
+        var region = config.scope;
+        region.selectionLoading = true;
+        _updateSelectedCountMessage(region);
+
         var params = {};
         if (!config.url) {
             // DataRegion doesn't have selectAllURL so generate url and query parameters manually
@@ -4485,8 +4524,8 @@ if (!LABKEY.DataRegions) {
             url: config.url,
             method: 'POST',
             params: params,
-            success: LABKEY.Utils.getCallbackWrapper(LABKEY.Utils.getOnSuccess(config), config.scope),
-            failure: LABKEY.Utils.getCallbackWrapper(LABKEY.Utils.getOnFailure(config), config.scope, true)
+            success: LABKEY.Utils.getCallbackWrapper(LABKEY.Utils.getOnSuccess(config), region),
+            failure: LABKEY.Utils.getCallbackWrapper(LABKEY.Utils.getOnFailure(config), region, true)
         });
     };