Skip to content

Commit f3f4b63

Browse files
labkey-adamlabkey-adam
authored
Improvements after first dry run attempt (#7140)
1 parent 0b8d3cc commit f3f4b63

4 files changed

Lines changed: 175 additions & 91 deletions

File tree

‎api/src/org/labkey/api/data/DatabaseMigrationConfiguration.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
public interface DatabaseMigrationConfiguration
1212
{
1313
boolean shouldInsertData();
14+
default void beforeMigration(){};
1415
DbScope getSourceScope();
1516
DbScope getTargetScope();
1617
@NotNull Set<String> getSkipSchemas();

‎api/src/org/labkey/api/data/DatabaseMigrationService.java‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public interface DatabaseMigrationService
3030
{
3131
Logger LOG = LogHelper.getLogger(DatabaseMigrationService.class, "Information about database migration");
3232

33-
record DomainFilter(String column, FilterClause condition) {}
33+
record DomainFilter(Set<GUID> containers, String column, FilterClause condition) {}
3434

3535
static @NotNull DatabaseMigrationService get()
3636
{
@@ -87,7 +87,7 @@ interface MigrationSchemaHandler
8787

8888
@Nullable FieldKey getContainerFieldKey(TableInfo sourceTable);
8989

90-
void addDomainDataFilter(OrClause orClause, DomainFilter filter, TableInfo sourceTable, FieldKey fKey, GUID guid, Set<String> selectColumnNames);
90+
void addDomainDataFilter(OrClause orClause, DomainFilter filter, TableInfo sourceTable, FieldKey fKey, Set<String> selectColumnNames);
9191

9292
// Do any necessary clean up after the target table has been populated. notCopiedFilter selects all rows in the
9393
// source table that were NOT copied to the target table. (For example, they were filtered out due to container
@@ -191,27 +191,27 @@ public FilterClause getContainerClause(TableInfo sourceTable, FieldKey container
191191
}
192192

193193
@Override
194-
public void addDomainDataFilter(OrClause orClause, DomainFilter filter, TableInfo sourceTable, FieldKey fKey, GUID guid, Set<String> selectColumnNames)
194+
public void addDomainDataFilter(OrClause orClause, DomainFilter filter, TableInfo sourceTable, FieldKey fKey, Set<String> selectColumnNames)
195195
{
196-
addDomainDataStandardFilter(orClause, filter, sourceTable, fKey, guid, selectColumnNames);
196+
addDomainDataStandardFilter(orClause, filter, sourceTable, fKey, selectColumnNames);
197197
}
198198

199-
private void addDomainDataStandardFilter(OrClause orClause, DomainFilter filter, TableInfo sourceTable, FieldKey fKey, GUID guid, Set<String> selectColumnNames)
199+
protected void addDomainDataStandardFilter(OrClause orClause, DomainFilter filter, TableInfo sourceTable, FieldKey fKey, Set<String> selectColumnNames)
200200
{
201201
if (selectColumnNames.contains(filter.column()))
202202
{
203203
// Select all rows in this domain-filtered container that meet its criteria
204204
orClause.addClause(
205205
new AndClause(
206-
getContainerClause(sourceTable, fKey, Set.of(guid)),
206+
getContainerClause(sourceTable, fKey, filter.containers()),
207207
filter.condition()
208208
)
209209
);
210210
}
211211
}
212212

213213
// Special domain data filter method for provisioned tables that have a built-in Flag field
214-
protected void addDomainDataFlagFilter(OrClause orClause, DomainFilter filter, TableInfo sourceTable, FieldKey fKey, GUID guid, Set<String> selectColumnNames)
214+
protected void addDomainDataFlagFilter(OrClause orClause, DomainFilter filter, TableInfo sourceTable, FieldKey fKey, Set<String> selectColumnNames)
215215
{
216216
if (filter.column().equalsIgnoreCase("Flag"))
217217
{
@@ -220,20 +220,20 @@ protected void addDomainDataFlagFilter(OrClause orClause, DomainFilter filter, T
220220
// Select all rows where the built-in flag column equals the filter value
221221
orClause.addClause(
222222
new AndClause(
223-
getContainerClause(sourceTable, fKey, Set.of(guid)),
223+
getContainerClause(sourceTable, fKey, filter.containers()),
224224
new SimpleFilter.SQLClause(flagWhere)
225225
)
226226
);
227227
}
228228
else
229229
{
230-
addDomainDataStandardFilter(orClause, filter, sourceTable, fKey, guid, selectColumnNames);
230+
addDomainDataStandardFilter(orClause, filter, sourceTable, fKey, selectColumnNames);
231231
}
232232
}
233233

234234
private Integer _commentPropertyId = null;
235235

236-
private synchronized int getCommentPropertyId(TableInfo sourceTable)
236+
protected synchronized int getCommentPropertyId(TableInfo sourceTable)
237237
{
238238
if (_commentPropertyId == null)
239239
{
@@ -297,6 +297,7 @@ public TableInfo getTableInfo()
297297
interface MigrationFilter
298298
{
299299
String getName();
300-
void saveFilter(GUID guid, String value);
300+
// Implementations should validate guid nullity
301+
void saveFilter(@Nullable GUID guid, String value);
301302
}
302303
}

‎core/src/org/labkey/core/CoreModule.java‎

Lines changed: 135 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import org.labkey.api.data.DataRegion;
6262
import org.labkey.api.data.DatabaseMigrationService;
6363
import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationSchemaHandler;
64+
import org.labkey.api.data.DatabaseMigrationService.MigrationFilter;
6465
import org.labkey.api.data.DbSchema;
6566
import org.labkey.api.data.DbSchemaType;
6667
import org.labkey.api.data.DbScope;
@@ -71,8 +72,10 @@
7172
import org.labkey.api.data.PropertySchema;
7273
import org.labkey.api.data.SQLFragment;
7374
import org.labkey.api.data.SchemaTableInfoFactory;
75+
import org.labkey.api.data.SimpleFilter.AndClause;
7476
import org.labkey.api.data.SimpleFilter.FilterClause;
7577
import org.labkey.api.data.SimpleFilter.OrClause;
78+
import org.labkey.api.data.SimpleFilter.SQLClause;
7679
import org.labkey.api.data.SqlExecutor;
7780
import org.labkey.api.data.SqlSelector;
7881
import org.labkey.api.data.TSVWriter;
@@ -175,6 +178,7 @@
175178
import org.labkey.api.thumbnail.ThumbnailService;
176179
import org.labkey.api.usageMetrics.SimpleMetricsService;
177180
import org.labkey.api.usageMetrics.UsageMetricsService;
181+
import org.labkey.api.util.ConfigurationException;
178182
import org.labkey.api.util.ContextListener;
179183
import org.labkey.api.util.ExceptionUtil;
180184
import org.labkey.api.util.FileUtil;
@@ -409,6 +413,30 @@ public class CoreModule extends SpringModule implements SearchService.DocumentPr
409413
private CoreWarningProvider _warningProvider;
410414
private ServletRegistration.Dynamic _webdavServletDynamic;
411415

416+
private SQLFragment _groupFilterCondition = null;
417+
418+
public CoreModule()
419+
{
420+
// Must be registered very early
421+
DatabaseMigrationService.get().registerMigrationFilter(new MigrationFilter()
422+
{
423+
@Override
424+
public String getName()
425+
{
426+
return "GroupFilter";
427+
}
428+
429+
@Override
430+
public void saveFilter(@Nullable GUID guid, String groupFilter)
431+
{
432+
if (guid != null)
433+
throw new ConfigurationException("GUID should not be provided to GroupFilter");
434+
435+
_groupFilterCondition = new SQLFragment(groupFilter);
436+
}
437+
});
438+
}
439+
412440
@Override
413441
public boolean hasScripts()
414442
{
@@ -1279,76 +1307,7 @@ public void moduleStartupComplete(ServletContext servletContext)
12791307
ContainerManager.addContainerListener(new EmailPreferenceContainerListener());
12801308
UserManager.addUserListener(new EmailPreferenceUserListener());
12811309

1282-
DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(CoreSchema.getInstance().getSchema())
1283-
{
1284-
@Override
1285-
public void beforeVerification()
1286-
{
1287-
super.beforeVerification();
1288-
1289-
// Delete root and shared containers that were needed for bootstrapping
1290-
TableInfo containers = CoreSchema.getInstance().getTableInfoContainers();
1291-
Table.delete(containers);
1292-
DbScope targetScope = DbScope.getLabKeyScope();
1293-
new SqlExecutor(targetScope).execute("ALTER SEQUENCE core.containers_rowid_seq RESTART"); // Reset Containers sequence
1294-
}
1295-
1296-
@Override
1297-
public void beforeSchema()
1298-
{
1299-
new SqlExecutor(getSchema()).execute("ALTER TABLE core.Containers DROP CONSTRAINT FK_Containers_Containers");
1300-
new SqlExecutor(getSchema()).execute("ALTER TABLE core.ViewCategory DROP CONSTRAINT FK_ViewCategory_Parent");
1301-
}
1302-
1303-
@Override
1304-
public List<TableInfo> getTablesToCopy()
1305-
{
1306-
List<TableInfo> tablesToCopy = super.getTablesToCopy();
1307-
tablesToCopy.remove(CoreSchema.getInstance().getTableInfoModules());
1308-
tablesToCopy.remove(CoreSchema.getInstance().getTableInfoSqlScripts());
1309-
tablesToCopy.remove(CoreSchema.getInstance().getTableInfoUpgradeSteps());
1310-
1311-
return tablesToCopy;
1312-
}
1313-
1314-
@Override
1315-
public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable)
1316-
{
1317-
return switch (sourceTable.getName())
1318-
{
1319-
case "ContainerAliases" -> FieldKey.fromParts("ContainerRowId", "EntityId");
1320-
case "Containers" -> FieldKey.fromParts("EntityId");
1321-
case "Report" -> FieldKey.fromParts("ContainerId");
1322-
// Note: DataStates is not really site-wide, but there seem to be exp.Materials referencing DataStates with conflicting containers
1323-
case "APIKeys", "AuthenticationConfigurations", "DataStates", "EmailOptions", "Logins", "ReportEngines", "ShortURL", "UsersData" -> SITE_WIDE_TABLE;
1324-
default -> super.getContainerFieldKey(sourceTable);
1325-
};
1326-
}
1327-
1328-
@Override
1329-
public FilterClause getContainerClause(TableInfo sourceTable, FieldKey containerFieldKey, Set<GUID> containers)
1330-
{
1331-
FilterClause containerClause = super.getContainerClause(sourceTable, containerFieldKey, containers);
1332-
1333-
// Users and root groups have container == null, so add that as an OR clause
1334-
if (sourceTable.getName().equals("Principals") || sourceTable.getName().equals("Members"))
1335-
{
1336-
OrClause orClause = new OrClause();
1337-
orClause.addClause(containerClause);
1338-
orClause.addClause(new CompareClause(containerFieldKey, CompareType.ISBLANK, null));
1339-
containerClause = orClause;
1340-
}
1341-
1342-
return containerClause;
1343-
}
1344-
1345-
@Override
1346-
public void afterSchema()
1347-
{
1348-
new SqlExecutor(getSchema()).execute("ALTER TABLE core.Containers ADD CONSTRAINT FK_Containers_Containers FOREIGN KEY (Parent) REFERENCES core.Containers(EntityId)");
1349-
new SqlExecutor(getSchema()).execute("ALTER TABLE core.ViewCategory ADD CONSTRAINT FK_ViewCategory_Parent FOREIGN KEY (Parent) REFERENCES core.ViewCategory(RowId)");
1350-
}
1351-
});
1310+
DatabaseMigrationService.get().registerSchemaHandler(new CoreMigrationSchemaHandler());
13521311

13531312
DatabaseMigrationService.get().registerSchemaHandler(new DefaultMigrationSchemaHandler(PropertySchema.getInstance().getSchema()){
13541313
@Override
@@ -1382,6 +1341,112 @@ public List<TableInfo> getTablesToCopy()
13821341
Encryption.checkMigration();
13831342
}
13841343

1344+
private class CoreMigrationSchemaHandler extends DefaultMigrationSchemaHandler
1345+
{
1346+
public CoreMigrationSchemaHandler()
1347+
{
1348+
super(CoreSchema.getInstance().getSchema());
1349+
}
1350+
1351+
@Override
1352+
public void beforeVerification()
1353+
{
1354+
super.beforeVerification();
1355+
1356+
// Delete root and shared containers that were needed for bootstrapping
1357+
TableInfo containers = CoreSchema.getInstance().getTableInfoContainers();
1358+
Table.delete(containers);
1359+
DbScope targetScope = DbScope.getLabKeyScope();
1360+
new SqlExecutor(targetScope).execute("ALTER SEQUENCE core.containers_rowid_seq RESTART"); // Reset Containers sequence
1361+
}
1362+
1363+
@Override
1364+
public void beforeSchema()
1365+
{
1366+
new SqlExecutor(getSchema()).execute("ALTER TABLE core.Containers DROP CONSTRAINT FK_Containers_Containers");
1367+
new SqlExecutor(getSchema()).execute("ALTER TABLE core.ViewCategory DROP CONSTRAINT FK_ViewCategory_Parent");
1368+
}
1369+
1370+
@Override
1371+
public List<TableInfo> getTablesToCopy()
1372+
{
1373+
List<TableInfo> tablesToCopy = super.getTablesToCopy();
1374+
tablesToCopy.remove(CoreSchema.getInstance().getTableInfoModules());
1375+
tablesToCopy.remove(CoreSchema.getInstance().getTableInfoSqlScripts());
1376+
tablesToCopy.remove(CoreSchema.getInstance().getTableInfoUpgradeSteps());
1377+
1378+
return tablesToCopy;
1379+
}
1380+
1381+
@Override
1382+
public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable)
1383+
{
1384+
return switch (sourceTable.getName())
1385+
{
1386+
case "ContainerAliases" -> FieldKey.fromParts("ContainerRowId", "EntityId");
1387+
case "Containers" -> FieldKey.fromParts("EntityId");
1388+
case "Report" -> FieldKey.fromParts("ContainerId");
1389+
// Note: DataStates is not really site-wide, but there seem to be exp.Materials referencing DataStates with conflicting containers
1390+
case "APIKeys", "AuthenticationConfigurations", "DataStates", "EmailOptions", "Logins", "ReportEngines", "ShortURL", "UsersData" -> SITE_WIDE_TABLE;
1391+
default -> super.getContainerFieldKey(sourceTable);
1392+
};
1393+
}
1394+
1395+
@Override
1396+
public FilterClause getContainerClause(TableInfo sourceTable, FieldKey containerFieldKey, Set<GUID> containers)
1397+
{
1398+
FilterClause containerClause = super.getContainerClause(sourceTable, containerFieldKey, containers);
1399+
String tableName = sourceTable.getName();
1400+
1401+
// Users and root groups have container == null, so add that as an OR clause
1402+
if ("Principals".equals(tableName) || "Members".equals(tableName))
1403+
{
1404+
OrClause orClause = new OrClause();
1405+
orClause.addClause(containerClause);
1406+
orClause.addClause(new CompareClause(containerFieldKey, CompareType.ISBLANK, null));
1407+
containerClause = orClause;
1408+
1409+
if (_groupFilterCondition != null)
1410+
{
1411+
SQLFragment groupFilterFragment = new SQLFragment();
1412+
1413+
if ("Principals".equals(tableName))
1414+
{
1415+
groupFilterFragment
1416+
.append("Type <> 'g' OR (type = 'g' AND UserId ")
1417+
.append(_groupFilterCondition)
1418+
.append(")");
1419+
}
1420+
else
1421+
{
1422+
groupFilterFragment
1423+
.append("GroupId ")
1424+
.append(_groupFilterCondition);
1425+
}
1426+
1427+
containerClause = new AndClause(containerClause, new SQLClause(groupFilterFragment));
1428+
}
1429+
}
1430+
1431+
if ("RoleAssignments".equals(tableName) && _groupFilterCondition != null)
1432+
{
1433+
SQLFragment groupFilterFragment = new SQLFragment("UserId IN (SELECT UserId FROM core.Principals WHERE Type <> 'g' OR (type = 'g' AND UserId ")
1434+
.append(_groupFilterCondition)
1435+
.append("))");
1436+
containerClause = new AndClause(containerClause, new SQLClause(groupFilterFragment));
1437+
}
1438+
1439+
return containerClause;
1440+
}
1441+
1442+
@Override
1443+
public void afterSchema()
1444+
{
1445+
new SqlExecutor(getSchema()).execute("ALTER TABLE core.Containers ADD CONSTRAINT FK_Containers_Containers FOREIGN KEY (Parent) REFERENCES core.Containers(EntityId)");
1446+
new SqlExecutor(getSchema()).execute("ALTER TABLE core.ViewCategory ADD CONSTRAINT FK_ViewCategory_Parent FOREIGN KEY (Parent) REFERENCES core.ViewCategory(RowId)");
1447+
}
1448+
}
1449+
13851450
// Issue 7527: Auto-detect missing SQL views and attempt to recreate
13861451
private void checkForMissingDbViews()
13871452
{

0 commit comments

Comments
 (0)