From 76c72fbcca7e4bcae706a5ca1ffcf31a8b3dcfbc Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Thu, 13 Aug 2026 16:10:02 -0700 Subject: [PATCH 01/12] BDP-108403: Add entityType discriminator and isolate tables from views Tables and views share one (databaseId, objectId) pointer key space, so a name must resolve to exactly one catalog object. This adds a nullable entityType discriminator end-to-end and makes every table path aware of it. Semantics: NULL and any case spelling of TABLE mean table; any case spelling of VIEW means view; any other non-null value fails closed. The column is nullable with no backfill, so existing rows and existing table writes are untouched -- ordinary commits still write no discriminator. Read paths filter in the query, never by post-filtering a returned Page. A fetch-then-filter implementation returns short pages and inflated totals; the predicate and its countQuery are the same shared String constant, so content and count cannot diverge. Applied to both /hts query families, the internal catalog listings, listHouseTables, searchTables, and database enumeration. Write paths separate typed load from name occupancy. findById and findTableRefById answer "can this be loaded as a table?" and hide non-table rows; the new findOccupyingEntityTypeById answers "is this name taken, and by what?" without parsing metadata. CREATE and rename-destination consult occupancy before authorization, storage allocation, metadata writes, and pointer saves, so a collision is an accurate 409 rather than a misleading concurrency error. HTS errors propagate rather than reading as a free name. The drop guard lives in findTableRefById and OpenHouseInternalCatalog rather than doRefresh, because deleteTable deliberately bypasses loadTable so drops survive corrupted metadata; a doRefresh-only guard would be inert there. Wrong-type read and drop return 404; collisions return 409. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../catalog/OpenHouseInternalCatalog.java | 28 +- .../OpenHouseInternalTableOperations.java | 26 +- .../catalog/mapper/HouseTableMapper.java | 9 + .../catalog/mapper/HouseTableSerdeUtils.java | 18 + .../internal/catalog/model/HouseTable.java | 7 + .../repository/HouseTableRepository.java | 34 +- .../catalog/utils/MetadataLocationUtils.java | 31 ++ .../catalog/OpenHouseInternalCatalogTest.java | 156 ++++++++ .../OpenHouseInternalTableOperationsTest.java | 150 ++++++++ .../catalog/mapper/HouseTableMapperTest.java | 56 +++ .../catalog/model/HouseTableTest.java | 79 ++++ .../HouseTableRepositoryImplTest.java | 60 +++ .../utils/MetadataLocationUtilsTest.java | 87 +++++ .../api/validator/ValidatorConstants.java | 6 + .../housetables/api/spec/model/UserTable.java | 11 + .../OpenHouseUserTableHtsApiValidator.java | 11 +- .../housetables/dto/model/UserTableDto.java | 2 + .../housetables/model/UserTableRow.java | 3 + .../impl/jdbc/UserTableHtsJdbcRepository.java | 120 ++++-- .../services/UserTablesServiceImpl.java | 14 +- .../housetables/src/main/resources/schema.sql | 1 + .../e2e/usertable/HtsControllerTest.java | 343 +++++++++++++++++ .../e2e/usertable/HtsRepositoryTest.java | 345 +++++++++++++++++ .../e2e/usertable/UserTablesServiceTest.java | 195 ++++++++++ .../api/OpenHouseUserTablesValidatorTest.java | 95 +++++ .../mock/mapper/UserTablesMapperTest.java | 77 ++++ .../OpenHouseInternalRepository.java | 11 + .../impl/OpenHouseInternalRepositoryImpl.java | 47 ++- .../tables/services/TablesServiceImpl.java | 42 ++- .../e2e/h2/DatabasesControllerTest.java | 139 +++++++ .../tables/e2e/h2/RepositoryTest.java | 354 ++++++++++++++++++ .../tables/e2e/h2/TablesControllerTest.java | 304 +++++++++++++++ .../tables/e2e/h2/TablesServiceTest.java | 322 ++++++++++++++++ .../OpenHouseInternalRepositoryImplTest.java | 171 +++++++++ 34 files changed, 3298 insertions(+), 56 deletions(-) create mode 100644 iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtils.java create mode 100644 iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtilsTest.java diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java index 57c216b2b..095804f9e 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java @@ -13,6 +13,7 @@ import com.linkedin.openhouse.internal.catalog.cache.TableMetadataCache; import com.linkedin.openhouse.internal.catalog.fileio.FileIOManager; import com.linkedin.openhouse.internal.catalog.mapper.HouseTableMapper; +import com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils; import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTableDto; @@ -157,8 +158,11 @@ public Optional findHouseTable(TableIdentifier identifier) { public boolean dropTable(TableIdentifier identifier, boolean purge) { // Look up the HouseTable row directly instead of calling loadTable(), so drop works even when // the table's metadata.json is corrupted and cannot be parsed by TableMetadataParser. + // This path bypasses loadTable(), so the doRefresh guard is inert here and the discriminator + // must be checked explicitly — otherwise a view could be dropped and purged. HouseTable houseTable = findHouseTable(identifier) + .filter(row -> HouseTableSerdeUtils.isTableEntityType(row.getEntityType())) .orElseThrow(() -> new NoSuchTableException("Table does not exist: %s", identifier)); HouseTablePrimaryKey primaryKey = @@ -210,6 +214,24 @@ private static String getTableBaseLocation(HouseTable houseTable, TableIdentifie @Override public void renameTable(TableIdentifier from, TableIdentifier to) { + // Defense in depth for direct catalog callers; both checks run before loadTable(), so a + // rejection reads no metadata, opens no transaction and writes no pointer. A wrong-type source + // is "no such table"; an occupied destination of ANY type is a collision. + findHouseTable(from) + .filter(row -> HouseTableSerdeUtils.isTableEntityType(row.getEntityType())) + .orElseThrow(() -> new NoSuchTableException("Table does not exist: %s", from)); + + findHouseTable(to) + .ifPresent( + occupant -> { + throw new AlreadyExistsException( + "Table", + to.namespace().toString() + "." + to.name(), + String.format( + "Cannot rename %s to %s because that name is already occupied", from, to), + null); + }); + Table fromTable = loadTable(from); String tableClusterId = fromTable.properties().get(CatalogConstants.OPENHOUSE_CLUSTERID_KEY); @@ -314,8 +336,12 @@ protected FileIO resolveFileIO(TableIdentifier tableIdentifier) { tableIdentifier.namespace().toString(), tableIdentifier.name()); } + // A non-table pointer is invisible here for the same reason it is invisible to doRefresh, so + // storage resolution falls back to the selector exactly as for an absent row. StorageType.Type type = - houseTable.isPresent() + houseTable + .filter(row -> HouseTableSerdeUtils.isTableEntityType(row.getEntityType())) + .isPresent() ? storageType.fromString(houseTable.get().getStorageType()) : storageSelector .selectStorage(tableIdentifier.namespace().toString(), tableIdentifier.name()) diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java index b99915696..98771d313 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java @@ -17,12 +17,14 @@ import com.linkedin.openhouse.internal.catalog.exception.InvalidIcebergSnapshotException; import com.linkedin.openhouse.internal.catalog.fileio.FileIOManager; import com.linkedin.openhouse.internal.catalog.mapper.HouseTableMapper; +import com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils; import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableCallerException; import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableConcurrentUpdateException; import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableNotFoundException; +import com.linkedin.openhouse.internal.catalog.utils.MetadataLocationUtils; import com.linkedin.openhouse.internal.catalog.utils.MetadataUpdateUtils; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.Span; @@ -39,7 +41,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.AllArgsConstructor; @@ -123,6 +124,18 @@ protected void doRefresh() { tableIdentifier.name()); metricsReporter.count(InternalCatalogMetricsConstant.NO_TABLE_WHEN_REFRESH); } + // A non-table row must act absent and never reach TableMetadataParser: view metadata.json is + // not parseable as table metadata, and an unknown type must fail closed. + if (houseTable.isPresent() + && !HouseTableSerdeUtils.isTableEntityType(houseTable.get().getEntityType())) { + log.debug( + "Key {}.{} is occupied by a non-table entity of type {}; treating it as absent for the " + + "table path", + tableIdentifier.namespace().toString(), + tableIdentifier.name(), + houseTable.get().getEntityType()); + houseTable = Optional.empty(); + } if (!houseTable.isPresent() && currentMetadataLocation() != null) { throw new IllegalStateException( String.format( @@ -184,6 +197,9 @@ protected void refreshMetadata(final String metadataLoc) { * List Files and Manifest Files. Finally, the data sub-directory ./table_directory/data holds all * the Data Files. * + *

Naming itself lives in the metadata-type-neutral {@link MetadataLocationUtils} so the + * sibling view commit path shares it; only the codec resolution is table-specific here. + * * @param metadata {@link TableMetadata} for which the metadata file location needs to be derived. * @param newVersion new table version. * @return path to the root table metadata location. @@ -192,12 +208,8 @@ private static String rootMetadataFileLocation(TableMetadata metadata, int newVe String codecName = metadata.property( TableProperties.METADATA_COMPRESSION, TableProperties.METADATA_COMPRESSION_DEFAULT); - return String.format( - "%s/%s", - metadata.location(), - String.format( - "%05d-%s%s", - newVersion, UUID.randomUUID(), TableMetadataParser.getFileExtension(codecName))); + return MetadataLocationUtils.rootMetadataFileLocation( + metadata.location(), codecName, newVersion); } /** diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableMapper.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableMapper.java index 075538df2..5ddfea4a2 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableMapper.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableMapper.java @@ -62,7 +62,16 @@ private static boolean isHtsField(String key) { && HouseTableSerdeUtils.HTS_FIELD_NAMES.contains(stripOhNamespace(key)); } + /** + * MapStruct picks this up as an implicit {@code String -> String} conversion for every String + * property on the generated mappers, so it must tolerate null. {@code entityType} is the first + * genuinely nullable String on {@link HouseTable} (it stays null for legacy tables), which is + * what surfaced this; a null {@code tableVersion} would have hit it too. + */ static String stripOhNamespace(String key) { + if (key == null) { + return null; + } return IS_OH_PREFIXED.test(key) ? key.substring(OPENHOUSE_NAMESPACE.length()) : key; } } diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableSerdeUtils.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableSerdeUtils.java index 6a303f797..0155bae31 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableSerdeUtils.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableSerdeUtils.java @@ -35,6 +35,24 @@ private HouseTableSerdeUtils() { // no-op for util class constructor } + @VisibleForTesting public static final String ENTITY_TYPE_FIELD_NAME = "entityType"; + + public static final String TABLE_ENTITY_TYPE = "TABLE"; + + public static final String VIEW_ENTITY_TYPE = "VIEW"; + + /** + * {@code null} means table: the column is nullable and not backfilled. Any other unrecognized + * value is neither a table nor a view, so table operations fail closed on it. + */ + public static boolean isTableEntityType(String entityType) { + return entityType == null || TABLE_ENTITY_TYPE.equalsIgnoreCase(entityType); + } + + public static boolean isViewEntityType(String entityType) { + return VIEW_ENTITY_TYPE.equalsIgnoreCase(entityType); + } + @VisibleForTesting public static String getCanonicalFieldName(String htsField) { return OPENHOUSE_NAMESPACE + htsField; diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/model/HouseTable.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/model/HouseTable.java index dcc9acc80..70b55a92b 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/model/HouseTable.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/model/HouseTable.java @@ -51,4 +51,11 @@ public class HouseTable { * with this table. */ private String storageType; + + /** + * As a private non-static field this is picked up automatically by {@link + * com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils#HTS_FIELD_NAMES}, so it + * serializes as the {@code openhouse.entityType} table property. + */ + private String entityType; } diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepository.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepository.java index 065c9e681..babfbe74d 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepository.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepository.java @@ -5,7 +5,9 @@ import java.util.List; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; /** @@ -16,7 +18,18 @@ public interface HouseTableRepository extends PagingAndSortingRepository { - List findAllByDatabaseId(String databaseId); + /** + * Excludes views from table listings. The predicate lives in the query — never in a stream over a + * returned {@link Page} — so content and counts agree. {@code IS NULL} is mandatory because the + * discriminator is nullable with no backfill; {@code upper(...)} avoids depending on collation. + * + *

Spring Data proxies in services/tables and in the published tables-test-fixtures module + * inherit these predicates without an edit. + */ + String TABLE_ROW_PREDICATE = "(h.entityType IS NULL OR upper(h.entityType) = 'TABLE')"; + + @Query("SELECT h FROM HouseTable h WHERE h.databaseId = :databaseId AND " + TABLE_ROW_PREDICATE) + List findAllByDatabaseId(@Param("databaseId") String databaseId); /** * Delete a table by its primary key with purge option @@ -26,7 +39,24 @@ public interface HouseTableRepository */ void deleteById(HouseTablePrimaryKey houseTablePrimaryKey, boolean purge); - Page findAllByDatabaseId(String databaseId, Pageable pageable); + @Query( + value = + "SELECT h FROM HouseTable h WHERE h.databaseId = :databaseId AND " + TABLE_ROW_PREDICATE, + countQuery = + "SELECT COUNT(h) FROM HouseTable h WHERE h.databaseId = :databaseId AND " + + TABLE_ROW_PREDICATE) + Page findAllByDatabaseId(@Param("databaseId") String databaseId, Pageable pageable); + + /** Redeclared only to add the table-only predicate; cardinality and dedup are unchanged. */ + @Override + @Query("SELECT h FROM HouseTable h WHERE " + TABLE_ROW_PREDICATE) + Iterable findAll(); + + @Override + @Query( + value = "SELECT h FROM HouseTable h WHERE " + TABLE_ROW_PREDICATE, + countQuery = "SELECT COUNT(h) FROM HouseTable h WHERE " + TABLE_ROW_PREDICATE) + Page findAll(Pageable pageable); void rename( String fromDatabaseId, diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtils.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtils.java new file mode 100644 index 000000000..138d7fc30 --- /dev/null +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtils.java @@ -0,0 +1,31 @@ +package com.linkedin.openhouse.internal.catalog.utils; + +import java.util.UUID; +import org.apache.iceberg.TableMetadataParser; + +/** + * Shared naming for the root metadata file, used by both the table and view commit paths. + * + *

The codec is supplied by the caller rather than resolved here because Iceberg's table and view + * compression defaults differ ({@code none} vs {@code gzip}). + */ +public final class MetadataLocationUtils { + + private MetadataLocationUtils() { + // no-op for util class constructor + } + + /** + * The UUID lets concurrent writers at the same version stage metadata side by side; the + * zero-padded version keeps lexical ordering aligned with numeric ordering. + */ + public static String rootMetadataFileLocation( + String rootLocation, String codecName, int newVersion) { + return String.format( + "%s/%s", + rootLocation, + String.format( + "%05d-%s%s", + newVersion, UUID.randomUUID(), TableMetadataParser.getFileExtension(codecName))); + } +} diff --git a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalogTest.java b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalogTest.java index e9a8ae910..d42e11818 100644 --- a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalogTest.java +++ b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalogTest.java @@ -9,17 +9,22 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.withSettings; +import com.linkedin.openhouse.common.exception.AlreadyExistsException; import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableNotFoundException; import java.util.Optional; +import org.apache.iceberg.Table; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.SupportsPrefixOperations; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; public class OpenHouseInternalCatalogTest { @@ -161,4 +166,155 @@ boolean isValidBaseIdentifier(TableIdentifier identifier) { return isValidIdentifier(identifier); } } + + // --------------------------------------------------------------------------------------------- + // Table APIs must fail closed on non-table pointer rows + // --------------------------------------------------------------------------------------------- + + private static final String DEST_TABLE = "dest_table"; + private static final TableIdentifier DEST_IDENTIFIER = TableIdentifier.of(DB, DEST_TABLE); + + private static HouseTablePrimaryKey key(String tableId) { + return HouseTablePrimaryKey.builder().databaseId(DB).tableId(tableId).build(); + } + + private static HouseTable pointer(String tableId, String entityType) { + return HouseTable.builder() + .databaseId(DB) + .tableId(tableId) + .tableUUID("uuid") + .tableLocation("/data/openhouse/test_db/" + tableId + "-uuid/00001-aaa.metadata.json") + .entityType(entityType) + .build(); + } + + /** + * Records whether the expensive typed load / transaction path was reached. The guards under test + * must reject before any of it runs, so the recording overrides throw if invoked in a case where + * the test expects them not to be. + */ + private static class RecordingCatalog extends OpenHouseInternalCatalog { + private final FileIO fileIO; + boolean loadTableCalled = false; + + RecordingCatalog(FileIO fileIO) { + this.fileIO = fileIO; + } + + @Override + protected FileIO resolveFileIO(TableIdentifier identifier) { + return fileIO; + } + + @Override + public Table loadTable(TableIdentifier identifier) { + loadTableCalled = true; + throw new AssertionError( + "loadTable must not be reached for a rejected rename: " + identifier); + } + } + + /** + * A VIEW (any spelling) or unknown discriminator is not a table: drop must behave as "no such + * table" and must never delete the shared pointer row or purge the object's files. + */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + void dropTableRejectsNonTableValuesWithoutDeletingPointerOrFiles(String entityType) { + HouseTableRepository repo = mock(HouseTableRepository.class); + when(repo.findById(any(HouseTablePrimaryKey.class))) + .thenReturn(Optional.of(pointer(TABLE, entityType))); + FileIO fileIO = + mock(FileIO.class, withSettings().extraInterfaces(SupportsPrefixOperations.class)); + OpenHouseInternalCatalog catalog = new FixedFileIOCatalog(fileIO); + catalog.houseTableRepository = repo; + + Assertions.assertThrows(NoSuchTableException.class, () -> catalog.dropTable(IDENTIFIER, true)); + + verify(repo, never()).deleteById(any(), anyBoolean()); + verify((SupportsPrefixOperations) fileIO, never()).deletePrefix(any()); + } + + /** + * The complement of the guard above: null and every spelling of TABLE remain droppable. This is + * what proves the Java guard and the SQL predicate agree on {@code table} / {@code TaBlE} — a + * guard that only accepted the uppercase literal would make lower/mixed-case rows visible in + * listings yet undroppable. + */ + @ParameterizedTest + @CsvSource( + nullValues = "NULL", + value = {"NULL", "TABLE", "table", "TaBlE"}) + void dropTableAcceptsCaseVariantsOfTable(String entityType) { + HouseTableRepository repo = mock(HouseTableRepository.class); + when(repo.findById(any(HouseTablePrimaryKey.class))) + .thenReturn(Optional.of(pointer(TABLE, entityType))); + FileIO fileIO = + mock(FileIO.class, withSettings().extraInterfaces(SupportsPrefixOperations.class)); + OpenHouseInternalCatalog catalog = new FixedFileIOCatalog(fileIO); + catalog.houseTableRepository = repo; + + Assertions.assertTrue(catalog.dropTable(IDENTIFIER, false)); + + verify(repo).deleteById(any(HouseTablePrimaryKey.class), eq(false)); + verify((SupportsPrefixOperations) fileIO, never()).deletePrefix(any()); + } + + /** + * A wrong-type rename SOURCE is indistinguishable from "no such table" and must be rejected + * before the source table is loaded, before any transaction is opened, and before the pointer is + * renamed. + */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + void renameTableRejectsNonTableSourceBeforeLoadingMetadata(String entityType) { + HouseTableRepository repo = mock(HouseTableRepository.class); + when(repo.findById(key(TABLE))).thenReturn(Optional.of(pointer(TABLE, entityType))); + when(repo.findById(key(DEST_TABLE))).thenReturn(Optional.empty()); + FileIO fileIO = + mock(FileIO.class, withSettings().extraInterfaces(SupportsPrefixOperations.class)); + RecordingCatalog catalog = new RecordingCatalog(fileIO); + catalog.houseTableRepository = repo; + + Assertions.assertThrows( + NoSuchTableException.class, () -> catalog.renameTable(IDENTIFIER, DEST_IDENTIFIER)); + + Assertions.assertFalse(catalog.loadTableCalled, "Source table must not be loaded"); + verify(repo, never()).rename(any(), any(), any(), any(), any()); + verify(repo, never()).save(any()); + verify(repo, never()).deleteById(any(), anyBoolean()); + } + + /** + * Defense in depth for direct catalog callers: ANY occupied destination pointer — a table, a view + * in any spelling, or an unknown type — is a name collision, and it must be detected before the + * source is loaded or a transaction is opened. + * + *

Because the shared primary key would eventually reject the write anyway with the SAME + * exception type, the exception alone proves nothing. The load-bearing assertions are the + * never-verifications: correct code never loads the source, never opens a transaction, and never + * asks the repository to rename or save. + */ + @ParameterizedTest + @ValueSource(strings = {"TABLE", "VIEW", "view", "ViEw", "UNKNOWN"}) + void renameTableRejectsAnyOccupiedRawDestinationBeforeSourceLoad(String destinationEntityType) { + HouseTableRepository repo = mock(HouseTableRepository.class); + when(repo.findById(key(TABLE))).thenReturn(Optional.of(pointer(TABLE, null))); + when(repo.findById(key(DEST_TABLE))) + .thenReturn(Optional.of(pointer(DEST_TABLE, destinationEntityType))); + FileIO fileIO = + mock(FileIO.class, withSettings().extraInterfaces(SupportsPrefixOperations.class)); + RecordingCatalog catalog = new RecordingCatalog(fileIO); + catalog.houseTableRepository = repo; + + Assertions.assertThrows( + AlreadyExistsException.class, () -> catalog.renameTable(IDENTIFIER, DEST_IDENTIFIER)); + + Assertions.assertFalse( + catalog.loadTableCalled, "Destination occupancy must be checked before loading the source"); + verify(repo, never()).rename(any(), any(), any(), any(), any()); + verify(repo, never()).save(any()); + verify(repo, never()).deleteById(any(), anyBoolean()); + verify((SupportsPrefixOperations) fileIO, never()).deletePrefix(any()); + } } diff --git a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperationsTest.java b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperationsTest.java index 12092c160..b85d225cd 100644 --- a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperationsTest.java +++ b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperationsTest.java @@ -72,6 +72,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; @@ -2128,4 +2131,151 @@ void testRefreshMetadataMissingFileThrowsInvalidTableMetadataException() { InvalidTableMetadataException.class, () -> openHouseInternalTableOperations.refreshMetadata(nonExistentPath)); } + + // --------------------------------------------------------------------------------------------- + // Table point loading must fail closed on non-table pointer rows + // --------------------------------------------------------------------------------------------- + + private static final String TYPED_METADATA_LOCATION = "typed_metadata_location"; + + private static HouseTablePrimaryKey testTablePrimaryKey() { + return HouseTablePrimaryKey.builder() + .databaseId(TEST_TABLE_IDENTIFIER.namespace().toString()) + .tableId(TEST_TABLE_IDENTIFIER.name()) + .build(); + } + + private static HouseTable typedPointer(String entityType) { + return HouseTable.builder() + .databaseId(TEST_TABLE_IDENTIFIER.namespace().toString()) + .tableId(TEST_TABLE_IDENTIFIER.name()) + .tableLocation(TYPED_METADATA_LOCATION) + .entityType(entityType) + .build(); + } + + /** + * A shared-key row that is a VIEW (any spelling) or an unknown type is not a table. The table + * path must treat it as absent and must never hand its metadata location to {@link + * TableMetadataParser} — a view metadata.json is not parseable as table metadata, and parsing an + * unknown type would leak a foreign object into the table API. + */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + void doRefreshTreatsViewRowAsNoSuchTableWithoutOpeningMetadata(String entityType) { + when(mockHouseTableRepository.findById(testTablePrimaryKey())) + .thenReturn(Optional.of(typedPointer(entityType))); + + try (MockedStatic parserMock = + Mockito.mockStatic(TableMetadataParser.class, Mockito.CALLS_REAL_METHODS)) { + parserMock + .when( + () -> + TableMetadataParser.read( + Mockito.any(FileIO.class), Mockito.eq(TYPED_METADATA_LOCATION))) + .thenReturn(BASE_TABLE_METADATA); + + openHouseInternalTableOperations.refresh(); + + Assertions.assertNull( + openHouseInternalTableOperations.currentMetadataLocation(), + "A " + entityType + " pointer must not become the table's current metadata location"); + Assertions.assertNull( + openHouseInternalTableOperations.current(), + "A " + entityType + " pointer must not produce table metadata"); + + parserMock.verify( + () -> TableMetadataParser.read(Mockito.any(FileIO.class), Mockito.anyString()), never()); + } + } + + /** The complement: null and every spelling of TABLE still refresh normally. */ + @ParameterizedTest + @CsvSource( + nullValues = "NULL", + value = {"NULL", "TABLE", "table", "TaBlE"}) + void doRefreshAcceptsNullAndExplicitTableRows(String entityType) { + when(mockHouseTableRepository.findById(testTablePrimaryKey())) + .thenReturn(Optional.of(typedPointer(entityType))); + + try (MockedStatic parserMock = + Mockito.mockStatic(TableMetadataParser.class, Mockito.CALLS_REAL_METHODS)) { + parserMock + .when( + () -> + TableMetadataParser.read( + Mockito.any(FileIO.class), Mockito.eq(TYPED_METADATA_LOCATION))) + .thenReturn(BASE_TABLE_METADATA); + + openHouseInternalTableOperations.refresh(); + + Assertions.assertEquals( + TYPED_METADATA_LOCATION, + openHouseInternalTableOperations.currentMetadataLocation(), + "entityType=" + entityType + " must be treated as a table"); + Assertions.assertNotNull(openHouseInternalTableOperations.current()); + } + } + + /** + * Backward compatibility: ordinary table commits must NOT start stamping the discriminator. + * Adding {@code openhouse.entityType=TABLE} to every table commit would rewrite every table's + * metadata.json content and mask the "NULL means table" compatibility contract that all the + * legacy-row assertions depend on. + */ + @Test + void normalTableCommitDoesNotStampEntityType() { + AtomicReference savedHouseTable = new AtomicReference<>(); + when(mockHouseTableMapper.toHouseTable(Mockito.any(TableMetadata.class), Mockito.any())) + .thenAnswer( + invocation -> { + TableMetadata tableMetadata = invocation.getArgument(0); + // Mirror the real mapper: the pointer's entityType comes solely from the + // openhouse.entityType property, so a null here proves the commit did not stamp it. + HouseTable mapped = + HouseTable.builder() + .databaseId(TEST_TABLE_IDENTIFIER.namespace().toString()) + .tableId(TEST_TABLE_IDENTIFIER.name()) + .tableLocation( + tableMetadata.properties().get(getCanonicalFieldName("tableLocation"))) + .entityType( + tableMetadata.properties().get(getCanonicalFieldName("entityType"))) + .build(); + savedHouseTable.set(mapped); + return mapped; + }); + when(mockHouseTableRepository.save(Mockito.any(HouseTable.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + + Map properties = new HashMap<>(BASE_TABLE_METADATA.properties()); + properties.put(getCanonicalFieldName("tableLocation"), TEST_LOCATION); + TableMetadata metadata = BASE_TABLE_METADATA.replaceProperties(properties); + + try (MockedStatic parserMock = + Mockito.mockStatic(TableMetadataParser.class, Mockito.CALLS_REAL_METHODS)) { + parserMock + .when( + () -> + TableMetadataParser.write( + Mockito.any(TableMetadata.class), + Mockito.any(org.apache.iceberg.io.OutputFile.class))) + .thenAnswer(invocation -> null); + + openHouseInternalTableOperations.doCommit(BASE_TABLE_METADATA, metadata); + + Mockito.verify(mockHouseTableMapper).toHouseTable(tblMetadataCaptor.capture(), Mockito.any()); + Map committedProperties = tblMetadataCaptor.getValue().properties(); + Assertions.assertFalse( + committedProperties.containsKey(getCanonicalFieldName("entityType")), + "Ordinary table commits must not write openhouse.entityType, but properties were: " + + committedProperties); + + Assertions.assertNull( + savedHouseTable.get().getEntityType(), + "The saved pointer for an ordinary table commit must keep a null discriminator"); + // NOTE: the captured-properties assertion above is the load-bearing one. This second + // assertion runs through a hand-rolled mirror of the real mapper, so it is null largely + // because the captured properties are; it guards the mapping wiring, not the commit itself. + } + } } diff --git a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableMapperTest.java b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableMapperTest.java index 83bfc56e4..9c5e78b6b 100644 --- a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableMapperTest.java +++ b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableMapperTest.java @@ -7,6 +7,7 @@ import com.linkedin.openhouse.housetables.client.api.ToggleStatusApi; import com.linkedin.openhouse.housetables.client.api.UserTableApi; import com.linkedin.openhouse.housetables.client.invoker.ApiClient; +import com.linkedin.openhouse.housetables.client.model.UserTable; import com.linkedin.openhouse.internal.catalog.fileio.FileIOManager; import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; @@ -70,4 +71,59 @@ public void simpleMapperTest() { Assertions.assertEquals("table", houseTable.getTableId()); Assertions.assertEquals("local", houseTable.getStorageType()); } + + /** + * The discriminator must survive the generated HTS client model in both directions, otherwise the + * pointer would be written or read back without its type. + */ + @Test + public void houseTableGeneratedUserTableRoundTripPreservesEntityType() { + HouseTable viewHouseTable = + HouseTable.builder() + .databaseId("d1") + .tableId("v1") + .tableLocation("/base/d1/v1-uuid/00001-x.metadata.json") + .tableVersion("INITIAL_VERSION") + .entityType("VIEW") + .build(); + + UserTable viewUserTable = houseTableMapper.toUserTable(viewHouseTable); + Assertions.assertEquals("VIEW", viewUserTable.getEntityType()); + Assertions.assertEquals( + "/base/d1/v1-uuid/00001-x.metadata.json", viewUserTable.getMetadataLocation()); + + HouseTable viewBack = houseTableMapper.toHouseTable(viewUserTable); + Assertions.assertEquals("VIEW", viewBack.getEntityType()); + Assertions.assertEquals("/base/d1/v1-uuid/00001-x.metadata.json", viewBack.getTableLocation()); + + HouseTable tableHouseTable = + HouseTable.builder() + .databaseId("d1") + .tableId("t1") + .tableLocation("/base/d1/t1-uuid/00001-y.metadata.json") + .entityType("TABLE") + .build(); + UserTable tableUserTable = houseTableMapper.toUserTable(tableHouseTable); + Assertions.assertEquals("TABLE", tableUserTable.getEntityType()); + Assertions.assertEquals("TABLE", houseTableMapper.toHouseTable(tableUserTable).getEntityType()); + } + + /** Backward compatibility: an absent discriminator maps to null, never "TABLE". */ + @Test + public void missingEntityTypeMapsToLegacyTableNull() { + HadoopFileIO fileIO = new HadoopFileIO(new Configuration()); + LocalStorage localStorage = mock(LocalStorage.class); + when(fileIOManager.getStorage(fileIO)).thenReturn(localStorage); + when(localStorage.getType()).thenReturn(StorageType.LOCAL); + + HouseTable fromProperties = + houseTableMapper.toHouseTable(ImmutableMap.of("databaseId", "d1", "tableId", "t1"), fileIO); + Assertions.assertNull(fromProperties.getEntityType()); + + UserTable legacyUserTable = + houseTableMapper.toUserTable( + HouseTable.builder().databaseId("d1").tableId("t1").tableLocation("loc").build()); + Assertions.assertNull(legacyUserTable.getEntityType()); + Assertions.assertNull(houseTableMapper.toHouseTable(legacyUserTable).getEntityType()); + } } diff --git a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/model/HouseTableTest.java b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/model/HouseTableTest.java index 063bcea88..3eff0b985 100644 --- a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/model/HouseTableTest.java +++ b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/model/HouseTableTest.java @@ -10,6 +10,8 @@ import java.util.Set; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; public class HouseTableTest { @@ -60,4 +62,81 @@ public void testHouseTableDefaultValues() { Assertions.fail(e); } } + + /** + * {@code HTS_FIELD_NAMES} is derived reflectively from HouseTable's private fields, and {@code + * HouseTableMapper.extractRawHTSFields} only carries properties whose stripped key is in that + * set. So the discriminator is only serialized through Iceberg table properties if it is a real + * private field named exactly {@code entityType}, and its canonical property key must be {@code + * openhouse.entityType}. + */ + @Test + public void testEntityTypeDefaultAndSerdeRegistration() { + Assertions.assertNull( + HouseTable.builder().build().getEntityType(), + "entityType must default to null so ordinary table commits keep writing no discriminator"); + + Assertions.assertTrue( + HouseTableSerdeUtils.HTS_FIELD_NAMES.contains(HouseTableSerdeUtils.ENTITY_TYPE_FIELD_NAME), + "entityType must be reflected into HTS_FIELD_NAMES: " + + HouseTableSerdeUtils.HTS_FIELD_NAMES); + + Assertions.assertEquals("entityType", HouseTableSerdeUtils.ENTITY_TYPE_FIELD_NAME); + Assertions.assertEquals( + "openhouse.entityType", + HouseTableSerdeUtils.getCanonicalFieldName(HouseTableSerdeUtils.ENTITY_TYPE_FIELD_NAME)); + + Assertions.assertEquals("TABLE", HouseTableSerdeUtils.TABLE_ENTITY_TYPE); + Assertions.assertEquals("VIEW", HouseTableSerdeUtils.VIEW_ENTITY_TYPE); + } + + /** + * Authoritative case-sensitivity contract. H2 (MODE=MySQL) is case-sensitive while production + * MySQL default collation is not, so no SQL-level test can certify these semantics across + * providers. These Java guards are what every point read, drop, rename, and occupancy check + * actually consults, so they are pinned here independently of any database. + * + *

NULL and every spelling of TABLE classify as a table; every spelling of VIEW classifies as a + * view; anything else is neither, so table APIs fail closed rather than treating an unknown + * discriminator as a legacy table. + * + *

The empty-string row goes beyond the plan, which only named NULL/TABLE/VIEW/garbage. It is + * included deliberately because {@code entity_type} is a nullable {@code VARCHAR} that can hold + * {@code ''}, and "unknown non-null fails closed" must cover it. The natural implementation + * ({@code entityType == null || entityType.equalsIgnoreCase(TABLE)}) satisfies it for free — + * implementers must not special-case {@code ""} as blank/absent. + */ + @ParameterizedTest + @CsvSource( + nullValues = "NULL", + value = { + "NULL, true, false", + "TABLE, true, false", + "table, true, false", + "TaBlE, true, false", + "VIEW, false, true", + "view, false, true", + "ViEw, false, true", + "UNKNOWN, false, false", + "'', false, false" + }) + public void testEntityTypeClassification( + String entityType, boolean expectedTable, boolean expectedView) { + Assertions.assertEquals( + expectedTable, + HouseTableSerdeUtils.isTableEntityType(entityType), + "isTableEntityType(" + entityType + ")"); + Assertions.assertEquals( + expectedView, + HouseTableSerdeUtils.isViewEntityType(entityType), + "isViewEntityType(" + entityType + ")"); + + // The same classification must hold when read off a real pointer row. + HouseTable row = + HouseTable.builder().databaseId("d1").tableId("t1").entityType(entityType).build(); + Assertions.assertEquals( + expectedTable, HouseTableSerdeUtils.isTableEntityType(row.getEntityType())); + Assertions.assertEquals( + expectedView, HouseTableSerdeUtils.isViewEntityType(row.getEntityType())); + } } diff --git a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepositoryImplTest.java b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepositoryImplTest.java index a5c53d93a..627429318 100644 --- a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepositoryImplTest.java +++ b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepositoryImplTest.java @@ -754,4 +754,64 @@ public void testRestoreSoftDeletedTablesFailsWhenTableDoesNotExist() { htsRepo.restoreTable( HOUSE_TABLE.getDatabaseId(), HOUSE_TABLE.getTableId(), System.currentTimeMillis())); } + + /** + * This adapter is an HTTP client, not a query owner. It must faithfully carry the discriminator + * across the wire in both directions so the internal catalog's Java guards see the real stored + * value. It must NOT gain a local view filter of its own — filtering belongs in the HTS query so + * page counts stay correct. + */ + @Test + public void testRepoPointAndSavePreserveEntityType() { + HouseTable viewHouseTable = HOUSE_TABLE.toBuilder().entityType("VIEW").build(); + + EntityResponseBodyUserTable getResponse = new EntityResponseBodyUserTable(); + getResponse.entity(houseTableMapper.toUserTable(viewHouseTable)); + mockHtsServer.enqueue( + new MockResponse() + .setResponseCode(200) + .setBody((new Gson()).toJson(getResponse)) + .addHeader("Content-Type", "application/json")); + + HouseTable found = + htsRepo + .findById( + HouseTablePrimaryKey.builder() + .tableId(HOUSE_TABLE.getTableId()) + .databaseId(HOUSE_TABLE.getDatabaseId()) + .build()) + .get(); + Assertions.assertEquals("VIEW", found.getEntityType()); + Assertions.assertEquals(HOUSE_TABLE.getTableLocation(), found.getTableLocation()); + + EntityResponseBodyUserTable putResponse = new EntityResponseBodyUserTable(); + putResponse.entity(houseTableMapper.toUserTable(viewHouseTable)); + mockHtsServer.enqueue( + new MockResponse() + .setResponseCode(201) + .setBody((new Gson()).toJson(putResponse)) + .addHeader("Content-Type", "application/json")); + + HouseTable saved = htsRepo.save(viewHouseTable); + Assertions.assertEquals("VIEW", saved.getEntityType()); + Assertions.assertEquals(HOUSE_TABLE.getTableLocation(), saved.getTableLocation()); + + // A legacy table pointer with no discriminator still round trips as null. + EntityResponseBodyUserTable legacyResponse = new EntityResponseBodyUserTable(); + legacyResponse.entity(houseTableMapper.toUserTable(HOUSE_TABLE)); + mockHtsServer.enqueue( + new MockResponse() + .setResponseCode(200) + .setBody((new Gson()).toJson(legacyResponse)) + .addHeader("Content-Type", "application/json")); + Assertions.assertNull( + htsRepo + .findById( + HouseTablePrimaryKey.builder() + .tableId(HOUSE_TABLE.getTableId()) + .databaseId(HOUSE_TABLE.getDatabaseId()) + .build()) + .get() + .getEntityType()); + } } diff --git a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtilsTest.java b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtilsTest.java new file mode 100644 index 000000000..c3f4c8a47 --- /dev/null +++ b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtilsTest.java @@ -0,0 +1,87 @@ +package com.linkedin.openhouse.internal.catalog.utils; + +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Pattern; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.view.ViewProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * {@code rootMetadataFileLocation} is extracted out of {@link + * com.linkedin.openhouse.internal.catalog.OpenHouseInternalTableOperations} so the table commit + * path and the sibling view commit path can share metadata-file naming without sharing a codec + * default. The helper must stay metadata-type neutral: the caller supplies the codec. + */ +public class MetadataLocationUtilsTest { + + private static final String UUID_REGEX = + "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"; + + private static final Pattern UNCOMPRESSED = + Pattern.compile("^root/00007-" + UUID_REGEX + "\\.metadata\\.json$"); + + private static final Pattern GZIPPED = + Pattern.compile("^root/00007-" + UUID_REGEX + "\\.gz\\.metadata\\.json$"); + + @Test + public void rootMetadataFileLocationUsesFiveDigitVersionAndUuid() { + String location = MetadataLocationUtils.rootMetadataFileLocation("root", "none", 7); + + Assertions.assertTrue( + UNCOMPRESSED.matcher(location).matches(), + "Expected /00007-.metadata.json but was: " + location); + + // Version padding must be five digits so lexical ordering matches numeric ordering. + Assertions.assertTrue( + MetadataLocationUtils.rootMetadataFileLocation("root", "none", 1) + .startsWith("root/00001-")); + Assertions.assertTrue( + MetadataLocationUtils.rootMetadataFileLocation("root", "none", 12345) + .startsWith("root/12345-")); + + // The UUID is what lets concurrent writers stage the same version safely; it must differ. + Set generated = new HashSet<>(); + for (int i = 0; i < 5; i++) { + generated.add(MetadataLocationUtils.rootMetadataFileLocation("root", "none", 7)); + } + Assertions.assertEquals(5, generated.size(), "Each call must produce a distinct file name"); + } + + @Test + public void rootMetadataFileLocationUsesGzipExtension() { + String location = MetadataLocationUtils.rootMetadataFileLocation("root", "gzip", 7); + + Assertions.assertTrue( + GZIPPED.matcher(location).matches(), + "Expected /00007-.gz.metadata.json but was: " + location); + } + + /** + * Pinned deliberately: pinned Iceberg uses {@code none} as the table metadata-compression default + * but {@code gzip} as the view default. A helper that hard-coded the table default would silently + * change the view file extension, so the codec must be resolved by each caller and passed in. + */ + @Test + public void tableAndViewDefaultsArePassedExplicitly() { + Assertions.assertNotEquals( + TableProperties.METADATA_COMPRESSION_DEFAULT, + ViewProperties.METADATA_COMPRESSION_DEFAULT, + "This test is only meaningful while the table and view codec defaults differ"); + + String tableDefaultLocation = + MetadataLocationUtils.rootMetadataFileLocation( + "root", TableProperties.METADATA_COMPRESSION_DEFAULT, 7); + Assertions.assertTrue( + UNCOMPRESSED.matcher(tableDefaultLocation).matches(), + "Table default codec must yield .metadata.json but was: " + tableDefaultLocation); + + String viewDefaultLocation = + MetadataLocationUtils.rootMetadataFileLocation( + "root", ViewProperties.METADATA_COMPRESSION_DEFAULT, 7); + Assertions.assertTrue( + GZIPPED.matcher(viewDefaultLocation).matches(), + "View default codec must yield .gz.metadata.json but was: " + viewDefaultLocation); + } +} diff --git a/services/common/src/main/java/com/linkedin/openhouse/common/api/validator/ValidatorConstants.java b/services/common/src/main/java/com/linkedin/openhouse/common/api/validator/ValidatorConstants.java index 3f11694b4..3c0616858 100644 --- a/services/common/src/main/java/com/linkedin/openhouse/common/api/validator/ValidatorConstants.java +++ b/services/common/src/main/java/com/linkedin/openhouse/common/api/validator/ValidatorConstants.java @@ -19,4 +19,10 @@ private ValidatorConstants() {} "Only alphanumerics, hyphen and underscore supported"; public static final int MAX_ALLOWED_CLUSTERING_COLUMNS = 4; public static final String INITIAL_TABLE_VERSION = "INITIAL_VERSION"; + + /** A null/absent value is allowed and means a legacy table row. */ + public static final String ENTITY_TYPE_REGEX = "(?i)^(TABLE|VIEW)$"; + + public static final String ENTITY_TYPE_ERROR_MSG = + "Only TABLE and VIEW are supported entity types (case-insensitive)"; } diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/spec/model/UserTable.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/spec/model/UserTable.java index 9369fddc2..f7854a0ca 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/spec/model/UserTable.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/spec/model/UserTable.java @@ -2,6 +2,8 @@ import static com.linkedin.openhouse.common.api.validator.ValidatorConstants.ALPHA_NUM_UNDERSCORE_ERROR_MSG; import static com.linkedin.openhouse.common.api.validator.ValidatorConstants.ALPHA_NUM_UNDERSCORE_REGEX; +import static com.linkedin.openhouse.common.api.validator.ValidatorConstants.ENTITY_TYPE_ERROR_MSG; +import static com.linkedin.openhouse.common.api.validator.ValidatorConstants.ENTITY_TYPE_REGEX; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.gson.Gson; @@ -61,6 +63,15 @@ public class UserTable { @JsonProperty(value = "creationTime") private Long creationTime; + @Schema( + description = + "Type of the catalog object occupying this (databaseId, tableId) key. Null or 'TABLE' " + + "means a table; 'VIEW' means a view. Matched case-insensitively.", + example = "TABLE") + @JsonProperty(value = "entityType") + @Pattern(regexp = ENTITY_TYPE_REGEX, message = ENTITY_TYPE_ERROR_MSG) + private String entityType; + @Schema( description = "Timestamp in milliseconds when the table was soft-deleted. " diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/validator/impl/OpenHouseUserTableHtsApiValidator.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/validator/impl/OpenHouseUserTableHtsApiValidator.java index 087d928fd..dd3237cc4 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/validator/impl/OpenHouseUserTableHtsApiValidator.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/validator/impl/OpenHouseUserTableHtsApiValidator.java @@ -110,7 +110,16 @@ private void validateUserTable(UserTable userTable, List validationFailu && userTable.getMetadataLocation() == null && userTable.getStorageType() == null && userTable.getCreationTime() == null)) { - validationFailures.add("Only databaseId and tableId are supported for the query"); + validationFailures.add("Only databaseId, tableId and entityType are supported for the query"); + } + + // entityType is the one additional permitted query filter. Reject garbage here so + // an unknown discriminator fails as a validation error rather than as a silently empty result. + if (userTable.getEntityType() != null + && !userTable.getEntityType().matches(ENTITY_TYPE_REGEX)) { + validationFailures.add( + String.format( + "entityType provided: %s, %s", userTable.getEntityType(), ENTITY_TYPE_ERROR_MSG)); } if (userTable.getDatabaseId() != null diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/dto/model/UserTableDto.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/dto/model/UserTableDto.java index 68b603026..11e700a23 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/dto/model/UserTableDto.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/dto/model/UserTableDto.java @@ -23,6 +23,8 @@ public class UserTableDto { Long creationTime; + String entityType; + Long deletedAtMs; Long purgeAfterMs; diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/model/UserTableRow.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/model/UserTableRow.java index 7ee862d3e..01593f622 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/model/UserTableRow.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/model/UserTableRow.java @@ -32,4 +32,7 @@ public class UserTableRow { String storageType; Long creationTime; + + /** Nullable and without a default so existing rows need no backfill. */ + String entityType; } diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java index 1c6ae8c06..0465df64c 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java @@ -39,56 +39,112 @@ Optional findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( void deleteByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(String databaseId, String tableId); - @Query("SELECT DISTINCT databaseId FROM UserTableRow") + /** + * Excludes views from table listings. Applied in the query — never by filtering a returned {@link + * Page} — so content and counts agree. {@code IS NULL} is mandatory because the discriminator is + * nullable with no backfill; {@code upper(...)} avoids depending on collation, which differs + * between H2 in {@code MODE=MySQL} (case-sensitive) and production MySQL. + */ + String TABLE_ROW_PREDICATE = "(u.entityType IS NULL OR upper(u.entityType) = 'TABLE')"; + + @Query("SELECT DISTINCT u.databaseId FROM UserTableRow u WHERE " + TABLE_ROW_PREDICATE) Iterable findAllDistinctDatabaseIds(); - Iterable findAllByDatabaseIdIgnoreCase(String databaseId); + @Query( + "SELECT u FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + TABLE_ROW_PREDICATE) + Iterable findAllByDatabaseIdIgnoreCase(@Param("databaseId") String databaseId); + @Query( + "SELECT u FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " + + TABLE_ROW_PREDICATE) Iterable findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - String databaseId, String tableIdPattern); + @Param("databaseId") String databaseId, @Param("tableIdPattern") String tableIdPattern); @Query( - "SELECT DISTINCT databaseId FROM UserTableRow u where " - + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId))") - Page findAllDistinctDatabaseIds(String databaseId, Pageable pageable); + value = + "SELECT DISTINCT u.databaseId FROM UserTableRow u WHERE " + + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " + + TABLE_ROW_PREDICATE, + countQuery = + "SELECT COUNT(DISTINCT u.databaseId) FROM UserTableRow u WHERE " + + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " + + TABLE_ROW_PREDICATE) + Page findAllDistinctDatabaseIds( + @Param("databaseId") String databaseId, Pageable pageable); - Page findAllByDatabaseIdIgnoreCase(String databaseId, Pageable pageable); + @Query( + value = + "SELECT u FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + TABLE_ROW_PREDICATE, + countQuery = + "SELECT COUNT(u) FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + TABLE_ROW_PREDICATE) + Page findAllByDatabaseIdIgnoreCase( + @Param("databaseId") String databaseId, Pageable pageable); + @Query( + value = + "SELECT u FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " + + TABLE_ROW_PREDICATE, + countQuery = + "SELECT COUNT(u) FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " + + TABLE_ROW_PREDICATE) Page findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - String databaseId, String tableIdPattern, Pageable pageable); + @Param("databaseId") String databaseId, + @Param("tableIdPattern") String tableIdPattern, + Pageable pageable); - @Query( - "select DISTINCT u from UserTableRow u where " - + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " + /** + * A null or {@code TABLE} request means tables, including legacy null rows; {@code VIEW} means + * views only. An unknown value matches neither branch, so garbage fails closed here even if it + * bypasses API validation. + */ + String ENTITY_TYPE_FILTER_PREDICATE = + "(((:entityType IS NULL OR upper(:entityType) = 'TABLE') " + + "AND (u.entityType IS NULL OR upper(u.entityType) = 'TABLE')) " + + "OR (upper(:entityType) = 'VIEW' AND upper(u.entityType) = 'VIEW'))"; + + String GENERAL_FILTER_PREDICATE = + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " + "(:tableId IS NULL OR lower(u.tableId) = lower(:tableId)) AND " + "(:tableVersion IS NULL OR u.version = :tableVersion) AND " + "(:metadataLocation IS NULL OR u.metadataLocation = :metadataLocation) AND " + "(:storageType IS NULL OR u.storageType = :storageType) AND " - + "(:creationTime IS NULL OR u.creationTime = :creationTime)") + + "(:creationTime IS NULL OR u.creationTime = :creationTime) AND " + + ENTITY_TYPE_FILTER_PREDICATE; + + @Query( + value = "select DISTINCT u from UserTableRow u where " + GENERAL_FILTER_PREDICATE, + countQuery = "select COUNT(DISTINCT u) from UserTableRow u where " + GENERAL_FILTER_PREDICATE) Page findAllByFilters( - String databaseId, - String tableId, - String tableVersion, - String metadataLocation, - String storageType, - Long creationTime, + @Param("databaseId") String databaseId, + @Param("tableId") String tableId, + @Param("tableVersion") String tableVersion, + @Param("metadataLocation") String metadataLocation, + @Param("storageType") String storageType, + @Param("creationTime") Long creationTime, + @Param("entityType") String entityType, Pageable pageable); - @Query( - "select DISTINCT u from UserTableRow u where " - + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " - + "(:tableId IS NULL OR lower(u.tableId) = lower(:tableId)) AND " - + "(:tableVersion IS NULL OR u.version = :tableVersion) AND " - + "(:metadataLocation IS NULL OR u.metadataLocation = :metadataLocation) AND " - + "(:storageType IS NULL OR u.storageType = :storageType) AND " - + "(:creationTime IS NULL OR u.creationTime = :creationTime)") + @Query("select DISTINCT u from UserTableRow u where " + GENERAL_FILTER_PREDICATE) Iterable findAllByFilters( - String databaseId, - String tableId, - String tableVersion, - String metadataLocation, - String storageType, - Long creationTime); + @Param("databaseId") String databaseId, + @Param("tableId") String tableId, + @Param("tableVersion") String tableVersion, + @Param("metadataLocation") String metadataLocation, + @Param("storageType") String storageType, + @Param("creationTime") Long creationTime, + @Param("entityType") String entityType); /* * The following methods are required to maintain the generality of the interface {@link com.linkedin.openhouse.housetables.repository.HtsRepository} diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java index f9b150862..2b4d62fc6 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java @@ -302,7 +302,7 @@ private Page listTables(UserTable userTable, int page, int size, S return METRICS_REPORTER.executeWithStats( () -> htsJdbcRepository - .findAllByFilters(userTable.getDatabaseId(), null, null, null, null, null, pageable) + .findAllByDatabaseIdIgnoreCase(userTable.getDatabaseId(), pageable) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)), MetricsConstant.HTS_PAGE_TABLES_TIME); } @@ -350,6 +350,7 @@ private Page searchTables(UserTable userTable, int page, int size, userTable.getMetadataLocation(), userTable.getStorageType(), userTable.getCreationTime(), + userTable.getEntityType(), pageable) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)), MetricsConstant.HTS_PAGE_SEARCH_TABLES_TIME); @@ -369,7 +370,8 @@ private List searchTables(UserTable userTable) { userTable.getTableVersion(), userTable.getMetadataLocation(), userTable.getStorageType(), - userTable.getCreationTime()) + userTable.getCreationTime(), + userTable.getEntityType()) .spliterator(), false) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)) @@ -395,10 +397,16 @@ private boolean isListTablesWithPattern(UserTable userTable) { && userTable.getTableId() != null; } + /** + * The list/pattern queries hard-code the table predicate, so {@code entityType} must count as a + * non-key field — otherwise a {@code databaseId + entityType=VIEW} request would route there and + * silently return tables instead of going through {@code findAllByFilters}. + */ private boolean isNonKeyFieldsNullForUserTable(UserTable userTable) { return userTable.getTableVersion() == null && userTable.getMetadataLocation() == null && userTable.getStorageType() == null - && userTable.getCreationTime() == null; + && userTable.getCreationTime() == null + && userTable.getEntityType() == null; } } diff --git a/services/housetables/src/main/resources/schema.sql b/services/housetables/src/main/resources/schema.sql index 81317e873..eee3124b4 100644 --- a/services/housetables/src/main/resources/schema.sql +++ b/services/housetables/src/main/resources/schema.sql @@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS user_table_row ( metadata_location VARCHAR (512) , storage_type VARCHAR (128) DEFAULT 'hdfs' NOT NULL, creation_time BIGINT DEFAULT NULL, + entity_type VARCHAR (128) DEFAULT NULL, last_modified_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, ETL_TS DATETIME(6) DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), PRIMARY KEY (database_id, table_id) diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java index d77a256f4..f5b882525 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java @@ -1,7 +1,9 @@ package com.linkedin.openhouse.housetables.e2e.usertable; +import static com.linkedin.openhouse.common.api.validator.ValidatorConstants.INITIAL_TABLE_VERSION; import static com.linkedin.openhouse.housetables.model.TestHouseTableModelConstants.*; import static com.linkedin.openhouse.housetables.model.TestHtsApiConstants.*; +import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.*; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; @@ -760,4 +762,345 @@ public void testPurgeAllSoftDeletedTables() throws Exception { .andExpect(status().isOk()) .andExpect(jsonPath("$.pageResults.content", hasSize(0))); } + + // --------------------------------------------------------------------------------------------- + // entityType discriminator over HTTP + // --------------------------------------------------------------------------------------------- + + /** + * Canonical interleaved fixture, seeded in its own database so it does not disturb the {@code + * test_db0} counts asserted by the tests above. + */ + private static final String ENTITY_TYPE_DB = "entity_type_db"; + + private UserTableRow entityTypeRow(String databaseId, String tableId, String entityType) { + return UserTableRow.builder() + .databaseId(databaseId) + .tableId(tableId) + .version(null) + .metadataLocation(String.format("/openhouse/%s/%s/v0_metadata.json", databaseId, tableId)) + .storageType(TEST_DEFAULT_STORAGE_TYPE) + .creationTime(TEST_CREATION_TIME) + .entityType(entityType) + .build(); + } + + private void seedCanonicalRows(String prefix) { + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t00_legacy", null)); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t01_view", "VIEW")); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t02_explicit", "TABLE")); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t03_view", "VIEW")); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t04_legacy", null)); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t05_view", "VIEW")); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t06_explicit", "TABLE")); + } + + private static MultiValueMap queryParams(String... keyValues) { + Map> paramsInternal = new HashMap<>(); + for (int i = 0; i < keyValues.length; i += 2) { + paramsInternal.put(keyValues[i], Collections.singletonList(keyValues[i + 1])); + } + return new MultiValueMapAdapter(paramsInternal); + } + + /** Both v0 table query families exclude views and keep legacy NULL rows. */ + @Test + public void testTableQueriesExcludeViewsAndKeepLegacyRows() throws Exception { + seedCanonicalRows(""); + + mvc.perform( + MockMvcRequestBuilders.get("/hts/tables/query") + .params(queryParams("databaseId", ENTITY_TYPE_DB)) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.results", hasSize(4))) + .andExpect( + jsonPath( + "$.results[*].tableId", + containsInAnyOrder("t00_legacy", "t02_explicit", "t04_legacy", "t06_explicit"))) + .andExpect(jsonPath("$.results[*].tableId", not(hasItem("t01_view")))) + .andExpect(jsonPath("$.results[*].tableId", not(hasItem("t03_view")))) + .andExpect(jsonPath("$.results[*].tableId", not(hasItem("t05_view")))); + + mvc.perform( + MockMvcRequestBuilders.get("/hts/tables/query") + .params(queryParams("databaseId", ENTITY_TYPE_DB, "tableId", "t0%")) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.results", hasSize(4))) + .andExpect( + jsonPath( + "$.results[*].tableId", + containsInAnyOrder("t00_legacy", "t02_explicit", "t04_legacy", "t06_explicit"))); + } + + /** + * Anti-post-filter assertion over HTTP for the v1 paged query families: an implementation that + * filters the returned page would report totalElements=7/totalPages=4 and a 1-row first page. + */ + @Test + public void testPaginatedTableQueriesFilterBeforePaging() throws Exception { + seedCanonicalRows(""); + + mvc.perform( + MockMvcRequestBuilders.get("/v1/hts/tables/query") + .params(queryParams("databaseId", ENTITY_TYPE_DB)) + .param("page", "0") + .param("size", "2") + .param("sortBy", "tableId") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t00_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t02_explicit"))); + + mvc.perform( + MockMvcRequestBuilders.get("/v1/hts/tables/query") + .params(queryParams("databaseId", ENTITY_TYPE_DB)) + .param("page", "1") + .param("size", "2") + .param("sortBy", "tableId") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t04_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t06_explicit"))); + + // Same assertions on the pattern form. + mvc.perform( + MockMvcRequestBuilders.get("/v1/hts/tables/query") + .params(queryParams("databaseId", ENTITY_TYPE_DB, "tableId", "t0%")) + .param("page", "0") + .param("size", "2") + .param("sortBy", "tableId") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t00_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t02_explicit"))); + } + + /** A database whose only pointer is a view must not appear in either database listing. */ + @Test + public void testDatabaseQueriesExcludeViewOnlyDatabases() throws Exception { + // The @BeforeEach fixture row lives in test_db0; remove it so the database set is exactly the + // canonical seven. This is a deliberate mid-test global reset: the class-level @AfterEach + // deleteAll() restores order either way, but it does make this method order-fragile if + // @TestMethodOrder is ever added to this class. + htsRepository.deleteAll(); + htsRepository.save(entityTypeRow("db00_legacy", "t1", null)); + htsRepository.save(entityTypeRow("db01_view_only", "t1", "VIEW")); + htsRepository.save(entityTypeRow("db02_explicit", "t1", "TABLE")); + htsRepository.save(entityTypeRow("db03_view_only", "t1", "VIEW")); + htsRepository.save(entityTypeRow("db04_legacy", "t1", null)); + htsRepository.save(entityTypeRow("db05_view_only", "t1", "VIEW")); + htsRepository.save(entityTypeRow("db06_explicit", "t1", "TABLE")); + + mvc.perform(MockMvcRequestBuilders.get("/hts/tables/query").accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.results", hasSize(4))) + .andExpect( + jsonPath( + "$.results[*].databaseId", + containsInAnyOrder("db00_legacy", "db02_explicit", "db04_legacy", "db06_explicit"))) + .andExpect(jsonPath("$.results[*].databaseId", not(hasItem("db01_view_only")))); + + mvc.perform( + MockMvcRequestBuilders.get("/v1/hts/tables/query") + .param("page", "0") + .param("size", "2") + .param("sortBy", "databaseId") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].databaseId", is("db00_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].databaseId", is("db02_explicit"))); + + mvc.perform( + MockMvcRequestBuilders.get("/v1/hts/tables/query") + .param("page", "1") + .param("size", "2") + .param("sortBy", "databaseId") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].databaseId", is("db04_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].databaseId", is("db06_explicit"))); + } + + /** The discriminator survives the HTTP PUT/GET boundary, and legacy writers stay null. */ + @Test + public void testEntityTypePutAndGetRoundTrip() throws Exception { + UserTable viewEntity = + UserTable.builder() + .databaseId(ENTITY_TYPE_DB) + .tableId("put_view") + .tableVersion(INITIAL_TABLE_VERSION) + .metadataLocation("/openhouse/entity_type_db/put_view/v0_metadata.json") + .entityType("VIEW") + .build(); + + mvc.perform( + MockMvcRequestBuilders.put("/hts/tables") + .contentType(MediaType.APPLICATION_JSON) + .content( + CreateUpdateEntityRequestBody.builder() + .entity(viewEntity) + .build() + .toJson()) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()) + .andExpect(jsonPath("$.entity.entityType", is("VIEW"))); + + mvc.perform( + MockMvcRequestBuilders.get("/hts/tables") + .param("databaseId", ENTITY_TYPE_DB) + .param("tableId", "put_view") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.entity.entityType", is("VIEW"))); + + assertThat( + htsRepository + .findById( + UserTableRowPrimaryKey.builder() + .databaseId(ENTITY_TYPE_DB) + .tableId("put_view") + .build()) + .get() + .getEntityType()) + .isEqualTo("VIEW"); + + // A legacy PUT that omits the field must stay null end-to-end. + UserTable legacyEntity = + UserTable.builder() + .databaseId(ENTITY_TYPE_DB) + .tableId("put_legacy") + .tableVersion(INITIAL_TABLE_VERSION) + .metadataLocation("/openhouse/entity_type_db/put_legacy/v0_metadata.json") + .build(); + + mvc.perform( + MockMvcRequestBuilders.put("/hts/tables") + .contentType(MediaType.APPLICATION_JSON) + .content( + CreateUpdateEntityRequestBody.builder() + .entity(legacyEntity) + .build() + .toJson()) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isCreated()) + .andExpect(jsonPath("$.entity.entityType").doesNotExist()); + + mvc.perform( + MockMvcRequestBuilders.get("/hts/tables") + .param("databaseId", ENTITY_TYPE_DB) + .param("tableId", "put_legacy") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.entity.entityType").doesNotExist()); + + assertThat( + htsRepository + .findById( + UserTableRowPrimaryKey.builder() + .databaseId(ENTITY_TYPE_DB) + .tableId("put_legacy") + .build()) + .get() + .getEntityType()) + .isNull(); + } + + /** + * Pins validator + service routing over HTTP, not merely repository behavior: the request carries + * only databaseId and entityType=VIEW. It fails if the validator rejects the parameter or if the + * routing predicate still classifies this as a plain table listing. + */ + @Test + public void testEntityTypeOnlyViewQueryRoutesToGeneralSearch() throws Exception { + seedCanonicalRows(""); + + mvc.perform( + MockMvcRequestBuilders.get("/hts/tables/query") + .params(queryParams("databaseId", ENTITY_TYPE_DB, "entityType", "VIEW")) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.results", hasSize(3))) + .andExpect( + jsonPath( + "$.results[*].tableId", containsInAnyOrder("t01_view", "t03_view", "t05_view"))); + + mvc.perform( + MockMvcRequestBuilders.get("/v1/hts/tables/query") + .params(queryParams("databaseId", ENTITY_TYPE_DB, "entityType", "VIEW")) + .param("page", "0") + .param("size", "2") + .param("sortBy", "tableId") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(3))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t01_view"))) + .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t03_view"))); + } + + /** + * Publish-boundary defense in depth. Issues the exact HTS PUT a table create would emit at a key + * already occupied by a VIEW pointer: tableVersion=INITIAL_VERSION, no entityType, and a + * different candidate metadataLocation. The pointer must be rejected with 409 and left + * byte-identical — same numeric JPA {@code version}, {@code entityType} and {@code + * metadataLocation}. + * + *

The Tables Service occupancy tests prove a real CREATE never reaches this boundary; this + * test proves the boundary itself does not lose the view. + */ + @Test + public void testCreateTablePointerPublishCannotOverwriteView() throws Exception { + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, "occupied_by_view", "VIEW")); + + UserTableRowPrimaryKey key = + UserTableRowPrimaryKey.builder() + .databaseId(ENTITY_TYPE_DB) + .tableId("occupied_by_view") + .build(); + UserTableRow before = htsRepository.findById(key).get(); + + UserTable tableCreatePut = + UserTable.builder() + .databaseId(ENTITY_TYPE_DB) + .tableId("occupied_by_view") + .tableVersion(INITIAL_TABLE_VERSION) + .metadataLocation( + "/openhouse/entity_type_db/occupied_by_view-uuid/00001-candidate.metadata.json") + .build(); + + mvc.perform( + MockMvcRequestBuilders.put("/hts/tables") + .contentType(MediaType.APPLICATION_JSON) + .content( + CreateUpdateEntityRequestBody.builder() + .entity(tableCreatePut) + .build() + .toJson()) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isConflict()); + + UserTableRow after = htsRepository.findById(key).get(); + assertThat(after.getEntityType()).isEqualTo("VIEW"); + assertThat(after.getEntityType()).isEqualTo(before.getEntityType()); + assertThat(after.getVersion()).isEqualTo(before.getVersion()); + assertThat(after.getMetadataLocation()).isEqualTo(before.getMetadataLocation()); + } } diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java index 1a51289c3..ac4aba39b 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java @@ -10,6 +10,7 @@ import com.linkedin.openhouse.housetables.model.UserTableRow; import com.linkedin.openhouse.housetables.model.UserTableRowPrimaryKey; import com.linkedin.openhouse.housetables.repository.impl.jdbc.UserTableHtsJdbcRepository; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.AfterEach; @@ -18,6 +19,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; import org.springframework.orm.ObjectOptimisticLockingFailureException; import org.springframework.test.context.ContextConfiguration; @@ -25,6 +30,33 @@ @ContextConfiguration(initializers = PropertyOverrideContextInitializer.class) public class HtsRepositoryTest { + /** + * Canonical interleaved fixture. Four visible tables (two legacy NULL, two explicit TABLE) are + * interleaved with three VIEW rows so that a fetch-then-filter implementation returns a SHORT + * first page (1 row) and totalElements=7, while the correct pre-pagination predicate returns a + * full page (2 rows) and totalElements=4. + */ + private static final String ENTITY_TYPE_DB = "entity_type_db"; + + private static final String[] CANONICAL_TABLE_IDS = { + "t00_legacy", "t02_explicit", "t04_legacy", "t06_explicit" + }; + + private static final String[] CANONICAL_VIEW_IDS = {"t01_view", "t03_view", "t05_view"}; + + /** Case-normalization fixture; see {@code CASE_DB}. */ + private static final String CASE_DB = "entity_type_case_db"; + + private static final String[] CASE_VISIBLE_TABLE_IDS = { + "case00_null", "case01_upper_table", "case02_lower_table", "case03_mixed_table" + }; + + private static final String[] CASE_VIEW_IDS = { + "case04_upper_view", "case05_lower_view", "case06_mixed_view" + }; + + private static final String CASE_GARBAGE_ID = "case07_garbage"; + @Autowired UserTableHtsJdbcRepository htsRepository; @AfterEach @@ -32,6 +64,56 @@ public void tearDown() { htsRepository.deleteAll(); } + private UserTableRow row(String databaseId, String tableId, String entityType) { + return UserTableRow.builder() + .databaseId(databaseId) + .tableId(tableId) + .version(null) + .metadataLocation(String.format("/openhouse/%s/%s/v0_metadata.json", databaseId, tableId)) + .storageType(TEST_DEFAULT_STORAGE_TYPE) + .creationTime(TEST_CREATION_TIME) + .entityType(entityType) + .build(); + } + + /** Seeds the canonical 7-row interleaved fixture into {@code databaseId} under {@code prefix}. */ + private void seedCanonicalRows(String databaseId, String prefix) { + htsRepository.save(row(databaseId, prefix + "t00_legacy", null)); + htsRepository.save(row(databaseId, prefix + "t01_view", "VIEW")); + htsRepository.save(row(databaseId, prefix + "t02_explicit", "TABLE")); + htsRepository.save(row(databaseId, prefix + "t03_view", "VIEW")); + htsRepository.save(row(databaseId, prefix + "t04_legacy", null)); + htsRepository.save(row(databaseId, prefix + "t05_view", "VIEW")); + htsRepository.save(row(databaseId, prefix + "t06_explicit", "TABLE")); + } + + /** Seeds the 8-row case-normalization fixture into {@link #CASE_DB}. */ + private void seedCaseNormalizationRows() { + htsRepository.save(row(CASE_DB, "case00_null", null)); + htsRepository.save(row(CASE_DB, "case01_upper_table", "TABLE")); + htsRepository.save(row(CASE_DB, "case02_lower_table", "table")); + htsRepository.save(row(CASE_DB, "case03_mixed_table", "TaBlE")); + htsRepository.save(row(CASE_DB, "case04_upper_view", "VIEW")); + htsRepository.save(row(CASE_DB, "case05_lower_view", "view")); + htsRepository.save(row(CASE_DB, "case06_mixed_view", "ViEw")); + htsRepository.save(row(CASE_DB, CASE_GARBAGE_ID, "UNKNOWN")); + } + + private static List tableIds(Iterable rows) { + return Lists.newArrayList(rows).stream() + .map(UserTableRow::getTableId) + .sorted() + .collect(Collectors.toList()); + } + + private static List pageTableIds(Page page) { + return page.getContent().stream().map(UserTableRow::getTableId).collect(Collectors.toList()); + } + + private static Pageable sortedPage(int page) { + return PageRequest.of(page, 2, Sort.by("tableId")); + } + @Test public void testSaveFirstRecord() { UserTableRow testUserTableRow = @@ -246,4 +328,267 @@ public void testRenameCaseSensitivity() { // verify testTuple1_1 doesn't exist any more. assertThat(htsRepository.existsById(key)).isFalse(); } + + // --------------------------------------------------------------------------------------------- + // entityType discriminator + // --------------------------------------------------------------------------------------------- + + /** The discriminator must persist verbatim and must not perturb version/metadata behavior. */ + @Test + public void testEntityTypePersistenceRoundTrip() { + UserTableRow viewRow = htsRepository.save(row(ENTITY_TYPE_DB, "persist_view", "VIEW")); + UserTableRow tableRow = htsRepository.save(row(ENTITY_TYPE_DB, "persist_table", "TABLE")); + UserTableRow legacyRow = htsRepository.save(row(ENTITY_TYPE_DB, "persist_legacy", null)); + + // Insert still yields version 0 for all three; the discriminator is orthogonal to versioning. + assertThat(viewRow.getVersion()).isEqualTo(0L); + assertThat(tableRow.getVersion()).isEqualTo(0L); + assertThat(legacyRow.getVersion()).isEqualTo(0L); + + assertThat(findRow(ENTITY_TYPE_DB, "persist_view").getEntityType()).isEqualTo("VIEW"); + assertThat(findRow(ENTITY_TYPE_DB, "persist_table").getEntityType()).isEqualTo("TABLE"); + assertThat(findRow(ENTITY_TYPE_DB, "persist_legacy").getEntityType()).isNull(); + + assertThat(findRow(ENTITY_TYPE_DB, "persist_view").getMetadataLocation()) + .isEqualTo(viewRow.getMetadataLocation()); + + // An update at the correct version preserves the stored discriminator. + UserTableRow updated = + htsRepository.save( + findRow(ENTITY_TYPE_DB, "persist_view") + .toBuilder() + .metadataLocation("/openhouse/entity_type_db/persist_view/v1_metadata.json") + .build()); + assertThat(updated.getEntityType()).isEqualTo("VIEW"); + assertThat(findRow(ENTITY_TYPE_DB, "persist_view").getEntityType()).isEqualTo("VIEW"); + } + + /** SHOW-TABLES-equivalent plain listing hides views and keeps legacy NULL rows. */ + @Test + public void testFindAllByDatabaseIdFiltersViewsAndKeepsLegacyTables() { + seedCanonicalRows(ENTITY_TYPE_DB, ""); + // A table in another database must not leak in. + htsRepository.save(row("other_db", "t00_legacy", null)); + + List result = + Lists.newArrayList(htsRepository.findAllByDatabaseIdIgnoreCase(ENTITY_TYPE_DB)); + + assertThat(tableIds(result)).containsExactly(CANONICAL_TABLE_IDS); + assertThat(result) + .allSatisfy(r -> assertThat(r.getEntityType()).isNotEqualToIgnoringCase("VIEW")); + } + + /** + * The canonical anti-post-filter assertion for the per-database page. A fetch-then-filter + * implementation returns [t00_legacy] on page 0 with totalElements=7/totalPages=4; the correct + * pre-pagination predicate returns a full 2-row page with totalElements=4/totalPages=2. + */ + @Test + public void testFindAllByDatabaseIdFiltersBeforePagination() { + seedCanonicalRows(ENTITY_TYPE_DB, ""); + + Page page0 = + htsRepository.findAllByDatabaseIdIgnoreCase(ENTITY_TYPE_DB, sortedPage(0)); + assertThat(page0.getTotalElements()).isEqualTo(4); + assertThat(page0.getTotalPages()).isEqualTo(2); + assertThat(page0.getContent()).hasSize(2); + assertThat(pageTableIds(page0)).containsExactly("t00_legacy", "t02_explicit"); + + Page page1 = + htsRepository.findAllByDatabaseIdIgnoreCase(ENTITY_TYPE_DB, sortedPage(1)); + assertThat(page1.getTotalElements()).isEqualTo(4); + assertThat(page1.getTotalPages()).isEqualTo(2); + assertThat(page1.getContent()).hasSize(2); + assertThat(pageTableIds(page1)).containsExactly("t04_legacy", "t06_explicit"); + + assertThat(pageTableIds(page0)).doesNotContainAnyElementsOf(Arrays.asList(CANONICAL_VIEW_IDS)); + assertThat(pageTableIds(page1)).doesNotContainAnyElementsOf(Arrays.asList(CANONICAL_VIEW_IDS)); + } + + /** The pattern (LIKE) listing family applies the same table-only predicate. */ + @Test + public void testFindAllByPatternFiltersViewsAndKeepsLegacyTables() { + seedCanonicalRows(ENTITY_TYPE_DB, "match_"); + // Non-matching table in the same database must be excluded by the pattern, not by type. + htsRepository.save(row(ENTITY_TYPE_DB, "nomatch_table", "TABLE")); + + List result = + Lists.newArrayList( + htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + ENTITY_TYPE_DB, "match_%")); + + assertThat(tableIds(result)) + .containsExactly( + "match_t00_legacy", "match_t02_explicit", "match_t04_legacy", "match_t06_explicit"); + } + + /** Anti-post-filter assertion for the paged pattern listing. */ + @Test + public void testFindAllByPatternFiltersBeforePagination() { + seedCanonicalRows(ENTITY_TYPE_DB, "match_"); + htsRepository.save(row(ENTITY_TYPE_DB, "nomatch_table", "TABLE")); + + Page page0 = + htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + ENTITY_TYPE_DB, "match_%", sortedPage(0)); + assertThat(page0.getTotalElements()).isEqualTo(4); + assertThat(page0.getTotalPages()).isEqualTo(2); + assertThat(page0.getContent()).hasSize(2); + assertThat(pageTableIds(page0)).containsExactly("match_t00_legacy", "match_t02_explicit"); + + Page page1 = + htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + ENTITY_TYPE_DB, "match_%", sortedPage(1)); + assertThat(page1.getTotalElements()).isEqualTo(4); + assertThat(page1.getTotalPages()).isEqualTo(2); + assertThat(page1.getContent()).hasSize(2); + assertThat(pageTableIds(page1)).containsExactly("match_t04_legacy", "match_t06_explicit"); + } + + /** + * The general-filter query defaults to tables (null and any TABLE spelling) and can be asked + * explicitly for views. This is the only query family that can return VIEW rows. + */ + @Test + public void testFindAllByFiltersDefaultsToTablesAndCanSelectViews() { + seedCanonicalRows(ENTITY_TYPE_DB, ""); + + // entityType == null means "tables", not "everything". + assertThat( + tableIds( + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, (String) null))) + .containsExactly(CANONICAL_TABLE_IDS); + + for (String tableSpelling : new String[] {"TABLE", "table", "TaBlE"}) { + assertThat( + tableIds( + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, tableSpelling))) + .as("entityType=%s must resolve to the four visible tables", tableSpelling) + .containsExactly(CANONICAL_TABLE_IDS); + } + + for (String viewSpelling : new String[] {"VIEW", "view", "ViEw"}) { + assertThat( + tableIds( + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, viewSpelling))) + .as("entityType=%s must resolve to exactly the three views", viewSpelling) + .containsExactly(CANONICAL_VIEW_IDS); + } + + // Pageable overload: default (tables) and explicit VIEW both count in the database. + Page defaultPage0 = + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, (String) null, sortedPage(0)); + assertThat(defaultPage0.getTotalElements()).isEqualTo(4); + assertThat(defaultPage0.getTotalPages()).isEqualTo(2); + assertThat(pageTableIds(defaultPage0)).containsExactly("t00_legacy", "t02_explicit"); + + Page viewPage0 = + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, "VIEW", sortedPage(0)); + assertThat(viewPage0.getTotalElements()).isEqualTo(3); + assertThat(viewPage0.getTotalPages()).isEqualTo(2); + assertThat(pageTableIds(viewPage0)).containsExactly("t01_view", "t03_view"); + + Page viewPage1 = + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, "VIEW", sortedPage(1)); + assertThat(viewPage1.getTotalElements()).isEqualTo(3); + assertThat(pageTableIds(viewPage1)).containsExactly("t05_view"); + } + + /** A database whose only pointers are views must disappear from the database listing. */ + @Test + public void testFindDistinctDatabasesExcludesViewOnlyDatabases() { + htsRepository.save(row("db00_legacy", "t1", null)); + htsRepository.save(row("db01_view_only", "t1", "VIEW")); + htsRepository.save(row("db02_explicit", "t1", "TABLE")); + htsRepository.save(row("db03_view_only", "t1", "VIEW")); + htsRepository.save(row("db04_legacy", "t1", null)); + htsRepository.save(row("db05_view_only", "t1", "VIEW")); + htsRepository.save(row("db06_explicit", "t1", "TABLE")); + + assertThat(Lists.newArrayList(htsRepository.findAllDistinctDatabaseIds())) + .containsExactlyInAnyOrder("db00_legacy", "db02_explicit", "db04_legacy", "db06_explicit"); + + Pageable dbPage0 = PageRequest.of(0, 2, Sort.by("databaseId")); + Page page0 = htsRepository.findAllDistinctDatabaseIds(null, dbPage0); + assertThat(page0.getTotalElements()).isEqualTo(4); + assertThat(page0.getTotalPages()).isEqualTo(2); + assertThat(page0.getContent()).containsExactly("db00_legacy", "db02_explicit"); + + Page page1 = + htsRepository.findAllDistinctDatabaseIds(null, PageRequest.of(1, 2, Sort.by("databaseId"))); + assertThat(page1.getTotalElements()).isEqualTo(4); + assertThat(page1.getTotalPages()).isEqualTo(2); + assertThat(page1.getContent()).containsExactly("db04_legacy", "db06_explicit"); + } + + /** + * Case/garbage matrix at the SQL layer. + * + *

H2 runs in {@code MODE=MySQL} which is case-SENSITIVE for string comparison, whereas + * production MySQL's default collation is case-INSENSITIVE. This test therefore proves that the + * query normalizes explicitly (e.g. {@code upper(u.entityType) = 'TABLE'}) rather than leaning on + * a provider collation: an implementation using a bare {@code = 'TABLE'} comparison would hide + * {@code table}/{@code TaBlE} here and fail. It does NOT certify production MySQL behavior — the + * authoritative case-insensitivity contract is pinned at the Java guard layer in {@code + * HouseTableTest#testEntityTypeClassification} and the catalog guard tests, and a MySQL staging + * smoke test is still required before views are enabled. + */ + @Test + public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { + seedCaseNormalizationRows(); + + assertThat(tableIds(htsRepository.findAllByDatabaseIdIgnoreCase(CASE_DB))) + .containsExactly(CASE_VISIBLE_TABLE_IDS); + assertThat( + tableIds( + htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase(CASE_DB, "case%"))) + .containsExactly(CASE_VISIBLE_TABLE_IDS); + + Page dbPage0 = + htsRepository.findAllByDatabaseIdIgnoreCase(CASE_DB, sortedPage(0)); + assertThat(dbPage0.getTotalElements()).isEqualTo(4); + assertThat(dbPage0.getTotalPages()).isEqualTo(2); + assertThat(pageTableIds(dbPage0)).containsExactly("case00_null", "case01_upper_table"); + + Page patternPage0 = + htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + CASE_DB, "case%", sortedPage(0)); + assertThat(patternPage0.getTotalElements()).isEqualTo(4); + assertThat(patternPage0.getTotalPages()).isEqualTo(2); + assertThat(pageTableIds(patternPage0)).containsExactly("case00_null", "case01_upper_table"); + + // Every VIEW spelling is selectable and the garbage row is never one of them. + for (String viewSpelling : new String[] {"VIEW", "view", "ViEw"}) { + assertThat( + tableIds( + htsRepository.findAllByFilters( + CASE_DB, null, null, null, null, null, viewSpelling))) + .as("entityType=%s", viewSpelling) + .containsExactly(CASE_VIEW_IDS); + } + + // Garbage fails closed on the repository: it is neither a table nor a view. + assertThat( + Lists.newArrayList( + htsRepository.findAllByFilters(CASE_DB, null, null, null, null, null, "UNKNOWN"))) + .isEmpty(); + assertThat(tableIds(htsRepository.findAllByDatabaseIdIgnoreCase(CASE_DB))) + .doesNotContain(CASE_GARBAGE_ID); + + // The garbage row is still stored — it is hidden, not dropped. + assertThat(findRow(CASE_DB, CASE_GARBAGE_ID).getEntityType()).isEqualTo("UNKNOWN"); + } + + private UserTableRow findRow(String databaseId, String tableId) { + return htsRepository + .findById(UserTableRowPrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()) + .orElseThrow( + () -> new AssertionError("Expected row " + databaseId + "." + tableId + " to exist")); + } } diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java index bf97095bd..c7d6a9565 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java @@ -12,12 +12,14 @@ import com.linkedin.openhouse.housetables.e2e.SpringH2HtsApplication; import com.linkedin.openhouse.housetables.model.TestHouseTableModelConstants; import com.linkedin.openhouse.housetables.model.UserTableRow; +import com.linkedin.openhouse.housetables.model.UserTableRowPrimaryKey; import com.linkedin.openhouse.housetables.repository.impl.jdbc.SoftDeletedUserTableHtsJdbcRepository; import com.linkedin.openhouse.housetables.repository.impl.jdbc.UserTableHtsJdbcRepository; import com.linkedin.openhouse.housetables.services.UserTablesService; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -637,4 +639,197 @@ private Boolean isUserTableDtoEqual(UserTableDto expected, UserTableDto actual) .build() .equals(actual.toBuilder().tableVersion("").build()); } + + // --------------------------------------------------------------------------------------------- + // entityType discriminator at the service call sites + // --------------------------------------------------------------------------------------------- + + /** + * Canonical interleaved fixture. Seeded into its own database so it never perturbs the + * pre-existing per-database counts asserted by the tests above. + */ + private static final String ENTITY_TYPE_DB = "entity_type_db"; + + private static final List CANONICAL_TABLE_IDS = + Arrays.asList("t00_legacy", "t02_explicit", "t04_legacy", "t06_explicit"); + + private static final List CANONICAL_VIEW_IDS = + Arrays.asList("t01_view", "t03_view", "t05_view"); + + private UserTableRow entityTypeRow(String databaseId, String tableId, String entityType) { + return UserTableRow.builder() + .databaseId(databaseId) + .tableId(tableId) + .version(null) + .metadataLocation(String.format("/openhouse/%s/%s/v0_metadata.json", databaseId, tableId)) + .storageType(TEST_DEFAULT_STORAGE_TYPE) + .creationTime(TEST_CREATION_TIME) + .entityType(entityType) + .build(); + } + + private void seedCanonicalRows(String prefix) { + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t00_legacy", null)); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t01_view", "VIEW")); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t02_explicit", "TABLE")); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t03_view", "VIEW")); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t04_legacy", null)); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t05_view", "VIEW")); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, prefix + "t06_explicit", "TABLE")); + } + + private static List sortedIds(List dtos) { + return dtos.stream().map(UserTableDto::getTableId).sorted().collect(Collectors.toList()); + } + + private static List pageIds(Page page) { + return page.getContent().stream().map(UserTableDto::getTableId).collect(Collectors.toList()); + } + + /** Plain per-database listing through the service hides views and keeps legacy NULL rows. */ + @Test + public void testListTablesCallSiteFiltersViewsAndKeepsNullRows() { + seedCanonicalRows(""); + + List result = + userTablesService.getAllUserTables(UserTable.builder().databaseId(ENTITY_TYPE_DB).build()); + + assertThat(sortedIds(result)).isEqualTo(CANONICAL_TABLE_IDS); + } + + /** + * Anti-post-filter assertion at the service layer, and the pin for routing the paged per-database + * listing through the table-predicated query rather than the untyped {@code findAllByFilters}. A + * fetch-then-filter implementation yields a 1-row page 0 with totalElements=7/totalPages=4. + */ + @Test + public void testListTablesCallSiteFiltersBeforePagination() { + seedCanonicalRows(""); + UserTable searchBy = UserTable.builder().databaseId(ENTITY_TYPE_DB).build(); + + Page page0 = userTablesService.getAllUserTables(searchBy, 0, 2, "tableId"); + Assertions.assertEquals(4, page0.getTotalElements()); + Assertions.assertEquals(2, page0.getTotalPages()); + Assertions.assertEquals(2, page0.getContent().size()); + assertThat(pageIds(page0)).containsExactly("t00_legacy", "t02_explicit"); + + Page page1 = userTablesService.getAllUserTables(searchBy, 1, 2, "tableId"); + Assertions.assertEquals(4, page1.getTotalElements()); + Assertions.assertEquals(2, page1.getTotalPages()); + Assertions.assertEquals(2, page1.getContent().size()); + assertThat(pageIds(page1)).containsExactly("t04_legacy", "t06_explicit"); + + assertThat(pageIds(page0)).doesNotContainAnyElementsOf(CANONICAL_VIEW_IDS); + assertThat(pageIds(page1)).doesNotContainAnyElementsOf(CANONICAL_VIEW_IDS); + } + + /** The pattern-listing call sites (plain and paged) apply the same predicate. */ + @Test + public void testPatternCallSitesFilterViewsPlainAndPaged() { + seedCanonicalRows("match_"); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, "nomatch_table", "TABLE")); + UserTable searchBy = UserTable.builder().databaseId(ENTITY_TYPE_DB).tableId("match_%").build(); + + assertThat(sortedIds(userTablesService.getAllUserTables(searchBy))) + .containsExactly( + "match_t00_legacy", "match_t02_explicit", "match_t04_legacy", "match_t06_explicit"); + + Page page0 = userTablesService.getAllUserTables(searchBy, 0, 2, "tableId"); + Assertions.assertEquals(4, page0.getTotalElements()); + Assertions.assertEquals(2, page0.getTotalPages()); + assertThat(pageIds(page0)).containsExactly("match_t00_legacy", "match_t02_explicit"); + + Page page1 = userTablesService.getAllUserTables(searchBy, 1, 2, "tableId"); + Assertions.assertEquals(4, page1.getTotalElements()); + Assertions.assertEquals(2, page1.getTotalPages()); + assertThat(pageIds(page1)).containsExactly("match_t04_legacy", "match_t06_explicit"); + } + + /** + * Pins the routing predicate. The request carries only {@code databaseId} plus {@code + * entityType=VIEW} and no other filter, so if {@code isNonKeyFieldsNullForUserTable} is not + * extended to consider entityType, this request is classified as a plain "list tables" request + * and returns the four tables instead of the three views. + */ + @Test + public void testGeneralSearchHonorsEntityType() { + seedCanonicalRows(""); + + List views = + userTablesService.getAllUserTables( + UserTable.builder().databaseId(ENTITY_TYPE_DB).entityType("VIEW").build()); + assertThat(sortedIds(views)).isEqualTo(CANONICAL_VIEW_IDS); + + for (String tableSpelling : new String[] {"TABLE", "table", "TaBlE"}) { + List tables = + userTablesService.getAllUserTables( + UserTable.builder().databaseId(ENTITY_TYPE_DB).entityType(tableSpelling).build()); + assertThat(sortedIds(tables)) + .as("entityType=%s must resolve to the four visible tables", tableSpelling) + .isEqualTo(CANONICAL_TABLE_IDS); + } + + // Default (no entityType) still means tables. + assertThat( + sortedIds( + userTablesService.getAllUserTables( + UserTable.builder().databaseId(ENTITY_TYPE_DB).build()))) + .isEqualTo(CANONICAL_TABLE_IDS); + + // Paged entityType-only VIEW request routes the same way. + Page viewPage = + userTablesService.getAllUserTables( + UserTable.builder().databaseId(ENTITY_TYPE_DB).entityType("VIEW").build(), + 0, + 2, + "tableId"); + Assertions.assertEquals(3, viewPage.getTotalElements()); + Assertions.assertEquals(2, viewPage.getTotalPages()); + assertThat(pageIds(viewPage)).containsExactly("t01_view", "t03_view"); + } + + /** + * Defense in depth for the shared key space: if a rename ever reaches the HTS storage layer with + * an occupied destination, the primary-key violation must roll back cleanly and leave BOTH JPA + * rows byte-identical — same numeric {@code version}, {@code metadataLocation} and {@code + * entityType}. The table-service tests prove correct code never reaches this fallback; this pins + * that the fallback itself is non-mutating. + */ + @Test + public void testRenameCollisionLeavesJPARowsUnchanged() { + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, "rename_src_table", "TABLE")); + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, "rename_dst_view", "VIEW")); + + UserTableRow sourceBefore = findRow(ENTITY_TYPE_DB, "rename_src_table"); + UserTableRow destinationBefore = findRow(ENTITY_TYPE_DB, "rename_dst_view"); + + Assertions.assertThrows( + AlreadyExistsException.class, + () -> + userTablesService.renameUserTable( + ENTITY_TYPE_DB, + "rename_src_table", + ENTITY_TYPE_DB, + "rename_dst_view", + "/openhouse/entity_type_db/rename_dst_view/v1_metadata.json")); + + UserTableRow sourceAfter = findRow(ENTITY_TYPE_DB, "rename_src_table"); + UserTableRow destinationAfter = findRow(ENTITY_TYPE_DB, "rename_dst_view"); + + Assertions.assertEquals(sourceBefore.getVersion(), sourceAfter.getVersion()); + Assertions.assertEquals(sourceBefore.getMetadataLocation(), sourceAfter.getMetadataLocation()); + Assertions.assertEquals(sourceBefore.getEntityType(), sourceAfter.getEntityType()); + + Assertions.assertEquals(destinationBefore.getVersion(), destinationAfter.getVersion()); + Assertions.assertEquals( + destinationBefore.getMetadataLocation(), destinationAfter.getMetadataLocation()); + Assertions.assertEquals("VIEW", destinationAfter.getEntityType()); + } + + private UserTableRow findRow(String databaseId, String tableId) { + return htsRepository + .findById(UserTableRowPrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()) + .orElseThrow( + () -> new AssertionError("Expected row " + databaseId + "." + tableId + " to exist")); + } } diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/api/OpenHouseUserTablesValidatorTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/api/OpenHouseUserTablesValidatorTest.java index 53cfb3d95..b0c087774 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/api/OpenHouseUserTablesValidatorTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/api/OpenHouseUserTablesValidatorTest.java @@ -147,4 +147,99 @@ public void validateRenameEntityInvalidInput() { RequestValidationFailureException.class, () -> userTablesHtsApiValidator.validateRenameEntity(fromKey, toKey)); } + + /** + * A type-qualified query must reach the repository. NOTE: {@code validateUserTable} only rejects + * non-null tableVersion/metadataLocation/storageType/creationTime, so this case passes whether or + * not entityType validation exists. It guards against a future change that adds entityType to + * that unsupported-field list; the load-bearing assertions for entityType validation live in + * {@link #validateEntityTypeQueryRejectsGarbage} and {@link + * #validatePutEntityTypeCaseInsensitivelyAndRejectsGarbage}. + */ + @Test + public void validateEntityTypeOnlyQueriesCaseInsensitively() { + for (String entityType : new String[] {"VIEW", "view", "ViEw", "TABLE", "table", "TaBlE"}) { + UserTable userTable = UserTable.builder().databaseId("db1").entityType(entityType).build(); + + assertDoesNotThrow( + () -> userTablesHtsApiValidator.validateGetEntities(userTable), + "entityType=" + entityType + " should be an accepted unpaged query filter"); + assertDoesNotThrow( + () -> userTablesHtsApiValidator.validateGetEntities(userTable, 0, 2, "tableId"), + "entityType=" + entityType + " should be an accepted paged query filter"); + } + } + + /** + * Load-bearing: an unknown discriminator must be rejected before it ever reaches the repository, + * so callers get a validation error rather than a silently empty result set. + */ + @Test + public void validateEntityTypeQueryRejectsGarbage() { + UserTable garbage = UserTable.builder().databaseId("db1").entityType("UNKNOWN").build(); + + assertThrows( + RequestValidationFailureException.class, + () -> userTablesHtsApiValidator.validateGetEntities(garbage)); + assertThrows( + RequestValidationFailureException.class, + () -> userTablesHtsApiValidator.validateGetEntities(garbage, 0, 2, "tableId")); + + // The pre-existing unsupported-field rejection must not be weakened by adding entityType as + // a permitted filter. + UserTable unsupportedField = UserTable.builder().creationTime(1L).build(); + assertThrows( + RequestValidationFailureException.class, + () -> userTablesHtsApiValidator.validateGetEntities(unsupportedField)); + UserTable unsupportedFieldWithEntityType = + UserTable.builder().databaseId("db1").entityType("VIEW").creationTime(1L).build(); + assertThrows( + RequestValidationFailureException.class, + () -> userTablesHtsApiValidator.validateGetEntities(unsupportedFieldWithEntityType)); + } + + /** + * Load-bearing: the PUT path relies on Bean Validation on the transport model, so the pattern on + * {@code UserTable#entityType} must accept every TABLE/VIEW spelling and reject anything else. + */ + @Test + public void validatePutEntityTypeCaseInsensitivelyAndRejectsGarbage() { + for (String entityType : new String[] {"VIEW", "view", "ViEw", "TABLE", "table", "TaBlE"}) { + UserTable userTable = + UserTable.builder() + .tableId("tb1") + .databaseId("db1") + .tableVersion("/tmp/test/opt/metadata.json") + .metadataLocation("INITIAL_VERSION") + .entityType(entityType) + .build(); + + assertDoesNotThrow( + () -> userTablesHtsApiValidator.validatePutEntity(userTable), + "PUT with entityType=" + entityType + " should validate"); + } + + // Omitting the field entirely stays valid (legacy table writers). + assertDoesNotThrow( + () -> + userTablesHtsApiValidator.validatePutEntity( + UserTable.builder() + .tableId("tb1") + .databaseId("db1") + .tableVersion("/tmp/test/opt/metadata.json") + .metadataLocation("INITIAL_VERSION") + .build())); + + UserTable garbage = + UserTable.builder() + .tableId("tb1") + .databaseId("db1") + .tableVersion("/tmp/test/opt/metadata.json") + .metadataLocation("INITIAL_VERSION") + .entityType("UNKNOWN") + .build(); + assertThrows( + RequestValidationFailureException.class, + () -> userTablesHtsApiValidator.validatePutEntity(garbage)); + } } diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/mapper/UserTablesMapperTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/mapper/UserTablesMapperTest.java index 42103148b..0d6ba1a2a 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/mapper/UserTablesMapperTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/mapper/UserTablesMapperTest.java @@ -1,9 +1,12 @@ package com.linkedin.openhouse.housetables.mock.mapper; +import com.linkedin.openhouse.housetables.api.spec.model.UserTable; import com.linkedin.openhouse.housetables.dto.mapper.UserTablesMapper; import com.linkedin.openhouse.housetables.dto.model.UserTableDto; import com.linkedin.openhouse.housetables.model.TestHouseTableModelConstants; import com.linkedin.openhouse.housetables.model.UserTableRow; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -64,4 +67,78 @@ void fromUserTable() { TestHouseTableModelConstants.TEST_USER_TABLE_DTO, userTablesMapper.fromUserTable(TestHouseTableModelConstants.TEST_USER_TABLE)); } + + /** + * The entityType discriminator must survive every hop of the HTS mapping chain (API -> JPA row -> + * DTO -> API). A mapper that silently drops it would let a view pointer be persisted as a legacy + * table row. + */ + @Test + void entityTypeRoundTripsAcrossUserTableRowDtoAndApi() { + UserTable viewUserTable = + TestHouseTableModelConstants.TEST_USER_TABLE.toBuilder().entityType("VIEW").build(); + + UserTableRow row = userTablesMapper.toUserTableRow(viewUserTable, Optional.empty()); + Assertions.assertEquals("VIEW", row.getEntityType()); + + UserTableDto dto = userTablesMapper.toUserTableDto(row); + Assertions.assertEquals("VIEW", dto.getEntityType()); + + UserTable roundTripped = userTablesMapper.toUserTable(dto); + Assertions.assertEquals("VIEW", roundTripped.getEntityType()); + + // Every other field is untouched by the new discriminator. + Assertions.assertEquals(viewUserTable.getTableId(), roundTripped.getTableId()); + Assertions.assertEquals(viewUserTable.getDatabaseId(), roundTripped.getDatabaseId()); + Assertions.assertEquals( + viewUserTable.getMetadataLocation(), roundTripped.getMetadataLocation()); + Assertions.assertEquals(viewUserTable.getStorageType(), roundTripped.getStorageType()); + Assertions.assertEquals(viewUserTable.getCreationTime(), roundTripped.getCreationTime()); + + // fromUserTable is the other API -> DTO direction and must carry it too. + Assertions.assertEquals("VIEW", userTablesMapper.fromUserTable(viewUserTable).getEntityType()); + } + + /** + * Backward compatibility: legacy writers omit the field entirely. No layer may default it to + * "TABLE", because that would start stamping a value on every existing table write and mask the + * null-means-table compatibility contract. + */ + @Test + void nullEntityTypeRemainsNullAcrossLegacyMappings() { + UserTable legacyUserTable = TestHouseTableModelConstants.TEST_USER_TABLE; + Assertions.assertNull(legacyUserTable.getEntityType()); + + UserTableRow row = userTablesMapper.toUserTableRow(legacyUserTable, Optional.empty()); + Assertions.assertNull(row.getEntityType()); + + UserTableDto dto = userTablesMapper.toUserTableDto(row); + Assertions.assertNull(dto.getEntityType()); + + Assertions.assertNull(userTablesMapper.toUserTable(dto).getEntityType()); + Assertions.assertNull(userTablesMapper.fromUserTable(legacyUserTable).getEntityType()); + + // The legacy fixture row (built without the field) must still map cleanly. + UserTableRow legacyRow = new TestHouseTableModelConstants.TestTuple(0).get_userTableRow(); + Assertions.assertNull(legacyRow.getEntityType()); + Assertions.assertNull(userTablesMapper.toUserTableDto(legacyRow).getEntityType()); + } + + /** + * The /hts query endpoint hands raw request parameters to {@code mapToUserTable}. If entityType + * is not recognized there, an {@code entityType=VIEW} query silently degrades to an unfiltered + * table listing. + */ + @Test + void mapToUserTableRecognizesEntityType() { + Map parameters = new HashMap<>(); + parameters.put("databaseId", "test_db0"); + parameters.put("entityType", "VIEW"); + + UserTable mapped = userTablesMapper.mapToUserTable(parameters); + + Assertions.assertEquals("test_db0", mapped.getDatabaseId()); + Assertions.assertEquals("VIEW", mapped.getEntityType()); + Assertions.assertNull(mapped.getTableId()); + } } diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/OpenHouseInternalRepository.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/OpenHouseInternalRepository.java index d44d5dedf..166628592 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/OpenHouseInternalRepository.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/OpenHouseInternalRepository.java @@ -28,6 +28,17 @@ public interface OpenHouseInternalRepository */ Optional findTableRefById(TableDtoPrimaryKey tableDtoPrimaryKey); + /** + * Name occupancy, not table existence: unlike {@link #findById}/{@link #findTableRefById} this + * sees every pointer row and never parses metadata.json. Empty means no row exists; a null or + * {@code TABLE} discriminator returns {@code "TABLE"}, and an unrecognized value is returned as + * stored so an occupied name fails closed. + * + *

HTS errors must propagate — swallowing them into an empty result would read as "this name is + * free" and let a CREATE clobber an existing view. + */ + Optional findOccupyingEntityTypeById(TableDtoPrimaryKey tableDtoPrimaryKey); + List findAllIds(); Page findAllIds(Pageable pageable); diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImpl.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImpl.java index e894e10f5..1bd36c6fe 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImpl.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImpl.java @@ -22,6 +22,8 @@ import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; import com.linkedin.openhouse.internal.catalog.SnapshotsUtil; import com.linkedin.openhouse.internal.catalog.fileio.FileIOManager; +import com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils; +import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTableDto; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTablePrimaryKey; import com.linkedin.openhouse.tables.api.spec.v0.request.components.Policies; @@ -799,13 +801,10 @@ public Optional findById(TableDtoPrimaryKey tableDtoPrimaryKey) { @Override public Optional findTableRefById(TableDtoPrimaryKey tableDtoPrimaryKey) { - if (!(catalog instanceof OpenHouseInternalCatalog)) { - throw new UnsupportedOperationException( - "findTableRefById is not supported for catalog type: " + catalog.getClass().getName()); - } - return ((OpenHouseInternalCatalog) catalog) - .findHouseTable( - TableIdentifier.of(tableDtoPrimaryKey.getDatabaseId(), tableDtoPrimaryKey.getTableId())) + // Backs table-only operations, notably drop, which avoids loadTable so it survives corrupted + // metadata. That bypass is why the discriminator must be filtered explicitly here. + return findRawPointerById(tableDtoPrimaryKey) + .filter(houseTable -> HouseTableSerdeUtils.isTableEntityType(houseTable.getEntityType())) .map( houseTable -> TableDto.builder() @@ -816,6 +815,40 @@ public Optional findTableRefById(TableDtoPrimaryKey tableDtoPrimaryKey .build()); } + @Override + public Optional findOccupyingEntityTypeById(TableDtoPrimaryKey tableDtoPrimaryKey) { + // Unlike findTableRefById, this must see EVERY raw pointer: a name taken by a view or by an + // unrecognized type is still taken. Repository errors intentionally propagate. + return findRawPointerById(tableDtoPrimaryKey) + .map( + houseTable -> { + String entityType = houseTable.getEntityType(); + if (HouseTableSerdeUtils.isTableEntityType(entityType)) { + return HouseTableSerdeUtils.TABLE_ENTITY_TYPE; + } + if (HouseTableSerdeUtils.isViewEntityType(entityType)) { + return HouseTableSerdeUtils.VIEW_ENTITY_TYPE; + } + return entityType; + }); + } + + /** + * Single raw pointer lookup shared by the two public projections above, so the "can this be + * loaded as a table?" and "is this name taken?" answers cannot drift apart. Never calls + * loadTable. + */ + private Optional findRawPointerById(TableDtoPrimaryKey tableDtoPrimaryKey) { + if (!(catalog instanceof OpenHouseInternalCatalog)) { + throw new UnsupportedOperationException( + "Raw pointer lookup is not supported for catalog type: " + catalog.getClass().getName()); + } + return ((OpenHouseInternalCatalog) catalog) + .findHouseTable( + TableIdentifier.of( + tableDtoPrimaryKey.getDatabaseId(), tableDtoPrimaryKey.getTableId())); + } + // FIXME: Likely need a cache layer to avoid expensive tableScan. @Timed(metricKey = MetricsConstant.REPO_TABLE_EXISTS_TIME) @Override diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java index 89e48225f..eb63ed397 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java @@ -9,6 +9,7 @@ import com.linkedin.openhouse.common.exception.OpenHouseCommitStateUnknownException; import com.linkedin.openhouse.common.exception.RequestValidationFailureException; import com.linkedin.openhouse.common.exception.UnsupportedClientOperationException; +import com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTableDto; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTablePrimaryKey; import com.linkedin.openhouse.tables.api.spec.v0.request.CreateUpdateLockRequestBody; @@ -105,9 +106,14 @@ public Pair putTable( String databaseId = createUpdateTableRequestBody.getDatabaseId(); String tableId = createUpdateTableRequestBody.getTableId(); - Optional tableDto = - openHouseInternalRepository.findById( - TableDtoPrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()); + TableDtoPrimaryKey tableDtoPrimaryKey = + TableDtoPrimaryKey.builder().databaseId(databaseId).tableId(tableId).build(); + + // The typed load below hides non-table rows, so without this preflight a CREATE at a view's + // name would look free and fail only after writing a candidate metadata.json. + rejectNonTableNameOccupancy(tableDtoPrimaryKey); + + Optional tableDto = openHouseInternalRepository.findById(tableDtoPrimaryKey); // Special case handling if (tableDto.isPresent() && createUpdateTableRequestBody.isStageReplace()) { @@ -212,6 +218,29 @@ private boolean updateNeeded( return !tablesMapper.toTableDto(existingTableDto, requestBody).equals(existingTableDto); } + /** + * {@code TABLE} occupancy returns normally so that {@code failOnExist=false} updates still work + * and the existing table-collision handling downstream owns that message. + */ + private void rejectNonTableNameOccupancy(TableDtoPrimaryKey key) { + Optional occupyingEntityType = + openHouseInternalRepository.findOccupyingEntityTypeById(key); + if (!occupyingEntityType.isPresent()) { + return; + } + String entityType = occupyingEntityType.get(); + if (HouseTableSerdeUtils.TABLE_ENTITY_TYPE.equals(entityType)) { + return; + } + String qualifiedName = String.format("%s.%s", key.getDatabaseId(), key.getTableId()); + String reason = + HouseTableSerdeUtils.VIEW_ENTITY_TYPE.equals(entityType) + ? "is occupied by a view" + : String.format("is occupied by a catalog object of type %s", entityType); + throw new AlreadyExistsException( + "Table", qualifiedName, String.format("Table name %s %s", qualifiedName, reason), null); + } + @Override public void deleteTable(String databaseId, String tableId, String actingPrincipal) { TableDtoPrimaryKey tableDtoPrimaryKey = @@ -244,6 +273,13 @@ public void renameTable( throw new NoSuchUserTableException(fromDatabaseId, fromTableId); } + // Check raw destination occupancy after the source is known to exist, but before + // the typed destination load (which hides views), the lock check, all authorization, and any + // mutation. A TABLE destination continues through the existing collision check below so its + // message and behavior are unchanged. + rejectNonTableNameOccupancy( + TableDtoPrimaryKey.builder().databaseId(toDatabaseId).tableId(toTableId).build()); + Optional targetedTableDto = openHouseInternalRepository.findById( TableDtoPrimaryKey.builder().databaseId(toDatabaseId).tableId(toTableId).build()); diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/DatabasesControllerTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/DatabasesControllerTest.java index c12fbdfc0..d034986d2 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/DatabasesControllerTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/DatabasesControllerTest.java @@ -4,10 +4,14 @@ import static com.linkedin.openhouse.tables.model.DatabaseModelConstants.GET_DATABASE_RESPONSE_BODY; import static com.linkedin.openhouse.tables.model.DatabaseModelConstants.GET_DATABASE_RESPONSE_BODY_DIFF_DB; import static com.linkedin.openhouse.tables.model.TableModelConstants.*; +import static org.hamcrest.Matchers.*; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; import com.linkedin.openhouse.cluster.storage.StorageManager; import com.linkedin.openhouse.common.test.cluster.PropertyOverrideContextInitializer; +import com.linkedin.openhouse.internal.catalog.model.HouseTable; +import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; +import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; import com.linkedin.openhouse.tables.api.spec.v0.response.GetAllDatabasesResponseBody; import com.linkedin.openhouse.tables.api.spec.v0.response.GetDatabaseResponseBody; import com.linkedin.openhouse.tables.api.spec.v0.response.GetTableResponseBody; @@ -16,9 +20,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.iceberg.catalog.Catalog; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; @@ -185,4 +192,136 @@ private void cleanUpHelper(TestInfo info) { log.warn("Cleaning up process interrupted with exception: {}", exception); } } + + // --------------------------------------------------------------------------------------------- + // A view-only database must not appear in the database listing + // --------------------------------------------------------------------------------------------- + + /** + * Raw pointer rows must be seeded through the pointer repository directly, because a VIEW row is + * invisible to the table HTTP API and therefore cannot be created or cleaned up through it. Every + * seeded key is removed in {@link #deleteSeededPointers()}. + */ + @Autowired HouseTableRepository houseTablesRepository; + + private final List seededPointerKeys = new ArrayList<>(); + + @AfterEach + void deleteSeededPointers() { + for (HouseTablePrimaryKey key : seededPointerKeys) { + try { + houseTablesRepository.deleteById(key); + } catch (Exception e) { + log.warn("Failed to clean up raw pointer {}: {}", key.getTableId(), e.toString()); + } + } + seededPointerKeys.clear(); + } + + private void seedRawPointer(String databaseId, String tableId, String entityType) { + houseTablesRepository.save( + HouseTable.builder() + .databaseId(databaseId) + .tableId(tableId) + .clusterId("test-cluster") + .tableUri(String.format("test-cluster.%s.%s", databaseId, tableId)) + .tableUUID(UUID.randomUUID().toString()) + .tableLocation( + String.format("/base/%s/%s-uuid/00001-x.metadata.json", databaseId, tableId)) + .tableVersion("INITIAL_VERSION") + .entityType(entityType) + .build()); + seededPointerKeys.add( + HouseTablePrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()); + } + + /** + * Canonical database fixture: seven databases with exactly one pointer each; three of them hold + * only a view. Only the four table databases may be listed. + */ + private void seedCanonicalDatabases() { + seedRawPointer("db00_legacy", "t1", null); + seedRawPointer("db01_view_only", "t1", "VIEW"); + seedRawPointer("db02_explicit", "t1", "TABLE"); + seedRawPointer("db03_view_only", "t1", "VIEW"); + seedRawPointer("db04_legacy", "t1", null); + seedRawPointer("db05_view_only", "t1", "VIEW"); + seedRawPointer("db06_explicit", "t1", "TABLE"); + } + + /** + * The two database-listing tests below assert a GLOBAL result count, so a row leaked by another + * method in this class would make them fail for an unrelated reason. Asserting the precondition + * up front keeps that failure diagnosable as leakage rather than as a filtering bug. + */ + private void assertPointerTableIsEmpty() { + List existing = new ArrayList<>(); + houseTablesRepository.findAll().forEach(existing::add); + Assertions.assertTrue( + existing.isEmpty(), + "This test asserts a global database count and requires a clean pointer table; " + + "a previous test leaked rows: " + + existing.stream() + .map(h -> h.getDatabaseId() + "." + h.getTableId()) + .collect(Collectors.toList())); + } + + @Test + public void testGetAllDatabasesExcludesViewOnlyDatabases() throws Exception { + assertPointerTableIsEmpty(); + seedCanonicalDatabases(); + + mvc.perform( + MockMvcRequestBuilders.get(CURRENT_MAJOR_VERSION_PREFIX + "/databases") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.results", hasSize(4))) + .andExpect( + jsonPath( + "$.results[*].databaseId", + containsInAnyOrder("db00_legacy", "db02_explicit", "db04_legacy", "db06_explicit"))) + .andExpect(jsonPath("$.results[*].databaseId", not(hasItem("db01_view_only")))) + .andExpect(jsonPath("$.results[*].databaseId", not(hasItem("db03_view_only")))) + .andExpect(jsonPath("$.results[*].databaseId", not(hasItem("db05_view_only")))); + } + + /** + * Anti-post-filter assertion for the paginated database listing: filtering the returned page + * would report totalElements=7/totalPages=4 with a 1-row first page. + */ + @Test + public void testGetAllDatabasesFiltersBeforePagination() throws Exception { + assertPointerTableIsEmpty(); + seedCanonicalDatabases(); + + mvc.perform( + MockMvcRequestBuilders.get("/v2/databases") + .param("page", "0") + .param("size", "2") + .param("sortBy", "databaseId") + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].databaseId", is("db00_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].databaseId", is("db02_explicit"))); + + mvc.perform( + MockMvcRequestBuilders.get("/v2/databases") + .param("page", "1") + .param("size", "2") + .param("sortBy", "databaseId") + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].databaseId", is("db04_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].databaseId", is("db06_explicit"))); + } } diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java index 4ace198dd..1e08c354d 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java @@ -10,6 +10,7 @@ import com.linkedin.openhouse.common.exception.UnsupportedClientOperationException; import com.linkedin.openhouse.common.test.cluster.PropertyOverrideContextInitializer; import com.linkedin.openhouse.internal.catalog.CatalogConstants; +import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; @@ -29,12 +30,15 @@ import com.linkedin.openhouse.tables.repository.impl.InternalRepositoryUtils; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; +import java.util.UUID; import java.util.stream.Collectors; import org.apache.iceberg.BaseTable; import org.apache.iceberg.Schema; @@ -43,6 +47,7 @@ import org.apache.iceberg.TableProperties; import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NoSuchTableException; @@ -54,6 +59,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.util.AopTestUtils; @@ -1236,6 +1245,12 @@ public void testRenameTablePreserveExistingCase() { Assertions.assertEquals( renamedTable.get().getTableProperties().get("openhouse.tableUri"), "local-cluster.d1.t1_renamed"); + + // The rename destination is now guarded: an occupied destination pointer is a collision rather + // than something a later rename silently overwrites. Leaving d1.t1_renamed behind would + // therefore collide with other tests in this class, which share one Spring context. + openHouseInternalRepository.deleteById( + TableDtoPrimaryKey.builder().databaseId("d1").tableId("t1_renamed").build()); } @Test @@ -1592,4 +1607,343 @@ private void verifyTable(HouseTable table) { table.getTableId() + "-" + table.getTableUUID()); Assertions.assertTrue(table.getTableLocation().startsWith(path.toString())); } + + // --------------------------------------------------------------------------------------------- + // Table listings must exclude views in the query, never by post-filtering a Page + // --------------------------------------------------------------------------------------------- + + /** + * Canonical interleaved fixture: four visible tables (two legacy NULL, two explicit TABLE) + * interleaved with three VIEW rows. A fetch-then-filter implementation returns a SHORT first page + * (1 row) with totalElements=7/totalPages=4; the correct pre-pagination predicate returns a full + * 2-row page with totalElements=4/totalPages=2. + */ + private static final String ENTITY_TYPE_DB = "entity_type_db"; + + private static final List CANONICAL_TABLE_IDS = + Arrays.asList("t00_legacy", "t02_explicit", "t04_legacy", "t06_explicit"); + + private static final List CANONICAL_VIEW_IDS = + Arrays.asList("t01_view", "t03_view", "t05_view"); + + private static final String CASE_DB = "entity_type_case_db"; + + private HouseTable rawPointer(String databaseId, String tableId, String entityType) { + return HouseTable.builder() + .databaseId(databaseId) + .tableId(tableId) + .clusterId("test-cluster") + .tableUri(String.format("test-cluster.%s.%s", databaseId, tableId)) + .tableUUID(UUID.randomUUID().toString()) + .tableLocation(String.format("/base/%s/%s-uuid/00001-x.metadata.json", databaseId, tableId)) + .tableVersion(INITIAL_TABLE_VERSION) + .entityType(entityType) + .build(); + } + + /** Seeds raw pointer rows and returns their keys so the caller can delete them in a finally. */ + private List seedRawPointers(String databaseId, String[][] idAndType) { + List keys = new ArrayList<>(); + for (String[] entry : idAndType) { + houseTablesRepository.save(rawPointer(databaseId, entry[0], entry[1])); + keys.add(HouseTablePrimaryKey.builder().databaseId(databaseId).tableId(entry[0]).build()); + } + return keys; + } + + private List seedCanonicalPointers(String databaseId) { + return seedRawPointers( + databaseId, + new String[][] { + {"t00_legacy", null}, + {"t01_view", "VIEW"}, + {"t02_explicit", "TABLE"}, + {"t03_view", "VIEW"}, + {"t04_legacy", null}, + {"t05_view", "VIEW"}, + {"t06_explicit", "TABLE"} + }); + } + + private List seedCaseNormalizationPointers(String databaseId) { + return seedRawPointers( + databaseId, + new String[][] { + {"case00_null", null}, + {"case01_upper_table", "TABLE"}, + {"case02_lower_table", "table"}, + {"case03_mixed_table", "TaBlE"}, + {"case04_upper_view", "VIEW"}, + {"case05_lower_view", "view"}, + {"case06_mixed_view", "ViEw"}, + {"case07_garbage", "UNKNOWN"} + }); + } + + /** + * Raw pointer rows are invisible to the table APIs by design, so no table-API cleanup can remove + * them. Every test that seeds them MUST delete them explicitly, otherwise later tests in this + * class (which asserts exact database/table sets, and shares one Spring context across methods) + * are polluted. + */ + private void deleteRawPointers(List keys) { + for (HouseTablePrimaryKey key : keys) { + try { + houseTablesRepository.deleteById(key); + } catch (Exception e) { + // Best effort: a missing row must not mask the real assertion failure. + } + } + } + + private static OpenHouseInternalCatalog openHouseCatalog(Catalog catalog) { + return (OpenHouseInternalCatalog) AopTestUtils.getUltimateTargetObject(catalog); + } + + private static List identifierNames(List identifiers) { + return identifiers.stream().map(TableIdentifier::name).sorted().collect(Collectors.toList()); + } + + private static Pageable sortedPage(int page) { + return PageRequest.of(page, 2, Sort.by("tableId")); + } + + /** SHOW TABLES contract: a view never appears in the catalog's table listing. */ + @Test + public void testCatalogListTablesExcludesViewsAndKeepsNullAndTable() { + List keys = seedCanonicalPointers(ENTITY_TYPE_DB); + try { + List identifiers = catalog.listTables(Namespace.of(ENTITY_TYPE_DB)); + + Assertions.assertEquals(CANONICAL_TABLE_IDS, identifierNames(identifiers)); + Assertions.assertTrue( + identifierNames(identifiers).stream().noneMatch(CANONICAL_VIEW_IDS::contains), + "No VIEW row may appear in SHOW TABLES: " + identifierNames(identifiers)); + } finally { + deleteRawPointers(keys); + } + } + + /** Anti-post-filter assertion for the paginated catalog listing overload. */ + @Test + public void testCatalogListTablesFiltersBeforePagination() { + List keys = seedCanonicalPointers(ENTITY_TYPE_DB); + try { + OpenHouseInternalCatalog ohCatalog = openHouseCatalog(catalog); + + Page page0 = + ohCatalog.listTables(Namespace.of(ENTITY_TYPE_DB), sortedPage(0)); + Assertions.assertEquals(4, page0.getTotalElements()); + Assertions.assertEquals(2, page0.getTotalPages()); + Assertions.assertEquals(2, page0.getContent().size()); + Assertions.assertEquals( + Arrays.asList("t00_legacy", "t02_explicit"), + page0.getContent().stream().map(TableIdentifier::name).collect(Collectors.toList())); + + Page page1 = + ohCatalog.listTables(Namespace.of(ENTITY_TYPE_DB), sortedPage(1)); + Assertions.assertEquals(4, page1.getTotalElements()); + Assertions.assertEquals(2, page1.getTotalPages()); + Assertions.assertEquals(2, page1.getContent().size()); + Assertions.assertEquals( + Arrays.asList("t04_legacy", "t06_explicit"), + page1.getContent().stream().map(TableIdentifier::name).collect(Collectors.toList())); + } finally { + deleteRawPointers(keys); + } + } + + /** Anti-post-filter assertion for the HouseTable-preserving paginated listing. */ + @Test + public void testListHouseTablesFiltersBeforePagination() { + List keys = seedCanonicalPointers(ENTITY_TYPE_DB); + try { + OpenHouseInternalCatalog ohCatalog = openHouseCatalog(catalog); + + Page page0 = + ohCatalog.listHouseTables(Namespace.of(ENTITY_TYPE_DB), sortedPage(0)); + Assertions.assertEquals(4, page0.getTotalElements()); + Assertions.assertEquals(2, page0.getTotalPages()); + Assertions.assertEquals(2, page0.getContent().size()); + Assertions.assertEquals( + Arrays.asList("t00_legacy", "t02_explicit"), + page0.getContent().stream().map(HouseTable::getTableId).collect(Collectors.toList())); + Assertions.assertTrue( + page0.getContent().stream().noneMatch(h -> "VIEW".equalsIgnoreCase(h.getEntityType()))); + + Page page1 = + ohCatalog.listHouseTables(Namespace.of(ENTITY_TYPE_DB), sortedPage(1)); + Assertions.assertEquals(4, page1.getTotalElements()); + Assertions.assertEquals(2, page1.getTotalPages()); + Assertions.assertEquals(2, page1.getContent().size()); + Assertions.assertEquals( + Arrays.asList("t04_legacy", "t06_explicit"), + page1.getContent().stream().map(HouseTable::getTableId).collect(Collectors.toList())); + } finally { + deleteRawPointers(keys); + } + } + + /** All three {@code searchTables} overloads must filter identically and before paging. */ + @Test + public void testOpenHouseRepositorySearchTablesFiltersAllOverloads() { + List keys = seedCanonicalPointers(ENTITY_TYPE_DB); + try { + List plain = openHouseInternalRepository.searchTables(ENTITY_TYPE_DB); + Assertions.assertEquals( + CANONICAL_TABLE_IDS, + plain.stream().map(TableDto::getTableId).sorted().collect(Collectors.toList())); + + Page page0 = + openHouseInternalRepository.searchTables(ENTITY_TYPE_DB, sortedPage(0)); + Assertions.assertEquals(4, page0.getTotalElements()); + Assertions.assertEquals(2, page0.getTotalPages()); + Assertions.assertEquals( + Arrays.asList("t00_legacy", "t02_explicit"), + page0.getContent().stream().map(TableDto::getTableId).collect(Collectors.toList())); + + Page page1 = + openHouseInternalRepository.searchTables(ENTITY_TYPE_DB, sortedPage(1)); + Assertions.assertEquals(4, page1.getTotalElements()); + Assertions.assertEquals(2, page1.getTotalPages()); + Assertions.assertEquals( + Arrays.asList("t04_legacy", "t06_explicit"), + page1.getContent().stream().map(TableDto::getTableId).collect(Collectors.toList())); + + // The fields projection goes through listHouseTables, so it must filter identically and + // still populate the requested field. + Page fieldsPage0 = + openHouseInternalRepository.searchTables( + ENTITY_TYPE_DB, sortedPage(0), Collections.singletonList("tableLocation")); + Assertions.assertEquals(4, fieldsPage0.getTotalElements()); + Assertions.assertEquals(2, fieldsPage0.getTotalPages()); + Assertions.assertEquals( + Arrays.asList("t00_legacy", "t02_explicit"), + fieldsPage0.getContent().stream().map(TableDto::getTableId).collect(Collectors.toList())); + Assertions.assertTrue( + fieldsPage0.getContent().stream().allMatch(dto -> dto.getTableLocation() != null), + "fields=tableLocation must be projected for every returned table"); + } finally { + deleteRawPointers(keys); + } + } + + /** + * Database enumeration: a database whose only pointer is a view must disappear entirely. Note the + * global-scope precondition — {@code findAllIds} is not database-scoped, so this test asserts the + * pointer table is empty first to keep a failure here diagnosable as leakage rather than as a + * filtering bug. + */ + @Test + public void testFindAllIdsExcludesViewOnlyDatabases() { + Assertions.assertTrue( + Streams.stream(houseTablesRepository.findAll()).count() == 0, + "This test asserts global pointer counts and requires a clean pointer table; " + + "a previous test leaked rows"); + + List keys = new ArrayList<>(); + try { + keys.addAll(seedRawPointers("db00_legacy", new String[][] {{"t1", null}})); + keys.addAll(seedRawPointers("db01_view_only", new String[][] {{"t1", "VIEW"}})); + keys.addAll(seedRawPointers("db02_explicit", new String[][] {{"t1", "TABLE"}})); + keys.addAll(seedRawPointers("db03_view_only", new String[][] {{"t1", "VIEW"}})); + keys.addAll(seedRawPointers("db04_legacy", new String[][] {{"t1", null}})); + keys.addAll(seedRawPointers("db05_view_only", new String[][] {{"t1", "VIEW"}})); + keys.addAll(seedRawPointers("db06_explicit", new String[][] {{"t1", "TABLE"}})); + + List databaseIds = + openHouseInternalRepository.findAllIds().stream() + .map(TableDtoPrimaryKey::getDatabaseId) + .sorted() + .collect(Collectors.toList()); + Assertions.assertEquals( + Arrays.asList("db00_legacy", "db02_explicit", "db04_legacy", "db06_explicit"), + databaseIds); + + Pageable dbPage = PageRequest.of(0, 2, Sort.by("databaseId")); + Page page0 = openHouseInternalRepository.findAllIds(dbPage); + Assertions.assertEquals(4, page0.getTotalElements()); + Assertions.assertEquals(2, page0.getTotalPages()); + Assertions.assertEquals( + Arrays.asList("db00_legacy", "db02_explicit"), + page0.getContent().stream() + .map(TableDtoPrimaryKey::getDatabaseId) + .collect(Collectors.toList())); + + Page page1 = + openHouseInternalRepository.findAllIds(PageRequest.of(1, 2, Sort.by("databaseId"))); + Assertions.assertEquals(4, page1.getTotalElements()); + Assertions.assertEquals(2, page1.getTotalPages()); + Assertions.assertEquals( + Arrays.asList("db04_legacy", "db06_explicit"), + page1.getContent().stream() + .map(TableDtoPrimaryKey::getDatabaseId) + .collect(Collectors.toList())); + } finally { + deleteRawPointers(keys); + } + } + + /** + * Case/garbage matrix at the internal H2 query layer. + * + *

H2 (MODE=MySQL) is case-SENSITIVE while production MySQL's default collation is not, so this + * proves the query normalizes explicitly (e.g. {@code upper(h.entityType) = 'TABLE'}) rather than + * relying on the provider's collation — a bare {@code = 'TABLE'} comparison would hide the + * lower/mixed-case table rows here and fail. It does NOT certify production MySQL behavior; the + * authoritative case contract lives in the Java guards ({@code + * HouseTableTest#testEntityTypeClassification} and the catalog guard tests). + */ + @Test + public void testCaseInsensitiveTypePredicateAndGarbageFailClosed() { + List keys = seedCaseNormalizationPointers(CASE_DB); + try { + List expectedVisible = + Arrays.asList( + "case00_null", "case01_upper_table", "case02_lower_table", "case03_mixed_table"); + List expectedHidden = + Arrays.asList( + "case04_upper_view", "case05_lower_view", "case06_mixed_view", "case07_garbage"); + + List listed = identifierNames(catalog.listTables(Namespace.of(CASE_DB))); + Assertions.assertEquals(expectedVisible, listed); + Assertions.assertTrue( + listed.stream().noneMatch(expectedHidden::contains), + "Views (any spelling) and unknown types must fail closed out of SHOW TABLES: " + listed); + + OpenHouseInternalCatalog ohCatalog = openHouseCatalog(catalog); + + Page page0 = ohCatalog.listTables(Namespace.of(CASE_DB), sortedPage(0)); + Assertions.assertEquals(4, page0.getTotalElements()); + Assertions.assertEquals(2, page0.getTotalPages()); + Assertions.assertEquals( + Arrays.asList("case00_null", "case01_upper_table"), + page0.getContent().stream().map(TableIdentifier::name).collect(Collectors.toList())); + + Page housePage0 = ohCatalog.listHouseTables(Namespace.of(CASE_DB), sortedPage(0)); + Assertions.assertEquals(4, housePage0.getTotalElements()); + Assertions.assertEquals(2, housePage0.getTotalPages()); + Assertions.assertEquals( + Arrays.asList("case00_null", "case01_upper_table"), + housePage0.getContent().stream() + .map(HouseTable::getTableId) + .collect(Collectors.toList())); + + Page housePage1 = ohCatalog.listHouseTables(Namespace.of(CASE_DB), sortedPage(1)); + Assertions.assertEquals( + Arrays.asList("case02_lower_table", "case03_mixed_table"), + housePage1.getContent().stream() + .map(HouseTable::getTableId) + .collect(Collectors.toList())); + + // Hidden, not dropped: the raw rows are all still there. + for (HouseTablePrimaryKey key : keys) { + Assertions.assertTrue( + houseTablesRepository.findById(key).isPresent(), + "Raw pointer " + key.getTableId() + " must still exist; it is hidden, not deleted"); + } + } finally { + deleteRawPointers(keys); + } + } } diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java index 0e4b6d6cd..7e2432531 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java @@ -23,7 +23,9 @@ import com.linkedin.openhouse.housetables.client.model.ToggleStatus; import com.linkedin.openhouse.internal.catalog.CatalogConstants; import com.linkedin.openhouse.internal.catalog.model.HouseTable; +import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTablePrimaryKey; +import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; import com.linkedin.openhouse.tables.api.spec.v0.request.CreateUpdateLockRequestBody; import com.linkedin.openhouse.tables.api.spec.v0.request.CreateUpdateTableRequestBody; import com.linkedin.openhouse.tables.api.spec.v0.request.components.ClusteringColumn; @@ -57,6 +59,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import lombok.SneakyThrows; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -67,8 +70,11 @@ import org.apache.iceberg.hadoop.HadoopFileIO; import org.apache.iceberg.types.Types; import org.json.JSONObject; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mockito; @@ -2061,4 +2067,302 @@ private MvcResult getTable(String databaseId, String tableId) throws Exception { .andExpect(status().isOk()) .andReturn(); } + + // --------------------------------------------------------------------------------------------- + // View isolation and shared-key collisions over the table HTTP API + // --------------------------------------------------------------------------------------------- + + /** + * Raw pointer rows must be seeded through the pointer repository directly, because a VIEW row is + * invisible to the table HTTP API and therefore cannot be created — or cleaned up — through it. + * Every seeded key is removed in {@link #deleteSeededPointers()}. + */ + @Autowired HouseTableRepository houseTablesRepository; + + private final List seededPointerKeys = new ArrayList<>(); + + @AfterEach + void deleteSeededPointers() { + for (HouseTablePrimaryKey key : seededPointerKeys) { + try { + houseTablesRepository.deleteById(key); + } catch (Exception e) { + // Best effort: cleanup must not mask the real assertion failure. + } + } + seededPointerKeys.clear(); + } + + private static final String VIEW_MIX_DB = "viewmixdb"; + + private void seedRawPointer(String databaseId, String tableId, String entityType) { + houseTablesRepository.save( + HouseTable.builder() + .databaseId(databaseId) + .tableId(tableId) + .clusterId("test-cluster") + .tableUri(String.format("test-cluster.%s.%s", databaseId, tableId)) + .tableUUID(UUID.randomUUID().toString()) + .tableLocation( + String.format("/base/%s/%s-uuid/00001-x.metadata.json", databaseId, tableId)) + .tableVersion(INITIAL_TABLE_VERSION) + .entityType(entityType) + .build()); + seededPointerKeys.add( + HouseTablePrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()); + } + + /** + * Canonical interleaved fixture: four visible tables (two legacy NULL, two explicit TABLE) + * interleaved with three VIEW rows. A fetch-then-filter implementation returns a SHORT first page + * (1 row) with totalElements=7/totalPages=4. + */ + private void seedCanonicalPointers() { + seedRawPointer(VIEW_MIX_DB, "t00_legacy", null); + seedRawPointer(VIEW_MIX_DB, "t01_view", "VIEW"); + seedRawPointer(VIEW_MIX_DB, "t02_explicit", "TABLE"); + seedRawPointer(VIEW_MIX_DB, "t03_view", "VIEW"); + seedRawPointer(VIEW_MIX_DB, "t04_legacy", null); + seedRawPointer(VIEW_MIX_DB, "t05_view", "VIEW"); + seedRawPointer(VIEW_MIX_DB, "t06_explicit", "TABLE"); + } + + @Test + public void testSearchTablesExcludesInterleavedViews() throws Exception { + seedCanonicalPointers(); + + mvc.perform( + MockMvcRequestBuilders.post( + String.format( + ValidationUtilities.CURRENT_MAJOR_VERSION_PREFIX + + "/databases/%s/tables/search", + VIEW_MIX_DB)) + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.results", hasSize(4))) + .andExpect( + jsonPath( + "$.results[*].tableId", + containsInAnyOrder("t00_legacy", "t02_explicit", "t04_legacy", "t06_explicit"))) + .andExpect(jsonPath("$.results[*].tableId", not(hasItem("t01_view")))) + .andExpect(jsonPath("$.results[*].tableId", not(hasItem("t03_view")))) + .andExpect(jsonPath("$.results[*].tableId", not(hasItem("t05_view")))); + } + + @Test + public void testSearchTablesFiltersBeforePagination() throws Exception { + seedCanonicalPointers(); + + mvc.perform( + MockMvcRequestBuilders.post("/v2/databases/" + VIEW_MIX_DB + "/tables/search") + .param("page", "0") + .param("size", "2") + .param("sortBy", "tableId") + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t00_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t02_explicit"))); + + mvc.perform( + MockMvcRequestBuilders.post("/v2/databases/" + VIEW_MIX_DB + "/tables/search") + .param("page", "1") + .param("size", "2") + .param("sortBy", "tableId") + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t04_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t06_explicit"))); + } + + @Test + public void testSearchTablesWithFieldsFiltersBeforePagination() throws Exception { + seedCanonicalPointers(); + + mvc.perform( + MockMvcRequestBuilders.post("/v2/databases/" + VIEW_MIX_DB + "/tables/search") + .param("page", "0") + .param("size", "2") + .param("sortBy", "tableId") + .param("fields", "tableLocation") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t00_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t02_explicit"))) + .andExpect( + jsonPath( + "$.pageResults.content[0].tableLocation", + is("/base/" + VIEW_MIX_DB + "/t00_legacy-uuid/00001-x.metadata.json"))) + .andExpect( + jsonPath( + "$.pageResults.content[1].tableLocation", + is("/base/" + VIEW_MIX_DB + "/t02_explicit-uuid/00001-x.metadata.json"))); + + mvc.perform( + MockMvcRequestBuilders.post("/v2/databases/" + VIEW_MIX_DB + "/tables/search") + .param("page", "1") + .param("size", "2") + .param("sortBy", "tableId") + .param("fields", "tableLocation") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.pageResults.totalElements", is(4))) + .andExpect(jsonPath("$.pageResults.totalPages", is(2))) + .andExpect(jsonPath("$.pageResults.content", hasSize(2))) + .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t04_legacy"))) + .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t06_explicit"))); + } + + /** + * CREATE TABLE at a name already occupied by a view must be an accurate 409 with a message that + * names the real condition. A guard implemented only in the table {@code doRefresh} would let the + * create proceed all the way to the HTS publish boundary and surface a misleading concurrent + * modification error instead. + */ + @Test + public void testCreateTableOnViewNameReturnsTypedCollision() throws Exception { + seedRawPointer(VIEW_MIX_DB, "occupied_by_view", "VIEW"); + + GetTableResponseBody createBody = + buildGetTableResponseBodyWithDbTbl(VIEW_MIX_DB, "occupied_by_view"); + + mvc.perform( + MockMvcRequestBuilders.post( + String.format( + ValidationUtilities.CURRENT_MAJOR_VERSION_PREFIX + "/databases/%s/tables/", + VIEW_MIX_DB)) + .contentType(MediaType.APPLICATION_JSON) + .content( + buildCreateUpdateTableRequestBody(createBody) + .toBuilder() + .baseTableVersion(INITIAL_TABLE_VERSION) + .build() + .toJson()) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isConflict()) + .andExpect( + jsonPath( + "$.message", + is("Table name " + VIEW_MIX_DB + ".occupied_by_view is occupied by a view"))); + } + + /** Renaming a real table onto a view's name is the same accurate 409. */ + @Test + public void testRenameTableToViewNameReturnsTypedCollision() throws Exception { + GetTableResponseBody source = buildGetTableResponseBodyWithDbTbl(VIEW_MIX_DB, "rename_source"); + RequestAndValidateHelper.createTableAndValidateResponse(source, mvc, storageManager); + seedRawPointer(VIEW_MIX_DB, "rename_dest_view", "VIEW"); + + try { + mvc.perform( + MockMvcRequestBuilders.patch( + String.format( + ValidationUtilities.CURRENT_MAJOR_VERSION_PREFIX + + "/databases/%s/tables/%s/rename", + VIEW_MIX_DB, + "rename_source")) + .contentType(MediaType.APPLICATION_JSON) + .param("toTableId", "rename_dest_view") + .param("toDatabaseId", VIEW_MIX_DB) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isConflict()) + .andExpect( + jsonPath( + "$.message", + is("Table name " + VIEW_MIX_DB + ".rename_dest_view is occupied by a view"))); + + // The source table must still be there under its original name. + getTable(VIEW_MIX_DB, "rename_source"); + } finally { + RequestAndValidateHelper.deleteTableAndValidateResponse(mvc, source); + } + } + + /** + * HTTP contract for reading a view through the table API: 404, not 400. + * + *

The status code is the assertion, not the exception type. The Java/Spark client's {@code + * OpenHouseTableOperations.doRefresh} resumes as an empty {@code Optional} on both 404 + * and 400, so an implementation that surfaced a 400 (or a 200 with an empty body, or a 500 from + * an unguarded NPE) would look correct to every Spark/Java client while being wrong for the REST + * contract, curl, and the audit log. Only an explicit status assertion pins it. + * + *

UNKNOWN is included because an unrecognized discriminator must fail closed the same way, + * rather than being read as a legacy table. + */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + public void testGetTableOnViewNameReturnsNotFound(String entityType) throws Exception { + seedRawPointer(VIEW_MIX_DB, "read_as_table", entityType); + + mvc.perform( + MockMvcRequestBuilders.get( + String.format( + ValidationUtilities.CURRENT_MAJOR_VERSION_PREFIX + + "/databases/%s/tables/%s", + VIEW_MIX_DB, + "read_as_table")) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.status", is(equalTo(HttpStatus.NOT_FOUND.name())))) + .andExpect(jsonPath("$.error", is(equalTo(HttpStatus.NOT_FOUND.getReasonPhrase())))); + + // The pointer is hidden from the table API, not destroyed by reading it. + Assertions.assertTrue( + houseTablesRepository + .findById( + HouseTablePrimaryKey.builder() + .databaseId(VIEW_MIX_DB) + .tableId("read_as_table") + .build()) + .isPresent()); + } + + /** + * HTTP contract for dropping a view through the table API: 404, not 400, and the pointer + * plus its files survive. + * + *

This is the path where a {@code doRefresh}-only guard does nothing at all: {@code + * TablesServiceImpl.deleteTable} deliberately bypasses {@code loadTable} via {@code + * findTableRefById} so that drop still works on corrupted metadata. The guard therefore has to + * live in the table-ref projection, and this test is what proves it does. + */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + public void testDeleteTableOnViewNameReturnsNotFound(String entityType) throws Exception { + seedRawPointer(VIEW_MIX_DB, "drop_as_table", entityType); + + mvc.perform( + MockMvcRequestBuilders.delete( + String.format( + ValidationUtilities.CURRENT_MAJOR_VERSION_PREFIX + + "/databases/%s/tables/%s", + VIEW_MIX_DB, + "drop_as_table")) + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.status", is(equalTo(HttpStatus.NOT_FOUND.name())))) + .andExpect(jsonPath("$.error", is(equalTo(HttpStatus.NOT_FOUND.getReasonPhrase())))); + + Assertions.assertTrue( + houseTablesRepository + .findById( + HouseTablePrimaryKey.builder() + .databaseId(VIEW_MIX_DB) + .tableId("drop_as_table") + .build()) + .isPresent(), + "A rejected drop must leave the view pointer in place"); + } } diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java index 563162e61..114e3e92e 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java @@ -15,8 +15,10 @@ import com.linkedin.openhouse.common.test.schema.ResourceIoHelper; import com.linkedin.openhouse.internal.catalog.CatalogConstants; import com.linkedin.openhouse.internal.catalog.model.HouseTable; +import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTableDto; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTablePrimaryKey; +import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; import com.linkedin.openhouse.tables.api.spec.v0.request.CreateUpdateLockRequestBody; import com.linkedin.openhouse.tables.api.spec.v0.request.UpdateAclPoliciesRequestBody; import com.linkedin.openhouse.tables.api.spec.v0.request.components.TimePartitionSpec; @@ -28,19 +30,29 @@ import com.linkedin.openhouse.tables.repository.OpenHouseInternalRepository; import com.linkedin.openhouse.tables.services.TablesService; import com.linkedin.openhouse.tables.utils.AuthorizationUtils; +import java.io.File; import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.iceberg.Schema; import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -1017,4 +1029,314 @@ public void testRestoreTableNotFound() { tablesService.restoreTable( nonExistentDbId, "nonexistent_table", deletedAtMs, TEST_USER)); } + + // --------------------------------------------------------------------------------------------- + // Shared-key occupancy and wrong-type guards on the table service + // --------------------------------------------------------------------------------------------- + + /** + * Raw pointer rows must be seeded through the pointer repository directly, because a VIEW row is + * invisible to the table API and therefore cannot be created — or cleaned up — through it. Every + * seeded key is removed in {@link #deleteSeededPointers()}. + */ + @Autowired HouseTableRepository houseTablesRepository; + + private final List seededPointerKeys = new ArrayList<>(); + + private final List seededDirectories = new ArrayList<>(); + + @AfterEach + public void deleteSeededPointers() throws IOException { + for (HouseTablePrimaryKey key : seededPointerKeys) { + try { + houseTablesRepository.deleteById(key); + } catch (Exception e) { + // Best effort: cleanup must not mask the real assertion failure. + } + } + seededPointerKeys.clear(); + for (Path directory : seededDirectories) { + try (Stream paths = Files.walk(directory)) { + paths.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } catch (Exception e) { + // Best effort. + } + } + seededDirectories.clear(); + } + + private static final String OCCUPANCY_DB = "entity_type_occupancy_db"; + + /** + * Seeds a raw pointer whose {@code tableLocation} points at a real on-disk metadata.json under + * the storage root, so a purge attempt would be observable as a missing file. + */ + private HouseTablePrimaryKey seedRawPointer(String databaseId, String tableId, String entityType) + throws IOException { + Path tableDirectory = + Paths.get( + storageManager.getDefaultStorage().getClient().getRootPrefix(), + databaseId, + tableId + "-" + UUID.randomUUID()); + Files.createDirectories(tableDirectory); + Path metadataFile = tableDirectory.resolve("00001-seeded.metadata.json"); + Files.write(metadataFile, "{\"not\":\"parsed by these tests\"}".getBytes()); + seededDirectories.add(tableDirectory); + + houseTablesRepository.save( + HouseTable.builder() + .databaseId(databaseId) + .tableId(tableId) + .clusterId(TABLE_DTO.getClusterId()) + .tableUri(String.format("%s.%s.%s", TABLE_DTO.getClusterId(), databaseId, tableId)) + .tableUUID(UUID.randomUUID().toString()) + .tableLocation(metadataFile.toString()) + .tableVersion(INITIAL_TABLE_VERSION) + .entityType(entityType) + .build()); + + HouseTablePrimaryKey key = + HouseTablePrimaryKey.builder().databaseId(databaseId).tableId(tableId).build(); + seededPointerKeys.add(key); + return key; + } + + private HouseTable reloadPointer(HouseTablePrimaryKey key) { + return houseTablesRepository + .findById(key) + .orElseThrow( + () -> + new AssertionError( + "Raw pointer " + key.getDatabaseId() + "." + key.getTableId() + " is gone")); + } + + /** Snapshot of every metadata.json under a database's storage root. */ + private Set metadataFilesUnder(String databaseId) throws IOException { + Path databaseRoot = + Paths.get(storageManager.getDefaultStorage().getClient().getRootPrefix(), databaseId); + if (!Files.exists(databaseRoot)) { + return Collections.emptySet(); + } + try (Stream paths = Files.walk(databaseRoot)) { + return paths + .filter(p -> p.toString().endsWith(".metadata.json")) + .map(Path::toString) + .collect(Collectors.toSet()); + } + } + + /** + * CREATE TABLE at a view-occupied name must be rejected with an accurate typed 409 BEFORE any + * authorization decision and BEFORE any metadata file is written. + * + *

This is the test that fails against a naive design that only guards the table {@code + * doRefresh}: with that design the typed load reports "no table", so the create proceeds through + * authorization, allocates a location, and writes a candidate metadata.json — leaving an orphaned + * file and surfacing a misleading concurrency 409 from the HTS publish boundary. The load-bearing + * assertions here are therefore the unchanged metadata-file set and the never-authorized + * verification, not the exception type. + */ + @Test + public void testCreateTableRejectsViewOccupancyBeforeAuthorizationOrMetadata() + throws IOException { + HouseTablePrimaryKey viewKey = seedRawPointer(OCCUPANCY_DB, "occupied_by_view", "VIEW"); + HouseTable before = reloadPointer(viewKey); + Set metadataFilesBefore = metadataFilesUnder(OCCUPANCY_DB); + + // Nothing authorizes during raw-repository seeding, so a plain never() verification is enough. + Mockito.verify(authorizationHandler, Mockito.never()) + .checkAccessDecision(Mockito.any(), (DatabaseDto) Mockito.any(), Mockito.any()); + + TableDto createDto = + TABLE_DTO + .toBuilder() + .databaseId(OCCUPANCY_DB) + .tableId("occupied_by_view") + .tableUri(TABLE_DTO.getClusterId() + "." + OCCUPANCY_DB + ".occupied_by_view") + .tableVersion(INITIAL_TABLE_VERSION) + .build(); + + AlreadyExistsException thrown = + Assertions.assertThrows( + AlreadyExistsException.class, + () -> + tablesService.putTable( + buildCreateUpdateTableRequestBody(createDto), TEST_USER, true)); + Assertions.assertEquals( + "Table name " + OCCUPANCY_DB + ".occupied_by_view is occupied by a view", + thrown.getMessage()); + + HouseTable after = reloadPointer(viewKey); + Assertions.assertEquals("VIEW", after.getEntityType()); + Assertions.assertEquals(before.getEntityType(), after.getEntityType()); + Assertions.assertEquals(before.getTableLocation(), after.getTableLocation()); + Assertions.assertEquals(before.getTableUUID(), after.getTableUUID()); + + Assertions.assertEquals( + metadataFilesBefore, + metadataFilesUnder(OCCUPANCY_DB), + "A rejected create must not write a candidate metadata.json"); + + Mockito.verify(authorizationHandler, Mockito.never()) + .checkAccessDecision(Mockito.any(), (DatabaseDto) Mockito.any(), Mockito.any()); + Mockito.verify(authorizationHandler, Mockito.never()) + .checkAccessDecision(Mockito.any(), (TableDto) Mockito.any(), Mockito.any()); + } + + /** A view (any spelling) or unknown type can never be dropped through the table API. */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + public void testDeleteTableRejectsNonTableAndPreservesPointer(String entityType) + throws IOException { + HouseTablePrimaryKey key = seedRawPointer(OCCUPANCY_DB, "no_drop_target", entityType); + HouseTable before = reloadPointer(key); + Path metadataFile = Paths.get(before.getTableLocation()); + Assertions.assertTrue(Files.exists(metadataFile)); + + Assertions.assertThrows( + NoSuchUserTableException.class, + () -> tablesService.deleteTable(OCCUPANCY_DB, "no_drop_target", TEST_USER)); + + HouseTable after = reloadPointer(key); + Assertions.assertEquals(entityType, after.getEntityType()); + Assertions.assertEquals(before.getTableLocation(), after.getTableLocation()); + Assertions.assertTrue( + Files.exists(metadataFile), "A rejected drop must not purge the object's storage prefix"); + } + + /** A wrong-type rename SOURCE reads as "no such table" and nothing is created or moved. */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + public void testRenameTableRejectsNonTableSourceAndPreservesPointer(String entityType) + throws IOException { + HouseTablePrimaryKey sourceKey = seedRawPointer(OCCUPANCY_DB, "no_rename_source", entityType); + HouseTable before = reloadPointer(sourceKey); + + Assertions.assertThrows( + NoSuchUserTableException.class, + () -> + tablesService.renameTable( + OCCUPANCY_DB, "no_rename_source", OCCUPANCY_DB, "renamed_target", TEST_USER)); + + HouseTable after = reloadPointer(sourceKey); + Assertions.assertEquals(entityType, after.getEntityType()); + Assertions.assertEquals(before.getTableLocation(), after.getTableLocation()); + Assertions.assertFalse( + houseTablesRepository + .findById( + HouseTablePrimaryKey.builder() + .databaseId(OCCUPANCY_DB) + .tableId("renamed_target") + .build()) + .isPresent(), + "A rejected rename must not create the destination pointer"); + } + + /** + * Renaming a real table onto a view-occupied destination must fail with an accurate typed 409 + * BEFORE authorization, before any pointer mutation, and before any metadata is written. + * + *

The exception TYPE alone proves nothing here: the shared primary key would eventually raise + * the same {@link AlreadyExistsException} from the storage layer. What kills that accidental + * fallback is (a) the byte-identical destination pointer, (b) the unchanged source {@code + * *.metadata.json} file set — the fallback only triggers after a candidate file is written — and + * (c) the verification that no authorization decision was ever taken. + */ + @Test + public void testRenameTableRejectsViewDestinationBeforeAuthorizationOrMetadata() + throws IOException { + TableDto sourceDto = + TABLE_DTO + .toBuilder() + .databaseId(OCCUPANCY_DB) + .tableId("rename_source") + .tableUri(TABLE_DTO.getClusterId() + "." + OCCUPANCY_DB + ".rename_source") + .tableVersion(INITIAL_TABLE_VERSION) + .build(); + TableDto created = verifyPutTableRequest(sourceDto, null, true); + HouseTablePrimaryKey sourceKey = + HouseTablePrimaryKey.builder().databaseId(OCCUPANCY_DB).tableId("rename_source").build(); + Path sourceDirectory = Paths.get(URI.create(created.getTableLocation())).getParent(); + // Register the real source table for teardown immediately after creation, so it cannot survive + // the class if any assertion below fails. @AfterEach removes both the pointer and the files. + seededPointerKeys.add(sourceKey); + seededDirectories.add(sourceDirectory); + + HouseTablePrimaryKey destinationKey = seedRawPointer(OCCUPANCY_DB, "rename_dest_view", "VIEW"); + + HouseTable sourceBefore = reloadPointer(sourceKey); + HouseTable destinationBefore = reloadPointer(destinationKey); + Set sourceMetadataBefore = metadataFilesIn(sourceDirectory); + + // Source setup legitimately authorizes; clear the recorded invocations (but keep the stubs) so + // the never() verification below is about the rename only. + Mockito.clearInvocations(authorizationHandler); + + AlreadyExistsException thrown = + Assertions.assertThrows( + AlreadyExistsException.class, + () -> + tablesService.renameTable( + OCCUPANCY_DB, "rename_source", OCCUPANCY_DB, "rename_dest_view", TEST_USER)); + Assertions.assertEquals( + "Table name " + OCCUPANCY_DB + ".rename_dest_view is occupied by a view", + thrown.getMessage()); + + HouseTable destinationAfter = reloadPointer(destinationKey); + Assertions.assertEquals("VIEW", destinationAfter.getEntityType()); + Assertions.assertEquals( + destinationBefore.getTableLocation(), destinationAfter.getTableLocation()); + Assertions.assertEquals(destinationBefore.getTableUUID(), destinationAfter.getTableUUID()); + + HouseTable sourceAfter = reloadPointer(sourceKey); + Assertions.assertEquals(sourceBefore.getTableLocation(), sourceAfter.getTableLocation()); + Assertions.assertEquals(sourceBefore.getEntityType(), sourceAfter.getEntityType()); + + Assertions.assertEquals( + sourceMetadataBefore, + metadataFilesIn(sourceDirectory), + "A rejected rename must not write a new source metadata.json"); + + Mockito.verify(authorizationHandler, Mockito.never()) + .checkAccessDecision(Mockito.any(), (DatabaseDto) Mockito.any(), Mockito.any()); + Mockito.verify(authorizationHandler, Mockito.never()) + .checkAccessDecision(Mockito.any(), (TableDto) Mockito.any(), Mockito.any()); + // No explicit deleteTable here: sourceKey/sourceDirectory are registered for @AfterEach + // teardown above, so cleanup happens even if an assertion between here and there fails. + } + + /** + * Service-layer complement to the HTTP 404 tests: reading a view (any spelling) or an unknown + * discriminator through the table API is indistinguishable from "no such table", and the read + * itself must not disturb the pointer. + */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + public void testGetTableRejectsNonTableAndPreservesPointer(String entityType) throws IOException { + HouseTablePrimaryKey key = seedRawPointer(OCCUPANCY_DB, "read_as_table", entityType); + HouseTable before = reloadPointer(key); + + Assertions.assertThrows( + NoSuchUserTableException.class, + () -> tablesService.getTable(OCCUPANCY_DB, "read_as_table", TEST_USER)); + + HouseTable after = reloadPointer(key); + Assertions.assertEquals(entityType, after.getEntityType()); + Assertions.assertEquals(before.getTableLocation(), after.getTableLocation()); + Assertions.assertTrue( + Files.exists(Paths.get(before.getTableLocation())), + "A rejected read must not touch the object's files"); + } + + private Set metadataFilesIn(Path directory) throws IOException { + if (directory == null || !Files.exists(directory)) { + return Collections.emptySet(); + } + try (Stream paths = Files.walk(directory)) { + return paths + .filter(p -> p.toString().endsWith(".metadata.json")) + .map(Path::toString) + .collect(Collectors.toSet()); + } + } } diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImplTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImplTest.java index e8e4ad585..ca5d19cba 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImplTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImplTest.java @@ -3,12 +3,16 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.linkedin.openhouse.cluster.configs.ClusterProperties; import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; import com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils; import com.linkedin.openhouse.internal.catalog.model.HouseTable; +import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableCallerException; +import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableRepositoryStateUnknownException; import com.linkedin.openhouse.tables.common.TableType; import com.linkedin.openhouse.tables.dto.mapper.iceberg.PoliciesSpecMapper; import com.linkedin.openhouse.tables.model.TableDto; @@ -26,8 +30,12 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -160,6 +168,169 @@ void findTableRefByIdThrowsWhenCatalogIsNotOpenHouseInternalCatalog() { TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build())); } + // --------------------------------------------------------------------------------------------- + // Typed table load vs. shared-name occupancy + // --------------------------------------------------------------------------------------------- + + private static HouseTable pointer(String entityType) { + return HouseTable.builder() + .databaseId(DB_ID) + .tableId(TABLE_ID) + .tableUUID("uuid-1") + .tableLocation("/base/db/table-uuid-1/00001-x.metadata.json") + .entityType(entityType) + .build(); + } + + /** + * {@code findTableRefById} answers "can this key be operated on as a table?" — it backs drop. A + * VIEW or unknown pointer must read as absent so a view can never be dropped through the table + * API. + */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + void findTableRefByIdReturnsEmptyForNonTable(String entityType) { + when(catalog.findHouseTable(TableIdentifier.of(DB_ID, TABLE_ID))) + .thenReturn(Optional.of(pointer(entityType))); + + Assertions.assertFalse( + openHouseInternalRepository + .findTableRefById( + TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build()) + .isPresent(), + "entityType=" + entityType + " must not resolve to a table ref"); + } + + /** The complement: null and every spelling of TABLE keep their existing partial DTO mapping. */ + @ParameterizedTest + @CsvSource( + nullValues = "NULL", + value = {"NULL", "TABLE", "table", "TaBlE"}) + void findTableRefByIdAcceptsNullAndCaseInsensitiveTable(String entityType) { + when(catalog.findHouseTable(TableIdentifier.of(DB_ID, TABLE_ID))) + .thenReturn(Optional.of(pointer(entityType))); + + Optional result = + openHouseInternalRepository.findTableRefById( + TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build()); + + Assertions.assertTrue(result.isPresent(), "entityType=" + entityType + " must be a table"); + TableDto dto = result.get(); + Assertions.assertEquals(DB_ID, dto.getDatabaseId()); + Assertions.assertEquals(TABLE_ID, dto.getTableId()); + Assertions.assertEquals("uuid-1", dto.getTableUUID()); + Assertions.assertEquals("/base/db/table-uuid-1/00001-x.metadata.json", dto.getTableLocation()); + Assertions.assertNull(dto.getSchema()); + Assertions.assertNull(dto.getTableCreator()); + } + + /** + * Occupancy is deliberately NOT the same question as typed load. This method answers "is this + * shared key taken, and by what?" so CREATE and rename-destination can reject an occupied name + * accurately instead of seeing a view-hidden key as free. It must therefore see EVERY raw pointer + * — including unknown types, which stay present so callers fail closed — and must never parse + * metadata (never call loadTable). + */ + @ParameterizedTest + @CsvSource( + nullValues = "NULL", + value = { + "NULL, TABLE", + "TABLE, TABLE", + "table, TABLE", + "TaBlE, TABLE", + "VIEW, VIEW", + "view, VIEW", + "ViEw, VIEW", + "UNKNOWN, UNKNOWN" + }) + void findOccupyingEntityTypeSeesEveryRawPointerWithoutLoadingMetadata( + String storedEntityType, String expectedCanonical) { + when(catalog.findHouseTable(TableIdentifier.of(DB_ID, TABLE_ID))) + .thenReturn(Optional.of(pointer(storedEntityType))); + + Optional occupancy = + openHouseInternalRepository.findOccupyingEntityTypeById( + TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build()); + + Assertions.assertTrue( + occupancy.isPresent(), + "A stored pointer with entityType=" + storedEntityType + " occupies the name"); + Assertions.assertEquals(expectedCanonical, occupancy.get()); + + verify(catalog).findHouseTable(TableIdentifier.of(DB_ID, TABLE_ID)); + verify(catalog, never()).loadTable(any(TableIdentifier.class)); + } + + /** Only a genuinely absent pointer means the name is free. */ + @Test + void findOccupyingEntityTypeReturnsEmptyOnlyWhenNoPointerExists() { + when(catalog.findHouseTable(any(TableIdentifier.class))).thenReturn(Optional.empty()); + + Assertions.assertFalse( + openHouseInternalRepository + .findOccupyingEntityTypeById( + TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build()) + .isPresent()); + + verify(catalog, never()).loadTable(any(TableIdentifier.class)); + } + + /** + * HTS 4xx must PROPAGATE out of the occupancy lookup. Swallowing a repository error into "the + * name is free" would let a CREATE proceed over an existing view during an HTS incident, which is + * exactly the hole this occupancy check exists to close. + * + *

Stubbed with {@code doThrow(...).when(...)} rather than {@code when(...).thenThrow(...)}: + * the latter evaluates its argument, which invokes the mock and would blow up the test itself. + */ + @Test + void findOccupyingEntityTypeDoesNotSwallowClientErrors() { + Mockito.doThrow( + new HouseTableCallerException("HTS returned 400", new RuntimeException("bad request"))) + .when(catalog) + .findHouseTable(any(TableIdentifier.class)); + + Assertions.assertThrows( + HouseTableCallerException.class, + () -> + openHouseInternalRepository.findOccupyingEntityTypeById( + TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build())); + } + + /** + * HTS 5xx must PROPAGATE for the same reason: an outage must never read as an unoccupied name. + * This is the branch that a broad {@code catch (Exception e) { return Optional.empty(); }} would + * silently convert into "free", reopening the CREATE-over-VIEW hole. + */ + @Test + void findOccupyingEntityTypeDoesNotSwallowServerErrors() { + Mockito.doThrow( + new HouseTableRepositoryStateUnknownException( + "HTS returned 503", new RuntimeException("unavailable"))) + .when(catalog) + .findHouseTable(any(TableIdentifier.class)); + + Assertions.assertThrows( + HouseTableRepositoryStateUnknownException.class, + () -> + openHouseInternalRepository.findOccupyingEntityTypeById( + TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build())); + } + + /** Occupancy follows the same unsupported-catalog contract as {@code findTableRefById}. */ + @Test + void findOccupyingEntityTypeThrowsWhenCatalogIsNotOpenHouseInternalCatalog() { + OpenHouseInternalRepositoryImpl impl = new OpenHouseInternalRepositoryImpl(); + impl.catalog = mock(Catalog.class); + + Assertions.assertThrows( + UnsupportedOperationException.class, + () -> + impl.findOccupyingEntityTypeById( + TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build())); + } + private TableDto createTableDto(Map properties) { return TableDto.builder() .databaseId(DB_ID) From 9bb4d716790c6c68bfae996b982de5fb4c495576 Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 13:51:41 -0700 Subject: [PATCH 02/12] BDP-108403: Revert JPQL @Query annotations on HouseTableRepository The @Query annotations added to HouseTableRepository were inert in production and broke a universal convention in this repo, so this reverts that interface to its pre-change state and drops the tests that only exercised them. Why they were inert: TablesSpringApplication excludes DataSourceAutoConfiguration, so the tables service has no DataSource bean and the only @EnableJpaRepositories scan is HTS-scoped. No Spring Data proxy of HouseTableRepository can ever be created there. The sole bean behind that interface is the hand-written HouseTableRepositoryImpl, which ignores @Query entirely and talks to HTS over HTTP. HTS in turn already applies the same table-only predicate in SQL inside UserTableHtsJdbcRepository, so production filtering is complete without these annotations. Why they were wrong stylistically: only a handful of files in this repo carry @Query, and every one of them executes against a real database. The established precedent for exactly this shape is HtsRepository, an empty interface whose JPA semantics live entirely on its impl/jdbc class. Production interfaces declare the contract; implementations own behavior. Restoring the interface puts HouseTableRepository back in line with that, and leaves internalcatalog's main sources with no spring-data-jpa usage at all. Why the removed tests go with them: the eleven deleted listing tests in RepositoryTest, DatabasesControllerTest and TablesControllerTest ran against the H2 Spring Data double, where the annotations did take effect. The production methods they covered (listTables, listHouseTables, searchTables, findAllIds) are byte-for-byte unchanged by this change set, so those tests were verifying a test double rather than production code. The genuine coverage for the same acceptance criteria lives in services/housetables, where the predicate actually runs in SQL. Every view isolation guard test that exercises real production logic is kept. Adding the same filtering to the H2 doubles is deliberately left out; it belongs with the view-commit work, since nothing in main sources writes a VIEW discriminator yet, which would make the filter unreachable and untestable today. Verified: housetables 153, internalcatalog 124, tables 519 (was 530, exactly the 11 removed), tables-test-fixtures 8, spark-3.5 catalogTest 66 - all green, plus spotlessCheck. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../repository/HouseTableRepository.java | 34 +- .../e2e/h2/DatabasesControllerTest.java | 139 ------- .../tables/e2e/h2/RepositoryTest.java | 348 ------------------ .../tables/e2e/h2/TablesControllerTest.java | 112 ------ 4 files changed, 2 insertions(+), 631 deletions(-) diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepository.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepository.java index babfbe74d..065c9e681 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepository.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/repository/HouseTableRepository.java @@ -5,9 +5,7 @@ import java.util.List; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; -import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; /** @@ -18,18 +16,7 @@ public interface HouseTableRepository extends PagingAndSortingRepository { - /** - * Excludes views from table listings. The predicate lives in the query — never in a stream over a - * returned {@link Page} — so content and counts agree. {@code IS NULL} is mandatory because the - * discriminator is nullable with no backfill; {@code upper(...)} avoids depending on collation. - * - *

Spring Data proxies in services/tables and in the published tables-test-fixtures module - * inherit these predicates without an edit. - */ - String TABLE_ROW_PREDICATE = "(h.entityType IS NULL OR upper(h.entityType) = 'TABLE')"; - - @Query("SELECT h FROM HouseTable h WHERE h.databaseId = :databaseId AND " + TABLE_ROW_PREDICATE) - List findAllByDatabaseId(@Param("databaseId") String databaseId); + List findAllByDatabaseId(String databaseId); /** * Delete a table by its primary key with purge option @@ -39,24 +26,7 @@ public interface HouseTableRepository */ void deleteById(HouseTablePrimaryKey houseTablePrimaryKey, boolean purge); - @Query( - value = - "SELECT h FROM HouseTable h WHERE h.databaseId = :databaseId AND " + TABLE_ROW_PREDICATE, - countQuery = - "SELECT COUNT(h) FROM HouseTable h WHERE h.databaseId = :databaseId AND " - + TABLE_ROW_PREDICATE) - Page findAllByDatabaseId(@Param("databaseId") String databaseId, Pageable pageable); - - /** Redeclared only to add the table-only predicate; cardinality and dedup are unchanged. */ - @Override - @Query("SELECT h FROM HouseTable h WHERE " + TABLE_ROW_PREDICATE) - Iterable findAll(); - - @Override - @Query( - value = "SELECT h FROM HouseTable h WHERE " + TABLE_ROW_PREDICATE, - countQuery = "SELECT COUNT(h) FROM HouseTable h WHERE " + TABLE_ROW_PREDICATE) - Page findAll(Pageable pageable); + Page findAllByDatabaseId(String databaseId, Pageable pageable); void rename( String fromDatabaseId, diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/DatabasesControllerTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/DatabasesControllerTest.java index d034986d2..c12fbdfc0 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/DatabasesControllerTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/DatabasesControllerTest.java @@ -4,14 +4,10 @@ import static com.linkedin.openhouse.tables.model.DatabaseModelConstants.GET_DATABASE_RESPONSE_BODY; import static com.linkedin.openhouse.tables.model.DatabaseModelConstants.GET_DATABASE_RESPONSE_BODY_DIFF_DB; import static com.linkedin.openhouse.tables.model.TableModelConstants.*; -import static org.hamcrest.Matchers.*; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; import com.linkedin.openhouse.cluster.storage.StorageManager; import com.linkedin.openhouse.common.test.cluster.PropertyOverrideContextInitializer; -import com.linkedin.openhouse.internal.catalog.model.HouseTable; -import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; -import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; import com.linkedin.openhouse.tables.api.spec.v0.response.GetAllDatabasesResponseBody; import com.linkedin.openhouse.tables.api.spec.v0.response.GetDatabaseResponseBody; import com.linkedin.openhouse.tables.api.spec.v0.response.GetTableResponseBody; @@ -20,12 +16,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.UUID; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.iceberg.catalog.Catalog; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; @@ -192,136 +185,4 @@ private void cleanUpHelper(TestInfo info) { log.warn("Cleaning up process interrupted with exception: {}", exception); } } - - // --------------------------------------------------------------------------------------------- - // A view-only database must not appear in the database listing - // --------------------------------------------------------------------------------------------- - - /** - * Raw pointer rows must be seeded through the pointer repository directly, because a VIEW row is - * invisible to the table HTTP API and therefore cannot be created or cleaned up through it. Every - * seeded key is removed in {@link #deleteSeededPointers()}. - */ - @Autowired HouseTableRepository houseTablesRepository; - - private final List seededPointerKeys = new ArrayList<>(); - - @AfterEach - void deleteSeededPointers() { - for (HouseTablePrimaryKey key : seededPointerKeys) { - try { - houseTablesRepository.deleteById(key); - } catch (Exception e) { - log.warn("Failed to clean up raw pointer {}: {}", key.getTableId(), e.toString()); - } - } - seededPointerKeys.clear(); - } - - private void seedRawPointer(String databaseId, String tableId, String entityType) { - houseTablesRepository.save( - HouseTable.builder() - .databaseId(databaseId) - .tableId(tableId) - .clusterId("test-cluster") - .tableUri(String.format("test-cluster.%s.%s", databaseId, tableId)) - .tableUUID(UUID.randomUUID().toString()) - .tableLocation( - String.format("/base/%s/%s-uuid/00001-x.metadata.json", databaseId, tableId)) - .tableVersion("INITIAL_VERSION") - .entityType(entityType) - .build()); - seededPointerKeys.add( - HouseTablePrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()); - } - - /** - * Canonical database fixture: seven databases with exactly one pointer each; three of them hold - * only a view. Only the four table databases may be listed. - */ - private void seedCanonicalDatabases() { - seedRawPointer("db00_legacy", "t1", null); - seedRawPointer("db01_view_only", "t1", "VIEW"); - seedRawPointer("db02_explicit", "t1", "TABLE"); - seedRawPointer("db03_view_only", "t1", "VIEW"); - seedRawPointer("db04_legacy", "t1", null); - seedRawPointer("db05_view_only", "t1", "VIEW"); - seedRawPointer("db06_explicit", "t1", "TABLE"); - } - - /** - * The two database-listing tests below assert a GLOBAL result count, so a row leaked by another - * method in this class would make them fail for an unrelated reason. Asserting the precondition - * up front keeps that failure diagnosable as leakage rather than as a filtering bug. - */ - private void assertPointerTableIsEmpty() { - List existing = new ArrayList<>(); - houseTablesRepository.findAll().forEach(existing::add); - Assertions.assertTrue( - existing.isEmpty(), - "This test asserts a global database count and requires a clean pointer table; " - + "a previous test leaked rows: " - + existing.stream() - .map(h -> h.getDatabaseId() + "." + h.getTableId()) - .collect(Collectors.toList())); - } - - @Test - public void testGetAllDatabasesExcludesViewOnlyDatabases() throws Exception { - assertPointerTableIsEmpty(); - seedCanonicalDatabases(); - - mvc.perform( - MockMvcRequestBuilders.get(CURRENT_MAJOR_VERSION_PREFIX + "/databases") - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andExpect(jsonPath("$.results", hasSize(4))) - .andExpect( - jsonPath( - "$.results[*].databaseId", - containsInAnyOrder("db00_legacy", "db02_explicit", "db04_legacy", "db06_explicit"))) - .andExpect(jsonPath("$.results[*].databaseId", not(hasItem("db01_view_only")))) - .andExpect(jsonPath("$.results[*].databaseId", not(hasItem("db03_view_only")))) - .andExpect(jsonPath("$.results[*].databaseId", not(hasItem("db05_view_only")))); - } - - /** - * Anti-post-filter assertion for the paginated database listing: filtering the returned page - * would report totalElements=7/totalPages=4 with a 1-row first page. - */ - @Test - public void testGetAllDatabasesFiltersBeforePagination() throws Exception { - assertPointerTableIsEmpty(); - seedCanonicalDatabases(); - - mvc.perform( - MockMvcRequestBuilders.get("/v2/databases") - .param("page", "0") - .param("size", "2") - .param("sortBy", "databaseId") - .contentType(MediaType.APPLICATION_JSON) - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andExpect(jsonPath("$.pageResults.totalElements", is(4))) - .andExpect(jsonPath("$.pageResults.totalPages", is(2))) - .andExpect(jsonPath("$.pageResults.content", hasSize(2))) - .andExpect(jsonPath("$.pageResults.content[0].databaseId", is("db00_legacy"))) - .andExpect(jsonPath("$.pageResults.content[1].databaseId", is("db02_explicit"))); - - mvc.perform( - MockMvcRequestBuilders.get("/v2/databases") - .param("page", "1") - .param("size", "2") - .param("sortBy", "databaseId") - .contentType(MediaType.APPLICATION_JSON) - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.pageResults.totalElements", is(4))) - .andExpect(jsonPath("$.pageResults.totalPages", is(2))) - .andExpect(jsonPath("$.pageResults.content", hasSize(2))) - .andExpect(jsonPath("$.pageResults.content[0].databaseId", is("db04_legacy"))) - .andExpect(jsonPath("$.pageResults.content[1].databaseId", is("db06_explicit"))); - } } diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java index 1e08c354d..52f5c537e 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java @@ -10,7 +10,6 @@ import com.linkedin.openhouse.common.exception.UnsupportedClientOperationException; import com.linkedin.openhouse.common.test.cluster.PropertyOverrideContextInitializer; import com.linkedin.openhouse.internal.catalog.CatalogConstants; -import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; @@ -30,15 +29,12 @@ import com.linkedin.openhouse.tables.repository.impl.InternalRepositoryUtils; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; -import java.util.UUID; import java.util.stream.Collectors; import org.apache.iceberg.BaseTable; import org.apache.iceberg.Schema; @@ -47,7 +43,6 @@ import org.apache.iceberg.TableProperties; import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.NoSuchTableException; @@ -59,10 +54,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.SpyBean; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.domain.Pageable; -import org.springframework.data.domain.Sort; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.util.AopTestUtils; @@ -1607,343 +1598,4 @@ private void verifyTable(HouseTable table) { table.getTableId() + "-" + table.getTableUUID()); Assertions.assertTrue(table.getTableLocation().startsWith(path.toString())); } - - // --------------------------------------------------------------------------------------------- - // Table listings must exclude views in the query, never by post-filtering a Page - // --------------------------------------------------------------------------------------------- - - /** - * Canonical interleaved fixture: four visible tables (two legacy NULL, two explicit TABLE) - * interleaved with three VIEW rows. A fetch-then-filter implementation returns a SHORT first page - * (1 row) with totalElements=7/totalPages=4; the correct pre-pagination predicate returns a full - * 2-row page with totalElements=4/totalPages=2. - */ - private static final String ENTITY_TYPE_DB = "entity_type_db"; - - private static final List CANONICAL_TABLE_IDS = - Arrays.asList("t00_legacy", "t02_explicit", "t04_legacy", "t06_explicit"); - - private static final List CANONICAL_VIEW_IDS = - Arrays.asList("t01_view", "t03_view", "t05_view"); - - private static final String CASE_DB = "entity_type_case_db"; - - private HouseTable rawPointer(String databaseId, String tableId, String entityType) { - return HouseTable.builder() - .databaseId(databaseId) - .tableId(tableId) - .clusterId("test-cluster") - .tableUri(String.format("test-cluster.%s.%s", databaseId, tableId)) - .tableUUID(UUID.randomUUID().toString()) - .tableLocation(String.format("/base/%s/%s-uuid/00001-x.metadata.json", databaseId, tableId)) - .tableVersion(INITIAL_TABLE_VERSION) - .entityType(entityType) - .build(); - } - - /** Seeds raw pointer rows and returns their keys so the caller can delete them in a finally. */ - private List seedRawPointers(String databaseId, String[][] idAndType) { - List keys = new ArrayList<>(); - for (String[] entry : idAndType) { - houseTablesRepository.save(rawPointer(databaseId, entry[0], entry[1])); - keys.add(HouseTablePrimaryKey.builder().databaseId(databaseId).tableId(entry[0]).build()); - } - return keys; - } - - private List seedCanonicalPointers(String databaseId) { - return seedRawPointers( - databaseId, - new String[][] { - {"t00_legacy", null}, - {"t01_view", "VIEW"}, - {"t02_explicit", "TABLE"}, - {"t03_view", "VIEW"}, - {"t04_legacy", null}, - {"t05_view", "VIEW"}, - {"t06_explicit", "TABLE"} - }); - } - - private List seedCaseNormalizationPointers(String databaseId) { - return seedRawPointers( - databaseId, - new String[][] { - {"case00_null", null}, - {"case01_upper_table", "TABLE"}, - {"case02_lower_table", "table"}, - {"case03_mixed_table", "TaBlE"}, - {"case04_upper_view", "VIEW"}, - {"case05_lower_view", "view"}, - {"case06_mixed_view", "ViEw"}, - {"case07_garbage", "UNKNOWN"} - }); - } - - /** - * Raw pointer rows are invisible to the table APIs by design, so no table-API cleanup can remove - * them. Every test that seeds them MUST delete them explicitly, otherwise later tests in this - * class (which asserts exact database/table sets, and shares one Spring context across methods) - * are polluted. - */ - private void deleteRawPointers(List keys) { - for (HouseTablePrimaryKey key : keys) { - try { - houseTablesRepository.deleteById(key); - } catch (Exception e) { - // Best effort: a missing row must not mask the real assertion failure. - } - } - } - - private static OpenHouseInternalCatalog openHouseCatalog(Catalog catalog) { - return (OpenHouseInternalCatalog) AopTestUtils.getUltimateTargetObject(catalog); - } - - private static List identifierNames(List identifiers) { - return identifiers.stream().map(TableIdentifier::name).sorted().collect(Collectors.toList()); - } - - private static Pageable sortedPage(int page) { - return PageRequest.of(page, 2, Sort.by("tableId")); - } - - /** SHOW TABLES contract: a view never appears in the catalog's table listing. */ - @Test - public void testCatalogListTablesExcludesViewsAndKeepsNullAndTable() { - List keys = seedCanonicalPointers(ENTITY_TYPE_DB); - try { - List identifiers = catalog.listTables(Namespace.of(ENTITY_TYPE_DB)); - - Assertions.assertEquals(CANONICAL_TABLE_IDS, identifierNames(identifiers)); - Assertions.assertTrue( - identifierNames(identifiers).stream().noneMatch(CANONICAL_VIEW_IDS::contains), - "No VIEW row may appear in SHOW TABLES: " + identifierNames(identifiers)); - } finally { - deleteRawPointers(keys); - } - } - - /** Anti-post-filter assertion for the paginated catalog listing overload. */ - @Test - public void testCatalogListTablesFiltersBeforePagination() { - List keys = seedCanonicalPointers(ENTITY_TYPE_DB); - try { - OpenHouseInternalCatalog ohCatalog = openHouseCatalog(catalog); - - Page page0 = - ohCatalog.listTables(Namespace.of(ENTITY_TYPE_DB), sortedPage(0)); - Assertions.assertEquals(4, page0.getTotalElements()); - Assertions.assertEquals(2, page0.getTotalPages()); - Assertions.assertEquals(2, page0.getContent().size()); - Assertions.assertEquals( - Arrays.asList("t00_legacy", "t02_explicit"), - page0.getContent().stream().map(TableIdentifier::name).collect(Collectors.toList())); - - Page page1 = - ohCatalog.listTables(Namespace.of(ENTITY_TYPE_DB), sortedPage(1)); - Assertions.assertEquals(4, page1.getTotalElements()); - Assertions.assertEquals(2, page1.getTotalPages()); - Assertions.assertEquals(2, page1.getContent().size()); - Assertions.assertEquals( - Arrays.asList("t04_legacy", "t06_explicit"), - page1.getContent().stream().map(TableIdentifier::name).collect(Collectors.toList())); - } finally { - deleteRawPointers(keys); - } - } - - /** Anti-post-filter assertion for the HouseTable-preserving paginated listing. */ - @Test - public void testListHouseTablesFiltersBeforePagination() { - List keys = seedCanonicalPointers(ENTITY_TYPE_DB); - try { - OpenHouseInternalCatalog ohCatalog = openHouseCatalog(catalog); - - Page page0 = - ohCatalog.listHouseTables(Namespace.of(ENTITY_TYPE_DB), sortedPage(0)); - Assertions.assertEquals(4, page0.getTotalElements()); - Assertions.assertEquals(2, page0.getTotalPages()); - Assertions.assertEquals(2, page0.getContent().size()); - Assertions.assertEquals( - Arrays.asList("t00_legacy", "t02_explicit"), - page0.getContent().stream().map(HouseTable::getTableId).collect(Collectors.toList())); - Assertions.assertTrue( - page0.getContent().stream().noneMatch(h -> "VIEW".equalsIgnoreCase(h.getEntityType()))); - - Page page1 = - ohCatalog.listHouseTables(Namespace.of(ENTITY_TYPE_DB), sortedPage(1)); - Assertions.assertEquals(4, page1.getTotalElements()); - Assertions.assertEquals(2, page1.getTotalPages()); - Assertions.assertEquals(2, page1.getContent().size()); - Assertions.assertEquals( - Arrays.asList("t04_legacy", "t06_explicit"), - page1.getContent().stream().map(HouseTable::getTableId).collect(Collectors.toList())); - } finally { - deleteRawPointers(keys); - } - } - - /** All three {@code searchTables} overloads must filter identically and before paging. */ - @Test - public void testOpenHouseRepositorySearchTablesFiltersAllOverloads() { - List keys = seedCanonicalPointers(ENTITY_TYPE_DB); - try { - List plain = openHouseInternalRepository.searchTables(ENTITY_TYPE_DB); - Assertions.assertEquals( - CANONICAL_TABLE_IDS, - plain.stream().map(TableDto::getTableId).sorted().collect(Collectors.toList())); - - Page page0 = - openHouseInternalRepository.searchTables(ENTITY_TYPE_DB, sortedPage(0)); - Assertions.assertEquals(4, page0.getTotalElements()); - Assertions.assertEquals(2, page0.getTotalPages()); - Assertions.assertEquals( - Arrays.asList("t00_legacy", "t02_explicit"), - page0.getContent().stream().map(TableDto::getTableId).collect(Collectors.toList())); - - Page page1 = - openHouseInternalRepository.searchTables(ENTITY_TYPE_DB, sortedPage(1)); - Assertions.assertEquals(4, page1.getTotalElements()); - Assertions.assertEquals(2, page1.getTotalPages()); - Assertions.assertEquals( - Arrays.asList("t04_legacy", "t06_explicit"), - page1.getContent().stream().map(TableDto::getTableId).collect(Collectors.toList())); - - // The fields projection goes through listHouseTables, so it must filter identically and - // still populate the requested field. - Page fieldsPage0 = - openHouseInternalRepository.searchTables( - ENTITY_TYPE_DB, sortedPage(0), Collections.singletonList("tableLocation")); - Assertions.assertEquals(4, fieldsPage0.getTotalElements()); - Assertions.assertEquals(2, fieldsPage0.getTotalPages()); - Assertions.assertEquals( - Arrays.asList("t00_legacy", "t02_explicit"), - fieldsPage0.getContent().stream().map(TableDto::getTableId).collect(Collectors.toList())); - Assertions.assertTrue( - fieldsPage0.getContent().stream().allMatch(dto -> dto.getTableLocation() != null), - "fields=tableLocation must be projected for every returned table"); - } finally { - deleteRawPointers(keys); - } - } - - /** - * Database enumeration: a database whose only pointer is a view must disappear entirely. Note the - * global-scope precondition — {@code findAllIds} is not database-scoped, so this test asserts the - * pointer table is empty first to keep a failure here diagnosable as leakage rather than as a - * filtering bug. - */ - @Test - public void testFindAllIdsExcludesViewOnlyDatabases() { - Assertions.assertTrue( - Streams.stream(houseTablesRepository.findAll()).count() == 0, - "This test asserts global pointer counts and requires a clean pointer table; " - + "a previous test leaked rows"); - - List keys = new ArrayList<>(); - try { - keys.addAll(seedRawPointers("db00_legacy", new String[][] {{"t1", null}})); - keys.addAll(seedRawPointers("db01_view_only", new String[][] {{"t1", "VIEW"}})); - keys.addAll(seedRawPointers("db02_explicit", new String[][] {{"t1", "TABLE"}})); - keys.addAll(seedRawPointers("db03_view_only", new String[][] {{"t1", "VIEW"}})); - keys.addAll(seedRawPointers("db04_legacy", new String[][] {{"t1", null}})); - keys.addAll(seedRawPointers("db05_view_only", new String[][] {{"t1", "VIEW"}})); - keys.addAll(seedRawPointers("db06_explicit", new String[][] {{"t1", "TABLE"}})); - - List databaseIds = - openHouseInternalRepository.findAllIds().stream() - .map(TableDtoPrimaryKey::getDatabaseId) - .sorted() - .collect(Collectors.toList()); - Assertions.assertEquals( - Arrays.asList("db00_legacy", "db02_explicit", "db04_legacy", "db06_explicit"), - databaseIds); - - Pageable dbPage = PageRequest.of(0, 2, Sort.by("databaseId")); - Page page0 = openHouseInternalRepository.findAllIds(dbPage); - Assertions.assertEquals(4, page0.getTotalElements()); - Assertions.assertEquals(2, page0.getTotalPages()); - Assertions.assertEquals( - Arrays.asList("db00_legacy", "db02_explicit"), - page0.getContent().stream() - .map(TableDtoPrimaryKey::getDatabaseId) - .collect(Collectors.toList())); - - Page page1 = - openHouseInternalRepository.findAllIds(PageRequest.of(1, 2, Sort.by("databaseId"))); - Assertions.assertEquals(4, page1.getTotalElements()); - Assertions.assertEquals(2, page1.getTotalPages()); - Assertions.assertEquals( - Arrays.asList("db04_legacy", "db06_explicit"), - page1.getContent().stream() - .map(TableDtoPrimaryKey::getDatabaseId) - .collect(Collectors.toList())); - } finally { - deleteRawPointers(keys); - } - } - - /** - * Case/garbage matrix at the internal H2 query layer. - * - *

H2 (MODE=MySQL) is case-SENSITIVE while production MySQL's default collation is not, so this - * proves the query normalizes explicitly (e.g. {@code upper(h.entityType) = 'TABLE'}) rather than - * relying on the provider's collation — a bare {@code = 'TABLE'} comparison would hide the - * lower/mixed-case table rows here and fail. It does NOT certify production MySQL behavior; the - * authoritative case contract lives in the Java guards ({@code - * HouseTableTest#testEntityTypeClassification} and the catalog guard tests). - */ - @Test - public void testCaseInsensitiveTypePredicateAndGarbageFailClosed() { - List keys = seedCaseNormalizationPointers(CASE_DB); - try { - List expectedVisible = - Arrays.asList( - "case00_null", "case01_upper_table", "case02_lower_table", "case03_mixed_table"); - List expectedHidden = - Arrays.asList( - "case04_upper_view", "case05_lower_view", "case06_mixed_view", "case07_garbage"); - - List listed = identifierNames(catalog.listTables(Namespace.of(CASE_DB))); - Assertions.assertEquals(expectedVisible, listed); - Assertions.assertTrue( - listed.stream().noneMatch(expectedHidden::contains), - "Views (any spelling) and unknown types must fail closed out of SHOW TABLES: " + listed); - - OpenHouseInternalCatalog ohCatalog = openHouseCatalog(catalog); - - Page page0 = ohCatalog.listTables(Namespace.of(CASE_DB), sortedPage(0)); - Assertions.assertEquals(4, page0.getTotalElements()); - Assertions.assertEquals(2, page0.getTotalPages()); - Assertions.assertEquals( - Arrays.asList("case00_null", "case01_upper_table"), - page0.getContent().stream().map(TableIdentifier::name).collect(Collectors.toList())); - - Page housePage0 = ohCatalog.listHouseTables(Namespace.of(CASE_DB), sortedPage(0)); - Assertions.assertEquals(4, housePage0.getTotalElements()); - Assertions.assertEquals(2, housePage0.getTotalPages()); - Assertions.assertEquals( - Arrays.asList("case00_null", "case01_upper_table"), - housePage0.getContent().stream() - .map(HouseTable::getTableId) - .collect(Collectors.toList())); - - Page housePage1 = ohCatalog.listHouseTables(Namespace.of(CASE_DB), sortedPage(1)); - Assertions.assertEquals( - Arrays.asList("case02_lower_table", "case03_mixed_table"), - housePage1.getContent().stream() - .map(HouseTable::getTableId) - .collect(Collectors.toList())); - - // Hidden, not dropped: the raw rows are all still there. - for (HouseTablePrimaryKey key : keys) { - Assertions.assertTrue( - houseTablesRepository.findById(key).isPresent(), - "Raw pointer " + key.getTableId() + " must still exist; it is hidden, not deleted"); - } - } finally { - deleteRawPointers(keys); - } - } } diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java index 7e2432531..bc4a0538e 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java @@ -2112,118 +2112,6 @@ private void seedRawPointer(String databaseId, String tableId, String entityType HouseTablePrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()); } - /** - * Canonical interleaved fixture: four visible tables (two legacy NULL, two explicit TABLE) - * interleaved with three VIEW rows. A fetch-then-filter implementation returns a SHORT first page - * (1 row) with totalElements=7/totalPages=4. - */ - private void seedCanonicalPointers() { - seedRawPointer(VIEW_MIX_DB, "t00_legacy", null); - seedRawPointer(VIEW_MIX_DB, "t01_view", "VIEW"); - seedRawPointer(VIEW_MIX_DB, "t02_explicit", "TABLE"); - seedRawPointer(VIEW_MIX_DB, "t03_view", "VIEW"); - seedRawPointer(VIEW_MIX_DB, "t04_legacy", null); - seedRawPointer(VIEW_MIX_DB, "t05_view", "VIEW"); - seedRawPointer(VIEW_MIX_DB, "t06_explicit", "TABLE"); - } - - @Test - public void testSearchTablesExcludesInterleavedViews() throws Exception { - seedCanonicalPointers(); - - mvc.perform( - MockMvcRequestBuilders.post( - String.format( - ValidationUtilities.CURRENT_MAJOR_VERSION_PREFIX - + "/databases/%s/tables/search", - VIEW_MIX_DB)) - .contentType(MediaType.APPLICATION_JSON) - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.results", hasSize(4))) - .andExpect( - jsonPath( - "$.results[*].tableId", - containsInAnyOrder("t00_legacy", "t02_explicit", "t04_legacy", "t06_explicit"))) - .andExpect(jsonPath("$.results[*].tableId", not(hasItem("t01_view")))) - .andExpect(jsonPath("$.results[*].tableId", not(hasItem("t03_view")))) - .andExpect(jsonPath("$.results[*].tableId", not(hasItem("t05_view")))); - } - - @Test - public void testSearchTablesFiltersBeforePagination() throws Exception { - seedCanonicalPointers(); - - mvc.perform( - MockMvcRequestBuilders.post("/v2/databases/" + VIEW_MIX_DB + "/tables/search") - .param("page", "0") - .param("size", "2") - .param("sortBy", "tableId") - .contentType(MediaType.APPLICATION_JSON) - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.pageResults.totalElements", is(4))) - .andExpect(jsonPath("$.pageResults.totalPages", is(2))) - .andExpect(jsonPath("$.pageResults.content", hasSize(2))) - .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t00_legacy"))) - .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t02_explicit"))); - - mvc.perform( - MockMvcRequestBuilders.post("/v2/databases/" + VIEW_MIX_DB + "/tables/search") - .param("page", "1") - .param("size", "2") - .param("sortBy", "tableId") - .contentType(MediaType.APPLICATION_JSON) - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.pageResults.totalElements", is(4))) - .andExpect(jsonPath("$.pageResults.totalPages", is(2))) - .andExpect(jsonPath("$.pageResults.content", hasSize(2))) - .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t04_legacy"))) - .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t06_explicit"))); - } - - @Test - public void testSearchTablesWithFieldsFiltersBeforePagination() throws Exception { - seedCanonicalPointers(); - - mvc.perform( - MockMvcRequestBuilders.post("/v2/databases/" + VIEW_MIX_DB + "/tables/search") - .param("page", "0") - .param("size", "2") - .param("sortBy", "tableId") - .param("fields", "tableLocation") - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.pageResults.totalElements", is(4))) - .andExpect(jsonPath("$.pageResults.totalPages", is(2))) - .andExpect(jsonPath("$.pageResults.content", hasSize(2))) - .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t00_legacy"))) - .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t02_explicit"))) - .andExpect( - jsonPath( - "$.pageResults.content[0].tableLocation", - is("/base/" + VIEW_MIX_DB + "/t00_legacy-uuid/00001-x.metadata.json"))) - .andExpect( - jsonPath( - "$.pageResults.content[1].tableLocation", - is("/base/" + VIEW_MIX_DB + "/t02_explicit-uuid/00001-x.metadata.json"))); - - mvc.perform( - MockMvcRequestBuilders.post("/v2/databases/" + VIEW_MIX_DB + "/tables/search") - .param("page", "1") - .param("size", "2") - .param("sortBy", "tableId") - .param("fields", "tableLocation") - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.pageResults.totalElements", is(4))) - .andExpect(jsonPath("$.pageResults.totalPages", is(2))) - .andExpect(jsonPath("$.pageResults.content", hasSize(2))) - .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t04_legacy"))) - .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t06_explicit"))); - } - /** * CREATE TABLE at a name already occupied by a view must be an accurate 409 with a message that * names the real condition. A guard implemented only in the table {@code doRefresh} would let the From e13edb00e5940946978799a9d507c71ffd936c5f Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 14:12:17 -0700 Subject: [PATCH 03/12] BDP-108403: Stop filtering views out of the database-ID projection Reverts the table predicate on both findAllDistinctDatabaseIds overloads in UserTableHtsJdbcRepository to their pre-change form, and drops the two tests that only asserted the reverted behavior. The four table-row filters are untouched: findAllByDatabaseIdIgnoreCase, the tableId-pattern variant, their paginated forms, and the findAllByFilters entity-type clause remain exactly as they are. Those are the genuine production filtering for this ticket. These two methods return a projection of database-ID strings, not rows, so no view can appear in their output under any implementation. The filter did not hide a view; it only changed which database names get listed. That is outside the scope this change set set for itself. The design enumerates the queries that need the table predicate and this is not among them, the stated harm is that SHOW TABLES would return views, and the acceptance criterion is that no view appears in a table listing. A database listing is not a table listing. Filtering here also contradicts three other design statements taken together: a namespace maps to an already-existing database and is never created implicitly, the server never auto-creates databases, and HTS infers databases from object rows and has no way to represent an empty database. With the filter, a database holding only views becomes non-existent by the only existence mechanism OpenHouse has - while views may only be created in databases that already exist. Concretely this path is Spark's SHOW DATABASES via OpenHouseCatalog.listNamespaces(). With the filter, a view-only namespace would be missing from SHOW DATABASES while still being addressable at /v2/databases/foo/views/v1. The rule this restores: queries that enumerate objects must be type-scoped; queries that enumerate containers must not. Removed with it, as they asserted only the reverted behavior: HtsRepositoryTest#testFindDistinctDatabasesExcludesViewOnlyDatabases and HtsControllerTest#testDatabaseQueriesExcludeViewOnlyDatabases. No fixture, helper or import became unused. The pre-existing testFindDistinctDatabases and the entity-type case/garbage matrix are unaffected and stay. Verified: housetables 151 (was 153, exactly the 2 removed), internalcatalog 124, tables 519, tables-test-fixtures 8, spark-3.5 catalogTest 66 - all green, plus spotlessCheck. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../impl/jdbc/UserTableHtsJdbcRepository.java | 15 ++---- .../e2e/usertable/HtsControllerTest.java | 52 ------------------- .../e2e/usertable/HtsRepositoryTest.java | 27 ---------- 3 files changed, 4 insertions(+), 90 deletions(-) diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java index 0465df64c..55005346a 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java @@ -47,7 +47,7 @@ Optional findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( */ String TABLE_ROW_PREDICATE = "(u.entityType IS NULL OR upper(u.entityType) = 'TABLE')"; - @Query("SELECT DISTINCT u.databaseId FROM UserTableRow u WHERE " + TABLE_ROW_PREDICATE) + @Query("SELECT DISTINCT databaseId FROM UserTableRow") Iterable findAllDistinctDatabaseIds(); @Query( @@ -65,16 +65,9 @@ Iterable findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( @Param("databaseId") String databaseId, @Param("tableIdPattern") String tableIdPattern); @Query( - value = - "SELECT DISTINCT u.databaseId FROM UserTableRow u WHERE " - + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " - + TABLE_ROW_PREDICATE, - countQuery = - "SELECT COUNT(DISTINCT u.databaseId) FROM UserTableRow u WHERE " - + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " - + TABLE_ROW_PREDICATE) - Page findAllDistinctDatabaseIds( - @Param("databaseId") String databaseId, Pageable pageable); + "SELECT DISTINCT databaseId FROM UserTableRow u where " + + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId))") + Page findAllDistinctDatabaseIds(String databaseId, Pageable pageable); @Query( value = diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java index f5b882525..8654d7f01 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java @@ -886,58 +886,6 @@ public void testPaginatedTableQueriesFilterBeforePaging() throws Exception { .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t02_explicit"))); } - /** A database whose only pointer is a view must not appear in either database listing. */ - @Test - public void testDatabaseQueriesExcludeViewOnlyDatabases() throws Exception { - // The @BeforeEach fixture row lives in test_db0; remove it so the database set is exactly the - // canonical seven. This is a deliberate mid-test global reset: the class-level @AfterEach - // deleteAll() restores order either way, but it does make this method order-fragile if - // @TestMethodOrder is ever added to this class. - htsRepository.deleteAll(); - htsRepository.save(entityTypeRow("db00_legacy", "t1", null)); - htsRepository.save(entityTypeRow("db01_view_only", "t1", "VIEW")); - htsRepository.save(entityTypeRow("db02_explicit", "t1", "TABLE")); - htsRepository.save(entityTypeRow("db03_view_only", "t1", "VIEW")); - htsRepository.save(entityTypeRow("db04_legacy", "t1", null)); - htsRepository.save(entityTypeRow("db05_view_only", "t1", "VIEW")); - htsRepository.save(entityTypeRow("db06_explicit", "t1", "TABLE")); - - mvc.perform(MockMvcRequestBuilders.get("/hts/tables/query").accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.results", hasSize(4))) - .andExpect( - jsonPath( - "$.results[*].databaseId", - containsInAnyOrder("db00_legacy", "db02_explicit", "db04_legacy", "db06_explicit"))) - .andExpect(jsonPath("$.results[*].databaseId", not(hasItem("db01_view_only")))); - - mvc.perform( - MockMvcRequestBuilders.get("/v1/hts/tables/query") - .param("page", "0") - .param("size", "2") - .param("sortBy", "databaseId") - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.pageResults.totalElements", is(4))) - .andExpect(jsonPath("$.pageResults.totalPages", is(2))) - .andExpect(jsonPath("$.pageResults.content", hasSize(2))) - .andExpect(jsonPath("$.pageResults.content[0].databaseId", is("db00_legacy"))) - .andExpect(jsonPath("$.pageResults.content[1].databaseId", is("db02_explicit"))); - - mvc.perform( - MockMvcRequestBuilders.get("/v1/hts/tables/query") - .param("page", "1") - .param("size", "2") - .param("sortBy", "databaseId") - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.pageResults.totalElements", is(4))) - .andExpect(jsonPath("$.pageResults.totalPages", is(2))) - .andExpect(jsonPath("$.pageResults.content", hasSize(2))) - .andExpect(jsonPath("$.pageResults.content[0].databaseId", is("db04_legacy"))) - .andExpect(jsonPath("$.pageResults.content[1].databaseId", is("db06_explicit"))); - } - /** The discriminator survives the HTTP PUT/GET boundary, and legacy writers stay null. */ @Test public void testEntityTypePutAndGetRoundTrip() throws Exception { diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java index ac4aba39b..551912932 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java @@ -500,33 +500,6 @@ public void testFindAllByFiltersDefaultsToTablesAndCanSelectViews() { assertThat(pageTableIds(viewPage1)).containsExactly("t05_view"); } - /** A database whose only pointers are views must disappear from the database listing. */ - @Test - public void testFindDistinctDatabasesExcludesViewOnlyDatabases() { - htsRepository.save(row("db00_legacy", "t1", null)); - htsRepository.save(row("db01_view_only", "t1", "VIEW")); - htsRepository.save(row("db02_explicit", "t1", "TABLE")); - htsRepository.save(row("db03_view_only", "t1", "VIEW")); - htsRepository.save(row("db04_legacy", "t1", null)); - htsRepository.save(row("db05_view_only", "t1", "VIEW")); - htsRepository.save(row("db06_explicit", "t1", "TABLE")); - - assertThat(Lists.newArrayList(htsRepository.findAllDistinctDatabaseIds())) - .containsExactlyInAnyOrder("db00_legacy", "db02_explicit", "db04_legacy", "db06_explicit"); - - Pageable dbPage0 = PageRequest.of(0, 2, Sort.by("databaseId")); - Page page0 = htsRepository.findAllDistinctDatabaseIds(null, dbPage0); - assertThat(page0.getTotalElements()).isEqualTo(4); - assertThat(page0.getTotalPages()).isEqualTo(2); - assertThat(page0.getContent()).containsExactly("db00_legacy", "db02_explicit"); - - Page page1 = - htsRepository.findAllDistinctDatabaseIds(null, PageRequest.of(1, 2, Sort.by("databaseId"))); - assertThat(page1.getTotalElements()).isEqualTo(4); - assertThat(page1.getTotalPages()).isEqualTo(2); - assertThat(page1.getContent()).containsExactly("db04_legacy", "db06_explicit"); - } - /** * Case/garbage matrix at the SQL layer. * From adca264703185857914786fbf4e51ea07f3fb2fe Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 14:31:56 -0700 Subject: [PATCH 04/12] BDP-108403: Inline rootMetadataFileLocation again, drop MetadataLocationUtils Restores the private rootMetadataFileLocation in OpenHouseInternalTableOperations to its pre-change form, doing the naming work inline, and deletes MetadataLocationUtils along with its test. The stated goal was to move this into a shared helper so the table and view paths use one implementation. The view path is not part of this change, so the helper has exactly one production caller: the very method it was extracted from. That is indirection rather than sharing. The caller now hops through a private wrapper into a public util, and OpenHouseInternalTableOperations picked up an import and a delegation without getting any simpler. The codecName parameter exists only to serve a future view caller, since Iceberg's table and view compression defaults differ, and the helper's test covered a gzip path that no production caller passes today. An extraction is a refactor that a second caller justifies. The view commit work will have that second caller and can do the extraction then, with the real shape of both callers in hand. This is the same reasoning that deferred the HouseTableMapper ViewMetadata overload out of this change. Behavior is unchanged, as it was when the code was extracted: identical path format, five-digit zero-padded version, random UUID, and extension resolved from the same codec property. Every OpenHouseInternalTableOperations metadata-location test passes untouched. The plain-text javadoc reference to this method in InternalRepositoryUtils#getSchemeLessPath again describes the inline implementation it was written against. The doRefresh non-table guard in this file is untouched; that is real view isolation logic and stays. Verified: internalcatalog 121 (was 124, exactly the 3 MetadataLocationUtilsTest cases), housetables 151, tables 519, tables-test-fixtures 8, spark-3.5 catalogTest 66 - all green, plus spotlessCheck. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OpenHouseInternalTableOperations.java | 13 +-- .../catalog/utils/MetadataLocationUtils.java | 31 ------- .../utils/MetadataLocationUtilsTest.java | 87 ------------------- 3 files changed, 7 insertions(+), 124 deletions(-) delete mode 100644 iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtils.java delete mode 100644 iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtilsTest.java diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java index 98771d313..ff565cd64 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java @@ -24,7 +24,6 @@ import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableCallerException; import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableConcurrentUpdateException; import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableNotFoundException; -import com.linkedin.openhouse.internal.catalog.utils.MetadataLocationUtils; import com.linkedin.openhouse.internal.catalog.utils.MetadataUpdateUtils; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.Span; @@ -41,6 +40,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.AllArgsConstructor; @@ -197,9 +197,6 @@ protected void refreshMetadata(final String metadataLoc) { * List Files and Manifest Files. Finally, the data sub-directory ./table_directory/data holds all * the Data Files. * - *

Naming itself lives in the metadata-type-neutral {@link MetadataLocationUtils} so the - * sibling view commit path shares it; only the codec resolution is table-specific here. - * * @param metadata {@link TableMetadata} for which the metadata file location needs to be derived. * @param newVersion new table version. * @return path to the root table metadata location. @@ -208,8 +205,12 @@ private static String rootMetadataFileLocation(TableMetadata metadata, int newVe String codecName = metadata.property( TableProperties.METADATA_COMPRESSION, TableProperties.METADATA_COMPRESSION_DEFAULT); - return MetadataLocationUtils.rootMetadataFileLocation( - metadata.location(), codecName, newVersion); + return String.format( + "%s/%s", + metadata.location(), + String.format( + "%05d-%s%s", + newVersion, UUID.randomUUID(), TableMetadataParser.getFileExtension(codecName))); } /** diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtils.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtils.java deleted file mode 100644 index 138d7fc30..000000000 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtils.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.linkedin.openhouse.internal.catalog.utils; - -import java.util.UUID; -import org.apache.iceberg.TableMetadataParser; - -/** - * Shared naming for the root metadata file, used by both the table and view commit paths. - * - *

The codec is supplied by the caller rather than resolved here because Iceberg's table and view - * compression defaults differ ({@code none} vs {@code gzip}). - */ -public final class MetadataLocationUtils { - - private MetadataLocationUtils() { - // no-op for util class constructor - } - - /** - * The UUID lets concurrent writers at the same version stage metadata side by side; the - * zero-padded version keeps lexical ordering aligned with numeric ordering. - */ - public static String rootMetadataFileLocation( - String rootLocation, String codecName, int newVersion) { - return String.format( - "%s/%s", - rootLocation, - String.format( - "%05d-%s%s", - newVersion, UUID.randomUUID(), TableMetadataParser.getFileExtension(codecName))); - } -} diff --git a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtilsTest.java b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtilsTest.java deleted file mode 100644 index c3f4c8a47..000000000 --- a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/utils/MetadataLocationUtilsTest.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.linkedin.openhouse.internal.catalog.utils; - -import java.util.HashSet; -import java.util.Set; -import java.util.regex.Pattern; -import org.apache.iceberg.TableProperties; -import org.apache.iceberg.view.ViewProperties; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** - * {@code rootMetadataFileLocation} is extracted out of {@link - * com.linkedin.openhouse.internal.catalog.OpenHouseInternalTableOperations} so the table commit - * path and the sibling view commit path can share metadata-file naming without sharing a codec - * default. The helper must stay metadata-type neutral: the caller supplies the codec. - */ -public class MetadataLocationUtilsTest { - - private static final String UUID_REGEX = - "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"; - - private static final Pattern UNCOMPRESSED = - Pattern.compile("^root/00007-" + UUID_REGEX + "\\.metadata\\.json$"); - - private static final Pattern GZIPPED = - Pattern.compile("^root/00007-" + UUID_REGEX + "\\.gz\\.metadata\\.json$"); - - @Test - public void rootMetadataFileLocationUsesFiveDigitVersionAndUuid() { - String location = MetadataLocationUtils.rootMetadataFileLocation("root", "none", 7); - - Assertions.assertTrue( - UNCOMPRESSED.matcher(location).matches(), - "Expected /00007-.metadata.json but was: " + location); - - // Version padding must be five digits so lexical ordering matches numeric ordering. - Assertions.assertTrue( - MetadataLocationUtils.rootMetadataFileLocation("root", "none", 1) - .startsWith("root/00001-")); - Assertions.assertTrue( - MetadataLocationUtils.rootMetadataFileLocation("root", "none", 12345) - .startsWith("root/12345-")); - - // The UUID is what lets concurrent writers stage the same version safely; it must differ. - Set generated = new HashSet<>(); - for (int i = 0; i < 5; i++) { - generated.add(MetadataLocationUtils.rootMetadataFileLocation("root", "none", 7)); - } - Assertions.assertEquals(5, generated.size(), "Each call must produce a distinct file name"); - } - - @Test - public void rootMetadataFileLocationUsesGzipExtension() { - String location = MetadataLocationUtils.rootMetadataFileLocation("root", "gzip", 7); - - Assertions.assertTrue( - GZIPPED.matcher(location).matches(), - "Expected /00007-.gz.metadata.json but was: " + location); - } - - /** - * Pinned deliberately: pinned Iceberg uses {@code none} as the table metadata-compression default - * but {@code gzip} as the view default. A helper that hard-coded the table default would silently - * change the view file extension, so the codec must be resolved by each caller and passed in. - */ - @Test - public void tableAndViewDefaultsArePassedExplicitly() { - Assertions.assertNotEquals( - TableProperties.METADATA_COMPRESSION_DEFAULT, - ViewProperties.METADATA_COMPRESSION_DEFAULT, - "This test is only meaningful while the table and view codec defaults differ"); - - String tableDefaultLocation = - MetadataLocationUtils.rootMetadataFileLocation( - "root", TableProperties.METADATA_COMPRESSION_DEFAULT, 7); - Assertions.assertTrue( - UNCOMPRESSED.matcher(tableDefaultLocation).matches(), - "Table default codec must yield .metadata.json but was: " + tableDefaultLocation); - - String viewDefaultLocation = - MetadataLocationUtils.rootMetadataFileLocation( - "root", ViewProperties.METADATA_COMPRESSION_DEFAULT, 7); - Assertions.assertTrue( - GZIPPED.matcher(viewDefaultLocation).matches(), - "View default codec must yield .gz.metadata.json but was: " + viewDefaultLocation); - } -} From 25cc35b65472cee0908eede54b0507fd4e425df3 Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 14:52:28 -0700 Subject: [PATCH 05/12] BDP-108403: Revert resolveFileIO entity-type guard, fix the fixture instead Restores OpenHouseInternalCatalog#resolveFileIO to its pre-change form and gives the raw-pointer test fixtures the storage type they were missing. The guard was compensating for a malformed fixture, not for a production condition. seedRawPointer built a HouseTable with databaseId, tableId, clusterId, tableUri, tableUUID, tableLocation, tableVersion and entityType but no storageType, so storageType.fromString(null) threw. A row seeded that way would have thrown just the same with entityType TABLE; the discriminator was incidental to the failure. The HTS schema settles it: storage_type is VARCHAR(128) DEFAULT 'hdfs' NOT NULL, so a null storage type cannot exist in production, whereas entity_type is DEFAULT NULL and is null on every pre-existing row. The guard was also wrong on its own terms. A real view row carries a valid storage type, so the original code returns the view's actual storage; skipping the row instead consults storageSelector, which can resolve to a different storage than the one the object is really on. And it is unreachable for the purpose it claimed: dropTable rejects a view before reaching this line, and on the newTableOps path doRefresh already treats a view as absent while create-over-view is stopped by the occupancy check. So the fix belongs in the fixture. Both seedRawPointer helpers now set storageType from storageManager.getDefaultStorage(), the same value a real table gets through HouseTableMapper. That makes the seeded row well-formed rather than merely tolerated. Every view-isolation guard test still passes, and now passes because the pointer is realistic rather than because production skips it: drop-VIEW, rename source and destination, CREATE-over-VIEW occupancy, findTableRefById, and the 404/409 status assertions, including all four case and garbage parameterizations of each. The dropTable and renameTable entity-type guards in this file are untouched, and so is the stripOhNamespace null-safety in the mapper - entity_type is DEFAULT NULL, so MapStruct's implicit String conversion would NPE on the real production mapping path without it. Verified: internalcatalog 121, tables 519, housetables 151, tables-test-fixtures 8, spark-3.5 catalogTest 66 - all green with no count change from this commit, plus spotlessCheck. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../internal/catalog/OpenHouseInternalCatalog.java | 6 +----- .../openhouse/tables/e2e/h2/TablesControllerTest.java | 1 + .../linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java index 095804f9e..f25b4333a 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java @@ -336,12 +336,8 @@ protected FileIO resolveFileIO(TableIdentifier tableIdentifier) { tableIdentifier.namespace().toString(), tableIdentifier.name()); } - // A non-table pointer is invisible here for the same reason it is invisible to doRefresh, so - // storage resolution falls back to the selector exactly as for an absent row. StorageType.Type type = - houseTable - .filter(row -> HouseTableSerdeUtils.isTableEntityType(row.getEntityType())) - .isPresent() + houseTable.isPresent() ? storageType.fromString(houseTable.get().getStorageType()) : storageSelector .selectStorage(tableIdentifier.namespace().toString(), tableIdentifier.name()) diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java index bc4a0538e..e03b750df 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java @@ -2106,6 +2106,7 @@ private void seedRawPointer(String databaseId, String tableId, String entityType .tableLocation( String.format("/base/%s/%s-uuid/00001-x.metadata.json", databaseId, tableId)) .tableVersion(INITIAL_TABLE_VERSION) + .storageType(storageManager.getDefaultStorage().getType().getValue()) .entityType(entityType) .build()); seededPointerKeys.add( diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java index 114e3e92e..6ef9d0939 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java @@ -1092,6 +1092,7 @@ private HouseTablePrimaryKey seedRawPointer(String databaseId, String tableId, S .tableUUID(UUID.randomUUID().toString()) .tableLocation(metadataFile.toString()) .tableVersion(INITIAL_TABLE_VERSION) + .storageType(storageManager.getDefaultStorage().getType().getValue()) .entityType(entityType) .build()); From 29d28f45d7293fae211ab90b5914927b605eedd2 Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 15:10:52 -0700 Subject: [PATCH 06/12] BDP-108403: Consolidate listTables onto findAllByFilters, parameterize pattern queries Deletes both findAllByDatabaseIdIgnoreCase overloads and routes listTables through findAllByFilters, and gives the two findAllByDatabaseIdAndTableIdLikeAllIgnoreCase overloads an entityType parameter. The paginated listTables already called findAllByFilters(databaseId, null, null, null, null, null, pageable) before this change set; it was switched to findAllByDatabaseIdIgnoreCase along the way. Consolidating restores that shape with entityType added. The non-paginated overload now matches it. The two plain methods were redundant with the parameterized family. Compared clause by clause: databaseId uses the same lower() comparison, tableId is exact equality rather than LIKE so an unset value adds no constraint, every other filter is guarded by an IS NULL check, DISTINCT over a single PK'd root is a no-op, and a null entityType takes the same predicate branch that the old hard-coded table predicate expressed. Identical results, one query family instead of two. The pattern overloads keep their own query because folding pattern matching into findAllByFilters would mean either a second tableId parameter or turning its exact match into a LIKE - and OpenHouse identifiers routinely contain underscores, so a LIKE there would silently treat them as wildcards. They now take entityType instead, reusing the same predicate constant. No listing method has a type baked into its name any more, and the call sites pass the request's own entityType rather than a hard-coded value, so the view path needs no new query methods - only entityType=VIEW at a call site. Verified: housetables 151 and tables 519, both unchanged and green, as expected for a refactor with identical semantics. HtsControllerTest 26, HtsRepositoryTest 17 and UserTablesServiceTest 21 all pass, which covers the rerouted list and pattern paths. Plus spotlessCheck. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../impl/jdbc/UserTableHtsJdbcRepository.java | 63 ++++++------------- .../services/UserTablesServiceImpl.java | 28 +++++++-- .../e2e/usertable/HtsRepositoryTest.java | 35 ++++++----- 3 files changed, 65 insertions(+), 61 deletions(-) diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java index 55005346a..e426b5012 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java @@ -39,74 +39,51 @@ Optional findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( void deleteByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(String databaseId, String tableId); - /** - * Excludes views from table listings. Applied in the query — never by filtering a returned {@link - * Page} — so content and counts agree. {@code IS NULL} is mandatory because the discriminator is - * nullable with no backfill; {@code upper(...)} avoids depending on collation, which differs - * between H2 in {@code MODE=MySQL} (case-sensitive) and production MySQL. - */ - String TABLE_ROW_PREDICATE = "(u.entityType IS NULL OR upper(u.entityType) = 'TABLE')"; - @Query("SELECT DISTINCT databaseId FROM UserTableRow") Iterable findAllDistinctDatabaseIds(); @Query( - "SELECT u FROM UserTableRow u WHERE " - + "lower(u.databaseId) = lower(:databaseId) AND " - + TABLE_ROW_PREDICATE) - Iterable findAllByDatabaseIdIgnoreCase(@Param("databaseId") String databaseId); + "SELECT DISTINCT databaseId FROM UserTableRow u where " + + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId))") + Page findAllDistinctDatabaseIds(String databaseId, Pageable pageable); + + /** + * A null or {@code TABLE} request means tables, including legacy null rows; {@code VIEW} means + * views only. An unknown value matches neither branch, so garbage fails closed here even if it + * bypasses API validation. + */ + String ENTITY_TYPE_FILTER_PREDICATE = + "(((:entityType IS NULL OR upper(:entityType) = 'TABLE') " + + "AND (u.entityType IS NULL OR upper(u.entityType) = 'TABLE')) " + + "OR (upper(:entityType) = 'VIEW' AND upper(u.entityType) = 'VIEW'))"; @Query( "SELECT u FROM UserTableRow u WHERE " + "lower(u.databaseId) = lower(:databaseId) AND " + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " - + TABLE_ROW_PREDICATE) + + ENTITY_TYPE_FILTER_PREDICATE) Iterable findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - @Param("databaseId") String databaseId, @Param("tableIdPattern") String tableIdPattern); - - @Query( - "SELECT DISTINCT databaseId FROM UserTableRow u where " - + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId))") - Page findAllDistinctDatabaseIds(String databaseId, Pageable pageable); - - @Query( - value = - "SELECT u FROM UserTableRow u WHERE " - + "lower(u.databaseId) = lower(:databaseId) AND " - + TABLE_ROW_PREDICATE, - countQuery = - "SELECT COUNT(u) FROM UserTableRow u WHERE " - + "lower(u.databaseId) = lower(:databaseId) AND " - + TABLE_ROW_PREDICATE) - Page findAllByDatabaseIdIgnoreCase( - @Param("databaseId") String databaseId, Pageable pageable); + @Param("databaseId") String databaseId, + @Param("tableIdPattern") String tableIdPattern, + @Param("entityType") String entityType); @Query( value = "SELECT u FROM UserTableRow u WHERE " + "lower(u.databaseId) = lower(:databaseId) AND " + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " - + TABLE_ROW_PREDICATE, + + ENTITY_TYPE_FILTER_PREDICATE, countQuery = "SELECT COUNT(u) FROM UserTableRow u WHERE " + "lower(u.databaseId) = lower(:databaseId) AND " + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " - + TABLE_ROW_PREDICATE) + + ENTITY_TYPE_FILTER_PREDICATE) Page findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( @Param("databaseId") String databaseId, @Param("tableIdPattern") String tableIdPattern, + @Param("entityType") String entityType, Pageable pageable); - /** - * A null or {@code TABLE} request means tables, including legacy null rows; {@code VIEW} means - * views only. An unknown value matches neither branch, so garbage fails closed here even if it - * bypasses API validation. - */ - String ENTITY_TYPE_FILTER_PREDICATE = - "(((:entityType IS NULL OR upper(:entityType) = 'TABLE') " - + "AND (u.entityType IS NULL OR upper(u.entityType) = 'TABLE')) " - + "OR (upper(:entityType) = 'VIEW' AND upper(u.entityType) = 'VIEW'))"; - String GENERAL_FILTER_PREDICATE = "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " + "(:tableId IS NULL OR lower(u.tableId) = lower(:tableId)) AND " diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java index 2b4d62fc6..64082fb84 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java @@ -288,7 +288,14 @@ private List listTables(UserTable userTable) { () -> StreamSupport.stream( htsJdbcRepository - .findAllByDatabaseIdIgnoreCase(userTable.getDatabaseId()) + .findAllByFilters( + userTable.getDatabaseId(), + null, + null, + null, + null, + null, + userTable.getEntityType()) .spliterator(), false) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)) @@ -302,7 +309,15 @@ private Page listTables(UserTable userTable, int page, int size, S return METRICS_REPORTER.executeWithStats( () -> htsJdbcRepository - .findAllByDatabaseIdIgnoreCase(userTable.getDatabaseId(), pageable) + .findAllByFilters( + userTable.getDatabaseId(), + null, + null, + null, + null, + null, + userTable.getEntityType(), + pageable) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)), MetricsConstant.HTS_PAGE_TABLES_TIME); } @@ -314,7 +329,9 @@ private List listTablesWithPattern(UserTable userTable) { StreamSupport.stream( htsJdbcRepository .findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - userTable.getDatabaseId(), userTable.getTableId()) + userTable.getDatabaseId(), + userTable.getTableId(), + userTable.getEntityType()) .spliterator(), false) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)) @@ -330,7 +347,10 @@ private Page listTablesWithPattern( () -> htsJdbcRepository .findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - userTable.getDatabaseId(), userTable.getTableId(), pageable) + userTable.getDatabaseId(), + userTable.getTableId(), + userTable.getEntityType(), + pageable) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)), MetricsConstant.HTS_PAGE_TABLES_TIME); } diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java index 551912932..3899998eb 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java @@ -139,7 +139,8 @@ public void testFindAllByDatabaseId() { htsRepository.save(TEST_TUPLE_1_1.get_userTableRow()); htsRepository.save(TEST_TUPLE_2_0.get_userTableRow()); List result = - Lists.newArrayList(htsRepository.findAllByDatabaseIdIgnoreCase("test_db0")); + Lists.newArrayList( + htsRepository.findAllByFilters("test_db0", null, null, null, null, null, null)); Assertions.assertEquals( Lists.newArrayList("test_table1", "test_table2"), result.stream().map(UserTableRow::getTableId).collect(Collectors.toList())); @@ -153,7 +154,7 @@ public void testFindAllByTableIdPattern() { List result = Lists.newArrayList( htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - "test_db0", "test_table%")); + "test_db0", "test_table%", null)); Assertions.assertEquals( Lists.newArrayList("test_table1", "test_table2"), result.stream().map(UserTableRow::getTableId).collect(Collectors.toList())); @@ -167,7 +168,7 @@ public void testFindAllByTableId() { List result = Lists.newArrayList( htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - "test_db0", "test_table1")); + "test_db0", "test_table1", null)); Assertions.assertEquals( Lists.newArrayList("test_table1"), result.stream().map(UserTableRow::getTableId).collect(Collectors.toList())); @@ -371,7 +372,8 @@ public void testFindAllByDatabaseIdFiltersViewsAndKeepsLegacyTables() { htsRepository.save(row("other_db", "t00_legacy", null)); List result = - Lists.newArrayList(htsRepository.findAllByDatabaseIdIgnoreCase(ENTITY_TYPE_DB)); + Lists.newArrayList( + htsRepository.findAllByFilters(ENTITY_TYPE_DB, null, null, null, null, null, null)); assertThat(tableIds(result)).containsExactly(CANONICAL_TABLE_IDS); assertThat(result) @@ -388,14 +390,16 @@ public void testFindAllByDatabaseIdFiltersBeforePagination() { seedCanonicalRows(ENTITY_TYPE_DB, ""); Page page0 = - htsRepository.findAllByDatabaseIdIgnoreCase(ENTITY_TYPE_DB, sortedPage(0)); + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, null, sortedPage(0)); assertThat(page0.getTotalElements()).isEqualTo(4); assertThat(page0.getTotalPages()).isEqualTo(2); assertThat(page0.getContent()).hasSize(2); assertThat(pageTableIds(page0)).containsExactly("t00_legacy", "t02_explicit"); Page page1 = - htsRepository.findAllByDatabaseIdIgnoreCase(ENTITY_TYPE_DB, sortedPage(1)); + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, null, sortedPage(1)); assertThat(page1.getTotalElements()).isEqualTo(4); assertThat(page1.getTotalPages()).isEqualTo(2); assertThat(page1.getContent()).hasSize(2); @@ -415,7 +419,7 @@ public void testFindAllByPatternFiltersViewsAndKeepsLegacyTables() { List result = Lists.newArrayList( htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - ENTITY_TYPE_DB, "match_%")); + ENTITY_TYPE_DB, "match_%", null)); assertThat(tableIds(result)) .containsExactly( @@ -430,7 +434,7 @@ public void testFindAllByPatternFiltersBeforePagination() { Page page0 = htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - ENTITY_TYPE_DB, "match_%", sortedPage(0)); + ENTITY_TYPE_DB, "match_%", null, sortedPage(0)); assertThat(page0.getTotalElements()).isEqualTo(4); assertThat(page0.getTotalPages()).isEqualTo(2); assertThat(page0.getContent()).hasSize(2); @@ -438,7 +442,7 @@ public void testFindAllByPatternFiltersBeforePagination() { Page page1 = htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - ENTITY_TYPE_DB, "match_%", sortedPage(1)); + ENTITY_TYPE_DB, "match_%", null, sortedPage(1)); assertThat(page1.getTotalElements()).isEqualTo(4); assertThat(page1.getTotalPages()).isEqualTo(2); assertThat(page1.getContent()).hasSize(2); @@ -516,22 +520,24 @@ public void testFindAllByFiltersDefaultsToTablesAndCanSelectViews() { public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { seedCaseNormalizationRows(); - assertThat(tableIds(htsRepository.findAllByDatabaseIdIgnoreCase(CASE_DB))) + assertThat( + tableIds(htsRepository.findAllByFilters(CASE_DB, null, null, null, null, null, null))) .containsExactly(CASE_VISIBLE_TABLE_IDS); assertThat( tableIds( - htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase(CASE_DB, "case%"))) + htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + CASE_DB, "case%", null))) .containsExactly(CASE_VISIBLE_TABLE_IDS); Page dbPage0 = - htsRepository.findAllByDatabaseIdIgnoreCase(CASE_DB, sortedPage(0)); + htsRepository.findAllByFilters(CASE_DB, null, null, null, null, null, null, sortedPage(0)); assertThat(dbPage0.getTotalElements()).isEqualTo(4); assertThat(dbPage0.getTotalPages()).isEqualTo(2); assertThat(pageTableIds(dbPage0)).containsExactly("case00_null", "case01_upper_table"); Page patternPage0 = htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - CASE_DB, "case%", sortedPage(0)); + CASE_DB, "case%", null, sortedPage(0)); assertThat(patternPage0.getTotalElements()).isEqualTo(4); assertThat(patternPage0.getTotalPages()).isEqualTo(2); assertThat(pageTableIds(patternPage0)).containsExactly("case00_null", "case01_upper_table"); @@ -551,7 +557,8 @@ public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { Lists.newArrayList( htsRepository.findAllByFilters(CASE_DB, null, null, null, null, null, "UNKNOWN"))) .isEmpty(); - assertThat(tableIds(htsRepository.findAllByDatabaseIdIgnoreCase(CASE_DB))) + assertThat( + tableIds(htsRepository.findAllByFilters(CASE_DB, null, null, null, null, null, null))) .doesNotContain(CASE_GARBAGE_ID); // The garbage row is still stored — it is hidden, not dropped. From 576d53f3d73279376add9456cbc7a21c4afc2fec Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 15:52:12 -0700 Subject: [PATCH 07/12] BDP-108403: Ship the entityType discriminator as substrate, defer the guards Removes every Java-side entity-type check in the tables service and the internal catalog, along with the tests that exercised them. What remains is the discriminator itself and the SQL that filters on it. Point-read type filtering is deferred to the view-commit ticket, where it will be done at the query level in HTS - a table-scoped getUserTable plus a neutral entity endpoint - rather than as Java guards layered on top of a type-blind read. Shipping the guards here would mean writing them twice and migrating callers off them a ticket later. The epic's acceptance criteria are evaluated across all six tickets rather than per ticket. Nothing deploys until the whole epic ships, and substantial client work is still required before a view can be created at all, so there is no window in which views exist unprotected by this deferral. Removed: the doRefresh non-table guard; the dropTable guard; the renameTable source guard and occupied-destination preflight; the findTableRefById type filter; findOccupyingEntityTypeById and its interface declaration and shared raw-pointer helper; and rejectNonTableNameOccupancy with both call sites. The five production files affected are now byte-identical to their pre-change state. Newly dead with them: HouseTableSerdeUtils.isTableEntityType, isViewEntityType, TABLE_ENTITY_TYPE and VIEW_ENTITY_TYPE, which had no remaining main-source caller. ENTITY_TYPE_FIELD_NAME stays - it is @VisibleForTesting like its neighbours in that class and backs the serde registration test, which is substrate. Write validation keeps its own ENTITY_TYPE_REGEX in ValidatorConstants and never depended on the removed constants. Kept as substrate: the schema column; UserTableRow, UserTable, UserTableDto and UserTablesMapper plumbing; HouseTable.entityType with its serde registration and mapper handling; the entity-type SQL predicate and its four query users in HTS; write validation; the stripOhNamespace null-safety; and every HTS-layer test for the list predicates and the round trip. Verified: housetables 151, internalcatalog 87, tables 475, tables-test-fixtures 8, spark-3.5 catalogTest 66 - all green, no surviving test failed. Plus spotlessCheck and checkstyle. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../catalog/OpenHouseInternalCatalog.java | 22 -- .../OpenHouseInternalTableOperations.java | 13 - .../catalog/mapper/HouseTableSerdeUtils.java | 16 - .../catalog/OpenHouseInternalCatalogTest.java | 156 --------- .../OpenHouseInternalTableOperationsTest.java | 88 ----- .../catalog/model/HouseTableTest.java | 55 --- .../e2e/usertable/HtsControllerTest.java | 3 - .../e2e/usertable/HtsRepositoryTest.java | 6 +- .../OpenHouseInternalRepository.java | 11 - .../impl/OpenHouseInternalRepositoryImpl.java | 47 +-- .../tables/services/TablesServiceImpl.java | 42 +-- .../tables/e2e/h2/RepositoryTest.java | 5 +- .../tables/e2e/h2/TablesControllerTest.java | 193 ----------- .../tables/e2e/h2/TablesServiceTest.java | 323 ------------------ .../OpenHouseInternalRepositoryImplTest.java | 171 ---------- 15 files changed, 14 insertions(+), 1137 deletions(-) diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java index f25b4333a..57c216b2b 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalog.java @@ -13,7 +13,6 @@ import com.linkedin.openhouse.internal.catalog.cache.TableMetadataCache; import com.linkedin.openhouse.internal.catalog.fileio.FileIOManager; import com.linkedin.openhouse.internal.catalog.mapper.HouseTableMapper; -import com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils; import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTableDto; @@ -158,11 +157,8 @@ public Optional findHouseTable(TableIdentifier identifier) { public boolean dropTable(TableIdentifier identifier, boolean purge) { // Look up the HouseTable row directly instead of calling loadTable(), so drop works even when // the table's metadata.json is corrupted and cannot be parsed by TableMetadataParser. - // This path bypasses loadTable(), so the doRefresh guard is inert here and the discriminator - // must be checked explicitly — otherwise a view could be dropped and purged. HouseTable houseTable = findHouseTable(identifier) - .filter(row -> HouseTableSerdeUtils.isTableEntityType(row.getEntityType())) .orElseThrow(() -> new NoSuchTableException("Table does not exist: %s", identifier)); HouseTablePrimaryKey primaryKey = @@ -214,24 +210,6 @@ private static String getTableBaseLocation(HouseTable houseTable, TableIdentifie @Override public void renameTable(TableIdentifier from, TableIdentifier to) { - // Defense in depth for direct catalog callers; both checks run before loadTable(), so a - // rejection reads no metadata, opens no transaction and writes no pointer. A wrong-type source - // is "no such table"; an occupied destination of ANY type is a collision. - findHouseTable(from) - .filter(row -> HouseTableSerdeUtils.isTableEntityType(row.getEntityType())) - .orElseThrow(() -> new NoSuchTableException("Table does not exist: %s", from)); - - findHouseTable(to) - .ifPresent( - occupant -> { - throw new AlreadyExistsException( - "Table", - to.namespace().toString() + "." + to.name(), - String.format( - "Cannot rename %s to %s because that name is already occupied", from, to), - null); - }); - Table fromTable = loadTable(from); String tableClusterId = fromTable.properties().get(CatalogConstants.OPENHOUSE_CLUSTERID_KEY); diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java index ff565cd64..b99915696 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperations.java @@ -17,7 +17,6 @@ import com.linkedin.openhouse.internal.catalog.exception.InvalidIcebergSnapshotException; import com.linkedin.openhouse.internal.catalog.fileio.FileIOManager; import com.linkedin.openhouse.internal.catalog.mapper.HouseTableMapper; -import com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils; import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; @@ -124,18 +123,6 @@ protected void doRefresh() { tableIdentifier.name()); metricsReporter.count(InternalCatalogMetricsConstant.NO_TABLE_WHEN_REFRESH); } - // A non-table row must act absent and never reach TableMetadataParser: view metadata.json is - // not parseable as table metadata, and an unknown type must fail closed. - if (houseTable.isPresent() - && !HouseTableSerdeUtils.isTableEntityType(houseTable.get().getEntityType())) { - log.debug( - "Key {}.{} is occupied by a non-table entity of type {}; treating it as absent for the " - + "table path", - tableIdentifier.namespace().toString(), - tableIdentifier.name(), - houseTable.get().getEntityType()); - houseTable = Optional.empty(); - } if (!houseTable.isPresent() && currentMetadataLocation() != null) { throw new IllegalStateException( String.format( diff --git a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableSerdeUtils.java b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableSerdeUtils.java index 0155bae31..61d62d95a 100644 --- a/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableSerdeUtils.java +++ b/iceberg/openhouse/internalcatalog/src/main/java/com/linkedin/openhouse/internal/catalog/mapper/HouseTableSerdeUtils.java @@ -37,22 +37,6 @@ private HouseTableSerdeUtils() { @VisibleForTesting public static final String ENTITY_TYPE_FIELD_NAME = "entityType"; - public static final String TABLE_ENTITY_TYPE = "TABLE"; - - public static final String VIEW_ENTITY_TYPE = "VIEW"; - - /** - * {@code null} means table: the column is nullable and not backfilled. Any other unrecognized - * value is neither a table nor a view, so table operations fail closed on it. - */ - public static boolean isTableEntityType(String entityType) { - return entityType == null || TABLE_ENTITY_TYPE.equalsIgnoreCase(entityType); - } - - public static boolean isViewEntityType(String entityType) { - return VIEW_ENTITY_TYPE.equalsIgnoreCase(entityType); - } - @VisibleForTesting public static String getCanonicalFieldName(String htsField) { return OPENHOUSE_NAMESPACE + htsField; diff --git a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalogTest.java b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalogTest.java index d42e11818..e9a8ae910 100644 --- a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalogTest.java +++ b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalCatalogTest.java @@ -9,22 +9,17 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.withSettings; -import com.linkedin.openhouse.common.exception.AlreadyExistsException; import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableNotFoundException; import java.util.Optional; -import org.apache.iceberg.Table; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.SupportsPrefixOperations; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.ValueSource; public class OpenHouseInternalCatalogTest { @@ -166,155 +161,4 @@ boolean isValidBaseIdentifier(TableIdentifier identifier) { return isValidIdentifier(identifier); } } - - // --------------------------------------------------------------------------------------------- - // Table APIs must fail closed on non-table pointer rows - // --------------------------------------------------------------------------------------------- - - private static final String DEST_TABLE = "dest_table"; - private static final TableIdentifier DEST_IDENTIFIER = TableIdentifier.of(DB, DEST_TABLE); - - private static HouseTablePrimaryKey key(String tableId) { - return HouseTablePrimaryKey.builder().databaseId(DB).tableId(tableId).build(); - } - - private static HouseTable pointer(String tableId, String entityType) { - return HouseTable.builder() - .databaseId(DB) - .tableId(tableId) - .tableUUID("uuid") - .tableLocation("/data/openhouse/test_db/" + tableId + "-uuid/00001-aaa.metadata.json") - .entityType(entityType) - .build(); - } - - /** - * Records whether the expensive typed load / transaction path was reached. The guards under test - * must reject before any of it runs, so the recording overrides throw if invoked in a case where - * the test expects them not to be. - */ - private static class RecordingCatalog extends OpenHouseInternalCatalog { - private final FileIO fileIO; - boolean loadTableCalled = false; - - RecordingCatalog(FileIO fileIO) { - this.fileIO = fileIO; - } - - @Override - protected FileIO resolveFileIO(TableIdentifier identifier) { - return fileIO; - } - - @Override - public Table loadTable(TableIdentifier identifier) { - loadTableCalled = true; - throw new AssertionError( - "loadTable must not be reached for a rejected rename: " + identifier); - } - } - - /** - * A VIEW (any spelling) or unknown discriminator is not a table: drop must behave as "no such - * table" and must never delete the shared pointer row or purge the object's files. - */ - @ParameterizedTest - @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) - void dropTableRejectsNonTableValuesWithoutDeletingPointerOrFiles(String entityType) { - HouseTableRepository repo = mock(HouseTableRepository.class); - when(repo.findById(any(HouseTablePrimaryKey.class))) - .thenReturn(Optional.of(pointer(TABLE, entityType))); - FileIO fileIO = - mock(FileIO.class, withSettings().extraInterfaces(SupportsPrefixOperations.class)); - OpenHouseInternalCatalog catalog = new FixedFileIOCatalog(fileIO); - catalog.houseTableRepository = repo; - - Assertions.assertThrows(NoSuchTableException.class, () -> catalog.dropTable(IDENTIFIER, true)); - - verify(repo, never()).deleteById(any(), anyBoolean()); - verify((SupportsPrefixOperations) fileIO, never()).deletePrefix(any()); - } - - /** - * The complement of the guard above: null and every spelling of TABLE remain droppable. This is - * what proves the Java guard and the SQL predicate agree on {@code table} / {@code TaBlE} — a - * guard that only accepted the uppercase literal would make lower/mixed-case rows visible in - * listings yet undroppable. - */ - @ParameterizedTest - @CsvSource( - nullValues = "NULL", - value = {"NULL", "TABLE", "table", "TaBlE"}) - void dropTableAcceptsCaseVariantsOfTable(String entityType) { - HouseTableRepository repo = mock(HouseTableRepository.class); - when(repo.findById(any(HouseTablePrimaryKey.class))) - .thenReturn(Optional.of(pointer(TABLE, entityType))); - FileIO fileIO = - mock(FileIO.class, withSettings().extraInterfaces(SupportsPrefixOperations.class)); - OpenHouseInternalCatalog catalog = new FixedFileIOCatalog(fileIO); - catalog.houseTableRepository = repo; - - Assertions.assertTrue(catalog.dropTable(IDENTIFIER, false)); - - verify(repo).deleteById(any(HouseTablePrimaryKey.class), eq(false)); - verify((SupportsPrefixOperations) fileIO, never()).deletePrefix(any()); - } - - /** - * A wrong-type rename SOURCE is indistinguishable from "no such table" and must be rejected - * before the source table is loaded, before any transaction is opened, and before the pointer is - * renamed. - */ - @ParameterizedTest - @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) - void renameTableRejectsNonTableSourceBeforeLoadingMetadata(String entityType) { - HouseTableRepository repo = mock(HouseTableRepository.class); - when(repo.findById(key(TABLE))).thenReturn(Optional.of(pointer(TABLE, entityType))); - when(repo.findById(key(DEST_TABLE))).thenReturn(Optional.empty()); - FileIO fileIO = - mock(FileIO.class, withSettings().extraInterfaces(SupportsPrefixOperations.class)); - RecordingCatalog catalog = new RecordingCatalog(fileIO); - catalog.houseTableRepository = repo; - - Assertions.assertThrows( - NoSuchTableException.class, () -> catalog.renameTable(IDENTIFIER, DEST_IDENTIFIER)); - - Assertions.assertFalse(catalog.loadTableCalled, "Source table must not be loaded"); - verify(repo, never()).rename(any(), any(), any(), any(), any()); - verify(repo, never()).save(any()); - verify(repo, never()).deleteById(any(), anyBoolean()); - } - - /** - * Defense in depth for direct catalog callers: ANY occupied destination pointer — a table, a view - * in any spelling, or an unknown type — is a name collision, and it must be detected before the - * source is loaded or a transaction is opened. - * - *

Because the shared primary key would eventually reject the write anyway with the SAME - * exception type, the exception alone proves nothing. The load-bearing assertions are the - * never-verifications: correct code never loads the source, never opens a transaction, and never - * asks the repository to rename or save. - */ - @ParameterizedTest - @ValueSource(strings = {"TABLE", "VIEW", "view", "ViEw", "UNKNOWN"}) - void renameTableRejectsAnyOccupiedRawDestinationBeforeSourceLoad(String destinationEntityType) { - HouseTableRepository repo = mock(HouseTableRepository.class); - when(repo.findById(key(TABLE))).thenReturn(Optional.of(pointer(TABLE, null))); - when(repo.findById(key(DEST_TABLE))) - .thenReturn(Optional.of(pointer(DEST_TABLE, destinationEntityType))); - FileIO fileIO = - mock(FileIO.class, withSettings().extraInterfaces(SupportsPrefixOperations.class)); - RecordingCatalog catalog = new RecordingCatalog(fileIO); - catalog.houseTableRepository = repo; - - Assertions.assertThrows( - AlreadyExistsException.class, () -> catalog.renameTable(IDENTIFIER, DEST_IDENTIFIER)); - - Assertions.assertFalse( - catalog.loadTableCalled, "Destination occupancy must be checked before loading the source"); - verify(repo, never()).rename(any(), any(), any(), any(), any()); - verify(repo, never()).save(any()); - verify(repo, never()).deleteById(any(), anyBoolean()); - verify((SupportsPrefixOperations) fileIO, never()).deletePrefix(any()); - } } diff --git a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperationsTest.java b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperationsTest.java index b85d225cd..3c0d92145 100644 --- a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperationsTest.java +++ b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperationsTest.java @@ -72,9 +72,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; @@ -2132,91 +2129,6 @@ void testRefreshMetadataMissingFileThrowsInvalidTableMetadataException() { () -> openHouseInternalTableOperations.refreshMetadata(nonExistentPath)); } - // --------------------------------------------------------------------------------------------- - // Table point loading must fail closed on non-table pointer rows - // --------------------------------------------------------------------------------------------- - - private static final String TYPED_METADATA_LOCATION = "typed_metadata_location"; - - private static HouseTablePrimaryKey testTablePrimaryKey() { - return HouseTablePrimaryKey.builder() - .databaseId(TEST_TABLE_IDENTIFIER.namespace().toString()) - .tableId(TEST_TABLE_IDENTIFIER.name()) - .build(); - } - - private static HouseTable typedPointer(String entityType) { - return HouseTable.builder() - .databaseId(TEST_TABLE_IDENTIFIER.namespace().toString()) - .tableId(TEST_TABLE_IDENTIFIER.name()) - .tableLocation(TYPED_METADATA_LOCATION) - .entityType(entityType) - .build(); - } - - /** - * A shared-key row that is a VIEW (any spelling) or an unknown type is not a table. The table - * path must treat it as absent and must never hand its metadata location to {@link - * TableMetadataParser} — a view metadata.json is not parseable as table metadata, and parsing an - * unknown type would leak a foreign object into the table API. - */ - @ParameterizedTest - @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) - void doRefreshTreatsViewRowAsNoSuchTableWithoutOpeningMetadata(String entityType) { - when(mockHouseTableRepository.findById(testTablePrimaryKey())) - .thenReturn(Optional.of(typedPointer(entityType))); - - try (MockedStatic parserMock = - Mockito.mockStatic(TableMetadataParser.class, Mockito.CALLS_REAL_METHODS)) { - parserMock - .when( - () -> - TableMetadataParser.read( - Mockito.any(FileIO.class), Mockito.eq(TYPED_METADATA_LOCATION))) - .thenReturn(BASE_TABLE_METADATA); - - openHouseInternalTableOperations.refresh(); - - Assertions.assertNull( - openHouseInternalTableOperations.currentMetadataLocation(), - "A " + entityType + " pointer must not become the table's current metadata location"); - Assertions.assertNull( - openHouseInternalTableOperations.current(), - "A " + entityType + " pointer must not produce table metadata"); - - parserMock.verify( - () -> TableMetadataParser.read(Mockito.any(FileIO.class), Mockito.anyString()), never()); - } - } - - /** The complement: null and every spelling of TABLE still refresh normally. */ - @ParameterizedTest - @CsvSource( - nullValues = "NULL", - value = {"NULL", "TABLE", "table", "TaBlE"}) - void doRefreshAcceptsNullAndExplicitTableRows(String entityType) { - when(mockHouseTableRepository.findById(testTablePrimaryKey())) - .thenReturn(Optional.of(typedPointer(entityType))); - - try (MockedStatic parserMock = - Mockito.mockStatic(TableMetadataParser.class, Mockito.CALLS_REAL_METHODS)) { - parserMock - .when( - () -> - TableMetadataParser.read( - Mockito.any(FileIO.class), Mockito.eq(TYPED_METADATA_LOCATION))) - .thenReturn(BASE_TABLE_METADATA); - - openHouseInternalTableOperations.refresh(); - - Assertions.assertEquals( - TYPED_METADATA_LOCATION, - openHouseInternalTableOperations.currentMetadataLocation(), - "entityType=" + entityType + " must be treated as a table"); - Assertions.assertNotNull(openHouseInternalTableOperations.current()); - } - } - /** * Backward compatibility: ordinary table commits must NOT start stamping the discriminator. * Adding {@code openhouse.entityType=TABLE} to every table commit would rewrite every table's diff --git a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/model/HouseTableTest.java b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/model/HouseTableTest.java index 3eff0b985..8855f59db 100644 --- a/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/model/HouseTableTest.java +++ b/iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/model/HouseTableTest.java @@ -10,8 +10,6 @@ import java.util.Set; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; public class HouseTableTest { @@ -85,58 +83,5 @@ public void testEntityTypeDefaultAndSerdeRegistration() { Assertions.assertEquals( "openhouse.entityType", HouseTableSerdeUtils.getCanonicalFieldName(HouseTableSerdeUtils.ENTITY_TYPE_FIELD_NAME)); - - Assertions.assertEquals("TABLE", HouseTableSerdeUtils.TABLE_ENTITY_TYPE); - Assertions.assertEquals("VIEW", HouseTableSerdeUtils.VIEW_ENTITY_TYPE); - } - - /** - * Authoritative case-sensitivity contract. H2 (MODE=MySQL) is case-sensitive while production - * MySQL default collation is not, so no SQL-level test can certify these semantics across - * providers. These Java guards are what every point read, drop, rename, and occupancy check - * actually consults, so they are pinned here independently of any database. - * - *

NULL and every spelling of TABLE classify as a table; every spelling of VIEW classifies as a - * view; anything else is neither, so table APIs fail closed rather than treating an unknown - * discriminator as a legacy table. - * - *

The empty-string row goes beyond the plan, which only named NULL/TABLE/VIEW/garbage. It is - * included deliberately because {@code entity_type} is a nullable {@code VARCHAR} that can hold - * {@code ''}, and "unknown non-null fails closed" must cover it. The natural implementation - * ({@code entityType == null || entityType.equalsIgnoreCase(TABLE)}) satisfies it for free — - * implementers must not special-case {@code ""} as blank/absent. - */ - @ParameterizedTest - @CsvSource( - nullValues = "NULL", - value = { - "NULL, true, false", - "TABLE, true, false", - "table, true, false", - "TaBlE, true, false", - "VIEW, false, true", - "view, false, true", - "ViEw, false, true", - "UNKNOWN, false, false", - "'', false, false" - }) - public void testEntityTypeClassification( - String entityType, boolean expectedTable, boolean expectedView) { - Assertions.assertEquals( - expectedTable, - HouseTableSerdeUtils.isTableEntityType(entityType), - "isTableEntityType(" + entityType + ")"); - Assertions.assertEquals( - expectedView, - HouseTableSerdeUtils.isViewEntityType(entityType), - "isViewEntityType(" + entityType + ")"); - - // The same classification must hold when read off a real pointer row. - HouseTable row = - HouseTable.builder().databaseId("d1").tableId("t1").entityType(entityType).build(); - Assertions.assertEquals( - expectedTable, HouseTableSerdeUtils.isTableEntityType(row.getEntityType())); - Assertions.assertEquals( - expectedView, HouseTableSerdeUtils.isViewEntityType(row.getEntityType())); } } diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java index 8654d7f01..4e1063653 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java @@ -1010,9 +1010,6 @@ public void testEntityTypeOnlyViewQueryRoutesToGeneralSearch() throws Exception * different candidate metadataLocation. The pointer must be rejected with 409 and left * byte-identical — same numeric JPA {@code version}, {@code entityType} and {@code * metadataLocation}. - * - *

The Tables Service occupancy tests prove a real CREATE never reaches this boundary; this - * test proves the boundary itself does not lose the view. */ @Test public void testCreateTablePointerPublishCannotOverwriteView() throws Exception { diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java index 3899998eb..6b06c1ab5 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java @@ -511,10 +511,8 @@ public void testFindAllByFiltersDefaultsToTablesAndCanSelectViews() { * production MySQL's default collation is case-INSENSITIVE. This test therefore proves that the * query normalizes explicitly (e.g. {@code upper(u.entityType) = 'TABLE'}) rather than leaning on * a provider collation: an implementation using a bare {@code = 'TABLE'} comparison would hide - * {@code table}/{@code TaBlE} here and fail. It does NOT certify production MySQL behavior — the - * authoritative case-insensitivity contract is pinned at the Java guard layer in {@code - * HouseTableTest#testEntityTypeClassification} and the catalog guard tests, and a MySQL staging - * smoke test is still required before views are enabled. + * {@code table}/{@code TaBlE} here and fail. It does NOT certify production MySQL behavior — a + * MySQL staging smoke test is still required before views are enabled. */ @Test public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/OpenHouseInternalRepository.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/OpenHouseInternalRepository.java index 166628592..d44d5dedf 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/OpenHouseInternalRepository.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/OpenHouseInternalRepository.java @@ -28,17 +28,6 @@ public interface OpenHouseInternalRepository */ Optional findTableRefById(TableDtoPrimaryKey tableDtoPrimaryKey); - /** - * Name occupancy, not table existence: unlike {@link #findById}/{@link #findTableRefById} this - * sees every pointer row and never parses metadata.json. Empty means no row exists; a null or - * {@code TABLE} discriminator returns {@code "TABLE"}, and an unrecognized value is returned as - * stored so an occupied name fails closed. - * - *

HTS errors must propagate — swallowing them into an empty result would read as "this name is - * free" and let a CREATE clobber an existing view. - */ - Optional findOccupyingEntityTypeById(TableDtoPrimaryKey tableDtoPrimaryKey); - List findAllIds(); Page findAllIds(Pageable pageable); diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImpl.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImpl.java index 1bd36c6fe..e894e10f5 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImpl.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImpl.java @@ -22,8 +22,6 @@ import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; import com.linkedin.openhouse.internal.catalog.SnapshotsUtil; import com.linkedin.openhouse.internal.catalog.fileio.FileIOManager; -import com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils; -import com.linkedin.openhouse.internal.catalog.model.HouseTable; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTableDto; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTablePrimaryKey; import com.linkedin.openhouse.tables.api.spec.v0.request.components.Policies; @@ -801,10 +799,13 @@ public Optional findById(TableDtoPrimaryKey tableDtoPrimaryKey) { @Override public Optional findTableRefById(TableDtoPrimaryKey tableDtoPrimaryKey) { - // Backs table-only operations, notably drop, which avoids loadTable so it survives corrupted - // metadata. That bypass is why the discriminator must be filtered explicitly here. - return findRawPointerById(tableDtoPrimaryKey) - .filter(houseTable -> HouseTableSerdeUtils.isTableEntityType(houseTable.getEntityType())) + if (!(catalog instanceof OpenHouseInternalCatalog)) { + throw new UnsupportedOperationException( + "findTableRefById is not supported for catalog type: " + catalog.getClass().getName()); + } + return ((OpenHouseInternalCatalog) catalog) + .findHouseTable( + TableIdentifier.of(tableDtoPrimaryKey.getDatabaseId(), tableDtoPrimaryKey.getTableId())) .map( houseTable -> TableDto.builder() @@ -815,40 +816,6 @@ public Optional findTableRefById(TableDtoPrimaryKey tableDtoPrimaryKey .build()); } - @Override - public Optional findOccupyingEntityTypeById(TableDtoPrimaryKey tableDtoPrimaryKey) { - // Unlike findTableRefById, this must see EVERY raw pointer: a name taken by a view or by an - // unrecognized type is still taken. Repository errors intentionally propagate. - return findRawPointerById(tableDtoPrimaryKey) - .map( - houseTable -> { - String entityType = houseTable.getEntityType(); - if (HouseTableSerdeUtils.isTableEntityType(entityType)) { - return HouseTableSerdeUtils.TABLE_ENTITY_TYPE; - } - if (HouseTableSerdeUtils.isViewEntityType(entityType)) { - return HouseTableSerdeUtils.VIEW_ENTITY_TYPE; - } - return entityType; - }); - } - - /** - * Single raw pointer lookup shared by the two public projections above, so the "can this be - * loaded as a table?" and "is this name taken?" answers cannot drift apart. Never calls - * loadTable. - */ - private Optional findRawPointerById(TableDtoPrimaryKey tableDtoPrimaryKey) { - if (!(catalog instanceof OpenHouseInternalCatalog)) { - throw new UnsupportedOperationException( - "Raw pointer lookup is not supported for catalog type: " + catalog.getClass().getName()); - } - return ((OpenHouseInternalCatalog) catalog) - .findHouseTable( - TableIdentifier.of( - tableDtoPrimaryKey.getDatabaseId(), tableDtoPrimaryKey.getTableId())); - } - // FIXME: Likely need a cache layer to avoid expensive tableScan. @Timed(metricKey = MetricsConstant.REPO_TABLE_EXISTS_TIME) @Override diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java index eb63ed397..89e48225f 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java @@ -9,7 +9,6 @@ import com.linkedin.openhouse.common.exception.OpenHouseCommitStateUnknownException; import com.linkedin.openhouse.common.exception.RequestValidationFailureException; import com.linkedin.openhouse.common.exception.UnsupportedClientOperationException; -import com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTableDto; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTablePrimaryKey; import com.linkedin.openhouse.tables.api.spec.v0.request.CreateUpdateLockRequestBody; @@ -106,14 +105,9 @@ public Pair putTable( String databaseId = createUpdateTableRequestBody.getDatabaseId(); String tableId = createUpdateTableRequestBody.getTableId(); - TableDtoPrimaryKey tableDtoPrimaryKey = - TableDtoPrimaryKey.builder().databaseId(databaseId).tableId(tableId).build(); - - // The typed load below hides non-table rows, so without this preflight a CREATE at a view's - // name would look free and fail only after writing a candidate metadata.json. - rejectNonTableNameOccupancy(tableDtoPrimaryKey); - - Optional tableDto = openHouseInternalRepository.findById(tableDtoPrimaryKey); + Optional tableDto = + openHouseInternalRepository.findById( + TableDtoPrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()); // Special case handling if (tableDto.isPresent() && createUpdateTableRequestBody.isStageReplace()) { @@ -218,29 +212,6 @@ private boolean updateNeeded( return !tablesMapper.toTableDto(existingTableDto, requestBody).equals(existingTableDto); } - /** - * {@code TABLE} occupancy returns normally so that {@code failOnExist=false} updates still work - * and the existing table-collision handling downstream owns that message. - */ - private void rejectNonTableNameOccupancy(TableDtoPrimaryKey key) { - Optional occupyingEntityType = - openHouseInternalRepository.findOccupyingEntityTypeById(key); - if (!occupyingEntityType.isPresent()) { - return; - } - String entityType = occupyingEntityType.get(); - if (HouseTableSerdeUtils.TABLE_ENTITY_TYPE.equals(entityType)) { - return; - } - String qualifiedName = String.format("%s.%s", key.getDatabaseId(), key.getTableId()); - String reason = - HouseTableSerdeUtils.VIEW_ENTITY_TYPE.equals(entityType) - ? "is occupied by a view" - : String.format("is occupied by a catalog object of type %s", entityType); - throw new AlreadyExistsException( - "Table", qualifiedName, String.format("Table name %s %s", qualifiedName, reason), null); - } - @Override public void deleteTable(String databaseId, String tableId, String actingPrincipal) { TableDtoPrimaryKey tableDtoPrimaryKey = @@ -273,13 +244,6 @@ public void renameTable( throw new NoSuchUserTableException(fromDatabaseId, fromTableId); } - // Check raw destination occupancy after the source is known to exist, but before - // the typed destination load (which hides views), the lock check, all authorization, and any - // mutation. A TABLE destination continues through the existing collision check below so its - // message and behavior are unchanged. - rejectNonTableNameOccupancy( - TableDtoPrimaryKey.builder().databaseId(toDatabaseId).tableId(toTableId).build()); - Optional targetedTableDto = openHouseInternalRepository.findById( TableDtoPrimaryKey.builder().databaseId(toDatabaseId).tableId(toTableId).build()); diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java index 52f5c537e..ddcbf91bd 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/RepositoryTest.java @@ -1237,9 +1237,8 @@ public void testRenameTablePreserveExistingCase() { renamedTable.get().getTableProperties().get("openhouse.tableUri"), "local-cluster.d1.t1_renamed"); - // The rename destination is now guarded: an occupied destination pointer is a collision rather - // than something a later rename silently overwrites. Leaving d1.t1_renamed behind would - // therefore collide with other tests in this class, which share one Spring context. + // Leaving d1.t1_renamed behind would leak into other tests in this class, which share one + // Spring context. openHouseInternalRepository.deleteById( TableDtoPrimaryKey.builder().databaseId("d1").tableId("t1_renamed").build()); } diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java index e03b750df..0e4b6d6cd 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesControllerTest.java @@ -23,9 +23,7 @@ import com.linkedin.openhouse.housetables.client.model.ToggleStatus; import com.linkedin.openhouse.internal.catalog.CatalogConstants; import com.linkedin.openhouse.internal.catalog.model.HouseTable; -import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTablePrimaryKey; -import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; import com.linkedin.openhouse.tables.api.spec.v0.request.CreateUpdateLockRequestBody; import com.linkedin.openhouse.tables.api.spec.v0.request.CreateUpdateTableRequestBody; import com.linkedin.openhouse.tables.api.spec.v0.request.components.ClusteringColumn; @@ -59,7 +57,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import lombok.SneakyThrows; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -70,11 +67,8 @@ import org.apache.iceberg.hadoop.HadoopFileIO; import org.apache.iceberg.types.Types; import org.json.JSONObject; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mockito; @@ -2067,191 +2061,4 @@ private MvcResult getTable(String databaseId, String tableId) throws Exception { .andExpect(status().isOk()) .andReturn(); } - - // --------------------------------------------------------------------------------------------- - // View isolation and shared-key collisions over the table HTTP API - // --------------------------------------------------------------------------------------------- - - /** - * Raw pointer rows must be seeded through the pointer repository directly, because a VIEW row is - * invisible to the table HTTP API and therefore cannot be created — or cleaned up — through it. - * Every seeded key is removed in {@link #deleteSeededPointers()}. - */ - @Autowired HouseTableRepository houseTablesRepository; - - private final List seededPointerKeys = new ArrayList<>(); - - @AfterEach - void deleteSeededPointers() { - for (HouseTablePrimaryKey key : seededPointerKeys) { - try { - houseTablesRepository.deleteById(key); - } catch (Exception e) { - // Best effort: cleanup must not mask the real assertion failure. - } - } - seededPointerKeys.clear(); - } - - private static final String VIEW_MIX_DB = "viewmixdb"; - - private void seedRawPointer(String databaseId, String tableId, String entityType) { - houseTablesRepository.save( - HouseTable.builder() - .databaseId(databaseId) - .tableId(tableId) - .clusterId("test-cluster") - .tableUri(String.format("test-cluster.%s.%s", databaseId, tableId)) - .tableUUID(UUID.randomUUID().toString()) - .tableLocation( - String.format("/base/%s/%s-uuid/00001-x.metadata.json", databaseId, tableId)) - .tableVersion(INITIAL_TABLE_VERSION) - .storageType(storageManager.getDefaultStorage().getType().getValue()) - .entityType(entityType) - .build()); - seededPointerKeys.add( - HouseTablePrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()); - } - - /** - * CREATE TABLE at a name already occupied by a view must be an accurate 409 with a message that - * names the real condition. A guard implemented only in the table {@code doRefresh} would let the - * create proceed all the way to the HTS publish boundary and surface a misleading concurrent - * modification error instead. - */ - @Test - public void testCreateTableOnViewNameReturnsTypedCollision() throws Exception { - seedRawPointer(VIEW_MIX_DB, "occupied_by_view", "VIEW"); - - GetTableResponseBody createBody = - buildGetTableResponseBodyWithDbTbl(VIEW_MIX_DB, "occupied_by_view"); - - mvc.perform( - MockMvcRequestBuilders.post( - String.format( - ValidationUtilities.CURRENT_MAJOR_VERSION_PREFIX + "/databases/%s/tables/", - VIEW_MIX_DB)) - .contentType(MediaType.APPLICATION_JSON) - .content( - buildCreateUpdateTableRequestBody(createBody) - .toBuilder() - .baseTableVersion(INITIAL_TABLE_VERSION) - .build() - .toJson()) - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isConflict()) - .andExpect( - jsonPath( - "$.message", - is("Table name " + VIEW_MIX_DB + ".occupied_by_view is occupied by a view"))); - } - - /** Renaming a real table onto a view's name is the same accurate 409. */ - @Test - public void testRenameTableToViewNameReturnsTypedCollision() throws Exception { - GetTableResponseBody source = buildGetTableResponseBodyWithDbTbl(VIEW_MIX_DB, "rename_source"); - RequestAndValidateHelper.createTableAndValidateResponse(source, mvc, storageManager); - seedRawPointer(VIEW_MIX_DB, "rename_dest_view", "VIEW"); - - try { - mvc.perform( - MockMvcRequestBuilders.patch( - String.format( - ValidationUtilities.CURRENT_MAJOR_VERSION_PREFIX - + "/databases/%s/tables/%s/rename", - VIEW_MIX_DB, - "rename_source")) - .contentType(MediaType.APPLICATION_JSON) - .param("toTableId", "rename_dest_view") - .param("toDatabaseId", VIEW_MIX_DB) - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isConflict()) - .andExpect( - jsonPath( - "$.message", - is("Table name " + VIEW_MIX_DB + ".rename_dest_view is occupied by a view"))); - - // The source table must still be there under its original name. - getTable(VIEW_MIX_DB, "rename_source"); - } finally { - RequestAndValidateHelper.deleteTableAndValidateResponse(mvc, source); - } - } - - /** - * HTTP contract for reading a view through the table API: 404, not 400. - * - *

The status code is the assertion, not the exception type. The Java/Spark client's {@code - * OpenHouseTableOperations.doRefresh} resumes as an empty {@code Optional} on both 404 - * and 400, so an implementation that surfaced a 400 (or a 200 with an empty body, or a 500 from - * an unguarded NPE) would look correct to every Spark/Java client while being wrong for the REST - * contract, curl, and the audit log. Only an explicit status assertion pins it. - * - *

UNKNOWN is included because an unrecognized discriminator must fail closed the same way, - * rather than being read as a legacy table. - */ - @ParameterizedTest - @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) - public void testGetTableOnViewNameReturnsNotFound(String entityType) throws Exception { - seedRawPointer(VIEW_MIX_DB, "read_as_table", entityType); - - mvc.perform( - MockMvcRequestBuilders.get( - String.format( - ValidationUtilities.CURRENT_MAJOR_VERSION_PREFIX - + "/databases/%s/tables/%s", - VIEW_MIX_DB, - "read_as_table")) - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isNotFound()) - .andExpect(jsonPath("$.status", is(equalTo(HttpStatus.NOT_FOUND.name())))) - .andExpect(jsonPath("$.error", is(equalTo(HttpStatus.NOT_FOUND.getReasonPhrase())))); - - // The pointer is hidden from the table API, not destroyed by reading it. - Assertions.assertTrue( - houseTablesRepository - .findById( - HouseTablePrimaryKey.builder() - .databaseId(VIEW_MIX_DB) - .tableId("read_as_table") - .build()) - .isPresent()); - } - - /** - * HTTP contract for dropping a view through the table API: 404, not 400, and the pointer - * plus its files survive. - * - *

This is the path where a {@code doRefresh}-only guard does nothing at all: {@code - * TablesServiceImpl.deleteTable} deliberately bypasses {@code loadTable} via {@code - * findTableRefById} so that drop still works on corrupted metadata. The guard therefore has to - * live in the table-ref projection, and this test is what proves it does. - */ - @ParameterizedTest - @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) - public void testDeleteTableOnViewNameReturnsNotFound(String entityType) throws Exception { - seedRawPointer(VIEW_MIX_DB, "drop_as_table", entityType); - - mvc.perform( - MockMvcRequestBuilders.delete( - String.format( - ValidationUtilities.CURRENT_MAJOR_VERSION_PREFIX - + "/databases/%s/tables/%s", - VIEW_MIX_DB, - "drop_as_table")) - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isNotFound()) - .andExpect(jsonPath("$.status", is(equalTo(HttpStatus.NOT_FOUND.name())))) - .andExpect(jsonPath("$.error", is(equalTo(HttpStatus.NOT_FOUND.getReasonPhrase())))); - - Assertions.assertTrue( - houseTablesRepository - .findById( - HouseTablePrimaryKey.builder() - .databaseId(VIEW_MIX_DB) - .tableId("drop_as_table") - .build()) - .isPresent(), - "A rejected drop must leave the view pointer in place"); - } } diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java index 6ef9d0939..563162e61 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/TablesServiceTest.java @@ -15,10 +15,8 @@ import com.linkedin.openhouse.common.test.schema.ResourceIoHelper; import com.linkedin.openhouse.internal.catalog.CatalogConstants; import com.linkedin.openhouse.internal.catalog.model.HouseTable; -import com.linkedin.openhouse.internal.catalog.model.HouseTablePrimaryKey; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTableDto; import com.linkedin.openhouse.internal.catalog.model.SoftDeletedTablePrimaryKey; -import com.linkedin.openhouse.internal.catalog.repository.HouseTableRepository; import com.linkedin.openhouse.tables.api.spec.v0.request.CreateUpdateLockRequestBody; import com.linkedin.openhouse.tables.api.spec.v0.request.UpdateAclPoliciesRequestBody; import com.linkedin.openhouse.tables.api.spec.v0.request.components.TimePartitionSpec; @@ -30,29 +28,19 @@ import com.linkedin.openhouse.tables.repository.OpenHouseInternalRepository; import com.linkedin.openhouse.tables.services.TablesService; import com.linkedin.openhouse.tables.utils.AuthorizationUtils; -import java.io.File; import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Set; import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.iceberg.Schema; import org.apache.iceberg.types.Types; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -1029,315 +1017,4 @@ public void testRestoreTableNotFound() { tablesService.restoreTable( nonExistentDbId, "nonexistent_table", deletedAtMs, TEST_USER)); } - - // --------------------------------------------------------------------------------------------- - // Shared-key occupancy and wrong-type guards on the table service - // --------------------------------------------------------------------------------------------- - - /** - * Raw pointer rows must be seeded through the pointer repository directly, because a VIEW row is - * invisible to the table API and therefore cannot be created — or cleaned up — through it. Every - * seeded key is removed in {@link #deleteSeededPointers()}. - */ - @Autowired HouseTableRepository houseTablesRepository; - - private final List seededPointerKeys = new ArrayList<>(); - - private final List seededDirectories = new ArrayList<>(); - - @AfterEach - public void deleteSeededPointers() throws IOException { - for (HouseTablePrimaryKey key : seededPointerKeys) { - try { - houseTablesRepository.deleteById(key); - } catch (Exception e) { - // Best effort: cleanup must not mask the real assertion failure. - } - } - seededPointerKeys.clear(); - for (Path directory : seededDirectories) { - try (Stream paths = Files.walk(directory)) { - paths.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); - } catch (Exception e) { - // Best effort. - } - } - seededDirectories.clear(); - } - - private static final String OCCUPANCY_DB = "entity_type_occupancy_db"; - - /** - * Seeds a raw pointer whose {@code tableLocation} points at a real on-disk metadata.json under - * the storage root, so a purge attempt would be observable as a missing file. - */ - private HouseTablePrimaryKey seedRawPointer(String databaseId, String tableId, String entityType) - throws IOException { - Path tableDirectory = - Paths.get( - storageManager.getDefaultStorage().getClient().getRootPrefix(), - databaseId, - tableId + "-" + UUID.randomUUID()); - Files.createDirectories(tableDirectory); - Path metadataFile = tableDirectory.resolve("00001-seeded.metadata.json"); - Files.write(metadataFile, "{\"not\":\"parsed by these tests\"}".getBytes()); - seededDirectories.add(tableDirectory); - - houseTablesRepository.save( - HouseTable.builder() - .databaseId(databaseId) - .tableId(tableId) - .clusterId(TABLE_DTO.getClusterId()) - .tableUri(String.format("%s.%s.%s", TABLE_DTO.getClusterId(), databaseId, tableId)) - .tableUUID(UUID.randomUUID().toString()) - .tableLocation(metadataFile.toString()) - .tableVersion(INITIAL_TABLE_VERSION) - .storageType(storageManager.getDefaultStorage().getType().getValue()) - .entityType(entityType) - .build()); - - HouseTablePrimaryKey key = - HouseTablePrimaryKey.builder().databaseId(databaseId).tableId(tableId).build(); - seededPointerKeys.add(key); - return key; - } - - private HouseTable reloadPointer(HouseTablePrimaryKey key) { - return houseTablesRepository - .findById(key) - .orElseThrow( - () -> - new AssertionError( - "Raw pointer " + key.getDatabaseId() + "." + key.getTableId() + " is gone")); - } - - /** Snapshot of every metadata.json under a database's storage root. */ - private Set metadataFilesUnder(String databaseId) throws IOException { - Path databaseRoot = - Paths.get(storageManager.getDefaultStorage().getClient().getRootPrefix(), databaseId); - if (!Files.exists(databaseRoot)) { - return Collections.emptySet(); - } - try (Stream paths = Files.walk(databaseRoot)) { - return paths - .filter(p -> p.toString().endsWith(".metadata.json")) - .map(Path::toString) - .collect(Collectors.toSet()); - } - } - - /** - * CREATE TABLE at a view-occupied name must be rejected with an accurate typed 409 BEFORE any - * authorization decision and BEFORE any metadata file is written. - * - *

This is the test that fails against a naive design that only guards the table {@code - * doRefresh}: with that design the typed load reports "no table", so the create proceeds through - * authorization, allocates a location, and writes a candidate metadata.json — leaving an orphaned - * file and surfacing a misleading concurrency 409 from the HTS publish boundary. The load-bearing - * assertions here are therefore the unchanged metadata-file set and the never-authorized - * verification, not the exception type. - */ - @Test - public void testCreateTableRejectsViewOccupancyBeforeAuthorizationOrMetadata() - throws IOException { - HouseTablePrimaryKey viewKey = seedRawPointer(OCCUPANCY_DB, "occupied_by_view", "VIEW"); - HouseTable before = reloadPointer(viewKey); - Set metadataFilesBefore = metadataFilesUnder(OCCUPANCY_DB); - - // Nothing authorizes during raw-repository seeding, so a plain never() verification is enough. - Mockito.verify(authorizationHandler, Mockito.never()) - .checkAccessDecision(Mockito.any(), (DatabaseDto) Mockito.any(), Mockito.any()); - - TableDto createDto = - TABLE_DTO - .toBuilder() - .databaseId(OCCUPANCY_DB) - .tableId("occupied_by_view") - .tableUri(TABLE_DTO.getClusterId() + "." + OCCUPANCY_DB + ".occupied_by_view") - .tableVersion(INITIAL_TABLE_VERSION) - .build(); - - AlreadyExistsException thrown = - Assertions.assertThrows( - AlreadyExistsException.class, - () -> - tablesService.putTable( - buildCreateUpdateTableRequestBody(createDto), TEST_USER, true)); - Assertions.assertEquals( - "Table name " + OCCUPANCY_DB + ".occupied_by_view is occupied by a view", - thrown.getMessage()); - - HouseTable after = reloadPointer(viewKey); - Assertions.assertEquals("VIEW", after.getEntityType()); - Assertions.assertEquals(before.getEntityType(), after.getEntityType()); - Assertions.assertEquals(before.getTableLocation(), after.getTableLocation()); - Assertions.assertEquals(before.getTableUUID(), after.getTableUUID()); - - Assertions.assertEquals( - metadataFilesBefore, - metadataFilesUnder(OCCUPANCY_DB), - "A rejected create must not write a candidate metadata.json"); - - Mockito.verify(authorizationHandler, Mockito.never()) - .checkAccessDecision(Mockito.any(), (DatabaseDto) Mockito.any(), Mockito.any()); - Mockito.verify(authorizationHandler, Mockito.never()) - .checkAccessDecision(Mockito.any(), (TableDto) Mockito.any(), Mockito.any()); - } - - /** A view (any spelling) or unknown type can never be dropped through the table API. */ - @ParameterizedTest - @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) - public void testDeleteTableRejectsNonTableAndPreservesPointer(String entityType) - throws IOException { - HouseTablePrimaryKey key = seedRawPointer(OCCUPANCY_DB, "no_drop_target", entityType); - HouseTable before = reloadPointer(key); - Path metadataFile = Paths.get(before.getTableLocation()); - Assertions.assertTrue(Files.exists(metadataFile)); - - Assertions.assertThrows( - NoSuchUserTableException.class, - () -> tablesService.deleteTable(OCCUPANCY_DB, "no_drop_target", TEST_USER)); - - HouseTable after = reloadPointer(key); - Assertions.assertEquals(entityType, after.getEntityType()); - Assertions.assertEquals(before.getTableLocation(), after.getTableLocation()); - Assertions.assertTrue( - Files.exists(metadataFile), "A rejected drop must not purge the object's storage prefix"); - } - - /** A wrong-type rename SOURCE reads as "no such table" and nothing is created or moved. */ - @ParameterizedTest - @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) - public void testRenameTableRejectsNonTableSourceAndPreservesPointer(String entityType) - throws IOException { - HouseTablePrimaryKey sourceKey = seedRawPointer(OCCUPANCY_DB, "no_rename_source", entityType); - HouseTable before = reloadPointer(sourceKey); - - Assertions.assertThrows( - NoSuchUserTableException.class, - () -> - tablesService.renameTable( - OCCUPANCY_DB, "no_rename_source", OCCUPANCY_DB, "renamed_target", TEST_USER)); - - HouseTable after = reloadPointer(sourceKey); - Assertions.assertEquals(entityType, after.getEntityType()); - Assertions.assertEquals(before.getTableLocation(), after.getTableLocation()); - Assertions.assertFalse( - houseTablesRepository - .findById( - HouseTablePrimaryKey.builder() - .databaseId(OCCUPANCY_DB) - .tableId("renamed_target") - .build()) - .isPresent(), - "A rejected rename must not create the destination pointer"); - } - - /** - * Renaming a real table onto a view-occupied destination must fail with an accurate typed 409 - * BEFORE authorization, before any pointer mutation, and before any metadata is written. - * - *

The exception TYPE alone proves nothing here: the shared primary key would eventually raise - * the same {@link AlreadyExistsException} from the storage layer. What kills that accidental - * fallback is (a) the byte-identical destination pointer, (b) the unchanged source {@code - * *.metadata.json} file set — the fallback only triggers after a candidate file is written — and - * (c) the verification that no authorization decision was ever taken. - */ - @Test - public void testRenameTableRejectsViewDestinationBeforeAuthorizationOrMetadata() - throws IOException { - TableDto sourceDto = - TABLE_DTO - .toBuilder() - .databaseId(OCCUPANCY_DB) - .tableId("rename_source") - .tableUri(TABLE_DTO.getClusterId() + "." + OCCUPANCY_DB + ".rename_source") - .tableVersion(INITIAL_TABLE_VERSION) - .build(); - TableDto created = verifyPutTableRequest(sourceDto, null, true); - HouseTablePrimaryKey sourceKey = - HouseTablePrimaryKey.builder().databaseId(OCCUPANCY_DB).tableId("rename_source").build(); - Path sourceDirectory = Paths.get(URI.create(created.getTableLocation())).getParent(); - // Register the real source table for teardown immediately after creation, so it cannot survive - // the class if any assertion below fails. @AfterEach removes both the pointer and the files. - seededPointerKeys.add(sourceKey); - seededDirectories.add(sourceDirectory); - - HouseTablePrimaryKey destinationKey = seedRawPointer(OCCUPANCY_DB, "rename_dest_view", "VIEW"); - - HouseTable sourceBefore = reloadPointer(sourceKey); - HouseTable destinationBefore = reloadPointer(destinationKey); - Set sourceMetadataBefore = metadataFilesIn(sourceDirectory); - - // Source setup legitimately authorizes; clear the recorded invocations (but keep the stubs) so - // the never() verification below is about the rename only. - Mockito.clearInvocations(authorizationHandler); - - AlreadyExistsException thrown = - Assertions.assertThrows( - AlreadyExistsException.class, - () -> - tablesService.renameTable( - OCCUPANCY_DB, "rename_source", OCCUPANCY_DB, "rename_dest_view", TEST_USER)); - Assertions.assertEquals( - "Table name " + OCCUPANCY_DB + ".rename_dest_view is occupied by a view", - thrown.getMessage()); - - HouseTable destinationAfter = reloadPointer(destinationKey); - Assertions.assertEquals("VIEW", destinationAfter.getEntityType()); - Assertions.assertEquals( - destinationBefore.getTableLocation(), destinationAfter.getTableLocation()); - Assertions.assertEquals(destinationBefore.getTableUUID(), destinationAfter.getTableUUID()); - - HouseTable sourceAfter = reloadPointer(sourceKey); - Assertions.assertEquals(sourceBefore.getTableLocation(), sourceAfter.getTableLocation()); - Assertions.assertEquals(sourceBefore.getEntityType(), sourceAfter.getEntityType()); - - Assertions.assertEquals( - sourceMetadataBefore, - metadataFilesIn(sourceDirectory), - "A rejected rename must not write a new source metadata.json"); - - Mockito.verify(authorizationHandler, Mockito.never()) - .checkAccessDecision(Mockito.any(), (DatabaseDto) Mockito.any(), Mockito.any()); - Mockito.verify(authorizationHandler, Mockito.never()) - .checkAccessDecision(Mockito.any(), (TableDto) Mockito.any(), Mockito.any()); - // No explicit deleteTable here: sourceKey/sourceDirectory are registered for @AfterEach - // teardown above, so cleanup happens even if an assertion between here and there fails. - } - - /** - * Service-layer complement to the HTTP 404 tests: reading a view (any spelling) or an unknown - * discriminator through the table API is indistinguishable from "no such table", and the read - * itself must not disturb the pointer. - */ - @ParameterizedTest - @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) - public void testGetTableRejectsNonTableAndPreservesPointer(String entityType) throws IOException { - HouseTablePrimaryKey key = seedRawPointer(OCCUPANCY_DB, "read_as_table", entityType); - HouseTable before = reloadPointer(key); - - Assertions.assertThrows( - NoSuchUserTableException.class, - () -> tablesService.getTable(OCCUPANCY_DB, "read_as_table", TEST_USER)); - - HouseTable after = reloadPointer(key); - Assertions.assertEquals(entityType, after.getEntityType()); - Assertions.assertEquals(before.getTableLocation(), after.getTableLocation()); - Assertions.assertTrue( - Files.exists(Paths.get(before.getTableLocation())), - "A rejected read must not touch the object's files"); - } - - private Set metadataFilesIn(Path directory) throws IOException { - if (directory == null || !Files.exists(directory)) { - return Collections.emptySet(); - } - try (Stream paths = Files.walk(directory)) { - return paths - .filter(p -> p.toString().endsWith(".metadata.json")) - .map(Path::toString) - .collect(Collectors.toSet()); - } - } } diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImplTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImplTest.java index ca5d19cba..e8e4ad585 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImplTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/repository/impl/OpenHouseInternalRepositoryImplTest.java @@ -3,16 +3,12 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.linkedin.openhouse.cluster.configs.ClusterProperties; import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; import com.linkedin.openhouse.internal.catalog.mapper.HouseTableSerdeUtils; import com.linkedin.openhouse.internal.catalog.model.HouseTable; -import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableCallerException; -import com.linkedin.openhouse.internal.catalog.repository.exception.HouseTableRepositoryStateUnknownException; import com.linkedin.openhouse.tables.common.TableType; import com.linkedin.openhouse.tables.dto.mapper.iceberg.PoliciesSpecMapper; import com.linkedin.openhouse.tables.model.TableDto; @@ -30,12 +26,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.ValueSource; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -168,169 +160,6 @@ void findTableRefByIdThrowsWhenCatalogIsNotOpenHouseInternalCatalog() { TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build())); } - // --------------------------------------------------------------------------------------------- - // Typed table load vs. shared-name occupancy - // --------------------------------------------------------------------------------------------- - - private static HouseTable pointer(String entityType) { - return HouseTable.builder() - .databaseId(DB_ID) - .tableId(TABLE_ID) - .tableUUID("uuid-1") - .tableLocation("/base/db/table-uuid-1/00001-x.metadata.json") - .entityType(entityType) - .build(); - } - - /** - * {@code findTableRefById} answers "can this key be operated on as a table?" — it backs drop. A - * VIEW or unknown pointer must read as absent so a view can never be dropped through the table - * API. - */ - @ParameterizedTest - @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) - void findTableRefByIdReturnsEmptyForNonTable(String entityType) { - when(catalog.findHouseTable(TableIdentifier.of(DB_ID, TABLE_ID))) - .thenReturn(Optional.of(pointer(entityType))); - - Assertions.assertFalse( - openHouseInternalRepository - .findTableRefById( - TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build()) - .isPresent(), - "entityType=" + entityType + " must not resolve to a table ref"); - } - - /** The complement: null and every spelling of TABLE keep their existing partial DTO mapping. */ - @ParameterizedTest - @CsvSource( - nullValues = "NULL", - value = {"NULL", "TABLE", "table", "TaBlE"}) - void findTableRefByIdAcceptsNullAndCaseInsensitiveTable(String entityType) { - when(catalog.findHouseTable(TableIdentifier.of(DB_ID, TABLE_ID))) - .thenReturn(Optional.of(pointer(entityType))); - - Optional result = - openHouseInternalRepository.findTableRefById( - TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build()); - - Assertions.assertTrue(result.isPresent(), "entityType=" + entityType + " must be a table"); - TableDto dto = result.get(); - Assertions.assertEquals(DB_ID, dto.getDatabaseId()); - Assertions.assertEquals(TABLE_ID, dto.getTableId()); - Assertions.assertEquals("uuid-1", dto.getTableUUID()); - Assertions.assertEquals("/base/db/table-uuid-1/00001-x.metadata.json", dto.getTableLocation()); - Assertions.assertNull(dto.getSchema()); - Assertions.assertNull(dto.getTableCreator()); - } - - /** - * Occupancy is deliberately NOT the same question as typed load. This method answers "is this - * shared key taken, and by what?" so CREATE and rename-destination can reject an occupied name - * accurately instead of seeing a view-hidden key as free. It must therefore see EVERY raw pointer - * — including unknown types, which stay present so callers fail closed — and must never parse - * metadata (never call loadTable). - */ - @ParameterizedTest - @CsvSource( - nullValues = "NULL", - value = { - "NULL, TABLE", - "TABLE, TABLE", - "table, TABLE", - "TaBlE, TABLE", - "VIEW, VIEW", - "view, VIEW", - "ViEw, VIEW", - "UNKNOWN, UNKNOWN" - }) - void findOccupyingEntityTypeSeesEveryRawPointerWithoutLoadingMetadata( - String storedEntityType, String expectedCanonical) { - when(catalog.findHouseTable(TableIdentifier.of(DB_ID, TABLE_ID))) - .thenReturn(Optional.of(pointer(storedEntityType))); - - Optional occupancy = - openHouseInternalRepository.findOccupyingEntityTypeById( - TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build()); - - Assertions.assertTrue( - occupancy.isPresent(), - "A stored pointer with entityType=" + storedEntityType + " occupies the name"); - Assertions.assertEquals(expectedCanonical, occupancy.get()); - - verify(catalog).findHouseTable(TableIdentifier.of(DB_ID, TABLE_ID)); - verify(catalog, never()).loadTable(any(TableIdentifier.class)); - } - - /** Only a genuinely absent pointer means the name is free. */ - @Test - void findOccupyingEntityTypeReturnsEmptyOnlyWhenNoPointerExists() { - when(catalog.findHouseTable(any(TableIdentifier.class))).thenReturn(Optional.empty()); - - Assertions.assertFalse( - openHouseInternalRepository - .findOccupyingEntityTypeById( - TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build()) - .isPresent()); - - verify(catalog, never()).loadTable(any(TableIdentifier.class)); - } - - /** - * HTS 4xx must PROPAGATE out of the occupancy lookup. Swallowing a repository error into "the - * name is free" would let a CREATE proceed over an existing view during an HTS incident, which is - * exactly the hole this occupancy check exists to close. - * - *

Stubbed with {@code doThrow(...).when(...)} rather than {@code when(...).thenThrow(...)}: - * the latter evaluates its argument, which invokes the mock and would blow up the test itself. - */ - @Test - void findOccupyingEntityTypeDoesNotSwallowClientErrors() { - Mockito.doThrow( - new HouseTableCallerException("HTS returned 400", new RuntimeException("bad request"))) - .when(catalog) - .findHouseTable(any(TableIdentifier.class)); - - Assertions.assertThrows( - HouseTableCallerException.class, - () -> - openHouseInternalRepository.findOccupyingEntityTypeById( - TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build())); - } - - /** - * HTS 5xx must PROPAGATE for the same reason: an outage must never read as an unoccupied name. - * This is the branch that a broad {@code catch (Exception e) { return Optional.empty(); }} would - * silently convert into "free", reopening the CREATE-over-VIEW hole. - */ - @Test - void findOccupyingEntityTypeDoesNotSwallowServerErrors() { - Mockito.doThrow( - new HouseTableRepositoryStateUnknownException( - "HTS returned 503", new RuntimeException("unavailable"))) - .when(catalog) - .findHouseTable(any(TableIdentifier.class)); - - Assertions.assertThrows( - HouseTableRepositoryStateUnknownException.class, - () -> - openHouseInternalRepository.findOccupyingEntityTypeById( - TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build())); - } - - /** Occupancy follows the same unsupported-catalog contract as {@code findTableRefById}. */ - @Test - void findOccupyingEntityTypeThrowsWhenCatalogIsNotOpenHouseInternalCatalog() { - OpenHouseInternalRepositoryImpl impl = new OpenHouseInternalRepositoryImpl(); - impl.catalog = mock(Catalog.class); - - Assertions.assertThrows( - UnsupportedOperationException.class, - () -> - impl.findOccupyingEntityTypeById( - TableDtoPrimaryKey.builder().databaseId(DB_ID).tableId(TABLE_ID).build())); - } - private TableDto createTableDto(Map properties) { return TableDto.builder() .databaseId(DB_ID) From fc36aaa9f53951d707bba52acd5bbff73a2ccf7a Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 16:27:43 -0700 Subject: [PATCH 08/12] BDP-108403: Filter the table point read in HTS instead of in Java callers Adds a table-scoped point read to HTS and wires getUserTable to it, so a view at a table's key is invisible to the table path because of the query rather than because every caller checks. getUserTable is the single HTS endpoint behind every table point read in the tables service, so filtering it there makes four call sites correct with no Java guard at all: doRefresh findById -> getUserTable -> 404 -> HouseTableNotFound, already caught, leaves Optional.empty, refreshes from a null location exactly as for an absent row findTableRefById findHouseTable catches the same exception and returns empty dropTable findHouseTable returns empty, so the existing orElseThrow raises NoSuchTableException rename source loadTable(from) -> doRefresh -> no metadata -> the same NoSuchTableException findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase stays neutral on purpose. HtsRepository.findById and existsById delegate to it and back putUserTable, deleteUserTable, restoreUserTable and renameUserTable inside HTS, which must see a row of any type to detect a collision at a shared key. Only the read serving getUserTable changed. TABLE_ROW_PREDICATE returns as the single statement of "null or TABLE", with ENTITY_TYPE_FILTER_PREDICATE now composed from it, so the row test is written once. No view-only method is added: nothing in this change reads views, and the list queries already reach them through the entityType parameter. Still deferred to the view-commit ticket, because they need the neutral fetcher: occupancy, the rename destination preflight, and reading a view back over HTTP. The tables-service guard tests could not follow this filter - those tests run the H2 double, which never goes through HTS - so the coverage moves to services/housetables where the query actually executes: the case and garbage matrix on the new point read, the neutral read still seeing every type, the service-level getUserTable behavior, and the HTTP 404. Replicating the predicate into the doubles was deliberately not done; that is the testing-the-fake pattern already reverted for the list queries. testEntityTypePutAndGetRoundTrip now asserts the view PUT is readable through the PUT response and the persisted row, and that the table-scoped GET returns 404. That is the deferred neutral read, not a regression. Verified: housetables 177, internalcatalog 87, tables 475, tables-test-fixtures 8, spark-3.5 catalogTest 66 - all green, plus spotlessCheck and checkstyle. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../impl/jdbc/UserTableHtsJdbcRepository.java | 22 +++++- .../services/UserTablesServiceImpl.java | 3 +- .../e2e/usertable/HtsControllerTest.java | 56 ++++++++++++++- .../e2e/usertable/HtsRepositoryTest.java | 68 +++++++++++++++++++ .../e2e/usertable/UserTablesServiceTest.java | 64 +++++++++++++++++ 5 files changed, 205 insertions(+), 8 deletions(-) diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java index e426b5012..0d4ac3119 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java @@ -39,6 +39,22 @@ Optional findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( void deleteByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(String databaseId, String tableId); + String TABLE_ROW_PREDICATE = "(u.entityType IS NULL OR upper(u.entityType) = 'TABLE')"; + + /** + * Table-scoped point read. Serves {@code getUserTable}, which is the single HTS endpoint behind + * every table point read in the tables service, so a view reads as absent there without any + * caller-side check. The neutral {@link #findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase} above + * stays unfiltered because the writers need to see a row of any type to detect a collision. + */ + @Query( + "SELECT u FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + "lower(u.tableId) = lower(:tableId) AND " + + TABLE_ROW_PREDICATE) + Optional findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( + @Param("databaseId") String databaseId, @Param("tableId") String tableId); + @Query("SELECT DISTINCT databaseId FROM UserTableRow") Iterable findAllDistinctDatabaseIds(); @@ -53,9 +69,9 @@ Optional findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( * bypasses API validation. */ String ENTITY_TYPE_FILTER_PREDICATE = - "(((:entityType IS NULL OR upper(:entityType) = 'TABLE') " - + "AND (u.entityType IS NULL OR upper(u.entityType) = 'TABLE')) " - + "OR (upper(:entityType) = 'VIEW' AND upper(u.entityType) = 'VIEW'))"; + "(((:entityType IS NULL OR upper(:entityType) = 'TABLE') AND " + + TABLE_ROW_PREDICATE + + ") OR (upper(:entityType) = 'VIEW' AND upper(u.entityType) = 'VIEW'))"; @Query( "SELECT u FROM UserTableRow u WHERE " diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java index 64082fb84..1f82a3eea 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java @@ -57,8 +57,7 @@ public UserTableDto getUserTable(String databaseId, String tableId) { try { userTableRow = htsJdbcRepository - .findById( - UserTableRowPrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()) + .findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(databaseId, tableId) .orElseThrow(NoSuchElementException::new); } catch (NoSuchElementException ne) { throw new NoSuchUserTableException(databaseId, tableId, ne); diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java index 4e1063653..990de00b6 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java @@ -30,6 +30,9 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.boot.test.context.SpringBootTest; @@ -773,6 +776,50 @@ public void testPurgeAllSoftDeletedTables() throws Exception { */ private static final String ENTITY_TYPE_DB = "entity_type_db"; + /** + * The HTTP contract the tables service actually consumes: a view at a table's key is a 404, the + * same response an absent row produces, so no client-side check is needed to hide it. + */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + public void testGetUserTableReturnsNotFoundForNonTableRow(String entityType) throws Exception { + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, "point_read", entityType)); + + mvc.perform( + MockMvcRequestBuilders.get("/hts/tables") + .param("databaseId", ENTITY_TYPE_DB) + .param("tableId", "point_read") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.status", is(equalTo(HttpStatus.NOT_FOUND.name())))); + + assertThat( + htsRepository + .findById( + UserTableRowPrimaryKey.builder() + .databaseId(ENTITY_TYPE_DB) + .tableId("point_read") + .build()) + .isPresent()) + .isTrue(); + } + + @ParameterizedTest + @CsvSource( + nullValues = "NULL", + value = {"NULL", "TABLE", "table", "TaBlE"}) + public void testGetUserTableReturnsNullAndTableRows(String entityType) throws Exception { + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, "point_read", entityType)); + + mvc.perform( + MockMvcRequestBuilders.get("/hts/tables") + .param("databaseId", ENTITY_TYPE_DB) + .param("tableId", "point_read") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.entity.tableId", is(equalTo("point_read")))); + } + private UserTableRow entityTypeRow(String databaseId, String tableId, String entityType) { return UserTableRow.builder() .databaseId(databaseId) @@ -886,7 +933,11 @@ public void testPaginatedTableQueriesFilterBeforePaging() throws Exception { .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t02_explicit"))); } - /** The discriminator survives the HTTP PUT/GET boundary, and legacy writers stay null. */ + /** + * The discriminator survives the HTTP write boundary, and legacy writers stay null. A view cannot + * be read back through {@code GET /hts/tables} because that read is table-scoped; the neutral + * entity read is deferred, so the PUT response and the persisted row are what pin the write. + */ @Test public void testEntityTypePutAndGetRoundTrip() throws Exception { UserTable viewEntity = @@ -915,8 +966,7 @@ public void testEntityTypePutAndGetRoundTrip() throws Exception { .param("databaseId", ENTITY_TYPE_DB) .param("tableId", "put_view") .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.entity.entityType", is("VIEW"))); + .andExpect(status().isNotFound()); assertThat( htsRepository diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java index 6b06c1ab5..7e9ed93d6 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java @@ -16,6 +16,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.dao.DataIntegrityViolationException; @@ -563,6 +565,72 @@ public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { assertThat(findRow(CASE_DB, CASE_GARBAGE_ID).getEntityType()).isEqualTo("UNKNOWN"); } + /** + * Table-scoped point read: the query, not any caller, is what makes a view unreadable through the + * table path. NULL and every spelling of TABLE resolve; every spelling of VIEW and an + * unrecognized value resolve to empty. + */ + @ParameterizedTest + @CsvSource( + nullValues = "NULL", + value = { + "case00_null, true", + "case01_upper_table, true", + "case02_lower_table, true", + "case03_mixed_table, true", + "case04_upper_view, false", + "case05_lower_view, false", + "case06_mixed_view, false", + "case07_garbage, false" + }) + public void testFindTableByKeyResolvesOnlyTableRows(String tableId, boolean expectedVisible) { + seedCaseNormalizationRows(); + + assertThat( + htsRepository + .findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(CASE_DB, tableId) + .isPresent()) + .as("findTableBy... for %s", tableId) + .isEqualTo(expectedVisible); + + // Case-insensitive on the key itself, exactly like the neutral read. + assertThat( + htsRepository + .findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( + CASE_DB.toUpperCase(), tableId.toUpperCase()) + .isPresent()) + .isEqualTo(expectedVisible); + } + + /** + * The neutral read must keep seeing every row type. It backs {@code findById}/{@code existsById}, + * which the HTS writers use to detect a collision at a key held by another entity type; filtering + * it would make a view invisible to the very code that must refuse to overwrite it. + */ + @Test + public void testNeutralPointReadStillSeesEveryEntityType() { + seedCaseNormalizationRows(); + + for (String tableId : + new String[] { + "case00_null", + "case01_upper_table", + "case04_upper_view", + "case06_mixed_view", + CASE_GARBAGE_ID + }) { + assertThat( + htsRepository + .findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(CASE_DB, tableId) + .isPresent()) + .as("neutral read must still see %s", tableId) + .isTrue(); + assertThat(htsRepository.existsByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(CASE_DB, tableId)) + .as("neutral exists must still see %s", tableId) + .isTrue(); + } + } + private UserTableRow findRow(String databaseId, String tableId) { return htsRepository .findById(UserTableRowPrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()) diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java index c7d6a9565..3eb721574 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java @@ -27,6 +27,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -656,6 +659,67 @@ private Boolean isUserTableDtoEqual(UserTableDto expected, UserTableDto actual) private static final List CANONICAL_VIEW_IDS = Arrays.asList("t01_view", "t03_view", "t05_view"); + /** + * {@code getUserTable} is the single HTS endpoint behind every table point read in the tables + * service, so filtering it here is what makes doRefresh, dropTable, the rename source and + * findTableRefById all treat a view as absent without a check of their own. + */ + @ParameterizedTest + @ValueSource(strings = {"VIEW", "view", "ViEw", "UNKNOWN"}) + public void testGetUserTableHidesNonTableRows(String entityType) { + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, "point_read", entityType)); + + Assertions.assertThrows( + NoSuchUserTableException.class, + () -> userTablesService.getUserTable(ENTITY_TYPE_DB, "point_read")); + + // Hidden from the table read, not deleted. + assertThat( + htsRepository + .findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(ENTITY_TYPE_DB, "point_read") + .isPresent()) + .isTrue(); + } + + @ParameterizedTest + @CsvSource( + nullValues = "NULL", + value = {"NULL", "TABLE", "table", "TaBlE"}) + public void testGetUserTableResolvesNullAndTableRows(String entityType) { + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, "point_read", entityType)); + + UserTableDto dto = userTablesService.getUserTable(ENTITY_TYPE_DB, "point_read"); + assertThat(dto.getTableId()).isEqualTo("point_read"); + assertThat(dto.getEntityType()).isEqualTo(entityType); + } + + /** + * The writers must still see a view at a shared key, otherwise a table create or delete would + * silently act on a name another entity already holds. + */ + @Test + public void testWritersStillSeeNonTableRowsAtTheSameKey() { + htsRepository.save(entityTypeRow(ENTITY_TYPE_DB, "shared_key", "VIEW")); + + UserTableRow seenByWriter = + htsRepository + .findById( + UserTableRowPrimaryKey.builder() + .databaseId(ENTITY_TYPE_DB) + .tableId("shared_key") + .build()) + .orElseThrow(() -> new AssertionError("writer read must see the view row")); + assertThat(seenByWriter.getEntityType()).isEqualTo("VIEW"); + + // deleteUserTable resolves through the same neutral read, so it can still remove the row. + userTablesService.deleteUserTable(ENTITY_TYPE_DB, "shared_key", false); + assertThat( + htsRepository + .findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(ENTITY_TYPE_DB, "shared_key") + .isPresent()) + .isFalse(); + } + private UserTableRow entityTypeRow(String databaseId, String tableId, String entityType) { return UserTableRow.builder() .databaseId(databaseId) From dd0c79a8c4d2727948415c297801d2f67fb32c8e Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 16:50:21 -0700 Subject: [PATCH 09/12] BDP-108403: Name table-scoped HTS queries for what they filter Applies the agreed query-level contract: a method whose name says "table" filters to tables, everything else stays neutral or takes entityType as a parameter. Renamed and filtered, because every caller assumes tables: findAllByDatabaseIdIgnoreCase -> findAllTablesByDatabaseIdIgnoreCase findAllByDatabaseIdIgnoreCase(Pageable) -> findAllTablesByDatabaseIdIgnoreCase(Pageable) findAllByDatabaseIdAndTableIdLikeAllIgnoreCase -> findAllTablesBy... findAllByDatabaseIdAndTableIdLikeAllIgnoreCase(Pageable) -> findAllTablesBy...(Pageable) "TableId" in those names is the column table_id, which under a shared key space holds a view's name too, so the old names were column-scoped and type-ambiguous rather than already table-scoped. Both findAllByDatabaseIdIgnoreCase overloads were removed earlier in this branch when listTables was consolidated onto findAllByFilters; they are restored under the new names and listTables routes back to them. The paged overload was declared but never called before this branch, so adopting it for paged listTables costs nothing. Added findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase, which getUserTable now calls. That is the single HTS endpoint behind every table point read in the tables service, so the guards removed earlier are correct by construction: findById maps a 404 to HouseTableNotFoundException, which doRefresh already catches to leave an empty Optional and refresh from a null location, and which findHouseTable already catches to return empty - so dropTable's existing orElseThrow raises NoSuchTableException, findTableRefById returns empty, and a rename whose source is a view fails in loadTable. findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase stays neutral and untouched. findById delegates to it and backs putUserTable, deleteUserTable and restoreUserTable, which must see a row of any type to detect a collision at a shared key. existsBy, deleteBy, renameTableId and both findAllDistinctDatabaseIds overloads are unchanged; findAllByFilters keeps entityType as a parameter because general search is caller-parameterized by design. No view-only method is added: nothing here reads views. TABLE_ROW_PREDICATE is the single statement of "null or TABLE" and is reused verbatim in every filtered query including the paged countQuery. With the list and pattern queries hard-coding the table predicate again, the entityType entry in isNonKeyFieldsNullForUserTable is load-bearing once more: it routes a databaseId + entityType=VIEW request to findAllByFilters instead of to a table-only listing. Tests live in services/housetables, where the query actually runs; the predicate was deliberately not replicated into the services/tables H2 doubles. Verified: housetables 177, internalcatalog 87, tables 475, tables-test-fixtures 8, spark-3.5 catalogTest 66 - all green, plus spotlessCheck and checkstyle. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../impl/jdbc/UserTableHtsJdbcRepository.java | 60 ++++++++++++------- .../services/UserTablesServiceImpl.java | 32 ++-------- .../e2e/usertable/HtsRepositoryTest.java | 48 +++++++-------- 3 files changed, 65 insertions(+), 75 deletions(-) diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java index 0d4ac3119..d54c30955 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java @@ -42,10 +42,10 @@ Optional findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( String TABLE_ROW_PREDICATE = "(u.entityType IS NULL OR upper(u.entityType) = 'TABLE')"; /** - * Table-scoped point read. Serves {@code getUserTable}, which is the single HTS endpoint behind - * every table point read in the tables service, so a view reads as absent there without any - * caller-side check. The neutral {@link #findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase} above - * stays unfiltered because the writers need to see a row of any type to detect a collision. + * Table-scoped point read serving {@code getUserTable}, the single HTS endpoint behind every + * table point read in the tables service. The neutral {@link + * #findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase} above stays unfiltered because the writers + * must see a row of any type to detect a collision at a shared key. */ @Query( "SELECT u FROM UserTableRow u WHERE " @@ -63,43 +63,59 @@ Optional findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId))") Page findAllDistinctDatabaseIds(String databaseId, Pageable pageable); - /** - * A null or {@code TABLE} request means tables, including legacy null rows; {@code VIEW} means - * views only. An unknown value matches neither branch, so garbage fails closed here even if it - * bypasses API validation. - */ - String ENTITY_TYPE_FILTER_PREDICATE = - "(((:entityType IS NULL OR upper(:entityType) = 'TABLE') AND " - + TABLE_ROW_PREDICATE - + ") OR (upper(:entityType) = 'VIEW' AND upper(u.entityType) = 'VIEW'))"; + @Query( + "SELECT u FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + TABLE_ROW_PREDICATE) + Iterable findAllTablesByDatabaseIdIgnoreCase( + @Param("databaseId") String databaseId); + + @Query( + value = + "SELECT u FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + TABLE_ROW_PREDICATE, + countQuery = + "SELECT COUNT(u) FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + TABLE_ROW_PREDICATE) + Page findAllTablesByDatabaseIdIgnoreCase( + @Param("databaseId") String databaseId, Pageable pageable); @Query( "SELECT u FROM UserTableRow u WHERE " + "lower(u.databaseId) = lower(:databaseId) AND " + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " - + ENTITY_TYPE_FILTER_PREDICATE) - Iterable findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - @Param("databaseId") String databaseId, - @Param("tableIdPattern") String tableIdPattern, - @Param("entityType") String entityType); + + TABLE_ROW_PREDICATE) + Iterable findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + @Param("databaseId") String databaseId, @Param("tableIdPattern") String tableIdPattern); @Query( value = "SELECT u FROM UserTableRow u WHERE " + "lower(u.databaseId) = lower(:databaseId) AND " + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " - + ENTITY_TYPE_FILTER_PREDICATE, + + TABLE_ROW_PREDICATE, countQuery = "SELECT COUNT(u) FROM UserTableRow u WHERE " + "lower(u.databaseId) = lower(:databaseId) AND " + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " - + ENTITY_TYPE_FILTER_PREDICATE) - Page findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + + TABLE_ROW_PREDICATE) + Page findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( @Param("databaseId") String databaseId, @Param("tableIdPattern") String tableIdPattern, - @Param("entityType") String entityType, Pageable pageable); + /** + * A null or {@code TABLE} request means tables, including legacy null rows; {@code VIEW} means + * views only. An unknown value matches neither branch, so garbage fails closed here even if it + * bypasses API validation. + */ + String ENTITY_TYPE_FILTER_PREDICATE = + "(((:entityType IS NULL OR upper(:entityType) = 'TABLE') AND " + + TABLE_ROW_PREDICATE + + ") OR (upper(:entityType) = 'VIEW' AND upper(u.entityType) = 'VIEW'))"; + String GENERAL_FILTER_PREDICATE = "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " + "(:tableId IS NULL OR lower(u.tableId) = lower(:tableId)) AND " diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java index 1f82a3eea..58802b745 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java @@ -287,14 +287,7 @@ private List listTables(UserTable userTable) { () -> StreamSupport.stream( htsJdbcRepository - .findAllByFilters( - userTable.getDatabaseId(), - null, - null, - null, - null, - null, - userTable.getEntityType()) + .findAllTablesByDatabaseIdIgnoreCase(userTable.getDatabaseId()) .spliterator(), false) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)) @@ -308,15 +301,7 @@ private Page listTables(UserTable userTable, int page, int size, S return METRICS_REPORTER.executeWithStats( () -> htsJdbcRepository - .findAllByFilters( - userTable.getDatabaseId(), - null, - null, - null, - null, - null, - userTable.getEntityType(), - pageable) + .findAllTablesByDatabaseIdIgnoreCase(userTable.getDatabaseId(), pageable) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)), MetricsConstant.HTS_PAGE_TABLES_TIME); } @@ -327,10 +312,8 @@ private List listTablesWithPattern(UserTable userTable) { () -> StreamSupport.stream( htsJdbcRepository - .findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - userTable.getDatabaseId(), - userTable.getTableId(), - userTable.getEntityType()) + .findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + userTable.getDatabaseId(), userTable.getTableId()) .spliterator(), false) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)) @@ -345,11 +328,8 @@ private Page listTablesWithPattern( return METRICS_REPORTER.executeWithStats( () -> htsJdbcRepository - .findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - userTable.getDatabaseId(), - userTable.getTableId(), - userTable.getEntityType(), - pageable) + .findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + userTable.getDatabaseId(), userTable.getTableId(), pageable) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)), MetricsConstant.HTS_PAGE_TABLES_TIME); } diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java index 7e9ed93d6..d90bbde6d 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java @@ -141,8 +141,7 @@ public void testFindAllByDatabaseId() { htsRepository.save(TEST_TUPLE_1_1.get_userTableRow()); htsRepository.save(TEST_TUPLE_2_0.get_userTableRow()); List result = - Lists.newArrayList( - htsRepository.findAllByFilters("test_db0", null, null, null, null, null, null)); + Lists.newArrayList(htsRepository.findAllTablesByDatabaseIdIgnoreCase("test_db0")); Assertions.assertEquals( Lists.newArrayList("test_table1", "test_table2"), result.stream().map(UserTableRow::getTableId).collect(Collectors.toList())); @@ -155,8 +154,8 @@ public void testFindAllByTableIdPattern() { htsRepository.save(TEST_TUPLE_2_0.get_userTableRow()); List result = Lists.newArrayList( - htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - "test_db0", "test_table%", null)); + htsRepository.findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + "test_db0", "test_table%")); Assertions.assertEquals( Lists.newArrayList("test_table1", "test_table2"), result.stream().map(UserTableRow::getTableId).collect(Collectors.toList())); @@ -169,8 +168,8 @@ public void testFindAllByTableId() { htsRepository.save(TEST_TUPLE_2_0.get_userTableRow()); List result = Lists.newArrayList( - htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - "test_db0", "test_table1", null)); + htsRepository.findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + "test_db0", "test_table1")); Assertions.assertEquals( Lists.newArrayList("test_table1"), result.stream().map(UserTableRow::getTableId).collect(Collectors.toList())); @@ -374,8 +373,7 @@ public void testFindAllByDatabaseIdFiltersViewsAndKeepsLegacyTables() { htsRepository.save(row("other_db", "t00_legacy", null)); List result = - Lists.newArrayList( - htsRepository.findAllByFilters(ENTITY_TYPE_DB, null, null, null, null, null, null)); + Lists.newArrayList(htsRepository.findAllTablesByDatabaseIdIgnoreCase(ENTITY_TYPE_DB)); assertThat(tableIds(result)).containsExactly(CANONICAL_TABLE_IDS); assertThat(result) @@ -392,16 +390,14 @@ public void testFindAllByDatabaseIdFiltersBeforePagination() { seedCanonicalRows(ENTITY_TYPE_DB, ""); Page page0 = - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, null, sortedPage(0)); + htsRepository.findAllTablesByDatabaseIdIgnoreCase(ENTITY_TYPE_DB, sortedPage(0)); assertThat(page0.getTotalElements()).isEqualTo(4); assertThat(page0.getTotalPages()).isEqualTo(2); assertThat(page0.getContent()).hasSize(2); assertThat(pageTableIds(page0)).containsExactly("t00_legacy", "t02_explicit"); Page page1 = - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, null, sortedPage(1)); + htsRepository.findAllTablesByDatabaseIdIgnoreCase(ENTITY_TYPE_DB, sortedPage(1)); assertThat(page1.getTotalElements()).isEqualTo(4); assertThat(page1.getTotalPages()).isEqualTo(2); assertThat(page1.getContent()).hasSize(2); @@ -420,8 +416,8 @@ public void testFindAllByPatternFiltersViewsAndKeepsLegacyTables() { List result = Lists.newArrayList( - htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - ENTITY_TYPE_DB, "match_%", null)); + htsRepository.findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + ENTITY_TYPE_DB, "match_%")); assertThat(tableIds(result)) .containsExactly( @@ -435,16 +431,16 @@ public void testFindAllByPatternFiltersBeforePagination() { htsRepository.save(row(ENTITY_TYPE_DB, "nomatch_table", "TABLE")); Page page0 = - htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - ENTITY_TYPE_DB, "match_%", null, sortedPage(0)); + htsRepository.findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + ENTITY_TYPE_DB, "match_%", sortedPage(0)); assertThat(page0.getTotalElements()).isEqualTo(4); assertThat(page0.getTotalPages()).isEqualTo(2); assertThat(page0.getContent()).hasSize(2); assertThat(pageTableIds(page0)).containsExactly("match_t00_legacy", "match_t02_explicit"); Page page1 = - htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - ENTITY_TYPE_DB, "match_%", null, sortedPage(1)); + htsRepository.findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + ENTITY_TYPE_DB, "match_%", sortedPage(1)); assertThat(page1.getTotalElements()).isEqualTo(4); assertThat(page1.getTotalPages()).isEqualTo(2); assertThat(page1.getContent()).hasSize(2); @@ -520,24 +516,23 @@ public void testFindAllByFiltersDefaultsToTablesAndCanSelectViews() { public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { seedCaseNormalizationRows(); - assertThat( - tableIds(htsRepository.findAllByFilters(CASE_DB, null, null, null, null, null, null))) + assertThat(tableIds(htsRepository.findAllTablesByDatabaseIdIgnoreCase(CASE_DB))) .containsExactly(CASE_VISIBLE_TABLE_IDS); assertThat( tableIds( - htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - CASE_DB, "case%", null))) + htsRepository.findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + CASE_DB, "case%"))) .containsExactly(CASE_VISIBLE_TABLE_IDS); Page dbPage0 = - htsRepository.findAllByFilters(CASE_DB, null, null, null, null, null, null, sortedPage(0)); + htsRepository.findAllTablesByDatabaseIdIgnoreCase(CASE_DB, sortedPage(0)); assertThat(dbPage0.getTotalElements()).isEqualTo(4); assertThat(dbPage0.getTotalPages()).isEqualTo(2); assertThat(pageTableIds(dbPage0)).containsExactly("case00_null", "case01_upper_table"); Page patternPage0 = - htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - CASE_DB, "case%", null, sortedPage(0)); + htsRepository.findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + CASE_DB, "case%", sortedPage(0)); assertThat(patternPage0.getTotalElements()).isEqualTo(4); assertThat(patternPage0.getTotalPages()).isEqualTo(2); assertThat(pageTableIds(patternPage0)).containsExactly("case00_null", "case01_upper_table"); @@ -557,8 +552,7 @@ public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { Lists.newArrayList( htsRepository.findAllByFilters(CASE_DB, null, null, null, null, null, "UNKNOWN"))) .isEmpty(); - assertThat( - tableIds(htsRepository.findAllByFilters(CASE_DB, null, null, null, null, null, null))) + assertThat(tableIds(htsRepository.findAllTablesByDatabaseIdIgnoreCase(CASE_DB))) .doesNotContain(CASE_GARBAGE_ID); // The garbage row is still stored — it is hidden, not dropped. From d908a03b7c0b3826ccb4182729a392324f34dbfe Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 17:16:33 -0700 Subject: [PATCH 10/12] BDP-108403: Make the HTS table endpoints table-scoped by path, not by parameter /hts/tables and /hts/tables/query are table endpoints, so the queries behind them hard-code the table predicate and entityType is no longer a query parameter anywhere. Views get mirror endpoints in the view-commit ticket. That removes the parameterized type clause entirely: ENTITY_TYPE_FILTER_PREDICATE is deleted and TABLE_ROW_PREDICATE is the single statement of "null or TABLE", appended to every table-named query and repeated verbatim in each paged countQuery through the same constant. No :entityType parameter remains in the repository. Table-scoped reads, all filtered, none parameterized: findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase new; getUserTable calls it findAllTablesByDatabaseIdIgnoreCase restored, renamed, filtered findAllTablesByDatabaseIdIgnoreCase(Pageable) restored, renamed, filtered findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase renamed, filtered ...(Pageable) renamed, filtered findAllTablesByFilters renamed, filtered, entityType param dropped ...(Pageable) renamed, filtered, entityType param dropped "TableId" in the pattern names is the column table_id, which under a shared key space holds a view's name too, so those names were column-scoped rather than already table-scoped. Neutral and untouched: findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase, which findById delegates to and which putUserTable, deleteUserTable and restoreUserTable need in order to see a row of any type at a shared key; plus existsBy, deleteBy, renameTableId and both findAllDistinctDatabaseIds overloads. No view-only method is added; nothing here reads views. With entityType gone from the query surface, isNonKeyFieldsNullForUserTable and the query branch of OpenHouseUserTableHtsApiValidator are restored to their pre-change form, so listDatabases, listTables, listTablesWithPattern and searchTables route exactly as at base. The transport-model @Pattern stays: entityType is still a valid PUT payload field. Because getUserTable is the one HTS endpoint behind every table point read in the tables service, the guards removed earlier are correct by construction. A 404 becomes HouseTableNotFoundException, which doRefresh already catches to leave an empty Optional and refresh from a null location, and which findHouseTable already catches to return empty - so dropTable's existing orElseThrow raises NoSuchTableException, findTableRefById returns empty, and a rename whose source is a view fails inside loadTable. Tests follow the surface: the type-selection tests are replaced by ones asserting the table-scoped families never return a view, and the entityType query parameter is now pinned as bound-but-ignored at the mapper, service and HTTP layers. The predicate was deliberately not replicated into the services/tables H2 doubles. Verified: housetables 175, internalcatalog 87, tables 475, tables-test-fixtures 8, spark-3.5 catalogTest 66 - all green, plus spotlessCheck and checkstyle. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OpenHouseUserTableHtsApiValidator.java | 11 +-- .../impl/jdbc/UserTableHtsJdbcRepository.java | 20 +---- .../services/UserTablesServiceImpl.java | 16 +--- .../e2e/usertable/HtsControllerTest.java | 27 ++---- .../e2e/usertable/HtsRepositoryTest.java | 87 ++++++------------- .../e2e/usertable/UserTablesServiceTest.java | 50 ++++------- .../api/OpenHouseUserTablesValidatorTest.java | 50 ----------- .../mock/mapper/UserTablesMapperTest.java | 8 +- 8 files changed, 64 insertions(+), 205 deletions(-) diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/validator/impl/OpenHouseUserTableHtsApiValidator.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/validator/impl/OpenHouseUserTableHtsApiValidator.java index dd3237cc4..087d928fd 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/validator/impl/OpenHouseUserTableHtsApiValidator.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/validator/impl/OpenHouseUserTableHtsApiValidator.java @@ -110,16 +110,7 @@ private void validateUserTable(UserTable userTable, List validationFailu && userTable.getMetadataLocation() == null && userTable.getStorageType() == null && userTable.getCreationTime() == null)) { - validationFailures.add("Only databaseId, tableId and entityType are supported for the query"); - } - - // entityType is the one additional permitted query filter. Reject garbage here so - // an unknown discriminator fails as a validation error rather than as a silently empty result. - if (userTable.getEntityType() != null - && !userTable.getEntityType().matches(ENTITY_TYPE_REGEX)) { - validationFailures.add( - String.format( - "entityType provided: %s, %s", userTable.getEntityType(), ENTITY_TYPE_ERROR_MSG)); + validationFailures.add("Only databaseId and tableId are supported for the query"); } if (userTable.getDatabaseId() != null diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java index d54c30955..df8eca21c 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java @@ -106,16 +106,6 @@ Page findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( @Param("tableIdPattern") String tableIdPattern, Pageable pageable); - /** - * A null or {@code TABLE} request means tables, including legacy null rows; {@code VIEW} means - * views only. An unknown value matches neither branch, so garbage fails closed here even if it - * bypasses API validation. - */ - String ENTITY_TYPE_FILTER_PREDICATE = - "(((:entityType IS NULL OR upper(:entityType) = 'TABLE') AND " - + TABLE_ROW_PREDICATE - + ") OR (upper(:entityType) = 'VIEW' AND upper(u.entityType) = 'VIEW'))"; - String GENERAL_FILTER_PREDICATE = "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " + "(:tableId IS NULL OR lower(u.tableId) = lower(:tableId)) AND " @@ -123,30 +113,28 @@ Page findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + "(:metadataLocation IS NULL OR u.metadataLocation = :metadataLocation) AND " + "(:storageType IS NULL OR u.storageType = :storageType) AND " + "(:creationTime IS NULL OR u.creationTime = :creationTime) AND " - + ENTITY_TYPE_FILTER_PREDICATE; + + TABLE_ROW_PREDICATE; @Query( value = "select DISTINCT u from UserTableRow u where " + GENERAL_FILTER_PREDICATE, countQuery = "select COUNT(DISTINCT u) from UserTableRow u where " + GENERAL_FILTER_PREDICATE) - Page findAllByFilters( + Page findAllTablesByFilters( @Param("databaseId") String databaseId, @Param("tableId") String tableId, @Param("tableVersion") String tableVersion, @Param("metadataLocation") String metadataLocation, @Param("storageType") String storageType, @Param("creationTime") Long creationTime, - @Param("entityType") String entityType, Pageable pageable); @Query("select DISTINCT u from UserTableRow u where " + GENERAL_FILTER_PREDICATE) - Iterable findAllByFilters( + Iterable findAllTablesByFilters( @Param("databaseId") String databaseId, @Param("tableId") String tableId, @Param("tableVersion") String tableVersion, @Param("metadataLocation") String metadataLocation, @Param("storageType") String storageType, - @Param("creationTime") Long creationTime, - @Param("entityType") String entityType); + @Param("creationTime") Long creationTime); /* * The following methods are required to maintain the generality of the interface {@link com.linkedin.openhouse.housetables.repository.HtsRepository} diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java index 58802b745..b6655aa4b 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java @@ -342,14 +342,13 @@ private Page searchTables(UserTable userTable, int page, int size, return METRICS_REPORTER.executeWithStats( () -> htsJdbcRepository - .findAllByFilters( + .findAllTablesByFilters( userTable.getDatabaseId(), userTable.getTableId(), userTable.getTableVersion(), userTable.getMetadataLocation(), userTable.getStorageType(), userTable.getCreationTime(), - userTable.getEntityType(), pageable) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)), MetricsConstant.HTS_PAGE_SEARCH_TABLES_TIME); @@ -363,14 +362,13 @@ private List searchTables(UserTable userTable) { () -> StreamSupport.stream( htsJdbcRepository - .findAllByFilters( + .findAllTablesByFilters( userTable.getDatabaseId(), userTable.getTableId(), userTable.getTableVersion(), userTable.getMetadataLocation(), userTable.getStorageType(), - userTable.getCreationTime(), - userTable.getEntityType()) + userTable.getCreationTime()) .spliterator(), false) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)) @@ -396,16 +394,10 @@ private boolean isListTablesWithPattern(UserTable userTable) { && userTable.getTableId() != null; } - /** - * The list/pattern queries hard-code the table predicate, so {@code entityType} must count as a - * non-key field — otherwise a {@code databaseId + entityType=VIEW} request would route there and - * silently return tables instead of going through {@code findAllByFilters}. - */ private boolean isNonKeyFieldsNullForUserTable(UserTable userTable) { return userTable.getTableVersion() == null && userTable.getMetadataLocation() == null && userTable.getStorageType() == null - && userTable.getCreationTime() == null - && userTable.getEntityType() == null; + && userTable.getCreationTime() == null; } } diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java index 990de00b6..2969574d0 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsControllerTest.java @@ -1021,12 +1021,12 @@ public void testEntityTypePutAndGetRoundTrip() throws Exception { } /** - * Pins validator + service routing over HTTP, not merely repository behavior: the request carries - * only databaseId and entityType=VIEW. It fails if the validator rejects the parameter or if the - * routing predicate still classifies this as a plain table listing. + * {@code /hts/tables/query} is table-scoped by path, so {@code entityType} is not a supported + * query parameter. It is mapped onto the request object but never reaches a predicate, so a + * client that sends one is silently answered with tables. */ @Test - public void testEntityTypeOnlyViewQueryRoutesToGeneralSearch() throws Exception { + public void testEntityTypeQueryParameterIsIgnored() throws Exception { seedCanonicalRows(""); mvc.perform( @@ -1034,24 +1034,11 @@ public void testEntityTypeOnlyViewQueryRoutesToGeneralSearch() throws Exception .params(queryParams("databaseId", ENTITY_TYPE_DB, "entityType", "VIEW")) .accept(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()) - .andExpect(jsonPath("$.results", hasSize(3))) + .andExpect(jsonPath("$.results", hasSize(4))) .andExpect( jsonPath( - "$.results[*].tableId", containsInAnyOrder("t01_view", "t03_view", "t05_view"))); - - mvc.perform( - MockMvcRequestBuilders.get("/v1/hts/tables/query") - .params(queryParams("databaseId", ENTITY_TYPE_DB, "entityType", "VIEW")) - .param("page", "0") - .param("size", "2") - .param("sortBy", "tableId") - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.pageResults.totalElements", is(3))) - .andExpect(jsonPath("$.pageResults.totalPages", is(2))) - .andExpect(jsonPath("$.pageResults.content", hasSize(2))) - .andExpect(jsonPath("$.pageResults.content[0].tableId", is("t01_view"))) - .andExpect(jsonPath("$.pageResults.content[1].tableId", is("t03_view"))); + "$.results[*].tableId", + containsInAnyOrder("t00_legacy", "t02_explicit", "t04_legacy", "t06_explicit"))); } /** diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java index d90bbde6d..5b99d7685 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java @@ -53,10 +53,6 @@ public class HtsRepositoryTest { "case00_null", "case01_upper_table", "case02_lower_table", "case03_mixed_table" }; - private static final String[] CASE_VIEW_IDS = { - "case04_upper_view", "case05_lower_view", "case06_mixed_view" - }; - private static final String CASE_GARBAGE_ID = "case07_garbage"; @Autowired UserTableHtsJdbcRepository htsRepository; @@ -447,59 +443,35 @@ public void testFindAllByPatternFiltersBeforePagination() { assertThat(pageTableIds(page1)).containsExactly("match_t04_legacy", "match_t06_explicit"); } - /** - * The general-filter query defaults to tables (null and any TABLE spelling) and can be asked - * explicitly for views. This is the only query family that can return VIEW rows. - */ + /** The general-filter query is table-scoped too: no overload can return a VIEW row. */ @Test - public void testFindAllByFiltersDefaultsToTablesAndCanSelectViews() { + public void testFindAllTablesByFiltersReturnsOnlyTables() { seedCanonicalRows(ENTITY_TYPE_DB, ""); - // entityType == null means "tables", not "everything". assertThat( tableIds( - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, (String) null))) + htsRepository.findAllTablesByFilters(ENTITY_TYPE_DB, null, null, null, null, null))) .containsExactly(CANONICAL_TABLE_IDS); - for (String tableSpelling : new String[] {"TABLE", "table", "TaBlE"}) { - assertThat( - tableIds( - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, tableSpelling))) - .as("entityType=%s must resolve to the four visible tables", tableSpelling) - .containsExactly(CANONICAL_TABLE_IDS); - } + Page page0 = + htsRepository.findAllTablesByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, sortedPage(0)); + assertThat(page0.getTotalElements()).isEqualTo(4); + assertThat(page0.getTotalPages()).isEqualTo(2); + assertThat(pageTableIds(page0)).containsExactly("t00_legacy", "t02_explicit"); - for (String viewSpelling : new String[] {"VIEW", "view", "ViEw"}) { - assertThat( - tableIds( - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, viewSpelling))) - .as("entityType=%s must resolve to exactly the three views", viewSpelling) - .containsExactly(CANONICAL_VIEW_IDS); - } + Page page1 = + htsRepository.findAllTablesByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, sortedPage(1)); + assertThat(page1.getTotalElements()).isEqualTo(4); + assertThat(pageTableIds(page1)).containsExactly("t04_legacy", "t06_explicit"); - // Pageable overload: default (tables) and explicit VIEW both count in the database. - Page defaultPage0 = - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, (String) null, sortedPage(0)); - assertThat(defaultPage0.getTotalElements()).isEqualTo(4); - assertThat(defaultPage0.getTotalPages()).isEqualTo(2); - assertThat(pageTableIds(defaultPage0)).containsExactly("t00_legacy", "t02_explicit"); - - Page viewPage0 = - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, "VIEW", sortedPage(0)); - assertThat(viewPage0.getTotalElements()).isEqualTo(3); - assertThat(viewPage0.getTotalPages()).isEqualTo(2); - assertThat(pageTableIds(viewPage0)).containsExactly("t01_view", "t03_view"); - - Page viewPage1 = - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, "VIEW", sortedPage(1)); - assertThat(viewPage1.getTotalElements()).isEqualTo(3); - assertThat(pageTableIds(viewPage1)).containsExactly("t05_view"); + // A view is unreachable through this family, by tableId as well as by database. + assertThat( + Lists.newArrayList( + htsRepository.findAllTablesByFilters( + ENTITY_TYPE_DB, "t01_view", null, null, null, null))) + .isEmpty(); } /** @@ -537,21 +509,12 @@ public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { assertThat(patternPage0.getTotalPages()).isEqualTo(2); assertThat(pageTableIds(patternPage0)).containsExactly("case00_null", "case01_upper_table"); - // Every VIEW spelling is selectable and the garbage row is never one of them. - for (String viewSpelling : new String[] {"VIEW", "view", "ViEw"}) { - assertThat( - tableIds( - htsRepository.findAllByFilters( - CASE_DB, null, null, null, null, null, viewSpelling))) - .as("entityType=%s", viewSpelling) - .containsExactly(CASE_VIEW_IDS); - } - - // Garbage fails closed on the repository: it is neither a table nor a view. + // The general filter family is table-scoped as well, so no view spelling leaks through it. assertThat( - Lists.newArrayList( - htsRepository.findAllByFilters(CASE_DB, null, null, null, null, null, "UNKNOWN"))) - .isEmpty(); + tableIds(htsRepository.findAllTablesByFilters(CASE_DB, null, null, null, null, null))) + .containsExactly(CASE_VISIBLE_TABLE_IDS); + + // Garbage fails closed everywhere: it is neither a table nor a view. assertThat(tableIds(htsRepository.findAllTablesByDatabaseIdIgnoreCase(CASE_DB))) .doesNotContain(CASE_GARBAGE_ID); diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java index 3eb721574..98ea078cd 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/UserTablesServiceTest.java @@ -762,9 +762,8 @@ public void testListTablesCallSiteFiltersViewsAndKeepsNullRows() { } /** - * Anti-post-filter assertion at the service layer, and the pin for routing the paged per-database - * listing through the table-predicated query rather than the untyped {@code findAllByFilters}. A - * fetch-then-filter implementation yields a 1-row page 0 with totalElements=7/totalPages=4. + * Anti-post-filter assertion at the service layer: a fetch-then-filter implementation yields a + * 1-row page 0 with totalElements=7/totalPages=4. */ @Test public void testListTablesCallSiteFiltersBeforePagination() { @@ -810,46 +809,35 @@ public void testPatternCallSitesFilterViewsPlainAndPaged() { } /** - * Pins the routing predicate. The request carries only {@code databaseId} plus {@code - * entityType=VIEW} and no other filter, so if {@code isNonKeyFieldsNullForUserTable} is not - * extended to consider entityType, this request is classified as a plain "list tables" request - * and returns the four tables instead of the three views. + * The query endpoint is table-scoped by path, so {@code entityType} is bound onto the request but + * never reaches a predicate: an {@code entityType=VIEW} request is answered with tables, and + * routing is unaffected by the field. */ @Test - public void testGeneralSearchHonorsEntityType() { + public void testEntityTypeOnQueryIsIgnoredAndAlwaysReturnsTables() { seedCanonicalRows(""); - List views = - userTablesService.getAllUserTables( - UserTable.builder().databaseId(ENTITY_TYPE_DB).entityType("VIEW").build()); - assertThat(sortedIds(views)).isEqualTo(CANONICAL_VIEW_IDS); - - for (String tableSpelling : new String[] {"TABLE", "table", "TaBlE"}) { - List tables = - userTablesService.getAllUserTables( - UserTable.builder().databaseId(ENTITY_TYPE_DB).entityType(tableSpelling).build()); - assertThat(sortedIds(tables)) - .as("entityType=%s must resolve to the four visible tables", tableSpelling) + for (String entityType : new String[] {"VIEW", "view", "TABLE", "TaBlE", null}) { + assertThat( + sortedIds( + userTablesService.getAllUserTables( + UserTable.builder() + .databaseId(ENTITY_TYPE_DB) + .entityType(entityType) + .build()))) + .as("entityType=%s must still resolve to the four visible tables", entityType) .isEqualTo(CANONICAL_TABLE_IDS); } - // Default (no entityType) still means tables. - assertThat( - sortedIds( - userTablesService.getAllUserTables( - UserTable.builder().databaseId(ENTITY_TYPE_DB).build()))) - .isEqualTo(CANONICAL_TABLE_IDS); - - // Paged entityType-only VIEW request routes the same way. - Page viewPage = + Page page0 = userTablesService.getAllUserTables( UserTable.builder().databaseId(ENTITY_TYPE_DB).entityType("VIEW").build(), 0, 2, "tableId"); - Assertions.assertEquals(3, viewPage.getTotalElements()); - Assertions.assertEquals(2, viewPage.getTotalPages()); - assertThat(pageIds(viewPage)).containsExactly("t01_view", "t03_view"); + Assertions.assertEquals(4, page0.getTotalElements()); + Assertions.assertEquals(2, page0.getTotalPages()); + assertThat(pageIds(page0)).containsExactly("t00_legacy", "t02_explicit"); } /** diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/api/OpenHouseUserTablesValidatorTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/api/OpenHouseUserTablesValidatorTest.java index b0c087774..8d038d2b8 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/api/OpenHouseUserTablesValidatorTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/api/OpenHouseUserTablesValidatorTest.java @@ -148,56 +148,6 @@ public void validateRenameEntityInvalidInput() { () -> userTablesHtsApiValidator.validateRenameEntity(fromKey, toKey)); } - /** - * A type-qualified query must reach the repository. NOTE: {@code validateUserTable} only rejects - * non-null tableVersion/metadataLocation/storageType/creationTime, so this case passes whether or - * not entityType validation exists. It guards against a future change that adds entityType to - * that unsupported-field list; the load-bearing assertions for entityType validation live in - * {@link #validateEntityTypeQueryRejectsGarbage} and {@link - * #validatePutEntityTypeCaseInsensitivelyAndRejectsGarbage}. - */ - @Test - public void validateEntityTypeOnlyQueriesCaseInsensitively() { - for (String entityType : new String[] {"VIEW", "view", "ViEw", "TABLE", "table", "TaBlE"}) { - UserTable userTable = UserTable.builder().databaseId("db1").entityType(entityType).build(); - - assertDoesNotThrow( - () -> userTablesHtsApiValidator.validateGetEntities(userTable), - "entityType=" + entityType + " should be an accepted unpaged query filter"); - assertDoesNotThrow( - () -> userTablesHtsApiValidator.validateGetEntities(userTable, 0, 2, "tableId"), - "entityType=" + entityType + " should be an accepted paged query filter"); - } - } - - /** - * Load-bearing: an unknown discriminator must be rejected before it ever reaches the repository, - * so callers get a validation error rather than a silently empty result set. - */ - @Test - public void validateEntityTypeQueryRejectsGarbage() { - UserTable garbage = UserTable.builder().databaseId("db1").entityType("UNKNOWN").build(); - - assertThrows( - RequestValidationFailureException.class, - () -> userTablesHtsApiValidator.validateGetEntities(garbage)); - assertThrows( - RequestValidationFailureException.class, - () -> userTablesHtsApiValidator.validateGetEntities(garbage, 0, 2, "tableId")); - - // The pre-existing unsupported-field rejection must not be weakened by adding entityType as - // a permitted filter. - UserTable unsupportedField = UserTable.builder().creationTime(1L).build(); - assertThrows( - RequestValidationFailureException.class, - () -> userTablesHtsApiValidator.validateGetEntities(unsupportedField)); - UserTable unsupportedFieldWithEntityType = - UserTable.builder().databaseId("db1").entityType("VIEW").creationTime(1L).build(); - assertThrows( - RequestValidationFailureException.class, - () -> userTablesHtsApiValidator.validateGetEntities(unsupportedFieldWithEntityType)); - } - /** * Load-bearing: the PUT path relies on Bean Validation on the transport model, so the pattern on * {@code UserTable#entityType} must accept every TABLE/VIEW spelling and reject anything else. diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/mapper/UserTablesMapperTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/mapper/UserTablesMapperTest.java index 0d6ba1a2a..f1b54ebef 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/mapper/UserTablesMapperTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/mock/mapper/UserTablesMapperTest.java @@ -125,12 +125,12 @@ void nullEntityTypeRemainsNullAcrossLegacyMappings() { } /** - * The /hts query endpoint hands raw request parameters to {@code mapToUserTable}. If entityType - * is not recognized there, an {@code entityType=VIEW} query silently degrades to an unfiltered - * table listing. + * {@code mapToUserTable} still binds an {@code entityType} request parameter onto the model, but + * the query endpoint is table-scoped by path so nothing consumes it. This pins where the value + * stops: bound here, never reaching a predicate. */ @Test - void mapToUserTableRecognizesEntityType() { + void mapToUserTableBindsButDoesNotConsumeEntityType() { Map parameters = new HashMap<>(); parameters.put("databaseId", "test_db0"); parameters.put("entityType", "VIEW"); From 07de437a737acfa4213ec3cf3a1c70fc6f4f6111 Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 17:38:23 -0700 Subject: [PATCH 11/12] BDP-108403: Keep the HTS queries general and add table-scoped delegates Restores findAllByFilters and findAllByDatabaseIdAndTableIdLikeAllIgnoreCase to general methods that take entityType, and adds table-scoped default methods that delegate to them. Nothing is renamed, and the general forms stay available for the view and neutral work. One shared ENTITY_TYPE_PREDICATE now spells all three branches out: null matches any type - genuinely general, not a table default TABLE matches TABLE and a stored null, because an absent discriminator means a table on a column that is nullable with no backfill VIEW matches VIEW An unrecognized request value matches no branch, so garbage fails closed. Note this changes what a null entityType means: it used to be a disguised table default, and it now returns both types, which is why every table caller pins TABLE explicitly. Added, all default and owning no JPQL: findAllTablesByFilters x2 findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase x2 findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase The pattern family keeps its own @Query because findAllByFilters matches tableId exactly; folding a LIKE into it would make _ a wildcard and OpenHouse identifiers routinely contain underscores. It shares the same predicate constant. The point read delegates rather than carrying its own query. The alternative was a dedicated three-clause @Query, which would read slightly more directly but would restate the table branch of a predicate that already exists. Since the key is the primary key, at most one row can match, so unwrapping the first element is exact. The tradeoff is that the hottest read in HTS now runs the general select DISTINCT; the key predicate is still exact, but say the word if you would rather pay a duplicated clause to avoid the DISTINCT. Untouched: findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase, which backs findById for putUserTable, deleteUserTable and restoreUserTable and must see a row of any type at a shared key; existsBy; deleteBy; renameTableId; and both findAllDistinctDatabaseIds overloads. No view-only method is added. Call sites: listTables and searchTables use findAllTablesByFilters, listTablesWithPattern uses the table-scoped pattern wrapper, and getUserTable uses the table-scoped point read. entityType is not read from the wire, so isNonKeyFieldsNullForUserTable and the validator's query branch stay at their pre-change form and all four routes behave as at base. Because getUserTable is the one HTS endpoint behind every table point read in the tables service, the guards removed earlier remain correct by construction: a 404 becomes HouseTableNotFoundException, which doRefresh already catches to leave an empty Optional and which findHouseTable already catches to return empty, so dropTable throws NoSuchTableException, findTableRefById returns empty, and a rename off a view source fails inside loadTable. Verified: housetables 177, internalcatalog 87, tables 475, tables-test-fixtures 8, spark-3.5 catalogTest 66 - all green, plus spotlessCheck and checkstyle. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../impl/jdbc/UserTableHtsJdbcRepository.java | 152 ++++++++++++------ .../services/UserTablesServiceImpl.java | 6 +- .../e2e/usertable/HtsRepositoryTest.java | 92 ++++++++++- 3 files changed, 189 insertions(+), 61 deletions(-) diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java index df8eca21c..0e3efc0c2 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java @@ -4,6 +4,7 @@ import com.linkedin.openhouse.housetables.model.UserTableRow; import com.linkedin.openhouse.housetables.model.UserTableRowPrimaryKey; import com.linkedin.openhouse.housetables.repository.HtsRepository; +import java.util.Iterator; import java.util.Optional; import org.jetbrains.annotations.NotNull; import org.springframework.data.domain.Page; @@ -39,21 +40,18 @@ Optional findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( void deleteByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(String databaseId, String tableId); - String TABLE_ROW_PREDICATE = "(u.entityType IS NULL OR upper(u.entityType) = 'TABLE')"; + String TABLE_ENTITY_TYPE = "TABLE"; /** - * Table-scoped point read serving {@code getUserTable}, the single HTS endpoint behind every - * table point read in the tables service. The neutral {@link - * #findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase} above stays unfiltered because the writers - * must see a row of any type to detect a collision at a shared key. + * {@code null} matches any type. {@code TABLE} also matches a stored null, because the column is + * nullable with no backfill and an absent discriminator means a table. An unrecognized request + * value matches neither branch, so garbage fails closed. */ - @Query( - "SELECT u FROM UserTableRow u WHERE " - + "lower(u.databaseId) = lower(:databaseId) AND " - + "lower(u.tableId) = lower(:tableId) AND " - + TABLE_ROW_PREDICATE) - Optional findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( - @Param("databaseId") String databaseId, @Param("tableId") String tableId); + String ENTITY_TYPE_PREDICATE = + "(:entityType IS NULL " + + "OR (upper(:entityType) = 'TABLE' " + + "AND (u.entityType IS NULL OR upper(u.entityType) = 'TABLE')) " + + "OR (upper(:entityType) = 'VIEW' AND upper(u.entityType) = 'VIEW'))"; @Query("SELECT DISTINCT databaseId FROM UserTableRow") Iterable findAllDistinctDatabaseIds(); @@ -63,47 +61,29 @@ Optional findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId))") Page findAllDistinctDatabaseIds(String databaseId, Pageable pageable); - @Query( - "SELECT u FROM UserTableRow u WHERE " - + "lower(u.databaseId) = lower(:databaseId) AND " - + TABLE_ROW_PREDICATE) - Iterable findAllTablesByDatabaseIdIgnoreCase( - @Param("databaseId") String databaseId); - - @Query( - value = - "SELECT u FROM UserTableRow u WHERE " - + "lower(u.databaseId) = lower(:databaseId) AND " - + TABLE_ROW_PREDICATE, - countQuery = - "SELECT COUNT(u) FROM UserTableRow u WHERE " - + "lower(u.databaseId) = lower(:databaseId) AND " - + TABLE_ROW_PREDICATE) - Page findAllTablesByDatabaseIdIgnoreCase( - @Param("databaseId") String databaseId, Pageable pageable); - - @Query( - "SELECT u FROM UserTableRow u WHERE " - + "lower(u.databaseId) = lower(:databaseId) AND " + String PATTERN_FILTER_PREDICATE = + "lower(u.databaseId) = lower(:databaseId) AND " + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " - + TABLE_ROW_PREDICATE) - Iterable findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( - @Param("databaseId") String databaseId, @Param("tableIdPattern") String tableIdPattern); + + ENTITY_TYPE_PREDICATE; + + /** + * Kept separate from {@link #findAllByFilters} because that query matches {@code tableId} + * exactly. Folding a LIKE into it would make {@code _} a wildcard, and OpenHouse identifiers + * routinely contain underscores. + */ + @Query("SELECT u FROM UserTableRow u WHERE " + PATTERN_FILTER_PREDICATE) + Iterable findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + @Param("databaseId") String databaseId, + @Param("tableIdPattern") String tableIdPattern, + @Param("entityType") String entityType); @Query( - value = - "SELECT u FROM UserTableRow u WHERE " - + "lower(u.databaseId) = lower(:databaseId) AND " - + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " - + TABLE_ROW_PREDICATE, - countQuery = - "SELECT COUNT(u) FROM UserTableRow u WHERE " - + "lower(u.databaseId) = lower(:databaseId) AND " - + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " - + TABLE_ROW_PREDICATE) - Page findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + value = "SELECT u FROM UserTableRow u WHERE " + PATTERN_FILTER_PREDICATE, + countQuery = "SELECT COUNT(u) FROM UserTableRow u WHERE " + PATTERN_FILTER_PREDICATE) + Page findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( @Param("databaseId") String databaseId, @Param("tableIdPattern") String tableIdPattern, + @Param("entityType") String entityType, Pageable pageable); String GENERAL_FILTER_PREDICATE = @@ -113,28 +93,96 @@ Page findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + "(:metadataLocation IS NULL OR u.metadataLocation = :metadataLocation) AND " + "(:storageType IS NULL OR u.storageType = :storageType) AND " + "(:creationTime IS NULL OR u.creationTime = :creationTime) AND " - + TABLE_ROW_PREDICATE; + + ENTITY_TYPE_PREDICATE; @Query( value = "select DISTINCT u from UserTableRow u where " + GENERAL_FILTER_PREDICATE, countQuery = "select COUNT(DISTINCT u) from UserTableRow u where " + GENERAL_FILTER_PREDICATE) - Page findAllTablesByFilters( + Page findAllByFilters( @Param("databaseId") String databaseId, @Param("tableId") String tableId, @Param("tableVersion") String tableVersion, @Param("metadataLocation") String metadataLocation, @Param("storageType") String storageType, @Param("creationTime") Long creationTime, + @Param("entityType") String entityType, Pageable pageable); @Query("select DISTINCT u from UserTableRow u where " + GENERAL_FILTER_PREDICATE) - Iterable findAllTablesByFilters( + Iterable findAllByFilters( @Param("databaseId") String databaseId, @Param("tableId") String tableId, @Param("tableVersion") String tableVersion, @Param("metadataLocation") String metadataLocation, @Param("storageType") String storageType, - @Param("creationTime") Long creationTime); + @Param("creationTime") Long creationTime, + @Param("entityType") String entityType); + + /* + * Table-scoped views onto the general queries above. They pin the discriminator and own no JPQL, + * so a table caller cannot drift from the general semantics. + */ + + default Page findAllTablesByFilters( + String databaseId, + String tableId, + String tableVersion, + String metadataLocation, + String storageType, + Long creationTime, + Pageable pageable) { + return findAllByFilters( + databaseId, + tableId, + tableVersion, + metadataLocation, + storageType, + creationTime, + TABLE_ENTITY_TYPE, + pageable); + } + + default Iterable findAllTablesByFilters( + String databaseId, + String tableId, + String tableVersion, + String metadataLocation, + String storageType, + Long creationTime) { + return findAllByFilters( + databaseId, + tableId, + tableVersion, + metadataLocation, + storageType, + creationTime, + TABLE_ENTITY_TYPE); + } + + default Iterable findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + String databaseId, String tableIdPattern) { + return findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + databaseId, tableIdPattern, TABLE_ENTITY_TYPE); + } + + default Page findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + String databaseId, String tableIdPattern, Pageable pageable) { + return findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + databaseId, tableIdPattern, TABLE_ENTITY_TYPE, pageable); + } + + /** + * Table-scoped point read serving {@code getUserTable}, the single HTS endpoint behind every + * table point read in the tables service. The key is the primary key, so at most one row can + * match. The neutral {@link #findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase} stays unfiltered + * because the writers must see a row of any type to detect a collision at a shared key. + */ + default Optional findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( + String databaseId, String tableId) { + Iterator matches = + findAllTablesByFilters(databaseId, tableId, null, null, null, null).iterator(); + return matches.hasNext() ? Optional.of(matches.next()) : Optional.empty(); + } /* * The following methods are required to maintain the generality of the interface {@link com.linkedin.openhouse.housetables.repository.HtsRepository} diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java index b6655aa4b..27f3f3227 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/UserTablesServiceImpl.java @@ -287,7 +287,8 @@ private List listTables(UserTable userTable) { () -> StreamSupport.stream( htsJdbcRepository - .findAllTablesByDatabaseIdIgnoreCase(userTable.getDatabaseId()) + .findAllTablesByFilters( + userTable.getDatabaseId(), null, null, null, null, null) .spliterator(), false) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)) @@ -301,7 +302,8 @@ private Page listTables(UserTable userTable, int page, int size, S return METRICS_REPORTER.executeWithStats( () -> htsJdbcRepository - .findAllTablesByDatabaseIdIgnoreCase(userTable.getDatabaseId(), pageable) + .findAllTablesByFilters( + userTable.getDatabaseId(), null, null, null, null, null, pageable) .map(userTableRow -> userTablesMapper.toUserTableDto(userTableRow)), MetricsConstant.HTS_PAGE_TABLES_TIME); } diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java index 5b99d7685..bafb0b7f3 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java @@ -10,7 +10,9 @@ import com.linkedin.openhouse.housetables.model.UserTableRow; import com.linkedin.openhouse.housetables.model.UserTableRowPrimaryKey; import com.linkedin.openhouse.housetables.repository.impl.jdbc.UserTableHtsJdbcRepository; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.AfterEach; @@ -137,7 +139,8 @@ public void testFindAllByDatabaseId() { htsRepository.save(TEST_TUPLE_1_1.get_userTableRow()); htsRepository.save(TEST_TUPLE_2_0.get_userTableRow()); List result = - Lists.newArrayList(htsRepository.findAllTablesByDatabaseIdIgnoreCase("test_db0")); + Lists.newArrayList( + htsRepository.findAllTablesByFilters("test_db0", null, null, null, null, null)); Assertions.assertEquals( Lists.newArrayList("test_table1", "test_table2"), result.stream().map(UserTableRow::getTableId).collect(Collectors.toList())); @@ -369,7 +372,8 @@ public void testFindAllByDatabaseIdFiltersViewsAndKeepsLegacyTables() { htsRepository.save(row("other_db", "t00_legacy", null)); List result = - Lists.newArrayList(htsRepository.findAllTablesByDatabaseIdIgnoreCase(ENTITY_TYPE_DB)); + Lists.newArrayList( + htsRepository.findAllTablesByFilters(ENTITY_TYPE_DB, null, null, null, null, null)); assertThat(tableIds(result)).containsExactly(CANONICAL_TABLE_IDS); assertThat(result) @@ -386,14 +390,16 @@ public void testFindAllByDatabaseIdFiltersBeforePagination() { seedCanonicalRows(ENTITY_TYPE_DB, ""); Page page0 = - htsRepository.findAllTablesByDatabaseIdIgnoreCase(ENTITY_TYPE_DB, sortedPage(0)); + htsRepository.findAllTablesByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, sortedPage(0)); assertThat(page0.getTotalElements()).isEqualTo(4); assertThat(page0.getTotalPages()).isEqualTo(2); assertThat(page0.getContent()).hasSize(2); assertThat(pageTableIds(page0)).containsExactly("t00_legacy", "t02_explicit"); Page page1 = - htsRepository.findAllTablesByDatabaseIdIgnoreCase(ENTITY_TYPE_DB, sortedPage(1)); + htsRepository.findAllTablesByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, sortedPage(1)); assertThat(page1.getTotalElements()).isEqualTo(4); assertThat(page1.getTotalPages()).isEqualTo(2); assertThat(page1.getContent()).hasSize(2); @@ -488,7 +494,8 @@ public void testFindAllTablesByFiltersReturnsOnlyTables() { public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { seedCaseNormalizationRows(); - assertThat(tableIds(htsRepository.findAllTablesByDatabaseIdIgnoreCase(CASE_DB))) + assertThat( + tableIds(htsRepository.findAllTablesByFilters(CASE_DB, null, null, null, null, null))) .containsExactly(CASE_VISIBLE_TABLE_IDS); assertThat( tableIds( @@ -497,7 +504,7 @@ public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { .containsExactly(CASE_VISIBLE_TABLE_IDS); Page dbPage0 = - htsRepository.findAllTablesByDatabaseIdIgnoreCase(CASE_DB, sortedPage(0)); + htsRepository.findAllTablesByFilters(CASE_DB, null, null, null, null, null, sortedPage(0)); assertThat(dbPage0.getTotalElements()).isEqualTo(4); assertThat(dbPage0.getTotalPages()).isEqualTo(2); assertThat(pageTableIds(dbPage0)).containsExactly("case00_null", "case01_upper_table"); @@ -515,7 +522,8 @@ public void testEntityTypePredicatesAreCaseInsensitiveAndGarbageFailsClosed() { .containsExactly(CASE_VISIBLE_TABLE_IDS); // Garbage fails closed everywhere: it is neither a table nor a view. - assertThat(tableIds(htsRepository.findAllTablesByDatabaseIdIgnoreCase(CASE_DB))) + assertThat( + tableIds(htsRepository.findAllTablesByFilters(CASE_DB, null, null, null, null, null))) .doesNotContain(CASE_GARBAGE_ID); // The garbage row is still stored — it is hidden, not dropped. @@ -588,6 +596,76 @@ public void testNeutralPointReadStillSeesEveryEntityType() { } } + /** + * The general query is genuinely general: a null discriminator means any type, not a table + * default. TABLE additionally absorbs legacy stored nulls, VIEW selects views, and an + * unrecognized request value matches nothing. + */ + @Test + public void testFindAllByFiltersTreatsNullEntityTypeAsAnyType() { + seedCanonicalRows(ENTITY_TYPE_DB, ""); + + List everything = new ArrayList<>(Arrays.asList(CANONICAL_TABLE_IDS)); + everything.addAll(Arrays.asList(CANONICAL_VIEW_IDS)); + Collections.sort(everything); + + assertThat( + tableIds( + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, (String) null))) + .as("entityType=null must return tables and views together") + .isEqualTo(everything); + + for (String tableSpelling : new String[] {"TABLE", "table", "TaBlE"}) { + assertThat( + tableIds( + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, tableSpelling))) + .as("entityType=%s must include legacy null rows", tableSpelling) + .containsExactly(CANONICAL_TABLE_IDS); + } + + for (String viewSpelling : new String[] {"VIEW", "view", "ViEw"}) { + assertThat( + tableIds( + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, viewSpelling))) + .as("entityType=%s must resolve to exactly the three views", viewSpelling) + .containsExactly(CANONICAL_VIEW_IDS); + } + + assertThat( + Lists.newArrayList( + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, "UNKNOWN"))) + .as("an unrecognized discriminator must fail closed") + .isEmpty(); + + // The paged overload agrees, including its count. + Page anyPage0 = + htsRepository.findAllByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, (String) null, sortedPage(0)); + assertThat(anyPage0.getTotalElements()).isEqualTo(7); + assertThat(pageTableIds(anyPage0)).containsExactly("t00_legacy", "t01_view"); + } + + /** The pattern family is general in the same way. */ + @Test + public void testFindAllByPatternHonorsEntityType() { + seedCanonicalRows(ENTITY_TYPE_DB, "match_"); + + assertThat( + tableIds( + htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + ENTITY_TYPE_DB, "match_%", null))) + .hasSize(7); + assertThat( + tableIds( + htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + ENTITY_TYPE_DB, "match_%", "VIEW"))) + .containsExactly("match_t01_view", "match_t03_view", "match_t05_view"); + } + private UserTableRow findRow(String databaseId, String tableId) { return htsRepository .findById(UserTableRowPrimaryKey.builder().databaseId(databaseId).tableId(tableId).build()) From d1a1567c41a795815d161ed79cb9bf7dd7c19d85 Mon Sep 17 00:00:00 2001 From: Ruolin Fan Date: Fri, 14 Aug 2026 18:00:40 -0700 Subject: [PATCH 12/12] BDP-108403: Select entity type by method, never by argument entityType is no longer a parameter anywhere in the query layer. A caller picks a type by picking a method: findAllByFilters returns both types, findAllTablesByFilters returns tables, and findAllViewsByFilters arrives with the view ticket. That drops the delegating-default idea: a typed wrapper cannot tell a parameterless general method what to filter, so each typed method carries its own @Query. To avoid restating the filter body, the six general clauses are extracted once into COMMON_FILTER_CLAUSES and the typed sibling composes that constant with TABLE_ROW_PREDICATE. The pattern family is split the same way through PATTERN_KEY_CLAUSES. The extraction is provably behavior-preserving: both findAllByFilters overloads now read "select DISTINCT u from UserTableRow u where " + COMMON_FILTER_CLAUSES, which expands byte-for-byte to the ba400b38 string. The pattern overloads are restored to their ba400b38 form exactly - derived queries with no @Query at all. Added, table-scoped, each with its own query composed from the shared constants: findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase findAllTablesByFilters x2 findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase x2 Nothing is renamed and no view method is added. Unchanged from ba400b38: findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase, which backs findById for putUserTable, deleteUserTable and restoreUserTable and must see a row of any type at a shared key; existsBy; deleteBy; renameTableId; and both findAllDistinctDatabaseIds overloads. The two findAllByDatabaseIdIgnoreCase overloads stay deleted, since findAllTablesByFilters(db, null, ...) covers them, which is what paged listTables already did at base. Call sites: listTables and searchTables use findAllTablesByFilters, listTablesWithPattern uses the table pattern methods, getUserTable uses the table point read. entityType is not read from the wire, so isNonKeyFieldsNullForUserTable and the validator's query branch remain at their pre-change form and all four routes behave as at base. Because getUserTable is the one HTS endpoint behind every table point read in the tables service, the guards removed earlier stay correct by construction: a 404 becomes HouseTableNotFoundException, which doRefresh already catches to leave an empty Optional and which findHouseTable already catches to return empty, so dropTable throws NoSuchTableException, findTableRefById returns empty, and a rename off a view source fails inside loadTable. Verified: housetables 177, internalcatalog 87, tables 475, tables-test-fixtures 8, spark-3.5 catalogTest 66 - all green, plus spotlessCheck and checkstyle. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../impl/jdbc/UserTableHtsJdbcRepository.java | 192 ++++++++---------- .../e2e/usertable/HtsRepositoryTest.java | 74 +++---- 2 files changed, 118 insertions(+), 148 deletions(-) diff --git a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java index 0e3efc0c2..099847976 100644 --- a/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java +++ b/services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/UserTableHtsJdbcRepository.java @@ -4,7 +4,6 @@ import com.linkedin.openhouse.housetables.model.UserTableRow; import com.linkedin.openhouse.housetables.model.UserTableRowPrimaryKey; import com.linkedin.openhouse.housetables.repository.HtsRepository; -import java.util.Iterator; import java.util.Optional; import org.jetbrains.annotations.NotNull; import org.springframework.data.domain.Page; @@ -40,149 +39,120 @@ Optional findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( void deleteByDatabaseIdIgnoreCaseAndTableIdIgnoreCase(String databaseId, String tableId); - String TABLE_ENTITY_TYPE = "TABLE"; + String COMMON_FILTER_CLAUSES = + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " + + "(:tableId IS NULL OR lower(u.tableId) = lower(:tableId)) AND " + + "(:tableVersion IS NULL OR u.version = :tableVersion) AND " + + "(:metadataLocation IS NULL OR u.metadataLocation = :metadataLocation) AND " + + "(:storageType IS NULL OR u.storageType = :storageType) AND " + + "(:creationTime IS NULL OR u.creationTime = :creationTime)"; + + String TABLE_ROW_PREDICATE = "(u.entityType IS NULL OR upper(u.entityType) = 'TABLE')"; + + String PATTERN_KEY_CLAUSES = + "lower(u.databaseId) = lower(:databaseId) AND " + + "lower(u.tableId) LIKE lower(:tableIdPattern)"; /** - * {@code null} matches any type. {@code TABLE} also matches a stored null, because the column is - * nullable with no backfill and an absent discriminator means a table. An unrecognized request - * value matches neither branch, so garbage fails closed. + * Table-scoped point read serving {@code getUserTable}, the single HTS endpoint behind every + * table point read in the tables service. The neutral {@link + * #findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase} above stays unfiltered because the writers + * must see a row of any type to detect a collision at a shared key. */ - String ENTITY_TYPE_PREDICATE = - "(:entityType IS NULL " - + "OR (upper(:entityType) = 'TABLE' " - + "AND (u.entityType IS NULL OR upper(u.entityType) = 'TABLE')) " - + "OR (upper(:entityType) = 'VIEW' AND upper(u.entityType) = 'VIEW'))"; + @Query( + "SELECT u FROM UserTableRow u WHERE " + + "lower(u.databaseId) = lower(:databaseId) AND " + + "lower(u.tableId) = lower(:tableId) AND " + + TABLE_ROW_PREDICATE) + Optional findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( + @Param("databaseId") String databaseId, @Param("tableId") String tableId); @Query("SELECT DISTINCT databaseId FROM UserTableRow") Iterable findAllDistinctDatabaseIds(); + Iterable findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + String databaseId, String tableIdPattern); + @Query( "SELECT DISTINCT databaseId FROM UserTableRow u where " + "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId))") Page findAllDistinctDatabaseIds(String databaseId, Pageable pageable); - String PATTERN_FILTER_PREDICATE = - "lower(u.databaseId) = lower(:databaseId) AND " - + "lower(u.tableId) LIKE lower(:tableIdPattern) AND " - + ENTITY_TYPE_PREDICATE; + Page findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + String databaseId, String tableIdPattern, Pageable pageable); - /** - * Kept separate from {@link #findAllByFilters} because that query matches {@code tableId} - * exactly. Folding a LIKE into it would make {@code _} a wildcard, and OpenHouse identifiers - * routinely contain underscores. - */ - @Query("SELECT u FROM UserTableRow u WHERE " + PATTERN_FILTER_PREDICATE) - Iterable findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - @Param("databaseId") String databaseId, - @Param("tableIdPattern") String tableIdPattern, - @Param("entityType") String entityType); + @Query("select DISTINCT u from UserTableRow u where " + COMMON_FILTER_CLAUSES) + Page findAllByFilters( + String databaseId, + String tableId, + String tableVersion, + String metadataLocation, + String storageType, + Long creationTime, + Pageable pageable); + + @Query("select DISTINCT u from UserTableRow u where " + COMMON_FILTER_CLAUSES) + Iterable findAllByFilters( + String databaseId, + String tableId, + String tableVersion, + String metadataLocation, + String storageType, + Long creationTime); @Query( - value = "SELECT u FROM UserTableRow u WHERE " + PATTERN_FILTER_PREDICATE, - countQuery = "SELECT COUNT(u) FROM UserTableRow u WHERE " + PATTERN_FILTER_PREDICATE) - Page findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( + "SELECT u FROM UserTableRow u WHERE " + PATTERN_KEY_CLAUSES + " AND " + TABLE_ROW_PREDICATE) + Iterable findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + @Param("databaseId") String databaseId, @Param("tableIdPattern") String tableIdPattern); + + @Query( + value = + "SELECT u FROM UserTableRow u WHERE " + + PATTERN_KEY_CLAUSES + + " AND " + + TABLE_ROW_PREDICATE, + countQuery = + "SELECT COUNT(u) FROM UserTableRow u WHERE " + + PATTERN_KEY_CLAUSES + + " AND " + + TABLE_ROW_PREDICATE) + Page findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( @Param("databaseId") String databaseId, @Param("tableIdPattern") String tableIdPattern, - @Param("entityType") String entityType, Pageable pageable); - String GENERAL_FILTER_PREDICATE = - "(:databaseId IS NULL OR lower(u.databaseId) = lower(:databaseId)) AND " - + "(:tableId IS NULL OR lower(u.tableId) = lower(:tableId)) AND " - + "(:tableVersion IS NULL OR u.version = :tableVersion) AND " - + "(:metadataLocation IS NULL OR u.metadataLocation = :metadataLocation) AND " - + "(:storageType IS NULL OR u.storageType = :storageType) AND " - + "(:creationTime IS NULL OR u.creationTime = :creationTime) AND " - + ENTITY_TYPE_PREDICATE; - @Query( - value = "select DISTINCT u from UserTableRow u where " + GENERAL_FILTER_PREDICATE, - countQuery = "select COUNT(DISTINCT u) from UserTableRow u where " + GENERAL_FILTER_PREDICATE) - Page findAllByFilters( + value = + "select DISTINCT u from UserTableRow u where " + + COMMON_FILTER_CLAUSES + + " AND " + + TABLE_ROW_PREDICATE, + countQuery = + "select COUNT(DISTINCT u) from UserTableRow u where " + + COMMON_FILTER_CLAUSES + + " AND " + + TABLE_ROW_PREDICATE) + Page findAllTablesByFilters( @Param("databaseId") String databaseId, @Param("tableId") String tableId, @Param("tableVersion") String tableVersion, @Param("metadataLocation") String metadataLocation, @Param("storageType") String storageType, @Param("creationTime") Long creationTime, - @Param("entityType") String entityType, Pageable pageable); - @Query("select DISTINCT u from UserTableRow u where " + GENERAL_FILTER_PREDICATE) - Iterable findAllByFilters( + @Query( + "select DISTINCT u from UserTableRow u where " + + COMMON_FILTER_CLAUSES + + " AND " + + TABLE_ROW_PREDICATE) + Iterable findAllTablesByFilters( @Param("databaseId") String databaseId, @Param("tableId") String tableId, @Param("tableVersion") String tableVersion, @Param("metadataLocation") String metadataLocation, @Param("storageType") String storageType, - @Param("creationTime") Long creationTime, - @Param("entityType") String entityType); - - /* - * Table-scoped views onto the general queries above. They pin the discriminator and own no JPQL, - * so a table caller cannot drift from the general semantics. - */ - - default Page findAllTablesByFilters( - String databaseId, - String tableId, - String tableVersion, - String metadataLocation, - String storageType, - Long creationTime, - Pageable pageable) { - return findAllByFilters( - databaseId, - tableId, - tableVersion, - metadataLocation, - storageType, - creationTime, - TABLE_ENTITY_TYPE, - pageable); - } - - default Iterable findAllTablesByFilters( - String databaseId, - String tableId, - String tableVersion, - String metadataLocation, - String storageType, - Long creationTime) { - return findAllByFilters( - databaseId, - tableId, - tableVersion, - metadataLocation, - storageType, - creationTime, - TABLE_ENTITY_TYPE); - } - - default Iterable findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( - String databaseId, String tableIdPattern) { - return findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - databaseId, tableIdPattern, TABLE_ENTITY_TYPE); - } - - default Page findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( - String databaseId, String tableIdPattern, Pageable pageable) { - return findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - databaseId, tableIdPattern, TABLE_ENTITY_TYPE, pageable); - } - - /** - * Table-scoped point read serving {@code getUserTable}, the single HTS endpoint behind every - * table point read in the tables service. The key is the primary key, so at most one row can - * match. The neutral {@link #findByDatabaseIdIgnoreCaseAndTableIdIgnoreCase} stays unfiltered - * because the writers must see a row of any type to detect a collision at a shared key. - */ - default Optional findTableByDatabaseIdIgnoreCaseAndTableIdIgnoreCase( - String databaseId, String tableId) { - Iterator matches = - findAllTablesByFilters(databaseId, tableId, null, null, null, null).iterator(); - return matches.hasNext() ? Optional.of(matches.next()) : Optional.empty(); - } + @Param("creationTime") Long creationTime); /* * The following methods are required to maintain the generality of the interface {@link com.linkedin.openhouse.housetables.repository.HtsRepository} diff --git a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java index bafb0b7f3..5819e6e9d 100644 --- a/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java +++ b/services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/usertable/HtsRepositoryTest.java @@ -597,12 +597,12 @@ public void testNeutralPointReadStillSeesEveryEntityType() { } /** - * The general query is genuinely general: a null discriminator means any type, not a table - * default. TABLE additionally absorbs legacy stored nulls, VIEW selects views, and an - * unrecognized request value matches nothing. + * The type is chosen by which method you call, not by an argument. {@code findAllByFilters} is + * the general query and returns both types; its table-scoped sibling adds only the table + * predicate, which also matches a legacy stored null. */ @Test - public void testFindAllByFiltersTreatsNullEntityTypeAsAnyType() { + public void testGeneralFiltersReturnBothTypesAndTableFiltersReturnOnlyTables() { seedCanonicalRows(ENTITY_TYPE_DB, ""); List everything = new ArrayList<>(Arrays.asList(CANONICAL_TABLE_IDS)); @@ -610,60 +610,60 @@ public void testFindAllByFiltersTreatsNullEntityTypeAsAnyType() { Collections.sort(everything); assertThat( - tableIds( - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, (String) null))) - .as("entityType=null must return tables and views together") + tableIds(htsRepository.findAllByFilters(ENTITY_TYPE_DB, null, null, null, null, null))) + .as("the general query must return tables and views together") .isEqualTo(everything); - for (String tableSpelling : new String[] {"TABLE", "table", "TaBlE"}) { - assertThat( - tableIds( - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, tableSpelling))) - .as("entityType=%s must include legacy null rows", tableSpelling) - .containsExactly(CANONICAL_TABLE_IDS); - } - - for (String viewSpelling : new String[] {"VIEW", "view", "ViEw"}) { - assertThat( - tableIds( - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, viewSpelling))) - .as("entityType=%s must resolve to exactly the three views", viewSpelling) - .containsExactly(CANONICAL_VIEW_IDS); - } + assertThat( + tableIds( + htsRepository.findAllTablesByFilters(ENTITY_TYPE_DB, null, null, null, null, null))) + .as("the table query must return tables and legacy nulls only") + .containsExactly(CANONICAL_TABLE_IDS); + // A view is unreachable through the table family, by tableId as well as by database. assertThat( Lists.newArrayList( - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, "UNKNOWN"))) - .as("an unrecognized discriminator must fail closed") + htsRepository.findAllTablesByFilters( + ENTITY_TYPE_DB, "t01_view", null, null, null, null))) .isEmpty(); - // The paged overload agrees, including its count. + // Paged overloads agree, counts included. Page anyPage0 = - htsRepository.findAllByFilters( - ENTITY_TYPE_DB, null, null, null, null, null, (String) null, sortedPage(0)); + htsRepository.findAllByFilters(ENTITY_TYPE_DB, null, null, null, null, null, sortedPage(0)); assertThat(anyPage0.getTotalElements()).isEqualTo(7); assertThat(pageTableIds(anyPage0)).containsExactly("t00_legacy", "t01_view"); + + Page tablePage0 = + htsRepository.findAllTablesByFilters( + ENTITY_TYPE_DB, null, null, null, null, null, sortedPage(0)); + assertThat(tablePage0.getTotalElements()).isEqualTo(4); + assertThat(tablePage0.getTotalPages()).isEqualTo(2); + assertThat(pageTableIds(tablePage0)).containsExactly("t00_legacy", "t02_explicit"); } - /** The pattern family is general in the same way. */ + /** The pattern family splits the same way. */ @Test - public void testFindAllByPatternHonorsEntityType() { + public void testGeneralPatternReturnsBothTypesAndTablePatternOnlyTables() { seedCanonicalRows(ENTITY_TYPE_DB, "match_"); assertThat( tableIds( htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - ENTITY_TYPE_DB, "match_%", null))) + ENTITY_TYPE_DB, "match_%"))) .hasSize(7); + assertThat( tableIds( - htsRepository.findAllByDatabaseIdAndTableIdLikeAllIgnoreCase( - ENTITY_TYPE_DB, "match_%", "VIEW"))) - .containsExactly("match_t01_view", "match_t03_view", "match_t05_view"); + htsRepository.findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + ENTITY_TYPE_DB, "match_%"))) + .containsExactly( + "match_t00_legacy", "match_t02_explicit", "match_t04_legacy", "match_t06_explicit"); + + Page tablePage0 = + htsRepository.findAllTablesByDatabaseIdAndTableIdLikeAllIgnoreCase( + ENTITY_TYPE_DB, "match_%", sortedPage(0)); + assertThat(tablePage0.getTotalElements()).isEqualTo(4); + assertThat(tablePage0.getTotalPages()).isEqualTo(2); } private UserTableRow findRow(String databaseId, String tableId) {