diff --git a/src/AasxServerBlazor/AasxServerBlazor.csproj b/src/AasxServerBlazor/AasxServerBlazor.csproj index 550035296..1cabdf57b 100644 --- a/src/AasxServerBlazor/AasxServerBlazor.csproj +++ b/src/AasxServerBlazor/AasxServerBlazor.csproj @@ -33,6 +33,7 @@ + diff --git a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs index f118c49c8..d1a427aa5 100644 --- a/src/AasxServerBlazor/Configuration/ServerConfiguration.cs +++ b/src/AasxServerBlazor/Configuration/ServerConfiguration.cs @@ -15,6 +15,7 @@ namespace AasxServerBlazor.Configuration; using System; using System.Collections.Generic; using System.IO; +using System.Linq; using AasSecurity; using AasxServerStandardBib.ServiceExtensions; using IO.Swagger.Controllers; @@ -34,8 +35,10 @@ namespace AasxServerBlazor.Configuration; using Microsoft.OpenApi.Models; using System.Text.Json; -using System.Text.Json.Serialization; -using AdminShellNS; +using System.Text.Json.Serialization; +using AdminShellNS; +using Contracts; +using IO.Swagger.Lib.V3.MCP; #if GRAPHQL using HotChocolate.AspNetCore; #endif @@ -77,7 +80,127 @@ public static void ConfigureServer(IServiceCollection services) IncludeExceptionDetails = true }); #endif - } + + // MCP-Server (Streamable HTTP) als dünner Adapter über die Query-Pipeline. Immer aktiv. + // Zwei Endpunkte: /mcp (voll, alle Tools) und /mcp-basic (nur aas_find_product, für schwache Modelle). + // Die Aufteilung erfolgt über Request-Filter, die den Request-Pfad prüfen. + services.AddHttpContextAccessor(); + services.AddMcpServer() + .WithHttpTransport() + .WithTools() + // Exportdateien (CSV/XLSX) der Export-Tools als MCP-Resource: aas-export://{token}. + .WithResources() + .WithRequestFilters(filters => + { + filters.AddListToolsFilter(next => async (context, ct) => + { + var result = await next(context, ct); + var allowed = AllowedMcpTools(context.Services); + if (result.Tools is not null) + { + if (allowed is not null) + { + result.Tools = result.Tools.Where(t => allowed.Contains(t.Name)).ToList(); + } + + // OpenAI Apps SDK / ChatGPT compatibility: declare each (read-only, public) tool as + // no-auth and give it invocation status text. securitySchemes is mirrored into _meta + // (the location ChatGPT reads; the typed SDK cannot emit a custom top-level field). + // title + readOnly/idempotent/etc. annotations come from the [McpServerTool] attributes. + foreach (var tool in result.Tools) + { + var meta = tool.Meta ?? new System.Text.Json.Nodes.JsonObject(); + meta["securitySchemes"] = new System.Text.Json.Nodes.JsonArray( + new System.Text.Json.Nodes.JsonObject { ["type"] = "noauth" }); + if (McpToolStatus.TryGetValue(tool.Name, out var status)) + { + meta["openai/toolInvocation/invoking"] = status.Invoking; + meta["openai/toolInvocation/invoked"] = status.Invoked; + } + tool.Meta = meta; + } + } + + return result; + }); + + filters.AddCallToolFilter(next => async (context, ct) => + { + var allowed = AllowedMcpTools(context.Services); + if (allowed is not null + && context.Params is not null + && !allowed.Contains(context.Params.Name)) + { + throw new InvalidOperationException( + $"Tool '{context.Params.Name}' ist auf diesem MCP-Endpunkt nicht verfügbar."); + } + + var logSession = ExplicitLogSession(context.Services); + if (!McpSessionLogStore.IsValidSessionId(logSession)) + { + return await next(context, ct); + } + + using var _ = DiagnosticsLog.BeginBrowserLogSession(logSession!); + return await next(context, ct); + }); + }); + } + + // Per-tool invocation status text (OpenAI Apps SDK _meta), kept <= 64 chars per the spec. + private static readonly IReadOnlyDictionary McpToolStatus = + new Dictionary(StringComparer.Ordinal) + { + ["aas_query"] = ("Searching Voyager…", "Voyager search complete"), + ["aas_query_export_csv"] = ("Exporting Voyager results…", "CSV export ready"), + ["aas_query_export_xlsx"] = ("Exporting Voyager results…", "Excel export ready"), + ["aas_count"] = ("Counting Voyager results…", "Voyager count complete"), + ["aas_get_submodel"] = ("Reading AAS submodel…", "AAS submodel loaded"), + ["aas_get_submodels"] = ("Reading AAS submodels…", "AAS submodels loaded"), + ["aas_get_shell"] = ("Reading AAS shell…", "AAS shell loaded"), + ["aas_get_shells"] = ("Reading AAS shells…", "AAS shells loaded"), + ["aas_get_product"] = ("Reading AAS product…", "AAS product loaded"), + ["aas_find_product"] = ("Finding AAS product…", "AAS product found"), + ["aas_find_product_simple"] = ("Finding product…", "Product found"), + ["aas_get_element"] = ("Reading AAS element…", "AAS element loaded"), + ["aas_find_concepts"] = ("Searching concept definitions…", "Concept definitions found"), + ["aas_describe_model"] = ("Analyzing data structures…", "Data structure overview ready"), + }; + + // Erlaubter Tool-Satz je nach MCP-Endpunkt-Pfad (null = alle Tools, voller Endpunkt /mcp). + // /mcp-simple zuerst prüfen (Pfad enthält nicht "mcp-basic"), dann /mcp-basic, sonst voll. + private static HashSet? AllowedMcpTools(IServiceProvider? services) + { + var path = services?.GetService()?.HttpContext?.Request.Path.Value; + if (path is null) + { + return null; + } + + if (path.Contains("mcp-simple", StringComparison.OrdinalIgnoreCase)) + { + return McpQueryTools.SimpleToolNames; + } + + if (path.Contains("mcp-basic", StringComparison.OrdinalIgnoreCase)) + { + return McpQueryTools.BasicToolNames; + } + + return null; + } + + private static string? ExplicitLogSession(IServiceProvider? services) + { + var request = services? + .GetService()? + .HttpContext? + .Request; + + return request?.Query.TryGetValue("logSession", out var value) == true + ? value.FirstOrDefault() + : null; + } /// /// Adds framework-specific services required by the application. @@ -169,6 +292,23 @@ private static void ConfigureEndpoints(IEndpointRouteBuilder endpoints) Tool = { Enable = true } }); #endif + // MCP-Endpoints (Streamable HTTP). PathBase ist "/api/v3.0", real also "/api/v3.0/mcp" bzw. "/api/v3.0/mcp-basic". + // /mcp = voller Toolsatz (starke Modelle), /mcp-basic = nur aas_find_product (schwache Modelle); Filter s. ConfigureMcp. + endpoints.MapMcp("/mcp"); + endpoints.MapMcp("/mcp-basic"); + endpoints.MapMcp("/mcp-simple"); + + // Download-Endpunkt für die von aas_query_export_csv/xlsx erzeugten Dateien. + // Das Token stammt aus der Tool-Antwort (downloadUrl); die Dateien verfallen nach ca. 60 Minuten. + endpoints.MapGet("/mcp-exports/{token}", (string token) => + { + if (!McpExportFileStore.TryGet(token, out var content, out var fileName, out var mimeType)) + { + return Microsoft.AspNetCore.Http.Results.NotFound(); + } + + return Microsoft.AspNetCore.Http.Results.File(content, mimeType, fileName); + }); } #endregion @@ -286,4 +426,4 @@ private static void AddSwaggerGen(IServiceCollection services) => }); #endregion -} \ No newline at end of file +} diff --git a/src/AasxServerBlazor/Pages/McpLog.razor b/src/AasxServerBlazor/Pages/McpLog.razor new file mode 100644 index 000000000..cc56e6048 --- /dev/null +++ b/src/AasxServerBlazor/Pages/McpLog.razor @@ -0,0 +1,180 @@ +@page "/McpLog" +@using Contracts +@using System.Threading +@using System.Threading.Tasks +@implements IDisposable +@inject NavigationManager Nav + + + +
+

MCP Log

+

+ Shows server-side MCP diagnostics only for the explicit logSession value. +

+ +
+ + + +
+ + @if (!HasSession) + { +

+ Open this page with ?logSession=your-token or enter the same token used on the MCP endpoint. +

+ } + else if (!McpSessionLogStore.IsValidSessionId(_session)) + { +

+ Invalid logSession. Use letters, digits, dash, underscore, dot or colon; max 128 characters. +

+ } + else + { +
+ MCP endpoint: /api/v3.0/mcp?logSession=@_session +
+
+@LogText +
+ } +
+ +@code { + private readonly List _entries = new(); + private PeriodicTimer? _timer; + private CancellationTokenSource? _cts; + private string? _session; + private string? _sessionInput; + private long _lastSequence; + + private bool HasSession => !string.IsNullOrWhiteSpace(_session); + + private string LogText => _entries.Count == 0 + ? "Waiting for MCP log lines..." + : string.Join(Environment.NewLine, _entries.Select(e => e.Line)); + + protected override void OnInitialized() + { + _session = ReadQueryValue("logSession"); + _sessionInput = _session; + _cts = new CancellationTokenSource(); + _timer = new PeriodicTimer(TimeSpan.FromMilliseconds(750)); + _ = PollAsync(_cts.Token); + } + + private void ApplySession() + { + _session = string.IsNullOrWhiteSpace(_sessionInput) ? null : _sessionInput.Trim(); + _entries.Clear(); + _lastSequence = 0; + } + + private void ClearView() + { + if (McpSessionLogStore.IsValidSessionId(_session)) + { + McpSessionLogStore.Clear(_session!); + } + + _entries.Clear(); + _lastSequence = 0; + } + + private async Task PollAsync(CancellationToken cancellationToken) + { + while (_timer is not null && await _timer.WaitForNextTickAsync(cancellationToken)) + { + if (!McpSessionLogStore.IsValidSessionId(_session)) + { + continue; + } + + var next = McpSessionLogStore.Read(_session!, _lastSequence, 500); + if (next.Count == 0) + { + continue; + } + + _entries.AddRange(next); + _lastSequence = _entries[^1].Sequence; + + if (_entries.Count > 2000) + { + _entries.RemoveRange(0, _entries.Count - 2000); + } + + await InvokeAsync(StateHasChanged); + } + } + + private string? ReadQueryValue(string key) + { + var query = new Uri(Nav.Uri).Query.TrimStart('?'); + if (query.Length == 0) + { + return null; + } + + foreach (var part in query.Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var pair = part.Split('=', 2); + if (pair.Length == 2 && string.Equals(Uri.UnescapeDataString(pair[0]), key, StringComparison.Ordinal)) + { + return Uri.UnescapeDataString(pair[1]); + } + } + + return null; + } + + public void Dispose() + { + _cts?.Cancel(); + _timer?.Dispose(); + _cts?.Dispose(); + } +} diff --git a/src/AasxServerBlazor/Properties/launchSettings.json b/src/AasxServerBlazor/Properties/launchSettings.json index 20d5557b6..48d1d1b67 100644 --- a/src/AasxServerBlazor/Properties/launchSettings.json +++ b/src/AasxServerBlazor/Properties/launchSettings.json @@ -10,7 +10,7 @@ }, "AasxServerBlazor": { "commandName": "Project", - "commandLineArgs": "--with-db --start-index 1000 --secret-string-api 1234 --data-path \"C:\\Development\\newdb1\" --edit --external-blazor http://localhost:5001", + "commandLineArgs": "--no-security --with-db --start-index 1000 --secret-string-api 1234 --data-path \"C:\\Development\\newdb1\" --edit --external-blazor http://localhost:5001", "launchBrowser": true, "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development", diff --git a/src/AasxServerDB.Tests/DatabaseFixture.cs b/src/AasxServerDB.Tests/DatabaseFixture.cs index 675139ac6..b5a5f71ab 100644 --- a/src/AasxServerDB.Tests/DatabaseFixture.cs +++ b/src/AasxServerDB.Tests/DatabaseFixture.cs @@ -67,6 +67,13 @@ public DatabaseFixture() VisitorAASX.ImportAASXIntoDB(file, createFilesOnly: false); } + // Production installs this during InitDB. Build it after fixture import so + // query execution exercises the same SQLite substring-search path. + using (var indexDb = new AasContext()) + { + SqliteTrigramIndex.Initialize(indexDb); + } + Console.WriteLine("[DatabaseFixture] Import done."); using var verify = new AasContext(); Console.WriteLine($" SMSets: {verify.SMSets.Count()}"); diff --git a/src/AasxServerDB.Tests/ProjectionOperatorTests.cs b/src/AasxServerDB.Tests/ProjectionOperatorTests.cs new file mode 100644 index 000000000..ee844adb9 --- /dev/null +++ b/src/AasxServerDB.Tests/ProjectionOperatorTests.cs @@ -0,0 +1,196 @@ +namespace AasxServerDB.Tests; + +using Contracts.DbRequests; +using FluentAssertions; + +/// +/// Tests der SQL-Batchprojektion (MCP-Fast-Path) gegen die importierten AASX-Testdaten: +/// exakte idShortPath-Treffer im Treffer-Submodel, Cross-Submodel-Auflösung über die AAS +/// und Not-Found-Verhalten. Die Erwartungswerte werden datengetrieben aus denselben +/// Tabellen (SMSets/SMESets/ValueSets) ermittelt. +/// +[Collection(DatabaseFixture.Collection)] +public sealed class ProjectionOperatorTests +{ + private readonly DatabaseFixture _fixture; + + public ProjectionOperatorTests(DatabaseFixture fixture) + { + _fixture = fixture; + } + + private sealed record PropCandidate(string SubmodelIdentifier, string IdShortPath, string SValue); + + private static PropCandidate FirstDottedProperty(AasContext db, int? smId = null) + { + var candidate = (from sme in db.SMESets + join v in db.ValueSets on sme.Id equals v.SMEId + join sm in db.SMSets on sme.SMId equals sm.Id + where sme.SMEType == "Prop" + && sme.IdShortPath != null && sme.IdShortPath.Contains(".") + && v.SValue != null && v.SValue != "" + && sm.Identifier != null + && (smId == null || sme.SMId == smId) + orderby sme.Id + select new { sm.Identifier, sme.IdShortPath, v.SValue }) + .FirstOrDefault(); + + candidate.Should().NotBeNull("the imported test data should contain a nested string property"); + return new PropCandidate(candidate!.Identifier!, candidate.IdShortPath!, candidate.SValue!); + } + + [Fact] + public void Project_SameSubmodelFullPath_ReturnsStoredValue() + { + using var db = _fixture.CreateDbContext(); + var candidate = FirstDottedProperty(db); + + var rows = ProjectionOperator.Project(db, new DbProjectionRequest + { + SubmodelIdentifiers = [candidate.SubmodelIdentifier], + Paths = [new DbProjectionPath { RawPath = candidate.IdShortPath, ElementIdShortPath = candidate.IdShortPath }], + }); + + rows.Should().HaveCount(1); + var cell = rows[0].Cells[candidate.IdShortPath]; + cell.Found.Should().BeTrue(); + cell.SmeType.Should().Be("Prop"); + cell.SourceSubmodelIdentifier.Should().Be(candidate.SubmodelIdentifier); + cell.Values.Should().Contain(v => v.SValue == candidate.SValue); + } + + [Fact] + public void Project_AbsolutePathToHitSubmodel_ReturnsStoredValue() + { + using var db = _fixture.CreateDbContext(); + var candidate = FirstDottedProperty(db); + var hitSubmodel = db.SMSets.Single(sm => sm.Identifier == candidate.SubmodelIdentifier); + hitSubmodel.IdShort.Should().NotBeNullOrWhiteSpace(); + + var rawPath = "/" + hitSubmodel.IdShort + "/" + candidate.IdShortPath; + var rows = ProjectionOperator.Project(db, new DbProjectionRequest + { + SubmodelIdentifiers = [candidate.SubmodelIdentifier], + Paths = + [ + new DbProjectionPath + { + RawPath = rawPath, + TargetSubmodelIdShort = hitSubmodel.IdShort, + ElementIdShortPath = candidate.IdShortPath, + }, + ], + }); + + rows.Should().HaveCount(1); + var cell = rows[0].Cells[rawPath]; + cell.Found.Should().BeTrue(); + cell.SmeType.Should().Be("Prop"); + cell.SourceSubmodelIdentifier.Should().Be(candidate.SubmodelIdentifier); + cell.Values.Should().Contain(v => v.SValue == candidate.SValue); + } + + [Fact] + public void Project_CrossSubmodelPath_ResolvesSiblingOfSameAas() + { + using var db = _fixture.CreateDbContext(); + + // Ein Treffer-Submodel plus ein Geschwister-Submodel derselben AAS (verknüpft über die + // Shell-Referenzen in SMRefSets — in den Testdaten ist SMSets.AASId nicht gesetzt), + // dessen IdShort innerhalb der AAS eindeutig ist und das ein verschachteltes + // String-Property enthält. + var pair = (from hitRef in db.SMRefSets + join siblingRef in db.SMRefSets on hitRef.AASId equals siblingRef.AASId + where hitRef.AASId != null && hitRef.Id != siblingRef.Id + && hitRef.Identifier != null && siblingRef.Identifier != null + join sibling in db.SMSets on siblingRef.Identifier equals sibling.Identifier + where sibling.IdShort != null + select new { HitIdentifier = hitRef.Identifier, hitRef.AASId, SiblingId = sibling.Id, SiblingIdShort = sibling.IdShort }) + .ToList() + .FirstOrDefault(p => + { + var refIdentifiers = db.SMRefSets + .Where(r => r.AASId == p.AASId && r.Identifier != null) + .Select(r => r.Identifier!) + .ToList(); + return db.SMSets.Count(sm => sm.Identifier != null && refIdentifiers.Contains(sm.Identifier) && sm.IdShort == p.SiblingIdShort) == 1 + && db.SMESets.Any(sme => sme.SMId == p.SiblingId && sme.SMEType == "Prop" + && sme.IdShortPath != null && sme.IdShortPath.Contains(".") + && sme.ValueSets.Any(v => v.SValue != null && v.SValue != "")); + }); + + pair.Should().NotBeNull("the test data should contain an AAS with at least two submodels"); + var target = FirstDottedProperty(db, pair!.SiblingId); + + var rawPath = "/" + pair.SiblingIdShort + "/" + target.IdShortPath; + var rows = ProjectionOperator.Project(db, new DbProjectionRequest + { + SubmodelIdentifiers = [pair.HitIdentifier!], + Paths = + [ + new DbProjectionPath + { + RawPath = rawPath, + TargetSubmodelIdShort = pair.SiblingIdShort, + ElementIdShortPath = target.IdShortPath, + }, + ], + }); + + rows.Should().HaveCount(1); + var cell = rows[0].Cells[rawPath]; + cell.Found.Should().BeTrue(); + cell.SourceSubmodelIdentifier.Should().Be(target.SubmodelIdentifier); + cell.Values.Should().Contain(v => v.SValue == target.SValue); + } + + [Fact] + public void Project_UnknownPathOrSubmodel_YieldsNotFoundCells() + { + using var db = _fixture.CreateDbContext(); + var candidate = FirstDottedProperty(db); + + var rows = ProjectionOperator.Project(db, new DbProjectionRequest + { + SubmodelIdentifiers = [candidate.SubmodelIdentifier, "urn:does:not:exist"], + Paths = + [ + new DbProjectionPath { RawPath = "No.Such.Path", ElementIdShortPath = "No.Such.Path" }, + new DbProjectionPath + { + RawPath = "/NoSuchSubmodel/Some.Path", + TargetSubmodelIdShort = "NoSuchSubmodel", + ElementIdShortPath = "Some.Path", + }, + ], + }); + + rows.Should().HaveCount(2); + rows[0].SubmodelIdentifier.Should().Be(candidate.SubmodelIdentifier); + rows[0].Cells["No.Such.Path"].Found.Should().BeFalse(); + rows[0].Cells["/NoSuchSubmodel/Some.Path"].Found.Should().BeFalse(); + rows[1].Cells["No.Such.Path"].Found.Should().BeFalse(); + } + + [Fact] + public void Project_MultipleIdentifiers_PreservesRequestOrder() + { + using var db = _fixture.CreateDbContext(); + var identifiers = db.SMSets + .Where(sm => sm.Identifier != null) + .OrderBy(sm => sm.Id) + .Select(sm => sm.Identifier!) + .Take(3) + .ToList(); + identifiers.Should().NotBeEmpty(); + identifiers.Reverse(); + + var rows = ProjectionOperator.Project(db, new DbProjectionRequest + { + SubmodelIdentifiers = identifiers, + Paths = [new DbProjectionPath { RawPath = "A.B", ElementIdShortPath = "A.B" }], + }); + + rows.Select(r => r.SubmodelIdentifier).Should().ContainInOrder(identifiers); + } +} diff --git a/src/AasxServerDB.Tests/QueryPathJoinOrderTests.cs b/src/AasxServerDB.Tests/QueryPathJoinOrderTests.cs new file mode 100644 index 000000000..3f730ce9d --- /dev/null +++ b/src/AasxServerDB.Tests/QueryPathJoinOrderTests.cs @@ -0,0 +1,65 @@ +namespace AasxServerDB.Tests; + +using Contracts; +using Contracts.QueryResult; +using FluentAssertions; + +/// +/// End-to-end shape check for the forced path-join order: equality/range path +/// conditions must drive from SMESets (IdShort B-tree) via CROSS JOIN, so plans +/// stay deterministic whether or not sqlite_stat1 exists (hot values like +/// SValue='24' must never become the outer loop of a nested AND arm). +/// +[Collection(DatabaseFixture.Collection)] +public sealed class QueryPathJoinOrderTests +{ + private readonly DatabaseFixture _fixture; + + public QueryPathJoinOrderTests(DatabaseFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public void ThreeAndPathConditions_ForceSmeFirstJoinOrder() + { + const string expression = """ + { + "Query": { + "$select": "id", + "$condition": + { "$and": [ + { "$eq": [ { "$field": "$sme.ProductClassifications.ProductClassificationItem.ProductClassId#value" }, { "$strVal": "27-04-07-01" } ] }, + { "$or": [ { "$eq": [ { "$field": "$sme.%.nominal_value_1__output_voltage#value" }, { "$strVal": "24" } ] }, + { "$eq": [ { "$field": "$sme.%.nominal_value_1__output_voltage#value" }, { "$numVal": 24 } ] } ] }, + { "$ge": [ { "$field": "$sme.%.Power_output#value" }, { "$numVal": 960 } ] } + ] } + } + } + """; + + using var db = _fixture.CreateDbContext(); + var result = new Query(_fixture.Grammar) + .GetQueryData( + noSecurity: true, + db, + pageFrom: 0, + pageSize: int.MaxValue, + ResultType.Submodel, + expression, + includeDebugSql: true, + securitySqlConditions: null); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql ?? new List()); + + // Every path CTE starts at SMESets and forces the ValueSets probe second. + sql.Should().Contain("FROM SMESets sme\r\nCROSS JOIN ValueSets v"); + sql.Should().NotContain("LEFT JOIN ValueSets v ON"); + + // The IdShort filters survived the rewrite (they are the outer-loop entry). + sql.Should().Contain("\"sme\".\"IdShort\" = 'nominal_value_1__output_voltage'"); + sql.Should().Contain("\"sme\".\"IdShort\" = 'Power_output'"); + sql.Should().Contain("\"sme\".\"IdShort\" = 'ProductClassId'"); + } +} diff --git a/src/AasxServerDB.Tests/QueryTests.cs b/src/AasxServerDB.Tests/QueryTests.cs index fcdf87808..28049aaa2 100644 --- a/src/AasxServerDB.Tests/QueryTests.cs +++ b/src/AasxServerDB.Tests/QueryTests.cs @@ -657,6 +657,267 @@ public void UnionOrTemp_WithSecurityFilter_DistributesUserOrAndMatchesDefault(st .Should().BeGreaterThan(1, "the user $or must split into separate temp-table INSERT branches"); } + /// + /// Without security the OR-distribution fallback must still split a fully-paren-wrapped top-level + /// OR ("(A OR B)") via StripEnclosingParens — otherwise $UNION/$TEMPTABLE would degenerate to one + /// monolithic branch (full SMSets scan) instead of value-/index-driven branches. + /// + [Theory] + [InlineData("$UNION")] + [InlineData("$TEMPTABLE")] + public void UnionOrTemp_NoSecurity_DistributesTopLevelOr(string flag) + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": { "$or": [ + { "$eq": [ { "$field": "$sm#idShort" }, { "$strVal": "Nameplate" } ] }, + { "$eq": [ { "$field": "$sm#idShort" }, { "$strVal": "TechnicalData" } ] } + ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var def = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + var flagged = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery + " " + flag, includeDebugSql: true); + + def.Should().NotBeNull(); + flagged.Should().NotBeNull(); + flagged!.Ids.Should().BeEquivalentTo(def!.Ids, "OR distribution must preserve query semantics"); + + var sql = string.Join("\n", flagged.Sql); + if (flag == "$UNION") + sql.Should().Contain("UNION", "a fully-wrapped top-level OR must split even without security"); + else + System.Text.RegularExpressions.Regex.Matches(sql, "INSERT OR IGNORE INTO union_ids").Count + .Should().BeGreaterThan(1, "a fully-wrapped top-level OR must split into temp-table branches even without security"); + } + + /// + /// A non-selective ("common") $contains is realized as a correlated EXISTS (early-stop under + /// ORDER BY/LIMIT), and the FTS candidate filter is NOT injected into it (which would re-materialize + /// the huge match set). Cap is lowered so any present term counts as common in the small fixture. + /// + [Fact] + public void Contains_CommonValue_RoutedToExists_WithoutFtsPrefilter() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$or": [ { "$contains": [ { "$field": "$sme#value" }, { "$strVal": "ZVEI" } ] } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // any present term is "common" + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sme_value.SMId = sm.Id", "a common contains must be realized as a correlated EXISTS"); + sql.Should().NotContain("ValueSets_fts", "the common contains EXISTS must not get the FTS candidate filter"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + + [Fact] + public void Contains_CommonValue_WithPagedLimit_RoutedToExists_WithoutFtsPrefilter() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$or": [ { "$contains": [ { "$field": "$sme#value" }, { "$strVal": "ZVEI" } ] } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // any present term is "common" + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: 20, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sme_value.SMId = sm.Id", "a common contains must also use EXISTS for paged queries"); + sql.Should().NotContain("sm.Id IN (", "a common contains must not materialize the full value-driven match set"); + sql.Should().NotContain("ValueSets_fts", "the common contains EXISTS must not get the FTS candidate filter"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + + /// + /// A rare/absent $contains stays on the value-driven IN path with the FTS candidate filter + /// (fast for small/empty match sets). + /// + [Fact] + public void Contains_RareValue_StaysValueDrivenWithFtsPrefilter() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$or": [ { "$contains": [ { "$field": "$sme#value" }, { "$strVal": "Zzqqxyz123" } ] } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // absent term still yields 0 matches -> rare + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sm.Id IN (", "a rare/absent contains stays value-driven"); + sql.Should().Contain("ValueSets_fts", "the rare contains keeps the FTS candidate filter"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + + /// + /// A common direct $sme#value equality must be realized as a correlated EXISTS (early-stop + /// under ORDER BY/LIMIT), NOT the value-driven IN — otherwise a frequent value (e.g. a manufacturer + /// name on hundreds of thousands of elements) materializes its whole match set even for LIMIT 3. + /// + [Fact] + public void Equality_CommonValue_RoutedToExists() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$eq": [ { "$field": "$sme#value" }, { "$strVal": "2500" } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // any present value is "common" + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sme_value.SMId = sm.Id", "a common equality must be realized as a correlated EXISTS"); + sql.Should().NotContain("sm.Id IN (", "a common equality must not materialize the full value-driven match set"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + + /// + /// A rare/absent $sme#value equality stays on the value-driven IN path (small match set). + /// + [Fact] + public void Equality_RareValue_StaysValueDriven() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$eq": [ { "$field": "$sme#value" }, { "$strVal": "Zzqqxyz-absent-value" } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // absent value yields 0 matches -> rare + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sm.Id IN (", "a rare/absent equality stays value-driven"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + + /// + /// A pure OR of direct $sme#value equalities represents MCP in and stays on the + /// value-driven path, instead of being downgraded to a correlated EXISTS just because it contains OR. + /// + [Fact] + public void EqualityOr_RareValues_StaysValueDriven() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$or": [ + { "$eq": [ { "$field": "$sme#value" }, { "$strVal": "Zzqqxyz-absent-a" } ] }, + { "$eq": [ { "$field": "$sme#value" }, { "$strVal": "Zzqqxyz-absent-b" } ] } + ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // absent values yield 0 matches -> rare + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sm.Id IN (", "a pure OR of value equalities should stay value-driven"); + sql.Should().NotContain("sme_value.SMId = sm.Id", "a pure OR of value equalities must not force a correlated EXISTS"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + + /// + /// A common direct $sme#value prefix (starts-with) must also route to the correlated + /// EXISTS (early-stop), not the value-driven IN that materializes the whole prefix range. + /// + [Fact] + public void Prefix_CommonValue_RoutedToExists() + { + const string userQuery = """ + { "Query": { "$select": "id", "$condition": + { "$starts-with": [ { "$field": "$sme#value" }, { "$strVal": "250" } ] } } } + """; + + using var db = _fixture.CreateDbContext(); + var originalCap = Query.ContainsCommonProbeCap; + Query.ContainsCommonProbeCap = 1; // any present prefix is "common" + try + { + var result = new Query(_fixture.Grammar).GetQueryData( + noSecurity: true, db, pageFrom: 0, pageSize: int.MaxValue, + ResultType.Submodel, userQuery, includeDebugSql: true); + + result.Should().NotBeNull(); + var sql = string.Join("\n", result!.Sql); + sql.Should().Contain("sme_value.SMId = sm.Id", "a common prefix must be realized as a correlated EXISTS"); + sql.Should().NotContain("sm.Id IN (", "a common prefix must not materialize the full value-driven match set"); + } + finally + { + Query.ContainsCommonProbeCap = originalCap; + } + } + private const string AnonymousSubmodelsAcl = """ { "AllAccessPermissionRules": { @@ -743,11 +1004,13 @@ public void ValueInequality_WithSecurity_StaysExists_AndFilterInsideMatch() } /// - /// A standalone $sme#value is realized via the value-join (not the EXISTS path). The element - /// FILTER must be applied inside that join too, otherwise a value in a hidden element could match. + /// A standalone $sme#value now routes through the value-match builder (a bare value + /// comparison is its own existential over the SM's elements), not a value join. The element + /// FILTER must be applied inside that value-match subquery too, otherwise a value in a hidden + /// element could match. /// [Fact] - public void StandaloneValue_WithSecurity_FilterInsideValueJoin() + public void StandaloneValue_WithSecurity_FilterInsideValueMatch() { const string userQuery = """ { "Query": { "$select": "id", "$condition": @@ -762,9 +1025,12 @@ public void StandaloneValue_WithSecurity_FilterInsideValueJoin() result.Should().NotBeNull(); var sql = string.Join("\n", result!.Sql); - // The FILTER must sit inside the value join (before its ") AS value" close). - sql.Should().MatchRegex(@"JOIN SMESets sme ON sme\.Id = v\.SMEId[\s\S]*?GLOB 'General\*'[\s\S]*?\) AS value", - "the element FILTER must sit inside the value join"); + // A selective bare value '=' is value-driven (IN), not a value join. The element-level + // security FILTER must sit inside that value-match subquery, applied to the very elements + // whose value is matched. + sql.Should().Contain("sm.Id IN (", "a selective bare value '=' is value-driven (IN), not a value join"); + sql.Should().MatchRegex(@"sm\.Id IN \([\s\S]*?v\.""SValue"" = '2500'[\s\S]*?GLOB 'General\*'[\s\S]*?\)\)", + "the element FILTER must sit inside the value-match subquery"); } // ------------------------------------------------------------------------- @@ -952,4 +1218,40 @@ public void TokenClaim_MissingClaim_ResolvesToEmptyLiteral() because: "missing claims must produce an empty SQL literal, not leave the sentinel in place"); perRequest.FormulaConditions["all"].Should().NotContain(SqlConditions.ClaimSentinelPrefix); } + + // ------------------------------------------------------------------------- + // Negative number literals — $numVal: -40 (e.g. minus-degree ambient + // temperatures) must parse; requires NumberOptions.AllowSign on the grammar. + // ------------------------------------------------------------------------- + [Fact] + public void NegativeNumVal_ParsesAndExecutes() + { + const string expression = """ + { + "Query": { + "$select": "id", + "$condition": + { "$le": [ + { "$field": "$sme#value" }, + { "$numVal": -40 } + ] } + } + } + """; + + using var db = _fixture.CreateDbContext(); + var result = new Query(_fixture.Grammar) + .GetQueryData( + noSecurity: true, + db, + pageFrom: 0, + pageSize: int.MaxValue, + ResultType.Submodel, + expression, + includeDebugSql: true, + securitySqlConditions: null); + + result.Should().NotBeNull("negative number literals must be accepted by the JSON grammar"); + result!.Sql.Should().Contain(sql => sql.Contains("-40")); + } } diff --git a/src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs b/src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs new file mode 100644 index 000000000..ca1311cb8 --- /dev/null +++ b/src/AasxServerDB.Tests/SqliteTrigramIndexTests.cs @@ -0,0 +1,461 @@ +namespace AasxServerDB.Tests; + +using AasxServerDB.Entities; +using Contracts; +using Contracts.QueryResult; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; + +[Collection(DatabaseFixture.Collection)] +public sealed class SqliteTrigramIndexTests +{ + private readonly DatabaseFixture _fixture; + + public SqliteTrigramIndexTests(DatabaseFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public void Initialize_CreatesFtsTableAndMaintenanceTriggers() + { + using var db = _fixture.CreateDbContext(); + + // A second initialization must be cheap and idempotent. + SqliteTrigramIndex.Initialize(db); + + using var command = db.Database.GetDbConnection().CreateCommand(); + db.Database.OpenConnection(); + command.CommandText = """ + SELECT COUNT(*) + FROM sqlite_master + WHERE name IN ( + 'ValueSets_fts', 'ValueSets_fts_ai', 'ValueSets_fts_ad', 'ValueSets_fts_au', + 'SMESets_fts', 'SMESets_fts_ai', 'SMESets_fts_ad', 'SMESets_fts_au' + ) + """; + + Convert.ToInt32(command.ExecuteScalar()).Should().Be(8); + } + + [Fact] + public void ApplySqliteTrigramIndex_AddsCandidateFilterForContains() + { + const string input = "SELECT * FROM ValueSets AS v WHERE v.\"SValue\" GLOB '*Netzteil*'"; + + var result = Query.ApplySqliteTrigramIndex(input); + + result.Should().Contain("v.\"Id\" IN"); + result.Should().Contain("FROM \"ValueSets_fts\""); + result.Should().Contain("v.\"SValue\" GLOB '*Netzteil*'"); + } + + [Fact] + public void ApplySqliteTrigramIndex_AddsIdShortCandidateFilterForContains() + { + const string input = + "SELECT * FROM SMESets AS sme WHERE \"sme\".\"IdShort\" GLOB '*output_power*'"; + + var result = Query.ApplySqliteTrigramIndex(input); + + result.Should().Contain("\"sme\".\"Id\" IN"); + result.Should().Contain("FROM \"SMESets_fts\""); + result.Should().Contain("\"IdShort\" GLOB '*output_power*'"); + } + + [Fact] + public void ApplySqliteTrigramIndex_SkipsPatternsInSkipSet() + { + const string input = "SELECT * FROM ValueSets AS v WHERE v.\"SValue\" GLOB '*Netzteil*'"; + var skip = new HashSet(StringComparer.Ordinal) { "'*Netzteil*'" }; + + var result = Query.ApplySqliteTrigramIndex(input, skip); + + result.Should().NotContain("ValueSets_fts", "patterns in the skip set must not get an FTS candidate filter"); + result.Should().Contain("v.\"SValue\" GLOB '*Netzteil*'", "the original GLOB stays unchanged"); + } + + [Theory] + [InlineData("v.\"SValue\" GLOB '*Phoenix*'", true)] // single indexable contains + [InlineData("(v.\"SValue\" GLOB '*Phoenix*')", true)] // wrapped in parens + [InlineData("v.\"SValue\" GLOB '*Phoenix*' AND v.\"NValue\" = 5", false)] // compound (extra selectivity) + [InlineData("v.\"SValue\" GLOB 'Phoenix*'", false)] // prefix, not a trigram contains + [InlineData("v.\"SValue\" = '2500'", false)] // equality, no contains + [InlineData("v.\"SValue\" GLOB '*ab*'", false)] // fewer than 3 literal chars + public void TryGetPureSValueContainsPattern_DetectsSingleIndexableContains(string predicate, bool expected) + { + Query.TryGetPureSValueContainsPattern(predicate, out _).Should().Be(expected); + } + + [Fact] + public void BuildRawSql_UsesUncorrelatedCandidateSetForIndexedValueContains() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "$$exists0$$"; + conditions.ExistsConditions.Add(new ExistsCondition + { + Placeholder = "exists0", + PredicateSql = + "((\"v\".\"SValue\" GLOB '*power supply*') AND " + + "(\"v\".\"SValue\" GLOB '*24*'))" + }); + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: false, + ResultType.Submodel, + pageFrom: 0, + pageSize: 50)!; + var sqliteSql = Query.ApplySqliteTrigramIndex(rawSql); + + sqliteSql.Should().Contain("sm.Id IN ("); + sqliteSql.Should().Contain("FROM ValueSets v"); + sqliteSql.Should().Contain("CROSS JOIN SMESets sme_value"); + sqliteSql.Should().Contain("FROM \"ValueSets_fts\""); + sqliteSql.Should().NotContain("WHERE sme_value.SMId = sm.Id"); + } + + [Fact] + public void ApplySqliteIndexedPathJoinOrder_DrivesPathSearchFromValueCandidates() + { + const string input = """ + SELECT sme.SMId + FROM SMESets sme + LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v."SValue" GLOB '*power*') + WHERE (v."SValue" GLOB '*power*') + AND "sme"."IdShort" = 'ManufacturerProductDesignation' + """; + + var reordered = Query.ApplySqliteIndexedPathJoinOrder(input); + var indexed = Query.ApplySqliteTrigramIndex(reordered); + + indexed.Should().Contain("FROM ValueSets v"); + indexed.Should().Contain("CROSS JOIN SMESets sme"); + indexed.Should().Contain("WHERE sme.Id = v.SMEId"); + (indexed.Split("v.\"SValue\" GLOB '*power*'", StringSplitOptions.None).Length - 1) + .Should().Be(1); + indexed.Should().Contain("FROM \"ValueSets_fts\""); + } + + [Fact] + public void ApplySqliteIndexedPathJoinOrder_DrivesPrefixSearchFromValueIndex() + { + const string input = """ + SELECT sme.SMId + FROM SMESets sme + LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v."SValue" GLOB 'TRIO-*') + WHERE (v."SValue" GLOB 'TRIO-*') + AND "sme"."IdShort" = 'Product_type' + """; + + var reordered = Query.ApplySqliteIndexedPathJoinOrder(input); + var indexed = Query.ApplySqliteTrigramIndex(reordered); + + indexed.Should().Contain("FROM ValueSets v"); + indexed.Should().Contain("CROSS JOIN SMESets sme"); + indexed.Should().Contain("WHERE sme.Id = v.SMEId"); + (indexed.Split("v.\"SValue\" GLOB 'TRIO-*'", StringSplitOptions.None).Length - 1) + .Should().Be(1); + indexed.Should().NotContain("ValueSets_fts"); + } + + [Fact] + public void ApplySqliteTrigramIndex_AcceleratesSuffixSearch() + { + const string input = """ + SELECT sme.SMId + FROM SMESets sme + LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v."SValue" GLOB '*supply') + WHERE (v."SValue" GLOB '*supply') + """; + + var reordered = Query.ApplySqliteIndexedPathJoinOrder(input); + var indexed = Query.ApplySqliteTrigramIndex(reordered); + + indexed.Should().Contain("FROM ValueSets v"); + indexed.Should().Contain("CROSS JOIN SMESets sme"); + indexed.Should().Contain("FROM \"ValueSets_fts\""); + indexed.Should().Contain("\"SValue\" GLOB '*supply'"); + } + + [Fact] + public void BuildRawSql_SinglePositivePathDoesNotScanOuterSubmodels() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "$$path0$$"; + conditions.Paths.Add(new PathJoin + { + Placeholder = "path0", + IdShortPath = "Product_type", + SubquerySql = "FROM SMESets sme\r\n" + + "LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v.\"SValue\" GLOB '*24DC/40*')\r\n" + + "WHERE (v.\"SValue\" GLOB '*24DC/40*')\r\n" + + "AND \"sme\".\"IdShort\" = 'Product_type'" + }); + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: false, + ResultType.Submodel, + pageFrom: 0, + pageSize: 200)!; + var indexed = Query.ApplySqliteTrigramIndex( + Query.ApplySqliteIndexedPathJoinOrder(rawSql)); + + indexed.Should().Contain("SELECT p1.SMId AS Id"); + indexed.Should().Contain("FROM ValueSets v"); + indexed.Should().Contain("FROM \"ValueSets_fts\""); + indexed.Should().NotContain("FROM SMSets"); + indexed.Should().NotContain("LEFT JOIN(\r\nSELECT sme.SMId"); + } + + [Fact] + public void BuildRawSql_PositivePathConjunctionIntersectsPathResultsDirectly() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "($$path0$$ AND $$path1$$)"; + conditions.Paths.Add(new PathJoin + { + Placeholder = "path0", + IdShortPath = "Product_type", + SubquerySql = "FROM SMESets sme\r\nWHERE \"sme\".\"IdShort\" = 'Product_type'" + }); + conditions.Paths.Add(new PathJoin + { + Placeholder = "path1", + IdShortPath = "Power_output", + SubquerySql = "FROM SMESets sme\r\nWHERE \"sme\".\"IdShort\" = 'Power_output'" + }); + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: false, + ResultType.Submodel, + pageFrom: 0, + pageSize: 200)!; + + rawSql.Should().Contain("p1 AS ("); + rawSql.Should().Contain("p2 AS ("); + rawSql.Should().Contain("INNER JOIN p2 ON p2.SMId = p1.SMId"); + rawSql.Should().NotContain("FROM SMSets"); + } + + [Fact] + public void BuildRawSql_PositiveAndOrPathsUseJoinAndUnionWithoutSubmodelScan() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "($$path0$$ AND ($$path1$$ OR $$path2$$))"; + for (var i = 0; i < 3; i++) + { + conditions.Paths.Add(new PathJoin + { + Placeholder = $"path{i}", + IdShortPath = $"Field{i}", + SubquerySql = $"FROM SMESets sme\r\nWHERE \"sme\".\"IdShort\" = 'Field{i}'" + }); + } + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: false, + ResultType.Submodel, + pageFrom: 0, + pageSize: 50)!; + + rawSql.Should().Contain("INNER JOIN p2 ON p2.SMId = p1.SMId"); + rawSql.Should().Contain("UNION"); + rawSql.Should().Contain("INNER JOIN p3 ON p3.SMId = p1.SMId"); + rawSql.Should().NotContain("FROM SMSets"); + rawSql.Should().NotContain("LEFT JOIN("); + } + + [Fact] + public void BuildRawSql_ShellPathSearchStartsAtMatchingSubmodels() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "$$path0$$"; + conditions.Paths.Add(new PathJoin + { + Placeholder = "path0", + IdShortPath = "Manufacturer_product_designation", + SubquerySql = "FROM SMESets sme\r\n" + + "LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v.\"SValue\" GLOB '*Power supply*')\r\n" + + "WHERE (v.\"SValue\" GLOB '*Power supply*')" + }); + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: true, + ResultType.AssetAdministrationShell, + pageFrom: 0, + pageSize: 500)!; + var indexed = Query.ApplySqliteTrigramIndex( + Query.ApplySqliteIndexedPathJoinOrder(rawSql)); + + indexed.Should().Contain("FROM matching_sm match"); + indexed.Should().Contain("INNER JOIN SMSets sm ON sm.Id = match.Id"); + indexed.Should().Contain("INNER JOIN SMRefSets sx ON sx.Identifier = sm.Identifier"); + indexed.Should().Contain("FROM \"ValueSets_fts\""); + indexed.Should().NotContain("FROM (\r\n SELECT Id, Identifier\r\n FROM AASSets"); + indexed.Should().NotContain("SCAN SMSets"); + } + + [Fact] + public void BuildRawSql_ShellValueSearchStartsAtValueCandidates() + { + var conditions = new SqlConditions(); + conditions.FormulaConditions["all"] = "\"value\".\"SValue\" GLOB '*Netzteil*'"; + + var rawSql = Query.BuildRawSqlFromSqlConditions( + conditions, + isWithAASTable: true, + ResultType.AssetAdministrationShell, + pageFrom: 0, + pageSize: 10)!; + var indexed = Query.ApplySqliteTrigramIndex(rawSql); + + indexed.Should().Contain("FROM ValueSets AS value"); + indexed.Should().Contain("INNER JOIN SMESets AS sme ON sme.Id = value.SMEId"); + indexed.Should().Contain("INNER JOIN SMRefSets AS sx ON sx.Identifier = sm.Identifier"); + indexed.Should().Contain("FROM \"ValueSets_fts\""); + indexed.Should().NotContain("FROM (\r\n SELECT Id, Identifier\r\n FROM AASSets"); + } + + [Fact] + public void MaintenanceTriggers_KeepSubstringIndexCurrent() + { + using var db = _fixture.CreateDbContext(); + using var transaction = db.Database.BeginTransaction(); + var smeId = db.SMESets.Select(sme => sme.Id).First(); + var value = new ValueSet + { + SMEId = smeId, + SValue = "Labornetzteil-4711", + Annotation = "test" + }; + + db.ValueSets.Add(value); + db.SaveChanges(); + FindIndexedRowId("*netzteil*").Should().Be(value.Id); + + value.SValue = "Trenntransformator-4711"; + db.SaveChanges(); + FindIndexedRowId("*netzteil*").Should().BeNull(); + FindIndexedRowId("*transformator*").Should().Be(value.Id); + + db.ValueSets.Remove(value); + db.SaveChanges(); + FindIndexedRowId("*transformator*").Should().BeNull(); + + transaction.Rollback(); + + int? FindIndexedRowId(string pattern) + { + using var command = db.Database.GetDbConnection().CreateCommand(); + command.Transaction = transaction.GetDbTransaction(); + command.CommandText = """ + SELECT rowid + FROM "ValueSets_fts" + WHERE rowid = $id AND "SValue" GLOB '__PATTERN__' + """.Replace("__PATTERN__", pattern.Replace("'", "''", StringComparison.Ordinal)); + var idParameter = command.CreateParameter(); + idParameter.ParameterName = "$id"; + idParameter.Value = value.Id; + command.Parameters.Add(idParameter); + var result = command.ExecuteScalar(); + return result is null ? null : Convert.ToInt32(result); + } + } + + [Fact] + public void IdShortMaintenanceTrigger_KeepsSubstringIndexCurrent() + { + using var db = _fixture.CreateDbContext(); + using var transaction = db.Database.BeginTransaction(); + var sme = db.SMESets.First(); + var originalIdShort = sme.IdShort; + + sme.IdShort = "CodexUniqueOutputPower"; + db.SaveChanges(); + FindIndexedRowId("*UniqueOutput*").Should().Be(sme.Id); + + sme.IdShort = "CodexUniqueVoltage"; + db.SaveChanges(); + FindIndexedRowId("*UniqueOutput*").Should().BeNull(); + FindIndexedRowId("*UniqueVoltage*").Should().Be(sme.Id); + + transaction.Rollback(); + sme.IdShort = originalIdShort; + + int? FindIndexedRowId(string pattern) + { + using var command = db.Database.GetDbConnection().CreateCommand(); + command.Transaction = transaction.GetDbTransaction(); + command.CommandText = """ + SELECT rowid + FROM "SMESets_fts" + WHERE rowid = $id AND "IdShort" GLOB '__PATTERN__' + """.Replace("__PATTERN__", pattern.Replace("'", "''", StringComparison.Ordinal)); + var idParameter = command.CreateParameter(); + idParameter.ParameterName = "$id"; + idParameter.Value = sme.Id; + command.Parameters.Add(idParameter); + var result = command.ExecuteScalar(); + return result is null ? null : Convert.ToInt32(result); + } + } + + [Theory] + [InlineData("'*ab*'")] + [InlineData("'Netzteil*'")] + [InlineData("'*Netz?eil*'")] + public void ApplySqliteTrigramIndex_LeavesUnsupportedPatternsUnchanged(string pattern) + { + var input = $"SELECT * FROM ValueSets AS v WHERE v.\"SValue\" GLOB {pattern}"; + + Query.ApplySqliteTrigramIndex(input).Should().Be(input); + } + + [Fact] + public void ApplySqliteIndexedPathJoinOrder_DrivesEqualityPathSearchFromSmeIdShort() + { + // Equality/range value predicates must be driven from the SMESets side (IdShort B-tree): + // hot values like SValue='24' would otherwise drive the loop when sqlite_stat1 averages + // underestimate them (measured 0.3 s -> 4.9 s after ANALYZE on the large corpus). + const string input = """ + SELECT sme.SMId + FROM SMESets sme + LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v."SValue" = '24') + WHERE (v."SValue" = '24') + AND "sme"."IdShort" = 'nominal_value_1__output_voltage' + AND 1=1 + """; + + var reordered = Query.ApplySqliteIndexedPathJoinOrder(input); + + reordered.Should().Contain("FROM SMESets sme"); + reordered.Should().Contain("CROSS JOIN ValueSets v"); + reordered.Should().Contain("WHERE v.SMEId = sme.Id"); + reordered.Should().Contain("AND \"sme\".\"IdShort\" = 'nominal_value_1__output_voltage'"); + reordered.Should().Contain("AND 1=1"); + (reordered.Split("v.\"SValue\" = '24'", StringSplitOptions.None).Length - 1) + .Should().Be(1, "the duplicated JOIN/WHERE predicate must collapse into one"); + } + + [Fact] + public void ApplySqliteIndexedPathJoinOrder_LeavesEqualityWithoutIdShortFilterUnchanged() + { + // Without an IdShort equality there is no selective SMESets entry point — + // forcing sme-first would scan the full SMESets table. + const string input = """ + SELECT sme.SMId + FROM SMESets sme + LEFT JOIN ValueSets v ON v.SMEId = sme.Id AND (v."SValue" = '24') + WHERE (v."SValue" = '24') + AND "sme"."IdShortPath" GLOB 'Documents[[]*[]].Title' + """; + + Query.ApplySqliteIndexedPathJoinOrder(input).Should().Be(input); + } +} diff --git a/src/AasxServerDB/CrudOperator.cs b/src/AasxServerDB/CrudOperator.cs index 886a363bb..27860d592 100644 --- a/src/AasxServerDB/CrudOperator.cs +++ b/src/AasxServerDB/CrudOperator.cs @@ -33,7 +33,11 @@ namespace AasxServerDB internal static class ReadDiag { - public static bool Enabled = true; + // Master switch for the VERBOSE query/read diagnostics (per-phase ReadSubmodel + // timings, generated SQL, query plan, SearchSMs/combinedCondition banners). + // Off by default; re-enable by setting AAS_QUERY_LOG=1 (AAS_QUERY_DIAG=1 is + // kept as a backwards-compatible alias). + public static bool Enabled => DiagnosticsLog.QueryEnabled; public static long ReadSubmodelTicks; public static int ReadSubmodelCount; @@ -65,14 +69,25 @@ public static void Reset() public static void Print(string label) { if (!Enabled) return; - Console.WriteLine($"[ReadDiag] {label}"); - Console.WriteLine($"[ReadDiag] ReadSubmodel : {ReadSubmodelCount,5} x, total {Ms(ReadSubmodelTicks),8:F1} ms, avg {(ReadSubmodelCount > 0 ? Ms(ReadSubmodelTicks) / ReadSubmodelCount : 0),6:F2} ms"); - Console.WriteLine($"[ReadDiag] IsSubmodelAllowed... : {AllowCheckCount,5} x, total {Ms(AllowCheckTicks),8:F1} ms, avg {(AllowCheckCount > 0 ? Ms(AllowCheckTicks) / AllowCheckCount : 0),6:F2} ms"); - Console.WriteLine($"[ReadDiag] ApplySmeSqlFilter... : {ApplyFilterCount,5} x, total {Ms(ApplyFilterTicks),8:F1} ms, avg {(ApplyFilterCount > 0 ? Ms(ApplyFilterTicks) / ApplyFilterCount : 0),6:F2} ms"); - Console.WriteLine($"[ReadDiag] GetSmeMerged : {GetSmeMergedCount,5} x, total {Ms(GetSmeMergedTicks),8:F1} ms, avg {(GetSmeMergedCount > 0 ? Ms(GetSmeMergedTicks) / GetSmeMergedCount : 0),6:F2} ms"); - Console.WriteLine($"[ReadDiag] join ValueSets : total {Ms(JoinSValueTicks),8:F1} ms"); - Console.WriteLine($"[ReadDiag] join OValueSets : total {Ms(JoinOValueTicks),8:F1} ms"); - Console.WriteLine($"[ReadDiag] no-value residual : total {Ms(NoValueTicks),8:F1} ms"); + Write($"[ReadDiag] {label}"); + Write($"[ReadDiag] ReadSubmodel : {ReadSubmodelCount,5} x, total {Ms(ReadSubmodelTicks),8:F1} ms, avg {(ReadSubmodelCount > 0 ? Ms(ReadSubmodelTicks) / ReadSubmodelCount : 0),6:F2} ms"); + Write($"[ReadDiag] IsSubmodelAllowed... : {AllowCheckCount,5} x, total {Ms(AllowCheckTicks),8:F1} ms, avg {(AllowCheckCount > 0 ? Ms(AllowCheckTicks) / AllowCheckCount : 0),6:F2} ms"); + Write($"[ReadDiag] ApplySmeSqlFilter... : {ApplyFilterCount,5} x, total {Ms(ApplyFilterTicks),8:F1} ms, avg {(ApplyFilterCount > 0 ? Ms(ApplyFilterTicks) / ApplyFilterCount : 0),6:F2} ms"); + Write($"[ReadDiag] GetSmeMerged : {GetSmeMergedCount,5} x, total {Ms(GetSmeMergedTicks),8:F1} ms, avg {(GetSmeMergedCount > 0 ? Ms(GetSmeMergedTicks) / GetSmeMergedCount : 0),6:F2} ms"); + Write($"[ReadDiag] join ValueSets : total {Ms(JoinSValueTicks),8:F1} ms"); + Write($"[ReadDiag] join OValueSets : total {Ms(JoinOValueTicks),8:F1} ms"); + Write($"[ReadDiag] no-value residual : total {Ms(NoValueTicks),8:F1} ms"); + } + + public static void Write(string message) => DiagnosticsLog.WriteQuery(message, IsStartMessage(message)); + + private static bool IsStartMessage(string message) + { + var line = message.TrimStart('\r', '\n'); + return line.StartsWith("SearchSM", StringComparison.Ordinal) || + (line.StartsWith("[ReadDiag] ReadSubmodel ", StringComparison.Ordinal) && + (line.EndsWith(": load SME rows and values...", StringComparison.Ordinal) || + line.EndsWith(": build SME tree...", StringComparison.Ordinal))); } } @@ -690,6 +705,11 @@ private static List GetSmeMerged(AasContext db, IQueryable? q if (querySME == null) return null; + // Materialize the SME rows once. The former no-value query translated + // all IDs which already had a value into a potentially huge SQL NOT IN + // expression. That becomes extremely expensive for large submodels. + var allSmes = querySME.ToList(); + IQueryable valueSets = db.ValueSets; var valueSql = sqlConditions?.FormulaConditions.GetValueOrDefault("value", ""); if (!string.IsNullOrWhiteSpace(valueSql)) @@ -721,8 +741,8 @@ private static List GetSmeMerged(AasContext db, IQueryable? q result.AddRange(joinOValue); var swNoVal = Stopwatch.StartNew(); - var smeIdList = result.Select(sme => sme.smeSet.Id).ToList(); - var noValue = querySME.Where(sme => !smeIdList.Contains(sme.Id)) + var smeIdsWithValue = result.Select(sme => sme.smeSet.Id).ToHashSet(); + var noValue = allSmes.Where(sme => !smeIdsWithValue.Contains(sme.Id)) .Select(sme => new SmeMerged {smSet = smSet, smeSet = sme, valueSet = null, oValueSet = null }) .ToList(); result.AddRange(noValue); @@ -769,7 +789,7 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, if (result == null) { if (ReadDiag.Enabled) - Console.WriteLine($"[ReadDiag] ReadPagedSubmodels: SearchSMs returned null after {swSearch.ElapsedMilliseconds} ms"); + ReadDiag.Write($"[ReadDiag] ReadPagedSubmodels: SearchSMs returned null after {swSearch.ElapsedMilliseconds} ms"); return output; } @@ -795,10 +815,10 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, swPage.Stop(); if (ReadDiag.Enabled) { - Console.WriteLine($"[ReadDiag] ReadPagedSubmodels page: pageSize={paginationParameters.Limit}, cursor={paginationParameters.Cursor}, returned={output.Count}, total={swPage.ElapsedMilliseconds} ms"); - Console.WriteLine($"[ReadDiag] prep mergedSqlConditions : {swPrep.ElapsedMilliseconds,5} ms"); - Console.WriteLine($"[ReadDiag] SearchSMs : {swSearch.ElapsedMilliseconds,5} ms (returned {result.Count} ids)"); - Console.WriteLine($"[ReadDiag] db.SMSets.Where(...).ToList : {swMaterialize.ElapsedMilliseconds,5} ms (loaded {smDBList.Count} rows)"); + ReadDiag.Write($"[ReadDiag] ReadPagedSubmodels page: pageSize={paginationParameters.Limit}, cursor={paginationParameters.Cursor}, returned={output.Count}, total={swPage.ElapsedMilliseconds} ms"); + ReadDiag.Write($"[ReadDiag] prep mergedSqlConditions : {swPrep.ElapsedMilliseconds,5} ms"); + ReadDiag.Write($"[ReadDiag] SearchSMs : {swSearch.ElapsedMilliseconds,5} ms (returned {result.Count} ids)"); + ReadDiag.Write($"[ReadDiag] db.SMSets.Where(...).ToList : {swMaterialize.ElapsedMilliseconds,5} ms (loaded {smDBList.Count} rows)"); ReadDiag.Print(" ReadSubmodel phases:"); } @@ -810,6 +830,7 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, bool skipAllowCheck = false) { var swRead = Stopwatch.StartNew(); + var mergedRowCount = 0; if (!submodelIdentifier.IsNullOrEmpty()) { @@ -877,16 +898,20 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, } var swMerged = Stopwatch.StartNew(); + if (ReadDiag.Enabled) + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: load SME rows and values..."); var smeMerged = GetSmeMerged( db, filteredSmeQuery, smDB, securitySqlConditions); swMerged.Stop(); + mergedRowCount = smeMerged?.Count ?? 0; if (ReadDiag.Enabled) { ReadDiag.GetSmeMergedTicks += swMerged.ElapsedTicks; ReadDiag.GetSmeMergedCount++; + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: loaded {mergedRowCount} merged rows in {swMerged.ElapsedMilliseconds} ms"); } if (smeMerged == null) @@ -896,7 +921,13 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, if (smeMerged.Count != 0) { + var swTree = Stopwatch.StartNew(); + if (ReadDiag.Enabled) + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: build SME tree..."); LoadSME(submodel, null, null, null, smeMerged); + swTree.Stop(); + if (ReadDiag.Enabled) + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: SME tree built in {swTree.ElapsedMilliseconds} ms"); } } @@ -904,11 +935,16 @@ public static List ReadPagedSubmodels(AasContext db, Query? querySM, submodel.TimeStamp = smDB.TimeStamp; submodel.TimeStampTree = smDB.TimeStampTree; submodel.TimeStampDelete = smDB.TimeStampDelete; + var swParents = Stopwatch.StartNew(); submodel.SetAllParents(); + swParents.Stop(); + if (ReadDiag.Enabled) + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: parents set in {swParents.ElapsedMilliseconds} ms"); swRead.Stop(); if (ReadDiag.Enabled) { + ReadDiag.Write($"[ReadDiag] ReadSubmodel {smDB.Identifier}: {mergedRowCount} rows in {swRead.ElapsedMilliseconds} ms"); ReadDiag.ReadSubmodelTicks += swRead.ElapsedTicks; ReadDiag.ReadSubmodelCount++; } @@ -1166,6 +1202,11 @@ internal static bool DeleteSubmodel(AasContext db, string submodelIdentifier) var smeFoundDB = db.SMESets.Where(sme => sme.SMId == smDBId && sme.IdShortPath == idShortPath); var smeFound = smeFoundDB.ToList(); + if (smeFound.Count != 1) + { + return null; + } + //for (int i = 1; i < idShortPathElements.Count; i++) //{ // idShort = idShortPathElements[i]; @@ -1179,7 +1220,15 @@ internal static bool DeleteSubmodel(AasContext db, string submodelIdentifier) // parentId = smeFound[0].Id; //} - var smeFoundTree = CrudOperator.GetTree(db, smDB[0], smeFound); + // Scalar data elements cannot contain child elements. Avoid walking a + // potentially huge or malformed descendant graph just to project one + // Property/MultiLanguageProperty value. + var containerTypes = new HashSet(StringComparer.Ordinal) + { "RelA", "SML", "SMC", "Ent", "Opr" }; + var normalizedType = smeFound[0].SMEType?.Split(VisitorAASX.OPERATION_SPLIT).LastOrDefault(); + var smeFoundTree = containerTypes.Contains(normalizedType ?? string.Empty) + ? CrudOperator.GetTree(db, smDB[0], smeFound) + : smeFound; var smeTreeIds = smeFoundTree?.Select(sme => sme.Id).ToList() ?? []; var smeTreeQuery = db.SMESets.Where(sme => smeTreeIds.Contains(sme.Id)); smeTreeQuery = ApplySmeSqlFilterConditions(db, smeTreeQuery, smDB[0], securitySqlConditions); @@ -1379,20 +1428,59 @@ internal static void LoadSME(Submodel submodel, ISubmodelElement? sme, SMESet? s var smeSets = SMEList; if (tree != null) { - smeSets = tree.Select(t => t.smeSet).Distinct().ToList(); - } - // smeSets = smeSets.Where(s => s.ParentSMEId == (smeSet != null ? smeSet.Id : null)).OrderBy(s => s.IdShort).ToList(); - smeSets = smeSets.Where(s => s.ParentSMEId == (smeSet != null ? smeSet.Id : null)).OrderBy(s => s.TimeStampTree).ToList(); + smeSets = tree.Select(t => t.smeSet).GroupBy(s => s.Id).Select(g => g.First()).ToList(); + } + + // Build the parent/child lookup once. Filtering the entire SME list + // again at every recursion level made large submodels O(n²). + var childrenByParent = smeSets + .GroupBy(s => s.ParentSMEId ?? 0) + .ToDictionary(g => g.Key, g => g.OrderBy(s => s.TimeStampTree).ToList()); + var valuesBySme = tree? + .Where(item => item.valueSet is not null) + .GroupBy(item => item.valueSet.SMEId) + .ToDictionary(g => g.Key, g => g.Select(item => item.valueSet).ToList()) + ?? new Dictionary>(); + var objectValuesBySme = tree? + .Where(item => item.oValueSet is not null) + .GroupBy(item => item.oValueSet.SMEId) + .ToDictionary( + g => g.Key, + g => g.ToDictionary(item => item.oValueSet.Attribute, item => item.oValueSet.Value)) + ?? new Dictionary>(); + var visited = new HashSet(); + LoadSMERecursive( + submodel, sme, smeSet, tree, childrenByParent, + valuesBySme, objectValuesBySme, visited); + } + + private static void LoadSMERecursive( + Submodel submodel, + ISubmodelElement? sme, + SMESet? smeSet, + List? tree, + IReadOnlyDictionary> childrenByParent, + IReadOnlyDictionary> valuesBySme, + IReadOnlyDictionary> objectValuesBySme, + HashSet visited) + { + var parentId = smeSet?.Id ?? 0; + if (!childrenByParent.TryGetValue(parentId, out var smeSets)) + return; foreach (var smel in smeSets) { + // Invalid cyclic or duplicate parent relations must not recurse forever. + if (!visited.Add(smel.Id)) + continue; + // prefix of operation var split = !smel.SMEType.IsNullOrEmpty() ? smel.SMEType.Split(VisitorAASX.OPERATION_SPLIT) : [string.Empty]; var oprPrefix = split.Length == 2 ? split[0] : string.Empty; smel.SMEType = split.Length == 2 ? split[1] : split[0]; // create SME from database - var nextSME = CreateSME(smel, tree); + var nextSME = CreateSME(smel, tree, valuesBySme, objectValuesBySme); // add sme to sm or sme if (sme == null) @@ -1436,7 +1524,9 @@ internal static void LoadSME(Submodel submodel, ISubmodelElement? sme, SMESet? s case "SMC": case "Ent": case "Opr": - LoadSME(submodel, nextSME, smel, SMEList, tree); + LoadSMERecursive( + submodel, nextSME, smel, tree, childrenByParent, + valuesBySme, objectValuesBySme, visited); break; } } @@ -1785,7 +1875,11 @@ public static Dictionary GetOValue(SMESet smeSet, List tree = null) + private static ISubmodelElement? CreateSME( + SMESet smeSet, + List tree = null, + IReadOnlyDictionary>? valuesBySme = null, + IReadOnlyDictionary>? objectValuesBySme = null) { ISubmodelElement? sme = null; @@ -1798,8 +1892,12 @@ public static Dictionary GetOValue(SMESet smeSet, List()) + : GetOValue(smeSet, tree); } switch (smeSet.SMEType) diff --git a/src/AasxServerDB/EntityFrameworkPersistenceService.cs b/src/AasxServerDB/EntityFrameworkPersistenceService.cs index 603bb8106..583f9c4ec 100644 --- a/src/AasxServerDB/EntityFrameworkPersistenceService.cs +++ b/src/AasxServerDB/EntityFrameworkPersistenceService.cs @@ -78,7 +78,7 @@ private static void LogSqliteVersion(string connectionString) } } - public void InitDB(bool reloadDB, string dataPath) + public void InitDB(bool reloadDB, string dataPath, bool deferSearchIndexes = false) { AasContext.DataPath = dataPath; @@ -165,6 +165,11 @@ public void InitDB(bool reloadDB, string dataPath) sqliteDb.Database.EnsureCreated(); + // During a bulk import, build the FTS indexes once afterwards. + // Creating their triggers here would index every imported row separately. + if (!deferSearchIndexes) + SqliteTrigramIndex.Initialize(sqliteDb); + //sqliteDb.Database.Migrate(); } catch (Exception ex) @@ -179,6 +184,15 @@ public void InitDB(bool reloadDB, string dataPath) } } + public void InitializeSearchIndexes() + { + if (AasContext.IsPostgres) + return; + + using var sqliteDb = new SqliteAasContext(); + SqliteTrigramIndex.Initialize(sqliteDb); + } + public void ImportAASXIntoDB(string filePath, bool createFilesOnly) { VisitorAASX.ImportAASXIntoDB(filePath, createFilesOnly); @@ -220,6 +234,8 @@ public async Task DoDbOperation(DbRequest dbRequest) case DbRequestOp.QuerySearchSMs: case DbRequestOp.QuerySearchSMEs: case DbRequestOp.QueryCountSMs: + case DbRequestOp.QueryProjectSMs: + case DbRequestOp.ReadSubmodelTemplates: isAllowed = InitSecurity(securityConfig, out accessRules, out securitySqlConditions, "/query"); if (!isAllowed) @@ -729,13 +745,18 @@ public async Task DoDbOperation(DbRequest dbRequest) // queryRequest.Identifier, queryRequest.Diff, queryRequest.PageFrom, queryRequest.PageSize, queryRequest.Expression); // result.QueryResult = qresult; // break; - //case DbRequestOp.QueryCountSMs: - // queryRequest = dbRequest.Context.Params.QueryRequest; - // query = new Query(_grammar); - // var count = query.CountSMs(securityConfig, db, queryRequest.SemanticId, queryRequest.Identifier, queryRequest.Diff, - // queryRequest.PageFrom, queryRequest.PageSize, queryRequest.Expression); - // result.Count = count; - // break; + case DbRequestOp.QueryCountSMs: + var countRequest = dbRequest.Context.Params.QueryRequest; + var countQuery = new Query(_grammar); + result.Count = countQuery.GetQueryDataCount( + securityConfig.NoSecurity, + db, + countRequest.PageFrom, + countRequest.PageSize, + countRequest.ResultType, + countRequest.Expression, + securitySqlConditions); + break; //case DbRequestOp.QuerySearchSMEs: // queryRequest = dbRequest.Context.Params.QueryRequest; // query = new Query(_grammar); @@ -751,6 +772,37 @@ public async Task DoDbOperation(DbRequest dbRequest) // queryRequest.Contains, queryRequest.Equal, queryRequest.Lower, queryRequest.Upper, queryRequest.PageFrom, queryRequest.PageSize, queryRequest.Expression); // result.Count = count; // break; + case DbRequestOp.QueryProjectSMs: + // The batch projection reads SMSets/SMESets/ValueSets directly and does NOT + // apply the object-level security filters; it is therefore only available + // with security disabled. Secured callers keep using the object read path + // (ReadSubmodelById/ReadSubmodelElementByPath). + if (securityConfig is not { NoSecurity: true }) + { + throw new NotAllowed("NOT ALLOWED: QueryProjectSMs is only available with security disabled."); + } + + result.ProjectionRows = ProjectionOperator.Project(db, dbRequest.Context.Params.ProjectionRequest); + break; + case DbRequestOp.ReadSubmodelTemplates: + // Deterministic template inventory: GROUP BY over SMSets only (2 columns, + // ~1 row per submodel — NOT the huge SMESets). Reads raw table rows without + // object-level security filters, therefore restricted to noSecurity mode. + if (securityConfig is not { NoSecurity: true }) + { + throw new NotAllowed("NOT ALLOWED: ReadSubmodelTemplates is only available with security disabled."); + } + + result.SubmodelTemplates = db.SMSets + .GroupBy(sm => new { sm.IdShort, sm.SemanticId }) + .Select(g => new DbSubmodelTemplateRow + { + IdShort = g.Key.IdShort, + SemanticId = g.Key.SemanticId, + Count = g.Count(), + }) + .ToList(); + break; case DbRequestOp.QueryGetSMs: var queryRequest = dbRequest.Context.Params.QueryRequest; var query = new Query(_grammar); diff --git a/src/AasxServerDB/ProjectionOperator.cs b/src/AasxServerDB/ProjectionOperator.cs new file mode 100644 index 000000000..0b9d01da9 --- /dev/null +++ b/src/AasxServerDB/ProjectionOperator.cs @@ -0,0 +1,310 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace AasxServerDB; + +using System; +using System.Collections.Generic; +using System.Linq; +using Contracts.DbRequests; + +/// +/// Batch projection of explicit full idShortPaths for a set of result submodels. +/// Reads the values directly from SMSets/SMRefSets/SMESets/ValueSets with a fixed +/// number of set-based queries — no per-hit submodel materialization. +/// Cross-submodel paths are resolved to sibling submodels of the same AAS via +/// SMSets.AASId and, for submodels only referenced by the shell, via SMRefSets. +/// +public static class ProjectionOperator +{ + // Keep IN() parameter lists comfortably below SQLite's default 999-variable limit + // (each chunk query additionally carries the path/idShort lists as parameters). + private const int ChunkSize = 400; + + public static List Project(AasContext db, DbProjectionRequest? request) + { + var identifiers = (request?.SubmodelIdentifiers ?? new List()) + .Where(id => !string.IsNullOrWhiteSpace(id)) + .ToList(); + var paths = (request?.Paths ?? new List()) + .Where(p => p != null + && !string.IsNullOrWhiteSpace(p.RawPath) + && !string.IsNullOrWhiteSpace(p.ElementIdShortPath)) + .ToList(); + + var rows = identifiers.Select(id => new DbProjectionRow { SubmodelIdentifier = id }).ToList(); + if (rows.Count == 0 || paths.Count == 0) + { + return rows; + } + + // 1) Result submodels: identifier -> (SMSets.Id, IdShort, AASId). Duplicate identifiers in the + // table are not expected; the row with the smallest Id wins for determinism. + var hitByIdentifier = new Dictionary(StringComparer.Ordinal); + foreach (var chunk in identifiers.Distinct(StringComparer.Ordinal).Chunk(ChunkSize)) + { + var found = db.SMSets + .Where(sm => sm.Identifier != null && chunk.Contains(sm.Identifier)) + .Select(sm => new { sm.Id, sm.Identifier, sm.IdShort, sm.AASId }) + .ToList(); + foreach (var sm in found.OrderBy(sm => sm.Id)) + { + if (sm.Identifier != null && !hitByIdentifier.ContainsKey(sm.Identifier)) + { + hitByIdentifier[sm.Identifier] = (sm.Id, sm.IdShort, sm.AASId); + } + } + } + + var samePathStrings = paths + .Where(p => p.TargetSubmodelIdShort == null) + .Select(p => p.ElementIdShortPath) + .Distinct(StringComparer.Ordinal) + .ToList(); + var crossPaths = paths.Where(p => p.TargetSubmodelIdShort != null).ToList(); + + // (SMId, IdShortPath) -> matched SME row; smallest Id wins when duplicated. + var smeByKey = new Dictionary<(int SmId, string Path), (int SmeId, string? SmeType, string? TValue)>(); + + // 2) Elements of the result submodels themselves. + var hitSmIds = hitByIdentifier.Values.Select(v => v.SmId).Distinct().ToList(); + var hitPathStrings = samePathStrings + .Concat(crossPaths.Select(p => p.ElementIdShortPath)) + .Distinct(StringComparer.Ordinal) + .ToList(); + if (hitPathStrings.Count > 0) + { + CollectSmes(db, hitSmIds, hitPathStrings, smeByKey); + } + + // 3) Cross-submodel paths: resolve the AAS of each hit, then the sibling submodels. + // siblingByAasAndIdShort: (AASId, submodel idShort) -> (sibling SMId, sibling identifier) + var aasIdByHitSmId = new Dictionary(); + var siblingByAasAndIdShort = new Dictionary<(int AasId, string IdShort), (int SmId, string Identifier)>(); + if (crossPaths.Count > 0) + { + foreach (var (smId, _, aasId) in hitByIdentifier.Values) + { + if (aasId.HasValue) + { + aasIdByHitSmId[smId] = aasId.Value; + } + } + + // Hits without a direct AASId link: resolve the AAS via the shell's submodel references. + var unresolved = hitByIdentifier + .Where(kv => !kv.Value.AasId.HasValue) + .Select(kv => (Identifier: kv.Key, kv.Value.SmId)) + .ToList(); + if (unresolved.Count > 0) + { + var refAasByIdentifier = new Dictionary(StringComparer.Ordinal); + foreach (var chunk in unresolved.Select(u => u.Identifier).Chunk(ChunkSize)) + { + var refs = db.SMRefSets + .Where(r => r.AASId != null && r.Identifier != null && chunk.Contains(r.Identifier)) + .Select(r => new { r.Identifier, r.AASId }) + .ToList(); + foreach (var r in refs) + { + if (r.Identifier != null && !refAasByIdentifier.ContainsKey(r.Identifier)) + { + refAasByIdentifier[r.Identifier] = r.AASId!.Value; + } + } + } + + foreach (var (identifier, smId) in unresolved) + { + if (refAasByIdentifier.TryGetValue(identifier, out var aasId)) + { + aasIdByHitSmId[smId] = aasId; + } + } + } + + var aasIds = aasIdByHitSmId.Values.Distinct().ToList(); + var targetIdShorts = crossPaths + .Select(p => p.TargetSubmodelIdShort!) + .Distinct(StringComparer.Ordinal) + .ToList(); + + // 3a) Siblings linked directly via SMSets.AASId. + foreach (var chunk in aasIds.Chunk(ChunkSize)) + { + var siblings = db.SMSets + .Where(sm => sm.AASId != null && chunk.Contains(sm.AASId.Value) + && sm.IdShort != null && targetIdShorts.Contains(sm.IdShort) + && sm.Identifier != null) + .Select(sm => new { sm.Id, sm.Identifier, sm.AASId, sm.IdShort }) + .ToList(); + foreach (var sm in siblings.OrderBy(sm => sm.Id)) + { + var key = (sm.AASId!.Value, sm.IdShort!); + if (!siblingByAasAndIdShort.ContainsKey(key)) + { + siblingByAasAndIdShort[key] = (sm.Id, sm.Identifier!); + } + } + } + + // 3b) Siblings only referenced by the shell (SMSets.AASId is null): follow SMRefSets. + var refPairs = new List<(int AasId, string Identifier)>(); + foreach (var chunk in aasIds.Chunk(ChunkSize)) + { + refPairs.AddRange(db.SMRefSets + .Where(r => r.AASId != null && chunk.Contains(r.AASId.Value) && r.Identifier != null) + .Select(r => new { r.AASId, r.Identifier }) + .ToList() + .Select(r => (r.AASId!.Value, r.Identifier!))); + } + + if (refPairs.Count > 0) + { + var aasByRefIdentifier = new Dictionary(StringComparer.Ordinal); + foreach (var (aasId, identifier) in refPairs) + { + if (!aasByRefIdentifier.ContainsKey(identifier)) + { + aasByRefIdentifier[identifier] = aasId; + } + } + + foreach (var chunk in aasByRefIdentifier.Keys.Chunk(ChunkSize)) + { + var siblings = db.SMSets + .Where(sm => sm.Identifier != null && chunk.Contains(sm.Identifier) + && sm.IdShort != null && targetIdShorts.Contains(sm.IdShort)) + .Select(sm => new { sm.Id, sm.Identifier, sm.IdShort }) + .ToList(); + foreach (var sm in siblings.OrderBy(sm => sm.Id)) + { + var key = (aasByRefIdentifier[sm.Identifier!], sm.IdShort!); + if (!siblingByAasAndIdShort.ContainsKey(key)) + { + siblingByAasAndIdShort[key] = (sm.Id, sm.Identifier!); + } + } + } + } + + // 3c) Elements of the sibling submodels. + var crossPathStrings = crossPaths + .Select(p => p.ElementIdShortPath) + .Distinct(StringComparer.Ordinal) + .ToList(); + var siblingSmIds = siblingByAasAndIdShort.Values.Select(v => v.SmId).Distinct().ToList(); + CollectSmes(db, siblingSmIds, crossPathStrings, smeByKey); + } + + // 4) Values of all matched elements in one pass (ordered for stable MLP language order). + var valuesBySmeId = new Dictionary>(); + var allSmeIds = smeByKey.Values.Select(v => v.SmeId).Distinct().ToList(); + foreach (var chunk in allSmeIds.Chunk(ChunkSize)) + { + var values = db.ValueSets + .Where(v => chunk.Contains(v.SMEId)) + .OrderBy(v => v.Id) + .Select(v => new { v.SMEId, v.SValue, v.NValue, v.Annotation }) + .ToList(); + foreach (var v in values) + { + if (!valuesBySmeId.TryGetValue(v.SMEId, out var list)) + { + list = new List(); + valuesBySmeId[v.SMEId] = list; + } + + list.Add(new DbProjectionValue { SValue = v.SValue, NValue = v.NValue, Annotation = v.Annotation }); + } + } + + // 5) Assemble the rows in the requested order. + var identifierBySmId = hitByIdentifier.ToDictionary(kv => kv.Value.SmId, kv => kv.Key); + foreach (var row in rows) + { + var hasHit = hitByIdentifier.TryGetValue(row.SubmodelIdentifier, out var hit); + foreach (var path in paths) + { + var cell = new DbProjectionCell(); + row.Cells[path.RawPath] = cell; + if (!hasHit) + { + continue; + } + + int targetSmId; + string targetIdentifier; + if (path.TargetSubmodelIdShort == null + || string.Equals(hit.IdShort, path.TargetSubmodelIdShort, StringComparison.Ordinal)) + { + targetSmId = hit.SmId; + targetIdentifier = row.SubmodelIdentifier; + } + else if (aasIdByHitSmId.TryGetValue(hit.SmId, out var aasId) + && siblingByAasAndIdShort.TryGetValue((aasId, path.TargetSubmodelIdShort), out var sibling)) + { + targetSmId = sibling.SmId; + targetIdentifier = sibling.Identifier; + } + else + { + continue; + } + + if (smeByKey.TryGetValue((targetSmId, path.ElementIdShortPath), out var sme)) + { + cell.Found = true; + cell.SmeType = sme.SmeType; + cell.TValue = sme.TValue; + cell.SourceSubmodelIdentifier = targetIdentifier; + if (valuesBySmeId.TryGetValue(sme.SmeId, out var values)) + { + cell.Values = values; + } + } + } + } + + return rows; + } + + private static void CollectSmes( + AasContext db, + List smIds, + List idShortPaths, + Dictionary<(int SmId, string Path), (int SmeId, string? SmeType, string? TValue)> smeByKey) + { + if (smIds.Count == 0 || idShortPaths.Count == 0) + { + return; + } + + foreach (var chunk in smIds.Chunk(ChunkSize)) + { + var smes = db.SMESets + .Where(sme => chunk.Contains(sme.SMId) + && sme.IdShortPath != null && idShortPaths.Contains(sme.IdShortPath)) + .Select(sme => new { sme.Id, sme.SMId, sme.IdShortPath, sme.SMEType, sme.TValue }) + .ToList(); + foreach (var sme in smes.OrderBy(sme => sme.Id)) + { + var key = (sme.SMId, sme.IdShortPath!); + if (!smeByKey.ContainsKey(key)) + { + smeByKey[key] = (sme.Id, sme.SMEType, sme.TValue); + } + } + } + } +} diff --git a/src/AasxServerDB/Query.cs b/src/AasxServerDB/Query.cs index 94465b7ad..6172b297b 100644 --- a/src/AasxServerDB/Query.cs +++ b/src/AasxServerDB/Query.cs @@ -97,7 +97,7 @@ public List SearchSMs(AasContext db, int pageFrom, int pageSize, string exp var text = string.Empty; var watch = Stopwatch.StartNew(); - Console.WriteLine("\nSearchSMs"); + if (ReadDiag.Enabled) ReadDiag.Write("\nSearchSMs"); watch.Restart(); @@ -262,6 +262,45 @@ public List SearchSMs(AasContext db, int pageFrom, int pageSize, string exp //} + // Count-only path: returns the number of matching ids WITHOUT materializing + // the full submodels (the "Collect results" phase). This is the fast, + // scalable way to answer aas_count for large databases. pageSize should be + // unbounded (e.g. int.MaxValue) so the count is the true total, not a page. + // A bounded pageSize is allowed and yields a capped count ("pageSize or more") — + // the subquery keeps its LIMIT, so huge match sets stop scanning early. + internal int GetQueryDataCount(bool noSecurity, AasContext db, + int pageFrom, int pageSize, ResultType resultType, string expression, + SqlConditions? securitySqlConditions = null) + { + var effectiveSecurity = noSecurity ? null : securitySqlConditions; + + var qResult = new QResult() + { + Count = 0, + TotalCount = 0, + PageFrom = 0, + PageSize = QResult.DefaultPageSize, + LastID = 0, + Messages = new List(), + SMResults = new List(), + SMEResults = new List(), + SQL = new List() + }; + + var watch = Stopwatch.StartNew(); + if (ReadDiag.Enabled) ReadDiag.Write("\nSearchSMs (count)"); + watch.Restart(); + + expression = "$JSONGRAMMAR " + expression; + + var result = GetSMs(out _, resultType, + qResult, watch, db, true, false, "", "", "", pageFrom, pageSize, expression, null, effectiveSecurity); + + var count = result?.FirstOrDefault() ?? 0; + if (ReadDiag.Enabled) ReadDiag.Write("Count query in " + watch.ElapsedMilliseconds + " ms (" + count + " matches)"); + return count; + } + internal QueryResult GetQueryData(bool noSecurity, AasContext db, int pageFrom, int pageSize, ResultType resultType, string expression, bool includeDebugSql = false, bool sqlOnly = false, SqlConditions? securitySqlConditions = null) @@ -285,7 +324,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, var text = string.Empty; var watch = Stopwatch.StartNew(); - Console.WriteLine("\nSearchSMs"); + if (ReadDiag.Enabled) ReadDiag.Write("\nSearchSMs"); watch.Restart(); @@ -303,7 +342,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, if (result == null) { text = "No query is generated."; - Console.WriteLine(text); + if (ReadDiag.Enabled) ReadDiag.Write(text); return null; } else @@ -318,7 +357,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, }; text = "Query data in " + watch.ElapsedMilliseconds + " ms"; - Console.WriteLine(text); + if (ReadDiag.Enabled) ReadDiag.Write(text); watch.Restart(); //var lastId = 0; @@ -501,7 +540,7 @@ internal QueryResult GetQueryData(bool noSecurity, AasContext db, } var collectResultText = "Collect results in " + watch.ElapsedMilliseconds + " ms"; - Console.WriteLine(collectResultText); + if (ReadDiag.Enabled) ReadDiag.Write(collectResultText); return queryDataResult; } @@ -621,7 +660,9 @@ public class SmDto // get data if (withExpression) // with expression { - comTable = CombineTablesLEFT(db, sqlConditions, pageFrom, pageSize, resultType, flags, generatedSql); + comTable = withCount + ? [CombineTablesLEFTCount(db, sqlConditions, resultType, flags, generatedSql, pageSize)] + : CombineTablesLEFT(db, sqlConditions, pageFrom, pageSize, resultType, flags, generatedSql); } if (comTable == null) @@ -1625,13 +1666,23 @@ private static List CombineTablesLEFT( bool isWithAASTable = restrictAAS || aasExistInCondition || resultType == ResultType.AssetAdministrationShell; + // Probe value-match selectivity (sets ExistsCondition.PreferExists for "common" contains/equality + // predicates) before building, so the value match can choose EXISTS vs the value-driven IN/FTS. + var commonContains = ClassifyCommonValueMatches(db, sqlConditions); + var swBuild = Stopwatch.StartNew(); var rawSql = BuildRawSqlFromSqlConditions(sqlConditions, isWithAASTable, resultType, pageFrom, pageSize, flags); swBuild.Stop(); - Console.WriteLine($"[ReadDiag] CombineTablesLEFT.BuildRawSql: {swBuild.ElapsedMilliseconds} ms"); + if (ReadDiag.Enabled) ReadDiag.Write($"[ReadDiag] CombineTablesLEFT.BuildRawSql: {swBuild.ElapsedMilliseconds} ms"); if (rawSql == null) throw new InvalidOperationException("BuildRawSqlFromSqlConditions returned null."); + if (db.Database.IsSqlite()) + { + rawSql = ApplySqliteIndexedPathJoinOrder(rawSql); + rawSql = ApplySqliteTrigramIndex(rawSql, commonContains); + } + // SQL-only debugging: emit SQL + query plan, but never execute the actual query. var sqlOnly = flags.Contains("$SQLONLY"); @@ -1644,7 +1695,7 @@ private static List CombineTablesLEFT( var swPlanOnly = Stopwatch.StartNew(); AddQueryPlan(generatedSql, GetQueryPlanForScript(db, rawSql)); swPlanOnly.Stop(); - Console.WriteLine($"[ReadDiag] CombineTablesLEFT.GetQueryPlan (sql-only, temptable): {swPlanOnly.ElapsedMilliseconds} ms"); + if (ReadDiag.Enabled) ReadDiag.Write($"[ReadDiag] CombineTablesLEFT.GetQueryPlan (sql-only, temptable): {swPlanOnly.ElapsedMilliseconds} ms"); return new List(); } @@ -1674,14 +1725,19 @@ ORDER BY 1 AddGeneratedSql(generatedSql, rawSql); - var swPlan = Stopwatch.StartNew(); - var qpRaw = GetQueryPlan(db, rawSql); - swPlan.Stop(); - Console.WriteLine($"[ReadDiag] CombineTablesLEFT.GetQueryPlan: {swPlan.ElapsedMilliseconds} ms"); + if (ReadDiag.Enabled) + { + var swPlan = Stopwatch.StartNew(); + var qpRaw = GetQueryPlan(db, rawSql); + swPlan.Stop(); + ReadDiag.Write($"[ReadDiag] CombineTablesLEFT.GetQueryPlan: {swPlan.ElapsedMilliseconds} ms"); + ReadDiag.Write($"[ReadDiag] SQL:\n{rawSql.TrimEnd()}"); + ReadDiag.Write($"[ReadDiag] QueryPlan:\n{qpRaw.TrimEnd()}"); + } if (sqlOnly) { - AddQueryPlan(generatedSql, qpRaw); + AddQueryPlan(generatedSql, GetQueryPlan(db, rawSql)); return new List(); } @@ -1701,10 +1757,66 @@ ORDER BY 1 ids.Add(reader.GetInt32(0)); } swExec.Stop(); - Console.WriteLine($"[ReadDiag] CombineTablesLEFT.Execute : {swExec.ElapsedMilliseconds} ms ({ids.Count} ids)"); + if (ReadDiag.Enabled) ReadDiag.Write($"[ReadDiag] CombineTablesLEFT.Execute : {swExec.ElapsedMilliseconds} ms ({ids.Count} ids)"); return ids; } + private static int CombineTablesLEFTCount( + AasContext db, + SqlConditions sqlConditions, + ResultType resultType, + List flags, + List? generatedSql = null, + int pageSize = int.MaxValue) + { + var sqlAasMerged = NormalizeSqlAliases(sqlConditions.FormulaConditions.GetValueOrDefault("aas", "")); + var sqlOverallCondition = NormalizeSqlAliases(sqlConditions.FormulaConditions.GetValueOrDefault("all", "")); + var restrictAAS = !SqlConditionIsPureTautology(sqlAasMerged); + var aasExistInCondition = !string.IsNullOrWhiteSpace(sqlOverallCondition) + && sqlOverallCondition.Contains("\"aas\"."); + var isWithAASTable = restrictAAS || aasExistInCondition || resultType == ResultType.AssetAdministrationShell; + + var commonContains = ClassifyCommonValueMatches(db, sqlConditions); + + var rawSql = BuildRawSqlFromSqlConditions( + sqlConditions, isWithAASTable, resultType, 0, int.MaxValue, flags) + ?? throw new InvalidOperationException("BuildRawSqlFromSqlConditions returned null."); + + if (db.Database.IsSqlite()) + { + rawSql = ApplySqliteIndexedPathJoinOrder(rawSql); + rawSql = ApplySqliteTrigramIndex(rawSql, commonContains); + } + + // Counting does not need stable result ordering or pagination. Removing + // both avoids a temp B-tree and materializing every matching identifier. + rawSql = Regex.Replace( + rawSql, + @"\r?\nORDER BY [^\r\n]+\r?\nLIMIT \d+ OFFSET \d+\s*$", + string.Empty, + RegexOptions.CultureInvariant); + + // Capped count: with a bounded pageSize the subquery keeps a LIMIT, so the scan + // stops after pageSize distinct ids. Callers get "pageSize or more" instead of the + // exact total — cheap existence/prevalence checks on huge match sets. + if (pageSize > 0 && pageSize < int.MaxValue) + { + rawSql = rawSql.TrimEnd() + $"\r\nLIMIT {pageSize.ToString(System.Globalization.CultureInfo.InvariantCulture)}"; + } + + var countSql = $"SELECT COUNT(*) AS Value\r\nFROM (\r\n{rawSql.TrimEnd()}\r\n) AS count_query"; + AddGeneratedSql(generatedSql, countSql); + + if (ReadDiag.Enabled) + { + var plan = GetQueryPlan(db, countSql); + ReadDiag.Write($"[ReadDiag] Count SQL:\n{countSql}"); + ReadDiag.Write($"[ReadDiag] Count QueryPlan:\n{plan.TrimEnd()}"); + } + + return db.Database.SqlQueryRaw(countSql).AsEnumerable().Single(); + } + // ------------------------------------------------------------------ // SQL assembly from SqlConditions (single source of truth for SM list SQL) // ------------------------------------------------------------------ @@ -1734,7 +1846,7 @@ ORDER BY 1 foreach (var path in sc.Paths) overall = overall.Replace($"$${path.Placeholder}$$", $"(p{pathNum++}.SMId IS NOT NULL)"); foreach (var match in sc.Matches) overall = overall.Replace($"$${match.Placeholder}$$", $"(m{matchNum++}.SMId IS NOT NULL)"); foreach (var exists in sc.ExistsConditions) - overall = overall.Replace($"$${exists.Placeholder}$$", BuildValueMatchSql(exists.PredicateSql, whereSme)); + overall = overall.Replace($"$${exists.Placeholder}$$", BuildValueMatchSql(exists.PredicateSql, whereSme, exists.PreferExists)); // ---------------------------------------------------------------- // Direct paths: no paths/matches, no AAS join, no SM filter, standalone SM list. @@ -1746,6 +1858,66 @@ ORDER BY 1 bool overallHasVRef = overall.Contains("\"value\"."); bool overallHasTRef = overall.Contains("\"sm\".") || overall.Contains("\"aas\"."); + // Positive path/match blocks already yield matching SMId values. Wrapping + // them in SMSets LEFT JOIN makes SQLite scan every submodel and may execute + // FTS path searches repeatedly. Intersect their result sets directly. + var positiveJoinAliases = Enumerable.Range(1, sc.Paths.Count).Select(i => $"p{i}") + .Concat(Enumerable.Range(1, sc.Matches.Count).Select(i => $"m{i}")) + .ToList(); + var allowedPositiveAliases = positiveJoinAliases.ToHashSet(StringComparer.Ordinal); + var isPositiveJoinExpression = TryBuildPositiveJoinDnf( + overall, allowedPositiveAliases, out var positiveJoinDnf); + var directSubmodelResult = resultType != ResultType.AssetAdministrationShell && !isWithAASTable; + var directShellResult = resultType == ResultType.AssetAdministrationShell + && string.IsNullOrWhiteSpace(whereAas); + bool canDirectPositiveJoins = positiveJoinAliases.Count > 0 + && string.IsNullOrWhiteSpace(whereSm) + && (directSubmodelResult || directShellResult) + && !unionOrTemp + && isPositiveJoinExpression; + + if (canDirectPositiveJoins) + { + var sources = new List<(string Alias, string Sql)>(); + for (var i = 0; i < sc.Paths.Count; i++) + sources.Add(($"p{i + 1}", $"SELECT sme.SMId AS SMId\r\n{sc.Paths[i].SubquerySql}")); + for (var i = 0; i < sc.Matches.Count; i++) + sources.Add(($"m{i + 1}", BuildMatchSubquerySql(sc.Matches[i]))); + + var raw = "WITH\r\n"; + raw += string.Join(",\r\n", sources.Select(source => + $"{source.Alias} AS (\r\n{source.Sql}\r\n)")); + raw += ",\r\nmatching_sm AS (\r\nSELECT DISTINCT Id\r\nFROM (\r\n"; + + for (var branchIndex = 0; branchIndex < positiveJoinDnf.Count; branchIndex++) + { + var branch = positiveJoinDnf[branchIndex]; + var firstAlias = branch[0]; + raw += $"SELECT {firstAlias}.SMId AS Id\r\nFROM {firstAlias}\r\n"; + foreach (var alias in branch.Skip(1)) + raw += $"INNER JOIN {alias} ON {alias}.SMId = {firstAlias}.SMId\r\n"; + if (branchIndex < positiveJoinDnf.Count - 1) + raw += "UNION\r\n"; + } + raw += ") AS positive_ids\r\n)\r\n"; + + if (directShellResult) + { + raw += "SELECT DISTINCT aas.Id AS Id\r\n"; + raw += "FROM matching_sm match\r\n"; + raw += "INNER JOIN SMSets sm ON sm.Id = match.Id\r\n"; + raw += "INNER JOIN SMRefSets sx ON sx.Identifier = sm.Identifier\r\n"; + raw += "INNER JOIN AASSets aas ON aas.Id = sx.AASId\r\n"; + raw += $"ORDER BY aas.Id\r\nLIMIT {pageSize} OFFSET {pageFrom}\r\n"; + } + else + { + raw += "SELECT Id FROM matching_sm\r\n"; + raw += $"ORDER BY Id\r\nLIMIT {pageSize} OFFSET {pageFrom}\r\n"; + } + return ApplyLikeToGlob(raw); + } + bool canDirectSme = !hasPathsOrMatches && !isWithAASTable && string.IsNullOrWhiteSpace(whereSm) && resultType != ResultType.AssetAdministrationShell @@ -1756,6 +1928,18 @@ ORDER BY 1 && resultType != ResultType.AssetAdministrationShell && overallHasVRef && !overallHasTRef; + bool canDirectShellSme = !hasPathsOrMatches + && resultType == ResultType.AssetAdministrationShell + && string.IsNullOrWhiteSpace(whereAas) + && string.IsNullOrWhiteSpace(whereSm) + && overallHasSmeRef && !overallHasVRef && !overallHasTRef; + + bool canDirectShellValue = !hasPathsOrMatches + && resultType == ResultType.AssetAdministrationShell + && string.IsNullOrWhiteSpace(whereAas) + && string.IsNullOrWhiteSpace(whereSm) + && overallHasVRef && !overallHasTRef; + if (canDirectSme) { if (unionOrTemp) @@ -1776,6 +1960,26 @@ ORDER BY 1 return ApplyLikeToGlob(raw); } + if (canDirectShellSme || canDirectShellValue) + { + var raw = "SELECT DISTINCT aas.Id AS Id\r\n"; + if (canDirectShellValue) + { + raw += "FROM ValueSets AS value\r\n"; + raw += "INNER JOIN SMESets AS sme ON sme.Id = value.SMEId\r\n"; + } + else + { + raw += "FROM SMESets AS sme\r\n"; + } + raw += "INNER JOIN SMSets AS sm ON sm.Id = sme.SMId\r\n"; + raw += "INNER JOIN SMRefSets AS sx ON sx.Identifier = sm.Identifier\r\n"; + raw += "INNER JOIN AASSets AS aas ON aas.Id = sx.AASId\r\n"; + raw += $"WHERE {overall}\r\n"; + raw += $"ORDER BY aas.Id\r\nLIMIT {pageSize} OFFSET {pageFrom}\r\n"; + return ApplyLikeToGlob(raw); + } + // ---------------------------------------------------------------- // Normal path: AAS/SM subqueries // ---------------------------------------------------------------- @@ -1858,6 +2062,9 @@ ORDER BY 1 var sbVal = new StringBuilder(); sbVal.Append("LEFT JOIN(\r\n SELECT DISTINCT\r\n sme.SMId AS \"SMId\""); + // Id lets the SQLite execution path correlate an outer value predicate + // with ValueSets_fts without changing the shape of the generated joins. + sbVal.Append(",\r\n v.\"Id\" AS \"Id\""); foreach (var f in valFields) sbVal.Append($",\r\n v.\"{f}\" AS \"{f}\""); sbVal.Append("\r\n FROM ValueSets v\r\n JOIN SMESets sme ON sme.Id = v.SMEId\r\n"); sbVal.Append($" WHERE {valWhere}\r\n) AS value ON value.\"SMId\" = sm.Id\r\n"); @@ -1909,27 +2116,365 @@ private static string ApplyLikeToGlob(string sql) return sql.Replace("%", "*"); } + private static readonly Regex SqliteSValueContainsRegex = new( + "(?(?\\\"(?:value|v)\\\"|v)\\.\\\"SValue\\\")\\s+GLOB\\s+(?'(?:''|[^'])*')", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static readonly Regex SqliteSValueEqualsRegex = new( + "(?(?\\\"(?:value|v)\\\"|v)\\.\\\"SValue\\\")\\s*=\\s*(?'(?:''|[^'])*')", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static readonly Regex SqliteIdShortContainsRegex = new( + "(?(?\\\"sme\\\"|sme)\\.\\\"IdShort\\\")\\s+GLOB\\s+(?'(?:''|[^'])*')", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static readonly Regex SqliteIndexedPathJoinRegex = new( + "FROM SMESets (?[A-Za-z][A-Za-z0-9_]*)(?\\r?\\n)" + + "(?:LEFT JOIN|JOIN) ValueSets (?[A-Za-z][A-Za-z0-9_]*) ON " + + "\\k\\.SMEId = \\k\\.Id AND (?[^\\r\\n]+)\\k" + + "WHERE \\k" + + "(?(?:\\kAND [^\\r\\n]+)*)", + RegexOptions.CultureInvariant | RegexOptions.Compiled); + + /// + /// Path searches normally start at SMESets and repeat the value predicate in + /// JOIN and WHERE. For an indexed contains or prefix predicate, start at the + /// reduced ValueSets candidates instead. For all other predicates (equality, + /// range) with an IdShort filter, start at SMESets: the IdShort B-tree is the + /// deterministic entry there, and hot values like SValue='24' must never drive. + /// Without the forced order the plan depends on sqlite_stat1 averages, which + /// underestimate hot keys (measured: 0.3 s -> 4.9 s after ANALYZE, 28 s in + /// nested AND arms without it). CROSS JOIN fixes the loop order either way. + /// + internal static string ApplySqliteIndexedPathJoinOrder(string sql) + { + if (string.IsNullOrWhiteSpace(sql)) + return sql; + + return SqliteIndexedPathJoinRegex.Replace(sql, match => + { + var predicate = match.Groups["predicate"].Value; + var smeAlias = match.Groups["sme"].Value; + var valueAlias = match.Groups["value"].Value; + var newline = match.Groups["nl"].Value; + var rest = match.Groups["rest"].Value; + + if (HasIndexableSValueTrigram(predicate) || HasIndexableSValuePrefix(predicate)) + { + return $"FROM ValueSets {valueAlias}{newline}" + + $"CROSS JOIN SMESets {smeAlias}{newline}" + + $"WHERE {smeAlias}.Id = {valueAlias}.SMEId{newline}" + + $"AND {predicate}" + + rest; + } + + // sme-first only when an IdShort equality narrows the outer loop; wildcard-only + // paths (no IdShort filter) would otherwise scan the full SMESets table. + if (!rest.Contains($"\"{smeAlias}\".\"IdShort\" = '", StringComparison.Ordinal)) + return match.Value; + + return $"FROM SMESets {smeAlias}{newline}" + + $"CROSS JOIN ValueSets {valueAlias}{newline}" + + $"WHERE {valueAlias}.SMEId = {smeAlias}.Id{newline}" + + $"AND {predicate}" + + rest; + }); + } + + /// + /// Adds FTS5 row-id candidate filters to ordinary SValue and IdShort contains predicates. + /// The original GLOB remains in place as the authoritative comparison. Patterns + /// that cannot benefit from a trigram (fewer than three literal characters or + /// embedded GLOB metacharacters) are deliberately left unchanged. + /// + internal static string ApplySqliteTrigramIndex(string sql, IReadOnlySet? skipSValuePatterns = null) + { + if (string.IsNullOrWhiteSpace(sql)) + return sql; + + var rewritten = SqliteSValueContainsRegex.Replace(sql, match => + { + if (!IsIndexableTrigramGlob(match)) + return match.Value; + + // Non-selective ("common") contains were routed to a correlated EXISTS; adding the FTS + // candidate filter there would re-materialize the huge match set, so skip those patterns. + if (skipSValuePatterns != null && skipSValuePatterns.Contains(match.Groups["pattern"].Value)) + return match.Value; + + var sqlLiteral = match.Groups["pattern"].Value; + var alias = match.Groups["alias"].Value; + return $"({match.Value} AND {alias}.\"Id\" IN " + + $"(SELECT rowid FROM \"{SqliteTrigramIndex.TableName}\" " + + $"WHERE \"SValue\" GLOB {sqlLiteral}))"; + }); + + return SqliteIdShortContainsRegex.Replace(rewritten, match => + { + if (!IsIndexableTrigramGlob(match)) + return match.Value; + + var sqlLiteral = match.Groups["pattern"].Value; + var alias = match.Groups["alias"].Value; + return $"({match.Value} AND {alias}.\"Id\" IN " + + $"(SELECT rowid FROM \"{SqliteTrigramIndex.IdShortTableName}\" " + + $"WHERE \"IdShort\" GLOB {sqlLiteral}))"; + }); + } + + private static bool HasIndexableSValueTrigram(string sql) + { + foreach (Match match in SqliteSValueContainsRegex.Matches(sql)) + { + if (IsIndexableTrigramGlob(match)) + return true; + } + + return false; + } + + private static bool HasIndexableSValuePrefix(string sql) + { + foreach (Match match in SqliteSValueContainsRegex.Matches(sql)) + { + var sqlLiteral = match.Groups["pattern"].Value; + var pattern = sqlLiteral[1..^1].Replace("''", "'", StringComparison.Ordinal); + if (pattern.Length < 2 || pattern[^1] != '*') + continue; + + var prefix = pattern[..^1]; + if (prefix.Length > 0 && prefix.IndexOfAny(['*', '?', '[', ']']) < 0) + return true; + } + + return false; + } + + private static bool IsIndexableTrigramGlob(Match match) + { + var sqlLiteral = match.Groups["pattern"].Value; + var pattern = sqlLiteral[1..^1].Replace("''", "'", StringComparison.Ordinal); + + // Prefix searches are handled more efficiently by SQLite's ordinary + // SValue/IdShort B-tree. Trigram is needed for leading-wildcard contains + // and suffix searches, provided the literal part has at least 3 chars. + if (pattern.Length < 4 || pattern[0] != '*') + return false; + + var searchText = pattern[1..]; + if (searchText.EndsWith('*')) + searchText = searchText[..^1]; + return searchText.Length >= 3 && searchText.IndexOfAny(['*', '?', '[', ']']) < 0; + } + + // A leading-wildcard contains matching at least this many value rows is treated as "common" and + // realized as a correlated EXISTS (early-stop) rather than the value-driven IN/FTS. Settable for tests. + // The EXISTS gamble only wins when hits are DENSE at submodel level (early-stop finds the LIMIT + // quickly). 1000 of ~50M value rows is far too sparse for that: terms like a product-family name + // match a few thousand rows concentrated in few submodels, and the EXISTS walk degenerated into a + // near-full corpus scan (measured 40 s) while the skipped FTS path would answer in milliseconds. + // 100000 keeps truly ubiquitous terms (e.g. the manufacturer name, present in half of all + // submodels, where FTS materialization would be the expensive side) on the EXISTS path. + internal static int ContainsCommonProbeCap = 100_000; + + // Classifies each direct $sme#value condition that is a *single* indexable value match (a + // leading-wildcard contains OR an exact equality) by a capped probe: if it matches at least + // ContainsCommonProbeCap rows it is non-selective ("common") and gets PreferExists (→ correlated + // EXISTS, which early-stops under ORDER BY/LIMIT) instead of materializing the huge match set on + // the value-driven IN/FTS path. For common contains the pattern is additionally returned so the + // trigram rewrite skips its FTS candidate filter (equality uses no FTS filter, so it is not + // returned). Rare/zero matches stay on the value-driven path (already fast). No-op for non-SQLite. + private static HashSet ClassifyCommonValueMatches(AasContext db, SqlConditions sc) + { + var common = new HashSet(StringComparer.Ordinal); + if (!db.Database.IsSqlite()) + return common; + + foreach (var exists in sc.ExistsConditions) + { + exists.PreferExists = false; + if (TryGetPureSValueContainsPattern(exists.PredicateSql, out var pattern) + && SValueContainsIsCommon(db, pattern, ContainsCommonProbeCap)) + { + exists.PreferExists = true; + common.Add(pattern); + } + else if (TryGetPureSValueEqualsLiteral(exists.PredicateSql, out var literal) + && SValueEqualsIsCommon(db, literal, ContainsCommonProbeCap)) + { + exists.PreferExists = true; + } + else if (TryGetPureSValuePrefixPattern(exists.PredicateSql, out var prefix) + && SValuePrefixIsCommon(db, prefix, ContainsCommonProbeCap)) + { + exists.PreferExists = true; + } + } + + return common; + } + + // True only when the predicate is exactly one indexable leading-wildcard SValue contains (nothing else + // ANDed that would add selectivity); returns the SQL literal pattern, e.g. '*Phoenix*'. + internal static bool TryGetPureSValueContainsPattern(string predicateSql, out string patternLiteral) + { + patternLiteral = string.Empty; + if (string.IsNullOrWhiteSpace(predicateSql)) + return false; + + var s = StripEnclosingParens(predicateSql).Trim(); + var matches = SqliteSValueContainsRegex.Matches(s); + if (matches.Count != 1) + return false; + + var m = matches[0]; + if (m.Index != 0 || m.Length != s.Length || !IsIndexableTrigramGlob(m)) + return false; + + patternLiteral = m.Groups["pattern"].Value; + return true; + } + + // Capped probe: counts up to `cap` FTS matches for the contains pattern; >= cap means "common". + // On any error (e.g. FTS table absent) returns false so the current value-driven path is kept. + private static bool SValueContainsIsCommon(AasContext db, string patternLiteral, int cap) + { + try + { + var connection = db.Database.GetDbConnection(); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = + $"SELECT COUNT(*) FROM (SELECT 1 FROM \"{SqliteTrigramIndex.TableName}\" " + + $"WHERE \"SValue\" GLOB {patternLiteral} LIMIT {cap})"; + var n = Convert.ToInt32(command.ExecuteScalar()); + return n >= cap; + } + catch + { + return false; + } + } + + // True only when the predicate is exactly one SValue equality (nothing else ANDed that would add + // selectivity); returns the SQL literal, e.g. 'Phoenix Contact GmbH & Co. KG'. + internal static bool TryGetPureSValueEqualsLiteral(string predicateSql, out string literal) + { + literal = string.Empty; + if (string.IsNullOrWhiteSpace(predicateSql)) + return false; + + var s = StripEnclosingParens(predicateSql).Trim(); + var matches = SqliteSValueEqualsRegex.Matches(s); + if (matches.Count != 1) + return false; + + var m = matches[0]; + if (m.Index != 0 || m.Length != s.Length) + return false; + + literal = m.Groups["literal"].Value; + return true; + } + + // Capped probe: counts up to `cap` exact-value matches via the SValue B-tree (covering index seek); + // >= cap means "common". On any error returns false so the current value-driven path is kept. + private static bool SValueEqualsIsCommon(AasContext db, string literal, int cap) + { + try + { + var connection = db.Database.GetDbConnection(); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = + $"SELECT COUNT(*) FROM (SELECT 1 FROM ValueSets WHERE \"SValue\" = {literal} LIMIT {cap})"; + var n = Convert.ToInt32(command.ExecuteScalar()); + return n >= cap; + } + catch + { + return false; + } + } + + // True only when the predicate is exactly one SValue prefix GLOB ('abc*' — trailing wildcard only, + // no leading wildcard, clean literal part); returns the SQL literal pattern, e.g. 'PXC-*'. + internal static bool TryGetPureSValuePrefixPattern(string predicateSql, out string patternLiteral) + { + patternLiteral = string.Empty; + if (string.IsNullOrWhiteSpace(predicateSql)) + return false; + + var s = StripEnclosingParens(predicateSql).Trim(); + var matches = SqliteSValueContainsRegex.Matches(s); + if (matches.Count != 1) + return false; + + var m = matches[0]; + if (m.Index != 0 || m.Length != s.Length) + return false; + + var sqlLiteral = m.Groups["pattern"].Value; + var pattern = sqlLiteral[1..^1].Replace("''", "'", StringComparison.Ordinal); + if (pattern.Length < 2 || pattern[^1] != '*' || pattern[0] == '*') + return false; + if (pattern[..^1].IndexOfAny(['*', '?', '[', ']']) >= 0) + return false; + + patternLiteral = sqlLiteral; + return true; + } + + // Capped probe: counts up to `cap` prefix matches via the SValue B-tree (GLOB 'abc*' is range-optimized + // on the BINARY index); >= cap means "common". On any error returns false so the value-driven path stays. + private static bool SValuePrefixIsCommon(AasContext db, string patternLiteral, int cap) + { + try + { + var connection = db.Database.GetDbConnection(); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = + $"SELECT COUNT(*) FROM (SELECT 1 FROM ValueSets WHERE \"SValue\" GLOB {patternLiteral} LIMIT {cap})"; + var n = Convert.ToInt32(command.ExecuteScalar()); + return n >= cap; + } + catch + { + return false; + } + } + // Realizes a direct $sme#value condition. The element-visibility FILTER (smeFilter, e.g. the // access-rule "$sme#idShort starts-with General/Manufacturer") is applied INSIDE the value match, // so a value sitting in a hidden element cannot satisfy the query (otherwise id-only queries would - // leak the existence of hidden-element values). Selective predicates (=, range, GLOB-prefix) become - // a value-index-driven, non-correlated subquery (sm.Id IN (...)); non-selective ones (<>, negation, - // OR of predicates) stay a correlated EXISTS that short-circuits per submodel. - // NOTE: interim operator heuristic — to be replaced by a structure-based decision in the planned - // condition-IR (where selectivity could come from ANALYZE statistics). - private static string BuildValueMatchSql(string predicateSql, string? smeFilter) + // leak the existence of hidden-element values). + // A correlated EXISTS starts at every SMSets row, which is disastrous when matches are rare. So for + // selective predicates we build the SM-id set once with ValueSets leading (CROSS JOIN forces that + // order, and lets its B-tree / FTS trigram row-id filter drive): mcp's trigram + SValue-prefix cases + // OR the operator heuristic (=, range, GLOB-prefix). Non-selective ones (<>, negation, OR) stay a + // correlated EXISTS that short-circuits per submodel. + // NOTE: interim heuristic — to be replaced by a structure-based decision in the planned condition-IR. + private static string BuildValueMatchSql(string predicateSql, string? smeFilter = null, bool preferExists = false) { var filter = string.IsNullOrWhiteSpace(smeFilter) ? string.Empty : $"\r\n AND ({smeFilter!.Replace("\"sme\".", "sme_value.", StringComparison.Ordinal)})"; - if (PredicatePrefersJoin(predicateSql)) + // preferExists: a capped FTS probe classified this contains as non-selective ("common") — force the + // correlated EXISTS (early-stop under ORDER BY/LIMIT) instead of materializing the huge match set. + if (!preferExists && + (HasIndexableSValueTrigram(predicateSql) || HasIndexableSValuePrefix(predicateSql) || PredicatePrefersJoin(predicateSql))) { return $@"sm.Id IN ( SELECT sme_value.SMId FROM ValueSets v - JOIN SMESets sme_value ON sme_value.Id = v.SMEId - WHERE ({predicateSql}){filter} + CROSS JOIN SMESets sme_value + WHERE sme_value.Id = v.SMEId + AND ({predicateSql}){filter} )"; } @@ -1943,8 +2488,8 @@ FROM ValueSets v } // Operator heuristic for BuildValueMatchSql: a value predicate is "join-worthy" when it is an - // index-usable positive comparison (=, <, >, <=, >=, or a GLOB prefix without leading wildcard) - // and carries no negation or top-level OR (one non-selective disjunct would force a scan). + // index-usable positive comparison (=, <, >, <=, >=, or a GLOB prefix without leading wildcard). + // Top-level OR is only join-worthy for a pure disjunction of value equalities, matching MCP "in". private static bool PredicatePrefersJoin(string predicateSql) { if (string.IsNullOrWhiteSpace(predicateSql)) @@ -1954,7 +2499,7 @@ private static bool PredicatePrefersJoin(string predicateSql) Regex.IsMatch(predicateSql, @"\bNOT\b", RegexOptions.IgnoreCase)) return false; if (Regex.IsMatch(predicateSql, @"\bOR\b", RegexOptions.IgnoreCase)) - return false; + return IsPureValueEqualityDisjunction(predicateSql); if (Regex.IsMatch(predicateSql, @"[=<>]", RegexOptions.CultureInvariant)) return true; if (Regex.IsMatch(predicateSql, @"GLOB\s+'[^*]")) @@ -1962,6 +2507,22 @@ private static bool PredicatePrefersJoin(string predicateSql) return false; } + private static bool IsPureValueEqualityDisjunction(string predicateSql) + { + var s = StripEnclosingParens(predicateSql).Trim(); + if (string.IsNullOrWhiteSpace(s)) + return false; + + var orParts = SplitTopLevelOr(s); + if (orParts.Count > 1) + return orParts.All(IsPureValueEqualityDisjunction); + + return Regex.IsMatch( + StripEnclosingParens(s).Trim(), + @"^v\.""(?:SValue|NValue)""\s*=\s*(?:'([^']|'')*'|-?\d+(?:\.\d+)?)$", + RegexOptions.CultureInvariant); + } + /// /// True if this top-level AND conjunct references "sme". but no other table aliases that would /// require staying in the outer WHERE (sm/aas/value/path/match). Used to gate SME EXISTS rewrite (unless $LEGACYSMEJOIN). @@ -2080,7 +2641,7 @@ private static string BuildDirectUnionOrTempSql( int pageFrom, int pageSize) { - var orParts = SplitTopLevelOr(overall); + var orParts = SplitTopLevelOr(StripEnclosingParens(overall)); if (orParts.Count == 0) orParts = new List { "1=1" }; var fromSql = isSmeTable @@ -2133,10 +2694,13 @@ private static List BuildUnionOrParts(SqlConditions sc, string combinedO { List NonEmpty(List parts) => parts.Count == 0 ? new List { "1=1" } : parts; + // Strip a redundant enclosing paren so a fully-wrapped top-level OR ("(A OR B OR C)") still + // splits — without it, the no-security path (this fallback) would emit a single monolithic + // branch (full SMSets scan) instead of distributing the OR. List Fallback() => NonEmpty(string.IsNullOrWhiteSpace(combinedOverall) || combinedOverall == "1=1" ? new List { "1=1" } - : SplitTopLevelOr(combinedOverall)); + : SplitTopLevelOr(StripEnclosingParens(combinedOverall))); var qRaw = sc.QueryOverallRaw; var secRaw = sc.SecurityOverallRaw; @@ -2149,7 +2713,7 @@ string Resolve(string raw) int pN = 1, mN = 1; foreach (var path in sc.Paths) s = s.Replace($"$${path.Placeholder}$$", $"(p{pN++}.SMId IS NOT NULL)"); foreach (var match in sc.Matches) s = s.Replace($"$${match.Placeholder}$$", $"(m{mN++}.SMId IS NOT NULL)"); - foreach (var exists in sc.ExistsConditions) s = s.Replace($"$${exists.Placeholder}$$", BuildValueMatchSql(exists.PredicateSql, smeFilter)); + foreach (var exists in sc.ExistsConditions) s = s.Replace($"$${exists.Placeholder}$$", BuildValueMatchSql(exists.PredicateSql, smeFilter, exists.PreferExists)); return s; } @@ -2442,6 +3006,67 @@ private static bool IsSingleBalancedParenthesisWrap(string t) return false; } + /// + /// Converts an expression made exclusively from positive path/match aliases + /// and AND/OR into disjunctive normal form. Each outer list item is one UNION + /// branch; aliases inside a branch are intersected by SMId joins. + /// + private static bool TryBuildPositiveJoinDnf( + string expression, + IReadOnlySet allowedAliases, + out List> dnf) + { + dnf = []; + var text = expression.Trim(); + while (IsSingleBalancedParenthesisWrap(text)) + text = text.Substring(1, text.Length - 2).Trim(); + + var orParts = SplitTopLevelOr(text); + if (orParts.Count > 1) + { + foreach (var part in orParts) + { + if (!TryBuildPositiveJoinDnf(part, allowedAliases, out var child)) + return false; + dnf.AddRange(child); + if (dnf.Count > 64) + return false; + } + return dnf.Count > 0; + } + + var andParts = SplitTopLevelAnd(text); + if (andParts.Count > 1) + { + var product = new List> { new List() }; + foreach (var part in andParts) + { + if (!TryBuildPositiveJoinDnf(part, allowedAliases, out var child)) + return false; + + var next = new List>(); + foreach (var left in product) + foreach (var right in child) + next.Add(left.Concat(right).Distinct(StringComparer.Ordinal).ToList()); + if (next.Count > 64) + return false; + product = next; + } + dnf = product; + return dnf.Count > 0; + } + + var leaf = Regex.Match( + text, + @"^(?[pm]\d+)\.SMId\s+IS\s+NOT\s+NULL$", + RegexOptions.CultureInvariant); + if (!leaf.Success || !allowedAliases.Contains(leaf.Groups["alias"].Value)) + return false; + + dnf.Add([leaf.Groups["alias"].Value]); + return true; + } + private static string AppendAndCondition(string? left, string? right) { var normalizedLeft = string.IsNullOrWhiteSpace(left) || SqlConditionIsPureTautology(left) ? "" : left.Trim(); @@ -2576,7 +3201,7 @@ private bool ConditionFromExpression(List messages, string expression, o { withQueryLanguage = 3; expression = expression.Replace("$JSONGRAMMAR", string.Empty); - Console.WriteLine("$JSONGRAMMAR"); + if (ReadDiag.Enabled) ReadDiag.Write("$JSONGRAMMAR"); messages.Add("$JSONGRAMMAR"); } if (expression.StartsWith("$JSON")) @@ -2718,7 +3343,7 @@ private bool ConditionFromExpression(List messages, string expression, o messages.Add(""); text = "combinedCondition: " + overallCondition; - Console.WriteLine(text); + if (ReadDiag.Enabled) ReadDiag.Write(text); messages.Add(text); } diff --git a/src/AasxServerDB/SqliteTrigramIndex.cs b/src/AasxServerDB/SqliteTrigramIndex.cs new file mode 100644 index 000000000..4298b3e5e --- /dev/null +++ b/src/AasxServerDB/SqliteTrigramIndex.cs @@ -0,0 +1,285 @@ +namespace AasxServerDB; + +using System; +using System.Data; +using System.Diagnostics; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; + +/// +/// Installs and maintains the SQLite FTS5 trigram index used for substring +/// searches on ValueSets.SValue and SMESets.IdShort. +/// +internal static class SqliteTrigramIndex +{ + internal const string TableName = "ValueSets_fts"; + internal const string IdShortTableName = "SMESets_fts"; + private const int BuildBatchSize = 250_000; + + /// + /// Adds the FTS table and its maintenance triggers to an existing database. + /// Existing values are indexed exactly once, when the FTS table is created. + /// + internal static void Initialize(DbContext db) + { + ArgumentNullException.ThrowIfNull(db); + + var connection = db.Database.GetDbConnection(); + var shouldCloseConnection = connection.State != ConnectionState.Open; + if (shouldCloseConnection) + db.Database.OpenConnection(); + try + { + using var existsCommand = connection.CreateCommand(); + existsCommand.CommandText = + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = $name LIMIT 1"; + var nameParameter = existsCommand.CreateParameter(); + nameParameter.ParameterName = "$name"; + nameParameter.Value = TableName; + existsCommand.Parameters.Add(nameParameter); + var indexAlreadyExists = existsCommand.ExecuteScalar() is not null; + nameParameter.Value = IdShortTableName; + var idShortIndexAlreadyExists = existsCommand.ExecuteScalar() is not null; + + using var transaction = db.Database.BeginTransaction(); + + db.Database.ExecuteSqlRaw($$""" + CREATE VIRTUAL TABLE IF NOT EXISTS "{{TableName}}" USING fts5( + "SValue", + content='ValueSets', + content_rowid='Id', + tokenize='trigram', + detail='none' + ); + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "ValueSets_fts_ai" + AFTER INSERT ON "ValueSets" + BEGIN + INSERT INTO "{{TableName}}"(rowid, "SValue") + VALUES (new."Id", new."SValue"); + END; + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "ValueSets_fts_ad" + AFTER DELETE ON "ValueSets" + BEGIN + INSERT INTO "{{TableName}}"("{{TableName}}", rowid, "SValue") + VALUES ('delete', old."Id", old."SValue"); + END; + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "ValueSets_fts_au" + AFTER UPDATE OF "SValue" ON "ValueSets" + BEGIN + INSERT INTO "{{TableName}}"("{{TableName}}", rowid, "SValue") + VALUES ('delete', old."Id", old."SValue"); + INSERT INTO "{{TableName}}"(rowid, "SValue") + VALUES (new."Id", new."SValue"); + END; + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE VIRTUAL TABLE IF NOT EXISTS "{{IdShortTableName}}" USING fts5( + "IdShort", + content='SMESets', + content_rowid='Id', + tokenize='trigram', + detail='none' + ); + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "SMESets_fts_ai" + AFTER INSERT ON "SMESets" + BEGIN + INSERT INTO "{{IdShortTableName}}"(rowid, "IdShort") + VALUES (new."Id", new."IdShort"); + END; + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "SMESets_fts_ad" + AFTER DELETE ON "SMESets" + BEGIN + INSERT INTO "{{IdShortTableName}}"("{{IdShortTableName}}", rowid, "IdShort") + VALUES ('delete', old."Id", old."IdShort"); + END; + """); + + db.Database.ExecuteSqlRaw($$""" + CREATE TRIGGER IF NOT EXISTS "SMESets_fts_au" + AFTER UPDATE OF "IdShort" ON "SMESets" + BEGIN + INSERT INTO "{{IdShortTableName}}"("{{IdShortTableName}}", rowid, "IdShort") + VALUES ('delete', old."Id", old."IdShort"); + INSERT INTO "{{IdShortTableName}}"(rowid, "IdShort") + VALUES (new."Id", new."IdShort"); + END; + """); + + if (!indexAlreadyExists) + { + BuildIndexWithProgress( + db, transaction, "ValueSets", TableName, "SValue", "ValueSets.SValue"); + } + + if (!idShortIndexAlreadyExists) + { + BuildIndexWithProgress( + db, transaction, "SMESets", IdShortTableName, "IdShort", "SMESets.IdShort"); + } + + transaction.Commit(); + } + catch (Exception ex) + { + throw new InvalidOperationException( + "Could not initialize the SQLite FTS5 trigram indexes. " + + "Verify that the selected SQLite library includes FTS5 and the trigram tokenizer.", + ex); + } + finally + { + if (shouldCloseConnection) + db.Database.CloseConnection(); + } + } + + private static void BuildIndexWithProgress( + DbContext db, + IDbContextTransaction transaction, + string contentTable, + string ftsTable, + string column, + string label) + { + var connection = db.Database.GetDbConnection(); + var adoTransaction = transaction.GetDbTransaction(); + + long ExecuteScalarLong(string sql, params (string Name, object Value)[] parameters) + { + using var command = connection.CreateCommand(); + command.Transaction = adoTransaction; + command.CommandText = sql; + foreach (var (name, value) in parameters) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = name; + parameter.Value = value; + command.Parameters.Add(parameter); + } + + return Convert.ToInt64(command.ExecuteScalar(), System.Globalization.CultureInfo.InvariantCulture); + } + + // MAX(Id) uses the INTEGER PRIMARY KEY b-tree and is effectively immediate, + // unlike COUNT(*) which has to visit every content-table row in SQLite. + var maximumId = ExecuteScalarLong( + $"SELECT COALESCE(MAX(\"Id\"), -1) FROM \"{contentTable}\""); + Console.WriteLine( + $"[SQLite] Building {label} trigram index up to Id {maximumId:N0} " + + $"in batches of {BuildBatchSize:N0} (progress by primary-key range)..."); + + if (maximumId < 0) + { + Console.WriteLine($"[SQLite] {label} trigram index built (database is empty)."); + return; + } + + var stopwatch = Stopwatch.StartNew(); + long processedRows = 0; + long lastId = -1; + + while (true) + { + long batchRows; + long batchLastId; + using (var boundaryCommand = connection.CreateCommand()) + { + boundaryCommand.Transaction = adoTransaction; + boundaryCommand.CommandText = $""" + SELECT COUNT(*), COALESCE(MAX("Id"), -1) + FROM ( + SELECT "Id" + FROM "{contentTable}" + WHERE "{column}" IS NOT NULL AND "Id" > $lastId + ORDER BY "Id" + LIMIT $batchSize + ) + """; + var lastIdParameter = boundaryCommand.CreateParameter(); + lastIdParameter.ParameterName = "$lastId"; + lastIdParameter.Value = lastId; + boundaryCommand.Parameters.Add(lastIdParameter); + var batchSizeParameter = boundaryCommand.CreateParameter(); + batchSizeParameter.ParameterName = "$batchSize"; + batchSizeParameter.Value = BuildBatchSize; + boundaryCommand.Parameters.Add(batchSizeParameter); + + using var reader = boundaryCommand.ExecuteReader(); + reader.Read(); + batchRows = reader.GetInt64(0); + batchLastId = reader.GetInt64(1); + } + + if (batchRows == 0) + break; + + using (var insertCommand = connection.CreateCommand()) + { + insertCommand.Transaction = adoTransaction; + insertCommand.CommandText = $""" + INSERT INTO "{ftsTable}"(rowid, "{column}") + SELECT "Id", "{column}" + FROM "{contentTable}" + WHERE "{column}" IS NOT NULL + AND "Id" > $lastId + AND "Id" <= $batchLastId + ORDER BY "Id" + """; + var lastIdParameter = insertCommand.CreateParameter(); + lastIdParameter.ParameterName = "$lastId"; + lastIdParameter.Value = lastId; + insertCommand.Parameters.Add(lastIdParameter); + var batchLastIdParameter = insertCommand.CreateParameter(); + batchLastIdParameter.ParameterName = "$batchLastId"; + batchLastIdParameter.Value = batchLastId; + insertCommand.Parameters.Add(batchLastIdParameter); + insertCommand.ExecuteNonQuery(); + } + + processedRows += batchRows; + lastId = batchLastId; + + var elapsedSeconds = Math.Max(stopwatch.Elapsed.TotalSeconds, 0.001); + var rowsPerSecond = processedRows / elapsedSeconds; + var idsPerSecond = (lastId + 1) / elapsedSeconds; + var remainingSeconds = idsPerSecond > 0 + ? Math.Max(0, maximumId - lastId) / idsPerSecond + : 0; + var percentage = maximumId > 0 + ? Math.Clamp(lastId * 100.0 / maximumId, 0, 100) + : 100; + Console.WriteLine( + $"[SQLite] {label}: {processedRows:N0} rows indexed, Id {lastId:N0} / {maximumId:N0} " + + $"(~{percentage:F1} %) - {rowsPerSecond:N0} rows/s - " + + $"elapsed {FormatDuration(stopwatch.Elapsed)} - " + + $"ETA {FormatDuration(TimeSpan.FromSeconds(remainingSeconds))}"); + } + + stopwatch.Stop(); + Console.WriteLine( + $"[SQLite] {label} trigram index built: {processedRows:N0} rows " + + $"in {FormatDuration(stopwatch.Elapsed)}."); + } + + private static string FormatDuration(TimeSpan duration) + { + var totalHours = (long)duration.TotalHours; + return $"{totalHours:D2}:{duration.Minutes:D2}:{duration.Seconds:D2}"; + } +} diff --git a/src/AasxServerDB/VisitorAASX.cs b/src/AasxServerDB/VisitorAASX.cs index ac8f2ee37..5307316a5 100644 --- a/src/AasxServerDB/VisitorAASX.cs +++ b/src/AasxServerDB/VisitorAASX.cs @@ -117,12 +117,81 @@ public static void EndBulkImport(bool analyze = false) Console.WriteLine($" SMESets: {_bulkDb.SMESets.Count()}"); Console.WriteLine($" ValueSets: {_bulkDb.ValueSets.Count()}"); Console.WriteLine($" OValueSets:{_bulkDb.OValueSets.Count()}"); + // Keep the FTS size diagnostics available, but do not run them during startup: + // counting the large FTS tables adds noticeable delay on production datasets. + // PrintTrigramIndexSizes(_bulkDb); _bulkDb.Database.ExecuteSqlRaw("PRAGMA foreign_keys = ON;"); _bulkDb.Database.ExecuteSqlRaw("PRAGMA synchronous = NORMAL;"); _bulkDb.Dispose(); _bulkDb = null; } + // Reports the on-disk size and row count of the two FTS5 trigram indexes (value SValue + + // element IdShort) next to the table row counts at startup. Best-effort: the size needs + // SQLite's dbstat vtab; if it is not available we still print the FTS row counts. Fully + // wrapped so it can never block startup. + private static void PrintTrigramIndexSizes(AasContext db) + { + if (!db.Database.IsSqlite()) + return; + try + { + var connection = db.Database.GetDbConnection(); + if (connection.State != System.Data.ConnectionState.Open) + connection.Open(); + + PrintOneTrigramIndex(connection, "ValueSets FTS (SValue)", SqliteTrigramIndex.TableName); + PrintOneTrigramIndex(connection, "SMESets FTS (IdShort)", SqliteTrigramIndex.IdShortTableName); + } + catch (Exception ex) + { + Console.WriteLine($" (FTS index sizes unavailable: {ex.Message})"); + } + } + + private static void PrintOneTrigramIndex(System.Data.Common.DbConnection connection, string label, string ftsTable) + { + long rows; + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = $"SELECT COUNT(*) FROM \"{ftsTable}\""; + rows = Convert.ToInt64(cmd.ExecuteScalar()); + } + + string sizeText; + try + { + using var cmd = connection.CreateCommand(); + // dbstat lists the FTS5 shadow tables (…_data/_idx/_docsize/…) individually; GLOB '*' + // rolls them up. GLOB treats '_' literally (LIKE would treat it as a wildcard). + cmd.CommandText = $"SELECT SUM(pgsize) FROM dbstat WHERE name GLOB '{ftsTable}*'"; + var result = cmd.ExecuteScalar(); + var bytes = result == null || result is DBNull ? 0L : Convert.ToInt64(result); + sizeText = FormatBytes(bytes); + } + catch + { + sizeText = "size n/a (dbstat off)"; + } + + Console.WriteLine($" {label,-22}: {rows,14:N0} rows, {sizeText}"); + } + + private static string FormatBytes(long bytes) + { + if (bytes <= 0) + return "0 B"; + string[] units = { "B", "KB", "MB", "GB", "TB" }; + double value = bytes; + var unit = 0; + while (value >= 1024 && unit < units.Length - 1) + { + value /= 1024; + unit++; + } + return $"{value:0.##} {units[unit]}"; + } + // Parse AASX without touching DB — for parallel producer threads public static EnvSet? ParseAASX(string filePath, bool createFilesOnly) { diff --git a/src/AasxServerStandardBib/DbRequestHandlerService.cs b/src/AasxServerStandardBib/DbRequestHandlerService.cs index b1f9b05b1..80ec435f7 100644 --- a/src/AasxServerStandardBib/DbRequestHandlerService.cs +++ b/src/AasxServerStandardBib/DbRequestHandlerService.cs @@ -1246,7 +1246,7 @@ public async Task QuerySearchSMs(ISecurityConfig securityConfig, bool w return tcs.QueryResult; } - public async Task QueryCountSMs(ISecurityConfig securityConfig, string semanticId, string identifier, string diff, IPaginationParameters paginationParameters, string expression) + public async Task QueryCountSMs(ISecurityConfig securityConfig, string semanticId, string identifier, string diff, IPaginationParameters paginationParameters, ResultType resultType, string expression) { var parameters = new DbRequestParams() { @@ -1257,6 +1257,7 @@ public async Task QueryCountSMs(ISecurityConfig securityConfig, string sema SemanticId = semanticId, Identifier = identifier, Diff = diff, + ResultType = resultType, Expression = expression } }; @@ -1343,6 +1344,45 @@ public async Task> QueryGetSMs(ISecurityConfig securityConfig, IPag return ConvertQueryItems(tcs); } + public async Task> QueryProjectSMs(ISecurityConfig securityConfig, DbProjectionRequest projectionRequest) + { + var parameters = new DbRequestParams() + { + ProjectionRequest = projectionRequest + }; + + var dbRequestContext = new DbRequestContext() + { + SecurityConfig = securityConfig, + Params = parameters + }; + var taskCompletionSource = new TaskCompletionSource(); + + var dbRequest = new DbRequest(DbRequestOp.QueryProjectSMs, DbRequestCrudType.Read, dbRequestContext, taskCompletionSource); + + _queryOperations.Add(dbRequest); + + var tcs = await taskCompletionSource.Task; + return tcs.ProjectionRows ?? []; + } + + public async Task> ReadSubmodelTemplates(ISecurityConfig securityConfig) + { + var dbRequestContext = new DbRequestContext() + { + SecurityConfig = securityConfig, + Params = new DbRequestParams() + }; + var taskCompletionSource = new TaskCompletionSource(); + + var dbRequest = new DbRequest(DbRequestOp.ReadSubmodelTemplates, DbRequestCrudType.Read, dbRequestContext, taskCompletionSource); + + _queryOperations.Add(dbRequest); + + var tcs = await taskCompletionSource.Task; + return tcs.SubmodelTemplates ?? []; + } + public async Task QueryGetSMsDebug(ISecurityConfig securityConfig, IPaginationParameters paginationParameters, ResultType resultType, string expression, bool sqlOnly = false) { var parameters = new DbRequestParams() diff --git a/src/AasxServerStandardBib/Program.cs b/src/AasxServerStandardBib/Program.cs index 6fda68abd..4ad22082d 100644 --- a/src/AasxServerStandardBib/Program.cs +++ b/src/AasxServerStandardBib/Program.cs @@ -392,7 +392,7 @@ public static void LoadAllPackages() public static Dictionary envVariables = new Dictionary(); public static bool withDb = false; - public static int startIndex = 0; + public static string startIndex = "0"; public static bool withPolicy = false; @@ -426,7 +426,7 @@ private class CommandLineArguments public int AasxInMemory { get; set; } public bool WithDb { get; set; } public bool NoDbFiles { get; set; } - public int StartIndex { get; set; } + public string StartIndex { get; set; } public bool Analyze { get; set; } #pragma warning restore 8618 // ReSharper enable UnusedAutoPropertyAccessor.Local @@ -583,8 +583,8 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv Program.htmlId = a.HtmlId; Program.withDb = a.WithDb; - if (a.StartIndex > 0) - startIndex = a.StartIndex; + if (!string.IsNullOrWhiteSpace(a.StartIndex)) + startIndex = a.StartIndex.Trim(); if (a.AasxInMemory > 0) envimax = a.AasxInMemory; if (a.SecretStringAPI != null && a.SecretStringAPI != "") @@ -772,7 +772,7 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv // Init DB if (withDb) { - persistenceService.InitDB(startIndex == 0 && !createFilesOnly, a.DataPath); + persistenceService.InitDB(startIndex == "0" && !createFilesOnly, a.DataPath, deferSearchIndexes: true); } string[] fileNames = null; @@ -780,9 +780,9 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv { var filesPath = Path.Combine(AasxHttpContextHelper.DataPath, "files"); - if (startIndex == 0) + if (startIndex == "0") { - persistenceService.InitDBFiles(startIndex == 0, AasxHttpContextHelper.DataPath); + persistenceService.InitDBFiles(true, AasxHttpContextHelper.DataPath); } var watch = System.Diagnostics.Stopwatch.StartNew(); @@ -808,23 +808,15 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv } // Phase 2: regular AASX files - var regularFiles = fileNames - .Skip(startIndex) - .Where(f => - { - var fl = f.ToLower(System.Globalization.CultureInfo.CurrentCulture); - return !fl.Contains("globalsecurity", StringComparison.InvariantCulture) && - !fl.Contains("registry", StringComparison.InvariantCulture); - }) - .ToArray(); - - // Full sorted list position (1-based) and --start-index (skip count) use fileNames order; parallel parse may complete out of order. - var fileIndexByPath = new Dictionary(StringComparer.OrdinalIgnoreCase); - for (var fileIdx = 0; fileIdx < fileNames.Length; fileIdx++) + static bool IsRegularImportFile(string file) { - fileIndexByPath[fileNames[fileIdx]] = fileIdx; + var lower = file.ToLower(System.Globalization.CultureInfo.CurrentCulture); + return !lower.Contains("globalsecurity", StringComparison.InvariantCulture) && + !lower.Contains("registry", StringComparison.InvariantCulture); } + var regularFiles = fileNames.Where(IsRegularImportFile).ToArray(); + if (withDb) { // Producer-consumer: parse in parallel, write to DB serially (one thread — same DbContext is not thread-safe). @@ -836,63 +828,121 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv // Increase (e.g. 200–500) for fewer SaveChanges rounds on large imports; uses more RAM until the next flush. const int importDbFlushEveryNPackages = 100; const int importParseQueueCapacity = 128; - var queue = new System.Collections.Concurrent.BlockingCollection(boundedCapacity: importParseQueueCapacity); + var queue = new System.Collections.Concurrent.BlockingCollection< + (int Lane, int Index, string Path, AasxServerDB.Entities.EnvSet? Env, bool LaneCompleted)>( + boundedCapacity: importParseQueueCapacity); + + var laneStarts = Enumerable.Range(0, parserThreads) + .Select(lane => lane * fileNames.Length / parserThreads).ToArray(); + var laneEnds = Enumerable.Range(0, parserThreads) + .Select(lane => (lane + 1) * fileNames.Length / parserThreads).ToArray(); + var resumeIndexes = new int[parserThreads]; + var startParts = startIndex.Split('/', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (startParts.Length == 1 && int.TryParse(startParts[0], out var globalStartIndex) && globalStartIndex >= 0) + { + for (var lane = 0; lane < parserThreads; lane++) + resumeIndexes[lane] = Math.Clamp(globalStartIndex, laneStarts[lane], laneEnds[lane]); + } + else if (startParts.Length == parserThreads) + { + for (var lane = 0; lane < parserThreads; lane++) + { + if (!int.TryParse(startParts[lane], out resumeIndexes[lane]) || + resumeIndexes[lane] < laneStarts[lane] || resumeIndexes[lane] > laneEnds[lane]) + { + Console.Error.WriteLine( + $"Invalid --start-index lane {lane + 1}: expected {laneStarts[lane]}..{laneEnds[lane]}, got '{startParts[lane]}'."); + return 1; + } + } + } + else + { + Console.Error.WriteLine( + $"Invalid --start-index '{startIndex}'. Use 0, one non-negative index, or four '/'-separated indexes."); + return 1; + } + + var regularBatchCount = Enumerable.Range(0, parserThreads).Sum(lane => + Enumerable.Range(resumeIndexes[lane], laneEnds[lane] - resumeIndexes[lane]) + .Count(index => IsRegularImportFile(fileNames[index]))); + Console.WriteLine($"Import resume: index {string.Join('/', resumeIndexes)}"); var parsedCount = 0; - var producer = Task.Run(() => + var producers = Enumerable.Range(0, parserThreads).Select(lane => Task.Run(() => { try { - Parallel.ForEach(regularFiles, - new ParallelOptions { MaxDegreeOfParallelism = parserThreads }, - f => + for (var index = resumeIndexes[lane]; index < laneEnds[lane]; index++) + { + var file = fileNames[index]; + if (!IsRegularImportFile(file)) + continue; + try { - try - { - var envDB = AasxServerDB.VisitorAASX.ParseAASX(f, createFilesOnly); - if (envDB != null) - { - queue.Add(envDB); - System.Threading.Interlocked.Increment(ref parsedCount); - } - } - catch (Exception ex) - { - Console.Error.WriteLine($"Error parsing {f}: {ex.Message} — skipping."); - } - }); + var envDB = AasxServerDB.VisitorAASX.ParseAASX(file, createFilesOnly); + queue.Add((lane, index, file, envDB, false)); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Error parsing {file}: {ex.Message} — skipping."); + queue.Add((lane, index, file, null, false)); + } + finally + { + System.Threading.Interlocked.Increment(ref parsedCount); + } + } } finally { - queue.CompleteAdding(); + queue.Add((lane, laneEnds[lane], string.Empty, null, true)); } + })).ToArray(); + var producer = Task.Run(() => + { + try { Task.WaitAll(producers); } + finally { queue.CompleteAdding(); } }); int count = 0; - var maxResumeStartIndex = startIndex; - foreach (var envDB in queue.GetConsumingEnumerable()) + var sinceLastFlush = 0; + var pendingResumeIndexes = resumeIndexes.ToArray(); + var committedResumeIndexes = resumeIndexes.ToArray(); + + void FlushAndReport() { - count++; - var aasxName = string.IsNullOrEmpty(envDB.Path) ? "?" : Path.GetFileName(envDB.Path); - var idx = string.IsNullOrEmpty(envDB.Path) ? -1 : fileIndexByPath.GetValueOrDefault(envDB.Path, -1); - if (idx >= 0) - { - maxResumeStartIndex = Math.Max(maxResumeStartIndex, idx + 1); - } + persistenceService.FlushBulkImport(); + Array.Copy(pendingResumeIndexes, committedResumeIndexes, parserThreads); + sinceLastFlush = 0; + Console.WriteLine( + $"DB Flush committed: written={count} (regular batch {regularBatchCount}) " + + $"index {string.Join('/', committedResumeIndexes)} parsed={parsedCount} " + + $"({watch.ElapsedMilliseconds / 1000}s)"); + } - // 1-based position in sorted directory; --start-index after this file finishes = (idx+1) — same as 0-based index of next file. - Console.WriteLine(idx >= 0 - ? $"Import to DB ({idx + 1}/{fileNames.Length}): {aasxName}" - : $"Import to DB (?/{fileNames.Length}): {aasxName}"); - AasxServerDB.VisitorAASX.AddEnvSetToBulk(envDB); - if (count % importDbFlushEveryNPackages == 0) + foreach (var item in queue.GetConsumingEnumerable()) + { + if (item.LaneCompleted) { - Console.WriteLine( - $"DB Flush: written={count} (regular batch {regularFiles.Length}) max --start-index={maxResumeStartIndex}/{fileNames.Length} parsed={parsedCount} ({watch.ElapsedMilliseconds / 1000}s)"); - persistenceService.FlushBulkImport(); + pendingResumeIndexes[item.Lane] = laneEnds[item.Lane]; + continue; } + + pendingResumeIndexes[item.Lane] = item.Index + 1; + if (item.Env == null) + continue; + + count++; + sinceLastFlush++; + Console.WriteLine( + $"Import to DB ({item.Index + 1}/{fileNames.Length}): {Path.GetFileName(item.Path)}"); + AasxServerDB.VisitorAASX.AddEnvSetToBulk(item.Env); + if (sinceLastFlush >= importDbFlushEveryNPackages) + FlushAndReport(); } producer.Wait(); + FlushAndReport(); } else { @@ -952,6 +1002,8 @@ private static async Task Run(CommandLineArguments a, IServiceProvider serv if (withDb) { persistenceService.EndBulkImport(a.Analyze); + Console.WriteLine("Building deferred SQLite substring-search indexes..."); + persistenceService.InitializeSearchIndexes(); // preload AASX from DB and keep in memory var packages = new List(); @@ -1280,9 +1332,9 @@ public static void Main(string[] args, IServiceProvider serviceProvider) new Option( new[] {"--no-db-files"}, "If set, do not export files from AASX into ZIP"), - new Option( + new Option( new[] {"--start-index"}, - "If set, start index in list of AASX files"), + "Start index: 0 for full import, one global index, or four '/'-separated resume indexes"), new Option( new[] {"--analyze"}, "If set, run ANALYZE after bulk import to update query planner statistics") diff --git a/src/Contracts/DbRequests/DbProjectionRequest.cs b/src/Contracts/DbRequests/DbProjectionRequest.cs new file mode 100644 index 000000000..1bd76da59 --- /dev/null +++ b/src/Contracts/DbRequests/DbProjectionRequest.cs @@ -0,0 +1,86 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace Contracts.DbRequests; + +using System; +using System.Collections.Generic; + +/// +/// Batch projection of explicit idShortPaths over a set of result submodels. +/// The values are read directly from SMSets/SMRefSets/SMESets/ValueSets without +/// materializing complete submodel objects (fast path for tabular exports). +/// +public class DbProjectionRequest +{ + /// Identifiers of the result submodels (one output row per entry, order preserved). + public List SubmodelIdentifiers { get; set; } = new(); + + /// Projected columns; every entry must be an explicit full idShortPath. + public List Paths { get; set; } = new(); +} + +/// One projected column. +public class DbProjectionPath +{ + /// Select entry exactly as given by the caller; used as cell key in the result rows. + public string RawPath { get; set; } = string.Empty; + + /// + /// IdShort of a sibling submodel of the same AAS (cross-submodel path "/SubmodelIdShort/idShortPath"); + /// null means the path refers to the result submodel itself. + /// + public string? TargetSubmodelIdShort { get; set; } + + /// Full dotted idShortPath inside the target submodel (matches SMESets.IdShortPath). + public string ElementIdShortPath { get; set; } = string.Empty; +} + +/// One ValueSets row of a projected element (MLP elements have one row per language). +public class DbProjectionValue +{ + public string? SValue { get; set; } + public double? NValue { get; set; } + + /// For MultiLanguageProperty rows this holds the language code. + public string? Annotation { get; set; } +} + +/// Projected cell: the SMESets row matched by the exact idShortPath plus its values. +public class DbProjectionCell +{ + /// True when an element with the exact idShortPath exists in the target submodel. + public bool Found { get; set; } + + /// SMESets.SMEType (may carry an operation prefix like "In-"). + public string? SmeType { get; set; } + + /// SMESets.TValue: "S" (string) or "D" (double); null when the element stores no value. + public string? TValue { get; set; } + + public List Values { get; set; } = new(); + + /// + /// Identifier of the submodel the element was read from + /// (differs from the row submodel for cross-submodel paths). + /// + public string? SourceSubmodelIdentifier { get; set; } +} + +/// One output row: the result submodel plus one cell per projected path (keyed by RawPath). +public class DbProjectionRow +{ + public string SubmodelIdentifier { get; set; } = string.Empty; + + public Dictionary Cells { get; set; } = new(StringComparer.Ordinal); +} diff --git a/src/Contracts/DbRequests/DbRequestOp.cs b/src/Contracts/DbRequests/DbRequestOp.cs index 8aa5b44cc..4e52d183a 100644 --- a/src/Contracts/DbRequests/DbRequestOp.cs +++ b/src/Contracts/DbRequests/DbRequestOp.cs @@ -71,6 +71,8 @@ public enum DbRequestOp QuerySearchSMEs, QueryGetSMs, + QueryProjectSMs, + ReadSubmodelTemplates, DeleteAASXByPackageId, ReadAASXByPackageId, diff --git a/src/Contracts/DbRequests/DbRequestParams.cs b/src/Contracts/DbRequests/DbRequestParams.cs index 287de0b1e..3f618bd09 100644 --- a/src/Contracts/DbRequests/DbRequestParams.cs +++ b/src/Contracts/DbRequests/DbRequestParams.cs @@ -66,5 +66,7 @@ public class DbRequestParams public DbEventRequest EventRequest { get; set; } public DbQueryRequest QueryRequest { get; set; } + + public DbProjectionRequest ProjectionRequest { get; set; } } diff --git a/src/Contracts/DbRequests/DbRequestResult.cs b/src/Contracts/DbRequests/DbRequestResult.cs index bc55788a2..6e5ace47b 100644 --- a/src/Contracts/DbRequests/DbRequestResult.cs +++ b/src/Contracts/DbRequests/DbRequestResult.cs @@ -48,6 +48,10 @@ public class DbRequestResult public QResult QueryResult { get; set; } public List RawSql { get; set; } + public List ProjectionRows { get; set; } + + public List SubmodelTemplates { get; set; } + public int Count { get; set; } } diff --git a/src/Contracts/DbRequests/DbSubmodelTemplateRow.cs b/src/Contracts/DbRequests/DbSubmodelTemplateRow.cs new file mode 100644 index 000000000..be569ecc2 --- /dev/null +++ b/src/Contracts/DbRequests/DbSubmodelTemplateRow.cs @@ -0,0 +1,29 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace Contracts.DbRequests; + +/// +/// One distinct submodel template of the repository: a (IdShort, SemanticId) group over +/// SMSets with its occurrence count. Deterministic inventory of which submodel types +/// exist, independent of any sampling. +/// +public class DbSubmodelTemplateRow +{ + public string? IdShort { get; set; } + + public string? SemanticId { get; set; } + + /// Number of submodels with this exact IdShort+SemanticId combination. + public int Count { get; set; } +} diff --git a/src/Contracts/DiagnosticsLog.cs b/src/Contracts/DiagnosticsLog.cs new file mode 100644 index 000000000..dbacf566d --- /dev/null +++ b/src/Contracts/DiagnosticsLog.cs @@ -0,0 +1,149 @@ +namespace Contracts; + +using System; +using System.Globalization; +using System.Linq; +using System.Threading; + +public static class DiagnosticsLog +{ + private static readonly AsyncLocal McpScopeDepth = new(); + private static readonly AsyncLocal BrowserLogSession = new(); + private static int GlobalMcpScopeDepth; + + public static bool QueryEnabled => + EnvEnabled("AAS_QUERY_LOG") || + EnvEnabled("AAS_QUERY_DIAG"); + + public static bool McpEnabled => + EnvEnabled("AAS_MCP_LOG"); + + public static IDisposable BeginMcpScope() + { + McpScopeDepth.Value++; + Interlocked.Increment(ref GlobalMcpScopeDepth); + return new Scope(() => + { + McpScopeDepth.Value = Math.Max(0, McpScopeDepth.Value - 1); + if (Interlocked.Decrement(ref GlobalMcpScopeDepth) < 0) + { + Volatile.Write(ref GlobalMcpScopeDepth, 0); + } + }); + } + + public static IDisposable BeginBrowserLogSession(string sessionId) + { + if (!McpSessionLogStore.IsValidSessionId(sessionId)) + { + return new Scope(() => { }); + } + + var previous = BrowserLogSession.Value; + BrowserLogSession.Value = sessionId; + return new Scope(() => BrowserLogSession.Value = previous); + } + + public static void WriteMcp(string message, bool timestamp = false) + { + if (!McpEnabled) + { + return; + } + + if (message.Length == 0) + { + Console.WriteLine(); + WriteBrowserLine(string.Empty); + return; + } + + var prefix = timestamp + ? $"[MCP {Timestamp()}] " + : "[MCP] "; + var line = prefix + message; + Console.WriteLine(line); + WriteBrowserLine(line); + } + + public static void WriteQuery(string message, bool timestamp = false) + { + if (!QueryEnabled) + { + return; + } + + var inMcpScope = McpEnabled && + (McpScopeDepth.Value > 0 || Volatile.Read(ref GlobalMcpScopeDepth) > 0); + if (inMcpScope) + { + message = message.TrimStart('\r', '\n'); + } + + var indent = inMcpScope ? " " : string.Empty; + var firstNonEmpty = true; + foreach (var line in SplitLines(message)) + { + if (line.Length == 0) + { + Console.WriteLine(); + WriteBrowserLine(string.Empty); + continue; + } + + var time = timestamp && firstNonEmpty + ? $"[{Timestamp()}] " + : string.Empty; + var output = indent + time + line; + Console.WriteLine(output); + WriteBrowserLine(output); + firstNonEmpty = false; + } + } + + private static void WriteBrowserLine(string line) + { + var sessionId = BrowserLogSession.Value; + if (McpSessionLogStore.IsValidSessionId(sessionId)) + { + McpSessionLogStore.Append(sessionId!, line); + } + } + + private static bool EnvEnabled(string name) + { + var value = Environment.GetEnvironmentVariable(name); + return value != null && + (string.Equals(value, "1", StringComparison.OrdinalIgnoreCase) || + string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) || + string.Equals(value, "yes", StringComparison.OrdinalIgnoreCase) || + string.Equals(value, "on", StringComparison.OrdinalIgnoreCase)); + } + + private static string[] SplitLines(string message) => + message.Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n') + .Split('\n'); + + private static string Timestamp() => + DateTime.Now.ToString("HH:mm:ss.fff", CultureInfo.InvariantCulture); + + private sealed class Scope : IDisposable + { + private readonly Action _dispose; + private bool _disposed; + + public Scope(Action dispose) => _dispose = dispose; + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _dispose(); + } + } +} diff --git a/src/Contracts/IDbRequestHandlerService.cs b/src/Contracts/IDbRequestHandlerService.cs index 2baa34e9d..d67555891 100644 --- a/src/Contracts/IDbRequestHandlerService.cs +++ b/src/Contracts/IDbRequestHandlerService.cs @@ -87,10 +87,12 @@ public interface IDbRequestHandlerService Task GenerateSerializationByIds(ISecurityConfig securityConfig, List aasIds = null, List submodelIds = null, bool includeCD = false, bool createAASXPackage = false); Task QuerySearchSMs(ISecurityConfig securityConfig, bool withTotalCount, bool withLastId, string semanticId, string identifier, string diff, IPaginationParameters paginationParameters, string expression); - Task QueryCountSMs(ISecurityConfig securityConfig, string semanticId, string identifier, string diff, IPaginationParameters paginationParameters, string expression); + Task QueryCountSMs(ISecurityConfig securityConfig, string semanticId, string identifier, string diff, IPaginationParameters paginationParameters, ResultType resultType, string expression); Task QuerySearchSMEs(ISecurityConfig securityConfig, string requested, bool withTotalCount, bool withLastId, string smSemanticId, string smIdentifier, string semanticId, string diff, string contains, string equal, string lower, string upper, IPaginationParameters paginationParameters, string expression); Task> QueryGetSMs(ISecurityConfig securityConfig, IPaginationParameters paginationParameters, ResultType resultType, string expression); + Task> QueryProjectSMs(ISecurityConfig securityConfig, DbRequests.DbProjectionRequest projectionRequest); + Task> ReadSubmodelTemplates(ISecurityConfig securityConfig); Task QueryGetSMsDebug(ISecurityConfig securityConfig, IPaginationParameters paginationParameters, ResultType resultType, string expression, bool sqlOnly = false); Task DeleteAASXByPackageId(ISecurityConfig securityConfig, string packageId); diff --git a/src/Contracts/IPersistenceService.cs b/src/Contracts/IPersistenceService.cs index f8f7003c6..df663659c 100644 --- a/src/Contracts/IPersistenceService.cs +++ b/src/Contracts/IPersistenceService.cs @@ -29,7 +29,9 @@ public class Running public interface IPersistenceService { - void InitDB(bool reloadDB, string dataPath); + void InitDB(bool reloadDB, string dataPath, bool deferSearchIndexes = false); + + void InitializeSearchIndexes(); void InitDBFiles(bool reloadDBFiles, string dataPath); diff --git a/src/Contracts/McpSessionLogStore.cs b/src/Contracts/McpSessionLogStore.cs new file mode 100644 index 000000000..3c7155841 --- /dev/null +++ b/src/Contracts/McpSessionLogStore.cs @@ -0,0 +1,120 @@ +namespace Contracts; + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; + +public sealed record McpSessionLogEntry(long Sequence, DateTimeOffset Timestamp, string Line); + +public static class McpSessionLogStore +{ + private const int MaxSessions = 64; + private const int MaxEntriesPerSession = 2000; + private const int MaxLineLength = 8000; + + private static readonly ConcurrentDictionary Sessions = new(StringComparer.Ordinal); + private static long NextSequence; + + public static bool IsValidSessionId(string? sessionId) + { + if (string.IsNullOrWhiteSpace(sessionId) || sessionId.Length > 128) + { + return false; + } + + foreach (var c in sessionId) + { + if (!(char.IsAsciiLetterOrDigit(c) || c is '-' or '_' or '.' or ':')) + { + return false; + } + } + + return true; + } + + public static void Append(string sessionId, string line) + { + if (!IsValidSessionId(sessionId)) + { + return; + } + + TrimSessionCount(); + + var buffer = Sessions.GetOrAdd(sessionId, _ => new SessionBuffer()); + var trimmedLine = line.Length <= MaxLineLength ? line : line[..MaxLineLength] + " ..."; + buffer.Append(new McpSessionLogEntry( + System.Threading.Interlocked.Increment(ref NextSequence), + DateTimeOffset.Now, + trimmedLine)); + } + + public static IReadOnlyList Read(string sessionId, long afterSequence = 0, int limit = 500) + { + if (!IsValidSessionId(sessionId) || !Sessions.TryGetValue(sessionId, out var buffer)) + { + return Array.Empty(); + } + + return buffer.Read(afterSequence, Math.Clamp(limit, 1, 2000)); + } + + public static void Clear(string sessionId) + { + if (IsValidSessionId(sessionId)) + { + Sessions.TryRemove(sessionId, out _); + } + } + + private static void TrimSessionCount() + { + if (Sessions.Count < MaxSessions) + { + return; + } + + foreach (var key in Sessions + .OrderBy(kv => kv.Value.LastWrite) + .Take(Math.Max(1, Sessions.Count - MaxSessions + 1)) + .Select(kv => kv.Key)) + { + Sessions.TryRemove(key, out _); + } + } + + private sealed class SessionBuffer + { + private readonly Queue _entries = new(); + private readonly object _gate = new(); + + public DateTimeOffset LastWrite { get; private set; } = DateTimeOffset.Now; + + public void Append(McpSessionLogEntry entry) + { + lock (_gate) + { + _entries.Enqueue(entry); + LastWrite = entry.Timestamp; + + while (_entries.Count > MaxEntriesPerSession) + { + _entries.Dequeue(); + } + } + } + + public IReadOnlyList Read(long afterSequence, int limit) + { + lock (_gate) + { + return _entries + .Where(entry => entry.Sequence > afterSequence) + .Take(limit) + .ToArray(); + } + } + } +} diff --git a/src/Contracts/QueryParserJSON.cs b/src/Contracts/QueryParserJSON.cs index 7edb9c6e9..77b11656a 100644 --- a/src/Contracts/QueryParserJSON.cs +++ b/src/Contracts/QueryParserJSON.cs @@ -50,7 +50,9 @@ public QueryGrammarJSON(IContractSecurityRules contractSecurityRules) : base(cas // Define terminals var stringLiteral = new StringLiteral("StringLiteral", "\"", StringOptions.AllowsAllEscapes); - var numberLiteral = new NumberLiteral("NumberLiteral"); + // AllowSign: JSON-Zahlen dürfen negativ sein ($numVal: -40, z.B. Minusgrade bei + // Umgebungstemperaturen); ohne die Option scheitert der Parser an führendem '-'. + var numberLiteral = new NumberLiteral("NumberLiteral", NumberOptions.AllowSign); var booleanLiteral = new RegexBasedTerminal("BooleanLiteral", "true|false"); // Define non-terminals: other Literals @@ -1499,15 +1501,13 @@ private static string BuildOverallSql(LogicalExpression le, SqlBuildContext ctx) case "$and": { - if (TryBuildDirectValueExistsPredicate(eList, type, out var existsPredicate)) - { - if (existsPredicate == SqlBoolTrue || existsPredicate == SqlBoolFalse) - return existsPredicate; - return AddExistsCondition(ctx, existsPredicate); - } - + // AND combines submodel-level facts. Each $sme#value comparison is its OWN existential + // ("∃ an element with this value"), so several bare sme conditions become + // EXISTS(A) AND EXISTS(B) — they may match DIFFERENT elements. Building one combined + // EXISTS(A AND B) would require a SINGLE element to satisfy all (wrong for "manufacturer X + // AND type Y") and is slow. Same-element correlation is expressed via idShortPath / path + // subqueries (handled separately). OR keeps a single combined EXISTS(A OR B), equivalent there. var parts = new List(); - var valueExistsPredicates = new List(); foreach (var e in eList) { if (TryBuildDirectValueExistsPredicate(e, out var valueExistsPredicate)) @@ -1515,7 +1515,7 @@ private static string BuildOverallSql(LogicalExpression le, SqlBuildContext ctx) if (valueExistsPredicate == SqlBoolFalse) return SqlBoolFalse; if (valueExistsPredicate != SqlBoolTrue) - valueExistsPredicates.Add(valueExistsPredicate); + parts.Add(AddExistsCondition(ctx, valueExistsPredicate)); // separate EXISTS per condition continue; } @@ -1529,14 +1529,6 @@ private static string BuildOverallSql(LogicalExpression le, SqlBuildContext ctx) parts.Add(part); } - if (valueExistsPredicates.Count > 0) - { - var valueExistsPredicate = valueExistsPredicates.Count == 1 - ? valueExistsPredicates[0] - : "(" + string.Join(" AND ", valueExistsPredicates) + ")"; - parts.Add(AddExistsCondition(ctx, valueExistsPredicate)); - } - if (parts.Count == 0) return "$SKIP"; parts = FoldBooleanAndParts(parts); @@ -1601,7 +1593,22 @@ private static string BuildOverallSql(LogicalExpression le, SqlBuildContext ctx) case "$eq": case "$ne": case "$gt": case "$ge": case "$lt": case "$le": case "$starts-with": case "$ends-with": case "$contains": + { + // A bare $sme#value comparison becomes its own existential (EXISTS over the SM's + // elements) instead of a ValueSets join, so the value-match builder + its + // common-contains→EXISTS probe handle it uniformly — including the common + // leading-wildcard case that a value join would materialize (slow). sm/aas/path + // comparisons (TryBuild… returns false for them) keep the direct comparison SQL. + if (TryBuildDirectValueExistsPredicate( + new LogicalExpression { ExpressionType = type, ExpressionValue = eList }, + out var directValuePredicate)) + { + if (directValuePredicate == SqlBoolTrue || directValuePredicate == SqlBoolFalse) + return directValuePredicate; + return AddExistsCondition(ctx, directValuePredicate); + } return BuildOverallComparisonSql(eList, type, ctx); + } } } @@ -1877,6 +1884,12 @@ private static bool TryBuildPathSubquerySql( private static string BuildIdShortPathSql(string smeAlias, string idShortPath) { + // Leading %. means a leaf at zero or more parent levels. Combining the + // top-level and nested variants avoids two otherwise identical path joins. + if (idShortPath.StartsWith("%.", StringComparison.Ordinal)) + { + return "1=1"; + } if (idShortPath.Contains("[]")) { var pattern = idShortPath.Replace("[]", "[[]*[]]"); diff --git a/src/Contracts/SqlConditions.cs b/src/Contracts/SqlConditions.cs index c433e486a..11eae82a6 100644 --- a/src/Contracts/SqlConditions.cs +++ b/src/Contracts/SqlConditions.cs @@ -250,6 +250,13 @@ public class ExistsCondition /// Predicate SQL inside the correlated EXISTS query. public string PredicateSql { get; set; } = ""; + + /// + /// Set at SQL-build time when a capped FTS probe found the (leading-wildcard contains) predicate to be + /// non-selective ("common"): realize it as a correlated EXISTS (early-stop under ORDER BY/LIMIT) instead + /// of the value-driven IN/FTS, which would materialize the huge match set. Default false. + /// + public bool PreferExists { get; set; } } /// diff --git a/src/IO.Swagger.Lib.V3.Tests/MCP/McpExportTests.cs b/src/IO.Swagger.Lib.V3.Tests/MCP/McpExportTests.cs new file mode 100644 index 000000000..a896027d7 --- /dev/null +++ b/src/IO.Swagger.Lib.V3.Tests/MCP/McpExportTests.cs @@ -0,0 +1,89 @@ +namespace IO.Swagger.Lib.V3.Tests.MCP; + +using IO.Swagger.Lib.V3.MCP; +using ModelContextProtocol.Protocol; +using System.IO.Compression; +using System.Text; +using System.Text.Json.Nodes; + +public class McpExportTests +{ + [Fact] + public void XlsxBuilder_ProducesWorkbookWithHeaderStringsAndNumbers() + { + var rows = new[] + { + new JsonObject + { + ["id"] = "submodel-1", + ["GeneralInformation.ManufacturerArticleNumber"] = "4711 <&> \"x\"", + ["TechnicalProperties.flowMax"] = 80.5, + }, + }; + + var bytes = XlsxBuilder.Build(new[] { "id", "GeneralInformation.ManufacturerArticleNumber", "TechnicalProperties.flowMax" }, rows); + + using var zip = new ZipArchive(new MemoryStream(bytes), ZipArchiveMode.Read); + Assert.NotNull(zip.GetEntry("[Content_Types].xml")); + Assert.NotNull(zip.GetEntry("xl/workbook.xml")); + + var sheetEntry = zip.GetEntry("xl/worksheets/sheet1.xml"); + Assert.NotNull(sheetEntry); + using var reader = new StreamReader(sheetEntry!.Open(), Encoding.UTF8); + var sheet = reader.ReadToEnd(); + + Assert.Contains("GeneralInformation.ManufacturerArticleNumber", sheet); + Assert.Contains("4711 <&> "x"", sheet); // XML-escaped + Assert.Contains("80.5", sheet); // numerische Zelle + Assert.Contains("t=\"inlineStr\"", sheet); + } + + [Fact] + public void XlsxBuilder_CellRefUsesExcelColumnLetters() + { + Assert.Equal("A1", XlsxBuilder.CellRef(0, 1)); + Assert.Equal("Z2", XlsxBuilder.CellRef(25, 2)); + Assert.Equal("AA3", XlsxBuilder.CellRef(26, 3)); + } + + [Fact] + public void McpExportFileStore_RoundtripsContentAndRejectsUnknownToken() + { + var content = Encoding.UTF8.GetBytes("id;value\r\nsm1;42\r\n"); + var token = McpExportFileStore.Add(content, "export.csv", "text/csv"); + + Assert.True(McpExportFileStore.TryGet(token, out var stored, out var fileName, out var mimeType)); + Assert.Equal(content, stored); + Assert.Equal("export.csv", fileName); + Assert.Equal("text/csv", mimeType); + + Assert.False(McpExportFileStore.TryGet("does-not-exist", out _, out _, out _)); + Assert.False(McpExportFileStore.TryGet(null, out _, out _, out _)); + } + + [Fact] + public void McpExportResources_GetExportFile_ReturnsBase64EncodedBlob() + { + // Binärinhalt mit ungültigen UTF-8-Sequenzen: würde der Blob nicht base64-kodiert, + // käme er auf dem Draht als zerstörter String an (U+FFFD-Ersatzzeichen). + var content = new byte[] { 0x50, 0x4B, 0x03, 0x04, 0xFF, 0x00, 0x10 }; + var token = McpExportFileStore.Add(content, "x.xlsx", XlsxBuilder.MimeType); + + var result = Assert.IsType(McpExportResources.GetExportFile(token)); + + Assert.Equal("aas-export://" + token, result.Uri); + Assert.Equal(XlsxBuilder.MimeType, result.MimeType); + Assert.Equal(content, result.DecodedData.ToArray()); + Assert.Equal(Convert.ToBase64String(content), Encoding.UTF8.GetString(result.Blob.ToArray())); + } + + [Fact] + public void NormalizeExportFileName_AppendsExtensionAndStripsPathSeparators() + { + Assert.Equal("aas_query_export.xlsx", McpQueryTools.NormalizeExportFileName(null, ".xlsx", "aas_query_export.xlsx")); + Assert.Equal("pumpen.xlsx", McpQueryTools.NormalizeExportFileName("pumpen", ".xlsx", "aas_query_export.xlsx")); + Assert.Equal("Pumpen.XLSX", McpQueryTools.NormalizeExportFileName("Pumpen.XLSX", ".xlsx", "aas_query_export.xlsx")); + Assert.Equal("a_b.csv", McpQueryTools.NormalizeExportFileName("a/b.csv", ".csv", "aas_query_export.csv")); + Assert.Equal("__secret.csv", McpQueryTools.NormalizeExportFileName("..\\secret.csv", ".csv", "aas_query_export.csv")); + } +} diff --git a/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsExportToolTests.cs b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsExportToolTests.cs new file mode 100644 index 000000000..f51400e1a --- /dev/null +++ b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsExportToolTests.cs @@ -0,0 +1,172 @@ +namespace IO.Swagger.Lib.V3.Tests.MCP; + +using AasCore.Aas3_1; +using Contracts; +using Contracts.DbRequests; +using Contracts.Pagination; +using Contracts.QueryResult; +using Contracts.Security; +using IO.Swagger.Lib.V3.MCP; +using IO.Swagger.Lib.V3.SerializationModifiers.Mappers; +using Moq; +using System.Text.Json; +using System.Text.Json.Nodes; + +/// +/// End-to-End-Tests der Export-Tools gegen einen gemockten : +/// Fast Path (SQL-Batchprojektion) für volle Pfade, Objektpfad-Fallback für Blattnamen +/// und die Datei-Metadaten der Antwort (downloadUrl/resourceUri/contentBase64). +/// Die Tests laufen sequenziell in einer Klasse, weil sie Program.noSecurity umschalten. +/// +public class McpQueryToolsExportToolTests +{ + private static JsonNode ToJson(object result) + => JsonNode.Parse(JsonSerializer.Serialize(result))!; + + private static McpQueryTools CreateTools( + out Mock dbMock, List queryIds, List? projectionRows = null) + { + dbMock = new Mock(MockBehavior.Loose); + dbMock + .Setup(m => m.QueryGetSMs(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(queryIds.Cast().ToList()); + if (projectionRows != null) + { + dbMock + .Setup(m => m.QueryProjectSMs(It.IsAny(), It.IsAny())) + .ReturnsAsync(projectionRows); + } + + return new McpQueryTools(dbMock.Object, new Mock().Object); + } + + private static McpQueryCondition[] AnyCondition() + => new[] { new McpQueryCondition { Scope = "sm", Field = "idShort", Op = "eq", Value = "TechnicalData" } }; + + [Fact] + public async Task ExportXlsx_UsesFastPathAndReturnsFileMetadata() + { + var previousNoSecurity = AasxServer.Program.noSecurity; + AasxServer.Program.noSecurity = true; + try + { + const string path = "GeneralInformation.ManufacturerArticleNumber"; + var projection = new List + { + new() + { + SubmodelIdentifier = "sm1", + Cells = + { + [path] = new DbProjectionCell + { + Found = true, + SmeType = "Prop", + TValue = "S", + Values = { new DbProjectionValue { SValue = "4711" } }, + SourceSubmodelIdentifier = "sm1", + }, + }, + }, + }; + var tools = CreateTools(out var dbMock, new List { "sm1" }, projection); + + var result = ToJson(await tools.AasQueryExportXlsx("submodels", AnyCondition(), new[] { path })); + + Assert.Equal(1, result["count"]!.GetValue()); + Assert.EndsWith(".xlsx", result["fileName"]!.GetValue()); + Assert.Equal(XlsxBuilder.MimeType, result["mimeType"]!.GetValue()); + Assert.Contains("/mcp-exports/", result["downloadUrl"]!.GetValue()); + Assert.StartsWith("aas-export://", result["resourceUri"]!.GetValue()); + Assert.False(result["hasMore"]!.GetValue()); + + // Kleine Datei -> contentBase64 inline; muss der gespeicherten Datei entsprechen. + var contentBase64 = result["contentBase64"]!.GetValue(); + var token = result["resourceUri"]!.GetValue()["aas-export://".Length..]; + Assert.True(McpExportFileStore.TryGet(token, out var stored, out _, out _)); + Assert.Equal(Convert.ToBase64String(stored), contentBase64); + + // Fast Path: Batchprojektion statt Submodel-Reads. + dbMock.Verify(m => m.QueryProjectSMs(It.IsAny(), It.IsAny()), Times.Once); + dbMock.Verify(m => m.ReadSubmodelById(It.IsAny(), It.IsAny(), It.IsAny(), null, null), Times.Never); + dbMock.Verify(m => m.ReadSubmodelElementByPath(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), null, null), Times.Never); + } + finally + { + AasxServer.Program.noSecurity = previousNoSecurity; + } + } + + [Fact] + public async Task ExportCsv_FallsBackToObjectPathForLeafNames() + { + var previousNoSecurity = AasxServer.Program.noSecurity; + AasxServer.Program.noSecurity = true; + try + { + var submodel = new Submodel( + "sm1", + submodelElements: new List + { + new Property(DataTypeDefXsd.String, idShort: "flowMax", value: "80"), + }); + + var tools = CreateTools(out var dbMock, new List { "sm1" }); + dbMock + .Setup(m => m.ReadSubmodelById(It.IsAny(), null, "sm1", null, null)) + .ReturnsAsync(submodel); + + // Blattname ohne Punkt -> kein Fast Path, Ranking-Suche über das geladene Submodel. + var result = ToJson(await tools.AasQueryExportCsv("submodels", AnyCondition(), new[] { "flowMax" })); + + Assert.Equal(1, result["count"]!.GetValue()); + Assert.Contains("flowMax", result["columns"]!.AsArray().Select(c => c!.GetValue())); + Assert.Contains("80", result["content"]!.GetValue()); + Assert.Contains("/mcp-exports/", result["downloadUrl"]!.GetValue()); + + dbMock.Verify(m => m.QueryProjectSMs(It.IsAny(), It.IsAny()), Times.Never); + dbMock.Verify(m => m.ReadSubmodelById(It.IsAny(), null, "sm1", null, null), Times.Once); + } + finally + { + AasxServer.Program.noSecurity = previousNoSecurity; + } + } + + [Fact] + public async Task ExportXlsx_WithSecurityEnabledUsesObjectPath() + { + var previousNoSecurity = AasxServer.Program.noSecurity; + AasxServer.Program.noSecurity = false; + try + { + const string path = "GeneralInformation.ManufacturerArticleNumber"; + var tools = CreateTools(out var dbMock, new List { "sm1" }); + dbMock + .Setup(m => m.ReadSubmodelElementByPath(It.IsAny(), null, "sm1", path, null, null)) + .ReturnsAsync(new Property(DataTypeDefXsd.String, idShort: "ManufacturerArticleNumber", value: "4711")); + + var result = ToJson(await tools.AasQueryExportXlsx("submodels", AnyCondition(), new[] { path })); + + Assert.Equal(1, result["count"]!.GetValue()); + + // Mit aktiver Security darf die SQL-Batchprojektion nicht verwendet werden. + dbMock.Verify(m => m.QueryProjectSMs(It.IsAny(), It.IsAny()), Times.Never); + dbMock.Verify(m => m.ReadSubmodelElementByPath(It.IsAny(), null, "sm1", path, null, null), Times.Once); + } + finally + { + AasxServer.Program.noSecurity = previousNoSecurity; + } + } + + [Fact] + public async Task ExportCsv_RequiresSelect() + { + var tools = CreateTools(out _, new List()); + + var result = ToJson(await tools.AasQueryExportCsv("submodels", AnyCondition(), Array.Empty())); + + Assert.NotNull(result["error"]); + } +} diff --git a/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsFastPathTests.cs b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsFastPathTests.cs new file mode 100644 index 000000000..0c5ddd066 --- /dev/null +++ b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsFastPathTests.cs @@ -0,0 +1,179 @@ +namespace IO.Swagger.Lib.V3.Tests.MCP; + +using Contracts.DbRequests; +using IO.Swagger.Lib.V3.MCP; +using System.Text.Json.Nodes; + +public class McpQueryToolsFastPathTests +{ + [Fact] + public void TryPlanFastProjection_AcceptsFullDottedPaths() + { + var planned = McpQueryTools.TryPlanFastProjection( + new[] { "GeneralInformation.ManufacturerArticleNumber", "TechnicalProperties.Manufacturer.Product_type" }, + out var plan); + + Assert.True(planned); + Assert.Equal(2, plan.Count); + Assert.Null(plan[0].TargetSubmodelIdShort); + Assert.Equal("GeneralInformation.ManufacturerArticleNumber", plan[0].ElementIdShortPath); + Assert.Equal("GeneralInformation.ManufacturerArticleNumber", plan[0].RawPath); + } + + [Fact] + public void TryPlanFastProjection_AcceptsCrossSubmodelPathsWithDot() + { + var planned = McpQueryTools.TryPlanFastProjection( + new[] { "/TechnicalData/GeneralInformation.ManufacturerArticleNumber" }, + out var plan); + + Assert.True(planned); + var path = Assert.Single(plan); + Assert.Equal("TechnicalData", path.TargetSubmodelIdShort); + Assert.Equal("GeneralInformation.ManufacturerArticleNumber", path.ElementIdShortPath); + Assert.Equal("/TechnicalData/GeneralInformation.ManufacturerArticleNumber", path.RawPath); + } + + [Fact] + public void TryPlanFastProjection_AcceptsCrossSubmodelLeafPaths() + { + var planned = McpQueryTools.TryPlanFastProjection( + new[] { "/Nameplate/ManufacturerName" }, + out var plan); + + Assert.True(planned); + var path = Assert.Single(plan); + Assert.Equal("Nameplate", path.TargetSubmodelIdShort); + Assert.Equal("ManufacturerName", path.ElementIdShortPath); + Assert.Equal("/Nameplate/ManufacturerName", path.RawPath); + } + + [Theory] + [InlineData("flowMax")] // Blattname ohne Punkt -> Ranking-Fallback + [InlineData("Documents[0].Title")] // Indexpfad ist in SMESets.IdShortPath nicht adressierbar + [InlineData("/TechnicalData")] // unvollständiger Cross-Pfad + public void TryPlanFastProjection_RejectsNonFullPaths(string path) + { + Assert.False(McpQueryTools.TryPlanFastProjection(new[] { path }, out _)); + } + + [Fact] + public void TryPlanFastProjection_RejectsMixedSelectListEntirely() + { + // Ein einziger Blattname im select deaktiviert den Fast Path komplett, + // damit alle Zellen konsistent über den Objektpfad aufgelöst werden. + var planned = McpQueryTools.TryPlanFastProjection( + new[] { "GeneralInformation.ManufacturerArticleNumber", "flowMax" }, + out _); + + Assert.False(planned); + } + + [Fact] + public void TryPlanFastProjection_RejectsEmptySelect() + { + Assert.False(McpQueryTools.TryPlanFastProjection(null, out _)); + Assert.False(McpQueryTools.TryPlanFastProjection(new[] { " " }, out _)); + } + + [Theory] + [InlineData("Prop", "Prop")] + [InlineData("In-Prop", "Prop")] + [InlineData("Out-MLP", "MLP")] + [InlineData(null, "")] + public void NormalizeSmeType_StripsOperationPrefix(string? smeType, string expected) + { + Assert.Equal(expected, McpQueryTools.NormalizeSmeType(smeType)); + } + + [Fact] + public void IsFastProjectableCell_OnlyForPropertyAndMlp() + { + Assert.True(McpQueryTools.IsFastProjectableCell(new DbProjectionCell { Found = true, SmeType = "Prop" })); + Assert.True(McpQueryTools.IsFastProjectableCell(new DbProjectionCell { Found = true, SmeType = "MLP" })); + Assert.False(McpQueryTools.IsFastProjectableCell(new DbProjectionCell { Found = true, SmeType = "SMC" })); + Assert.False(McpQueryTools.IsFastProjectableCell(new DbProjectionCell { Found = true, SmeType = "File" })); + Assert.False(McpQueryTools.IsFastProjectableCell(new DbProjectionCell { Found = false, SmeType = "Prop" })); + Assert.False(McpQueryTools.IsFastProjectableCell(null)); + } + + [Fact] + public void ProjectFastCellValue_ReturnsStringPropertyValue() + { + var cell = new DbProjectionCell + { + Found = true, + SmeType = "Prop", + TValue = "S", + Values = { new DbProjectionValue { SValue = "4711" } }, + }; + + var value = McpQueryTools.ProjectFastCellValue(cell, "en"); + + Assert.Equal("4711", value?.GetValue()); + } + + [Fact] + public void ProjectFastCellValue_ReturnsNumericPropertyValueAsNumber() + { + var cell = new DbProjectionCell + { + Found = true, + SmeType = "Prop", + TValue = "D", + Values = { new DbProjectionValue { NValue = 80.5 } }, + }; + + var value = McpQueryTools.ProjectFastCellValue(cell, "en"); + + Assert.Equal(80.5, value?.GetValue()); + } + + [Theory] + [InlineData(null)] + [InlineData("S")] + public void ProjectFastCellValue_ReturnsNumericValueEvenWhenTValueIsNotNumeric(string? tValue) + { + var cell = new DbProjectionCell + { + Found = true, + SmeType = "Prop", + TValue = tValue, + Values = { new DbProjectionValue { NValue = 9.002120415 } }, + }; + + var value = McpQueryTools.ProjectFastCellValue(cell, "en"); + + Assert.Equal(9.002120415, value?.GetValue()); + } + + [Fact] + public void ProjectFastCellValue_PicksRequestedLanguageForMlp() + { + var cell = new DbProjectionCell + { + Found = true, + SmeType = "MLP", + TValue = "S", + Values = + { + new DbProjectionValue { SValue = "Ventil", Annotation = "de" }, + new DbProjectionValue { SValue = "Valve", Annotation = "en" }, + }, + }; + + Assert.Equal("Valve", McpQueryTools.ProjectFastCellValue(cell, "en")?.GetValue()); + Assert.Equal("Ventil", McpQueryTools.ProjectFastCellValue(cell, "de")?.GetValue()); + + // Unbekannte Sprache -> erste vorhandene. + Assert.Equal("Ventil", McpQueryTools.ProjectFastCellValue(cell, "fr")?.GetValue()); + } + + [Fact] + public void ProjectFastCellValue_ReturnsNullWithoutValues() + { + var cell = new DbProjectionCell { Found = true, SmeType = "Prop", TValue = "S" }; + + Assert.Null(McpQueryTools.ProjectFastCellValue(cell, "en")); + } +} diff --git a/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs new file mode 100644 index 000000000..55b6c8d68 --- /dev/null +++ b/src/IO.Swagger.Lib.V3.Tests/MCP/McpQueryToolsProjectionTests.cs @@ -0,0 +1,173 @@ +namespace IO.Swagger.Lib.V3.Tests.MCP; + +using IO.Swagger.Lib.V3.MCP; +using System.Text.Json.Nodes; + +public class McpQueryToolsProjectionTests +{ + [Fact] + public void NormalizeLimit_DefaultsAndCapsAtMcpPageSize() + { + Assert.Equal(500, McpQueryTools.NormalizeLimit(null)); + Assert.Equal(500, McpQueryTools.NormalizeLimit(0)); + Assert.Equal(25, McpQueryTools.NormalizeLimit(25)); + Assert.Equal(1000, McpQueryTools.NormalizeLimit(1000)); + Assert.Equal(5000, McpQueryTools.NormalizeLimit(5000)); + Assert.Equal(5000, McpQueryTools.NormalizeLimit(6000)); + } + + [Fact] + public void NormalizeProductSubmodelLimit_DefaultsAndCapsAtProductPageSize() + { + Assert.Equal(50, McpQueryTools.NormalizeProductSubmodelLimit(null)); + Assert.Equal(50, McpQueryTools.NormalizeProductSubmodelLimit(-1)); + Assert.Equal(10, McpQueryTools.NormalizeProductSubmodelLimit(10)); + Assert.Equal(50, McpQueryTools.NormalizeProductSubmodelLimit(100)); + } + + [Fact] + public void NormalizeCursor_UsesZeroForInvalidOrNegativeValues() + { + Assert.Equal(0, McpQueryTools.NormalizeCursor(null)); + Assert.Equal(0, McpQueryTools.NormalizeCursor("")); + Assert.Equal(0, McpQueryTools.NormalizeCursor("abc")); + Assert.Equal(0, McpQueryTools.NormalizeCursor("-5")); + Assert.Equal(20, McpQueryTools.NormalizeCursor("20")); + } + + [Fact] + public void BuildCsv_EscapesExcelRelevantCells() + { + var rows = new[] + { + new JsonObject + { + ["id"] = "submodel-1", + ["name"] = "A;B", + ["text"] = "He said \"yes\"", + ["value"] = 42.5, + }, + }; + + var csv = McpQueryTools.BuildCsv(["id", "name", "text", "value"], rows, ";"); + + Assert.Contains("id;name;text;value", csv); + Assert.Contains("submodel-1;\"A;B\";\"He said \"\"yes\"\"\";42.5", csv); + } + + [Fact] + public void TryParseCrossSubmodelProjectionPath_ParsesExplicitSubmodelPrefix() + { + var parsed = McpQueryTools.TryParseCrossSubmodelProjectionPath( + "/TechnicalData/GeneralInformation.ManufacturerArticleNumber", + out var submodelIdShort, + out var elementPath); + + Assert.True(parsed); + Assert.Equal("TechnicalData", submodelIdShort); + Assert.Equal("GeneralInformation.ManufacturerArticleNumber", elementPath); + } + + [Theory] + [InlineData("GeneralInformation.ManufacturerArticleNumber")] + [InlineData("/TechnicalData")] + [InlineData("//GeneralInformation.ManufacturerArticleNumber")] + [InlineData("/TechnicalData/")] + public void TryParseCrossSubmodelProjectionPath_RejectsNonExplicitOrIncompletePaths(string path) + { + Assert.False(McpQueryTools.TryParseCrossSubmodelProjectionPath(path, out _, out _)); + } + + [Fact] + public void ValidateWildcardOperatorForField_AllowsValueIdShortAndIdShortPath() + { + McpQueryTools.ValidateWildcardOperatorForField("contains", "sme", "value"); + McpQueryTools.ValidateWildcardOperatorForField("starts-with", "sme", "idShort"); + McpQueryTools.ValidateWildcardOperatorForField("ends-with", "sme", "idShortPath"); + McpQueryTools.ValidateWildcardOperatorForField("contains", "sme", "idShort", "TechnicalData.Power_output"); + } + + [Fact] + public void ValidateWildcardOperatorForField_RejectsSemanticIdAndId() + { + var semanticId = Assert.Throws( + () => McpQueryTools.ValidateWildcardOperatorForField("contains", "sm", "semanticId")); + Assert.Contains("Wildcard search is not supported for field 'semanticId'", semanticId.Message); + + var id = Assert.Throws( + () => McpQueryTools.ValidateWildcardOperatorForField("starts-with", "sm", "id")); + Assert.Contains("Wildcard search is not supported for field 'id'", id.Message); + } + + [Fact] + public void SelectPreferredProjectionPath_PutsDeprioritizedBranchesLast() + { + var paths = new[] + { + "Part_relation.Associated_part.Product_type", + "GeneralInformation.Product_type", + }; + + var result = McpQueryTools.SelectPreferredProjectionPath(paths); + + Assert.Equal("GeneralInformation.Product_type", result); + } + + [Fact] + public void SelectPreferredProjectionPath_UsesConfiguredPriority() + { + var paths = new[] + { + "TechnicalProperties.Product_type", + "GeneralInformation.Product_type", + }; + + var result = McpQueryTools.SelectPreferredProjectionPath( + paths, new[] { "TechnicalProperties", "GeneralInformation" }); + + Assert.Equal("TechnicalProperties.Product_type", result); + } + + [Fact] + public void SelectPreferredProjectionPath_UsesShallowerPathForEqualPriority() + { + var paths = new[] + { + "Other.Nested.Product_type", + "Other.Product_type", + }; + + var result = McpQueryTools.SelectPreferredProjectionPath(paths); + + Assert.Equal("Other.Product_type", result); + } + + [Fact] + public void SelectPreferredProjectionPath_MatchesWholeSegmentsOnly() + { + var paths = new[] + { + "NotAssociated_part.Product_type", + "Associated_part.Product_type", + }; + + var result = McpQueryTools.SelectPreferredProjectionPath(paths); + + Assert.Equal("NotAssociated_part.Product_type", result); + } + + [Fact] + public void SelectPreferredProjectionPath_AllowsDisablingDefaultPenalty() + { + var paths = new[] + { + "Associated_part.Product_type", + "Other.Nested.Product_type", + }; + + var result = McpQueryTools.SelectPreferredProjectionPath( + paths, priority: Array.Empty(), deprioritize: Array.Empty()); + + Assert.Equal("Associated_part.Product_type", result); + } +} diff --git a/src/IO.Swagger.Lib.V3/Filters/GeneratePathParamsValidationFilter.cs b/src/IO.Swagger.Lib.V3/Filters/GeneratePathParamsValidationFilter.cs index ef2b6f13c..0d12bbc41 100644 --- a/src/IO.Swagger.Lib.V3/Filters/GeneratePathParamsValidationFilter.cs +++ b/src/IO.Swagger.Lib.V3/Filters/GeneratePathParamsValidationFilter.cs @@ -36,67 +36,72 @@ public void Apply(OpenApiOperation operation, OperationFilterContext context) foreach (var par in pars) { var swaggerParam = operation.Parameters.SingleOrDefault(p => p.Name == par.Name); - - var attributes = ((ControllerParameterDescriptor)par.ParameterDescriptor).ParameterInfo.CustomAttributes; - - if (attributes != null && attributes.Count() > 0 && swaggerParam != null) - { - // Required - [Required] - var requiredAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RequiredAttribute)); - if (requiredAttr != null) - { - swaggerParam.Required = true; - } - - // Regex Pattern [RegularExpression] - var regexAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RegularExpressionAttribute)); - if (regexAttr != null) - { - var regex = (string)regexAttr.ConstructorArguments[0].Value; - swaggerParam.Schema.Pattern = regex; - } - - // String Length [StringLength] - int? minLenght = null, maxLength = null; - var stringLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(StringLengthAttribute)); - if (stringLengthAttr != null) - { - if (stringLengthAttr.NamedArguments.Count == 1) - { - minLenght = (int)(stringLengthAttr.NamedArguments.Single(p => p.MemberName == "MinimumLength").TypedValue.Value ?? 0); - } - maxLength = (int)(stringLengthAttr.ConstructorArguments[0].Value ?? 1); - } - - var minLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(MinLengthAttribute)); - if (minLengthAttr != null) - { - minLenght = (int)(minLengthAttr.ConstructorArguments[0].Value ?? 0); - } - - var maxLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(MaxLengthAttribute)); - if (maxLengthAttr != null) - { - maxLength = (int)(maxLengthAttr.ConstructorArguments[0].Value ?? 1); - } - - if (swaggerParam is OpenApiParameter) - { - ((OpenApiParameter)swaggerParam).Schema.MinLength = minLenght; - ((OpenApiParameter)swaggerParam).Schema.MaxLength = maxLength; - } - - // Range [Range] - var rangeAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RangeAttribute)); - if (rangeAttr != null) - { - var rangeMin = (int)(rangeAttr.ConstructorArguments[0].Value ?? 0); - var rangeMax = (int)(rangeAttr.ConstructorArguments[1].Value ?? 1); - - swaggerParam.Schema.Minimum = rangeMin; - swaggerParam.Schema.Maximum = rangeMax; - } - } + + var controllerParameterDescriptor = par.ParameterDescriptor as ControllerParameterDescriptor; + + if (controllerParameterDescriptor != null) + { + var attributes = controllerParameterDescriptor.ParameterInfo.CustomAttributes; + + if (attributes != null && attributes.Count() > 0 && swaggerParam != null) + { + // Required - [Required] + var requiredAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RequiredAttribute)); + if (requiredAttr != null) + { + swaggerParam.Required = true; + } + + // Regex Pattern [RegularExpression] + var regexAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RegularExpressionAttribute)); + if (regexAttr != null) + { + var regex = (string)regexAttr.ConstructorArguments[0].Value; + swaggerParam.Schema.Pattern = regex; + } + + // String Length [StringLength] + int? minLenght = null, maxLength = null; + var stringLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(StringLengthAttribute)); + if (stringLengthAttr != null) + { + if (stringLengthAttr.NamedArguments.Count == 1) + { + minLenght = (int)(stringLengthAttr.NamedArguments.Single(p => p.MemberName == "MinimumLength").TypedValue.Value ?? 0); + } + maxLength = (int)(stringLengthAttr.ConstructorArguments[0].Value ?? 1); + } + + var minLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(MinLengthAttribute)); + if (minLengthAttr != null) + { + minLenght = (int)(minLengthAttr.ConstructorArguments[0].Value ?? 0); + } + + var maxLengthAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(MaxLengthAttribute)); + if (maxLengthAttr != null) + { + maxLength = (int)(maxLengthAttr.ConstructorArguments[0].Value ?? 1); + } + + if (swaggerParam is OpenApiParameter) + { + ((OpenApiParameter)swaggerParam).Schema.MinLength = minLenght; + ((OpenApiParameter)swaggerParam).Schema.MaxLength = maxLength; + } + + // Range [Range] + var rangeAttr = attributes.FirstOrDefault(p => p.AttributeType == typeof(RangeAttribute)); + if (rangeAttr != null) + { + var rangeMin = (int)(rangeAttr.ConstructorArguments[0].Value ?? 0); + var rangeMax = (int)(rangeAttr.ConstructorArguments[1].Value ?? 1); + + swaggerParam.Schema.Minimum = rangeMin; + swaggerParam.Schema.Maximum = rangeMax; + } + } + } } } } diff --git a/src/IO.Swagger.Lib.V3/GraphQL/GraphQLAPI.cs b/src/IO.Swagger.Lib.V3/GraphQL/GraphQLAPI.cs index 0b0c45d72..3240163fa 100644 --- a/src/IO.Swagger.Lib.V3/GraphQL/GraphQLAPI.cs +++ b/src/IO.Swagger.Lib.V3/GraphQL/GraphQLAPI.cs @@ -62,7 +62,7 @@ public async Task CountSMs(string semanticId = "", string identifier = "", { var paginationParameters = new PaginationParameters(pageFrom, MAX_PAGE_SIZE); var securityConfig = new SecurityConfig(Program.noSecurity, null); - var result = await _dbRequestHandlerService.QueryCountSMs(securityConfig, semanticId, identifier, diff, paginationParameters, expression); + var result = await _dbRequestHandlerService.QueryCountSMs(securityConfig, semanticId, identifier, diff, paginationParameters, Contracts.QueryResult.ResultType.Submodel, expression); return result; } diff --git a/src/IO.Swagger.Lib.V3/IO.Swagger.Lib.V3.csproj b/src/IO.Swagger.Lib.V3/IO.Swagger.Lib.V3.csproj index f7b688d23..a06f4461d 100644 --- a/src/IO.Swagger.Lib.V3/IO.Swagger.Lib.V3.csproj +++ b/src/IO.Swagger.Lib.V3/IO.Swagger.Lib.V3.csproj @@ -22,6 +22,7 @@ + diff --git a/src/IO.Swagger.Lib.V3/MCP/McpExportFileStore.cs b/src/IO.Swagger.Lib.V3/MCP/McpExportFileStore.cs new file mode 100644 index 000000000..fd958f461 --- /dev/null +++ b/src/IO.Swagger.Lib.V3/MCP/McpExportFileStore.cs @@ -0,0 +1,88 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace IO.Swagger.Lib.V3.MCP; + +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Security.Cryptography; + +/// +/// In-Memory-Ablage für MCP-Exportdateien (CSV/XLSX). Jede Datei bekommt ein zufälliges +/// URL-Token und ist darüber als HTTP-Download (/mcp-exports/{token}) und als +/// MCP-Resource (aas-export://{token}) abrufbar. Einträge verfallen nach ; +/// zusätzlich begrenzen / den Speicher. +/// +public static class McpExportFileStore +{ + private const int MaxEntries = 100; + private const long MaxTotalBytes = 256L * 1024 * 1024; + private static readonly TimeSpan Ttl = TimeSpan.FromMinutes(60); + + private sealed record Entry(byte[] Content, string FileName, string MimeType, DateTime CreatedUtc); + + private static readonly ConcurrentDictionary Files = new(StringComparer.Ordinal); + + public static string Add(byte[] content, string fileName, string mimeType) + { + Prune(); + var token = Convert.ToHexString(RandomNumberGenerator.GetBytes(16)).ToLowerInvariant(); + Files[token] = new Entry(content, fileName, mimeType, DateTime.UtcNow); + return token; + } + + public static bool TryGet(string? token, out byte[] content, out string fileName, out string mimeType) + { + content = []; + fileName = string.Empty; + mimeType = string.Empty; + if (string.IsNullOrWhiteSpace(token) || !Files.TryGetValue(token.Trim(), out var entry)) + { + return false; + } + + if (DateTime.UtcNow - entry.CreatedUtc > Ttl) + { + Files.TryRemove(token.Trim(), out _); + return false; + } + + content = entry.Content; + fileName = entry.FileName; + mimeType = entry.MimeType; + return true; + } + + private static void Prune() + { + var now = DateTime.UtcNow; + foreach (var kv in Files) + { + if (now - kv.Value.CreatedUtc > Ttl) + { + Files.TryRemove(kv.Key, out _); + } + } + + // Älteste Einträge entfernen, bis Anzahl- und Größenbudget wieder eingehalten sind. + while (Files.Count >= MaxEntries || Files.Sum(kv => (long)kv.Value.Content.Length) > MaxTotalBytes) + { + var oldest = Files.OrderBy(kv => kv.Value.CreatedUtc).FirstOrDefault(); + if (oldest.Key is null || !Files.TryRemove(oldest.Key, out _)) + { + break; + } + } + } +} diff --git a/src/IO.Swagger.Lib.V3/MCP/McpExportResources.cs b/src/IO.Swagger.Lib.V3/MCP/McpExportResources.cs new file mode 100644 index 000000000..85fde6eb7 --- /dev/null +++ b/src/IO.Swagger.Lib.V3/MCP/McpExportResources.cs @@ -0,0 +1,47 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace IO.Swagger.Lib.V3.MCP; + +using System; +using System.ComponentModel; +using ModelContextProtocol; +using ModelContextProtocol.Protocol; +using ModelContextProtocol.Server; + +/// +/// MCP-Resources für die von aas_query_export_csv/aas_query_export_xlsx erzeugten Dateien. +/// Clients, die MCP-Resources unterstützen, können die Datei über resources/read mit der +/// resourceUri aus der Tool-Antwort (aas-export://{token}) als Binärinhalt laden. +/// +[McpServerResourceType] +public sealed class McpExportResources +{ + public const string UriScheme = "aas-export"; + + public static string BuildResourceUri(string token) => $"{UriScheme}://{token}"; + + [McpServerResource(UriTemplate = "aas-export://{token}", Name = "aas_query_export_file", Title = "AAS Query Export File", MimeType = "application/octet-stream")] + [Description("Exportdatei (CSV/XLSX) eines aas_query_export-Aufrufs; token stammt aus dem Feld resourceUri der Tool-Antwort. Dateien verfallen nach ca. 60 Minuten.")] + public static ResourceContents GetExportFile(string token) + { + if (!McpExportFileStore.TryGet(token, out var content, out _, out var mimeType)) + { + throw new McpException($"Export \"{token}\" ist nicht (mehr) verfügbar. Bitte den Export erneut ausführen."); + } + + // FromBytes base64-kodiert die Rohdaten; Blob direkt zu setzen erwartet bereits + // base64-kodierte UTF-8-Bytes und würde die Datei auf dem Draht zerstören. + return BlobResourceContents.FromBytes(content, BuildResourceUri(token), mimeType); + } +} diff --git a/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs new file mode 100644 index 000000000..bbc3478af --- /dev/null +++ b/src/IO.Swagger.Lib.V3/MCP/McpQueryTools.cs @@ -0,0 +1,2929 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace IO.Swagger.Lib.V3.MCP; + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using AasCore.Aas3_1; +using AasxServer; +using Contracts; +using Contracts.DbRequests; +using Contracts.Exceptions; +using Contracts.Pagination; +using Contracts.QueryResult; +using DataTransferObjects.ValueDTOs; +using IO.Swagger.Lib.V3.Models; +using IO.Swagger.Lib.V3.SerializationModifiers.Mappers; +using IO.Swagger.Lib.V3.SerializationModifiers.Mappers.ValueMappers; +using ModelContextProtocol.Server; + +/// +/// Eine einzelne Suchbedingung. Der Server baut daraus eine gültige AASQL-Bedingung; +/// das Modell muss keine AASQL-Syntax erzeugen. +/// +public sealed class McpQueryCondition +{ + [Description("Worauf sich das Feld bezieht: \"aas\" (Shell), \"sm\" (Submodel) oder \"sme\" (Submodel-Element, nur als Filter). Default: \"sme\".")] + public string Scope { get; set; } = "sme"; + + [Description("Feldname je nach scope. aas: idShort|id|assetInformation.assetKind|assetInformation.assetType|assetInformation.globalAssetId|assetInformation.specificAssetIds[].name|... ; sm: semanticId|idShort|id ; sme: semanticId|idShort|idShortPath|value|valueType|language. Wildcard-/Teilstring-Operatoren contains, starts-with und ends-with sind nur für value, idShort und idShortPath erlaubt. Für semanticId und id ausschließlich eq oder in verwenden.")] + public string Field { get; set; } = ""; + + [Description("Optionaler idShortPath innerhalb des Submodels, nur für scope=\"sme\". Enthält der Wert einen Punkt, wird er als verschachtelter Pfad behandelt (z.B. \"TechnicalProperties.flowMax\" oder \"Documents[].DocumentVersion.Title\"). Ein einzelner Name ohne Punkt (z.B. \"flowMax\") wird als idShort des Elements gesucht und unabhängig von der Verschachtelungstiefe gefunden.")] + public string? IdShortPath { get; set; } + + [Description("Vergleichsoperator: eq|ne|gt|ge|lt|le|contains|starts-with|ends-with|regex|in. contains, starts-with und ends-with nur für value, idShort und idShortPath verwenden; semanticId und id nur mit eq oder in. regex ist derzeit nicht unterstützt. contains benötigt mindestens 3 Zeichen, damit der Trigrammindex genutzt werden kann. \"in\" prüft, ob der Wert in der Liste values vorkommt (ODER-Verknüpfung).")] + public string Op { get; set; } = "eq"; + + [Description("Vergleichswert als String (Zahlen als String angeben, z.B. \"100\"). Bei eq/ne/gt/ge/lt/le werden numerische Werte automatisch numerisch verglichen — \"eq 80\" findet also auch einen Double-Wert 80.")] + public string Value { get; set; } = ""; + + [Description("Werteliste für op=\"in\" (z.B. mehrere Artikelnummern). Trifft, wenn das Feld einem der Werte entspricht — ersetzt viele or-verknüpfte eq-Bedingungen.")] + public string[]? Values { get; set; } +} + +/// +/// MCP-Tools für die AASPE-Query-Pipeline. Dünner Adapter über +/// (analog zum GraphQL-Adapter). Das Modell füllt strukturierte Slots, der Server baut die AASQL +/// (JSON-Grammatik) und gibt nur die gefundenen Identifier zurück — keine Volldaten. +/// +[McpServerToolType] +public sealed class McpQueryTools +{ + private const int DefaultPageSize = 500; + private const int MaxPageSize = 5000; + + // Ab dieser Dateigröße wird der Inhalt nicht mehr inline (content/contentBase64) in die + // Tool-Antwort gelegt — große Tabellen laufen sonst durch das Kontextfenster des Modells. + // Der Abruf erfolgt dann über downloadUrl (HTTP) oder resourceUri (MCP-Resource). + private const int InlineExportContentMaxBytes = 200_000; + + // Tool-Sätze für die reduzierten Endpunkte (Pfad-Filter siehe ServerConfiguration). + // /mcp-basic: das mächtige conditions-Tool — ein Call, aber komplexe Suchen (AND/OR, Operatoren). Für mittlere Modelle (z.B. Gemini Flash). + public static readonly HashSet BasicToolNames = new(StringComparer.Ordinal) { "aas_find_product" }; + + // /mcp-simple: nur das Tool mit flachen Parametern (idShort/value) — für sehr kleine Modelle (z.B. Qwen 7B), + // die das conditions[]-Schema nicht korrekt füllen. + public static readonly HashSet SimpleToolNames = new(StringComparer.Ordinal) { "aas_find_product_simple" }; + + // Obergrenze für aas_get_product: max. so viele Submodelle pro AAS werden geladen (Schutz gegen + // pathologisch große Shells). Reale Produkte haben i.d.R. <10 Submodelle. + private const int MaxProductSubmodels = 50; + + // Default order for ambiguous leaf-name projections; callers can override both lists. + private static readonly string[] DefaultProjectionPriority = + { "Nameplate", "GeneralInformation", "TechnicalData", "TechnicalProperties", "HandoverDocumentation", "ProductCarbonFootprint" }; + + private static readonly string[] DefaultProjectionDeprioritize = + { "Associated_part", "Part_relation" }; + + // Submodel-idShort-Keywords, die in der simple-Stufe (coreOnly) weggelassen werden: sperrige Datei-/Doku- + // Submodelle (CAD, BoM_SpareParts, HandoverDocumentation), die sehr kleine Modelle nur ablenken/überfluten. + private static readonly HashSet NoiseSubmodelKeywords = new(StringComparer.OrdinalIgnoreCase) + { "handover", "documentation", "cad", "bom", "sparepart" }; + + private static readonly HashSet AllowedScopes = new(StringComparer.Ordinal) { "aas", "sm", "sme" }; + + // Slot-Operator -> AASQL-JSON-Schlüssel + private static readonly Dictionary OpMap = new(StringComparer.Ordinal) + { + ["eq"] = "$eq", + ["ne"] = "$ne", + ["gt"] = "$gt", + ["ge"] = "$ge", + ["lt"] = "$lt", + ["le"] = "$le", + ["contains"] = "$contains", + ["starts-with"] = "$starts-with", + ["ends-with"] = "$ends-with", + ["regex"] = "$regex", + }; + + // Operatoren, die bei einem als Zahl parsebaren Wert numerisch verglichen werden (sonst String). + // eq/ne sind bewusst dabei: ein Property mit valueType=Double speichert z.B. "80" nicht als exakten + // String, daher schlägt ein reiner String-eq fehl. Bei nicht-numerischen Werten greift automatisch $strVal. + private static readonly HashSet NumericCapableOps = new(StringComparer.Ordinal) { "eq", "ne", "gt", "ge", "lt", "le" }; + + private static readonly HashSet WildcardOps = new(StringComparer.Ordinal) { "contains", "starts-with", "ends-with", "regex" }; + + private static readonly HashSet WildcardFields = new(StringComparer.Ordinal) { "value", "idShort", "idShortPath" }; + + private readonly IDbRequestHandlerService _dbRequestHandlerService; + private readonly IMappingService _mappingService; + + public McpQueryTools(IDbRequestHandlerService dbRequestHandlerService, IMappingService mappingService) + { + _dbRequestHandlerService = dbRequestHandlerService; + _mappingService = mappingService; + } + + [McpServerTool(Name = "aas_query", Title = "Search AAS Repository", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Sucht im AAS-Repository und liefert nur die gefundenen Identifier zurück (keine Volldaten). " + + "Mehrere Bedingungen werden mit combine (\"and\"/\"or\") verknüpft. " + + "target=\"submodels\" gibt Submodel-Identifier zurück, target=\"shells\" gibt AAS-Identifier zurück. " + + "Wildcard-/Teilstring-Suchen (contains, starts-with, ends-with) dürfen ausschließlich auf value, idShort und idShortPath verwendet werden. Für semanticId und id ausschließlich eq oder in verwenden; regex ist derzeit nicht unterstützt. " + + "Wenn unklar ist, wie ein Merkmal heißt (z.B. \"Ausgangsspannung\"): ZUERST aas_find_concepts aufrufen und dann über field=\"semanticId\" suchen — das ist exakt und herstellerunabhängig, statt idShort-Schreibweisen zu raten. " + + "Für einen Überblick, welche Submodel-Typen und Feldpfade es überhaupt gibt: aas_describe_model. " + + "Beispiele: " + + "(1) eine Bedingung: target=\"submodels\", conditions=[{scope:\"sm\",field:\"idShort\",op:\"eq\",value:\"TechnicalData\"}]. " + + "(2) UND: combine=\"and\", conditions=[{scope:\"sm\",field:\"idShort\",op:\"eq\",value:\"TechnicalData\"},{scope:\"sme\",field:\"value\",op:\"lt\",value:\"100\"}]. " + + "(3) ODER: combine=\"or\", conditions=[{scope:\"sme\",field:\"value\",op:\"eq\",value:\"A\"},{scope:\"sme\",field:\"value\",op:\"eq\",value:\"B\"}]. " + + "Nicht automatisch aas_count vorschalten; aas_count nur verwenden, wenn der Benutzer explizit die exakte Gesamtzahl braucht. Wenn hasMore=true, mit cursor weiterblättern, nicht count aufrufen. " + + "Wichtig: aas_query liefert standardmäßig nur Identifier. Danach den Inhalt mit aas_get_submodel lesen — oder, wenn die Frage mehrere Submodelle eines Produkts betrifft (z.B. technische Daten + Hersteller + CO2), in EINEM Schritt mit aas_get_product(identifier). " + + "TIPP für Tabellen/Listen: Übergib select=[idShortPaths], dann liefert aas_query je Treffer direkt diese Feldwerte (Projektion) — das ersetzt viele Einzelabrufe. " + + "Für Daten aus einem anderen Submodel derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber. " + + "Wenn der Benutzer eine Tabelle, Excel- oder CSV-Datei möchte, stattdessen direkt aas_query_export_xlsx (bevorzugt) bzw. aas_query_export_csv verwenden. " + + "Bei vielen erwarteten Treffern limit explizit setzen (Maximum 5000).")] + public async Task AasQuery( + [Description("Zielobjekt: \"submodels\" oder \"shells\".")] string target, + [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\". Bei einer Bedingung ohne Bedeutung.")] string combine = "and", + [Description("Maximale Trefferzahl (Default 500, Maximum 5000).")] int? limit = null, + [Description("Cursor (Offset) zum Weiterblättern; aus nextCursor einer vorigen Antwort.")] string? cursor = null, + [Description("Optional: Liste von idShortPaths, deren Werte je Treffer direkt mitgeliefert werden (Projektion = Tabellenspalten), z.B. [\"GeneralInformation.ManufacturerArticleNumber\", \"TechnicalProperties.Power_output\"]. Pfade ohne / beziehen sich auf das Treffer-Submodel. Für andere Submodelle derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber. Nur für target=\"submodels\".")] string[]? select = null, + [Description("Sprache für mehrsprachige Felder (MultiLanguageProperty) in der Projektion: nur dieser Sprachwert kommt zurück (Default \"en\"; fehlt die Sprache, wird die erste vorhandene genommen). Nur relevant zusammen mit select.")] string lang = "en", + [Description("Optionale Prioritätsreihenfolge für mehrdeutige Blattnamen. Default: Nameplate, GeneralInformation, TechnicalData, TechnicalProperties, HandoverDocumentation, ProductCarbonFootprint.")] string[]? priority = null, + [Description("Optionale Pfadsegmente, die bei mehrdeutigen Blattnamen ans Ende gestellt werden. Default: Associated_part, Part_relation.")] string[]? deprioritize = null, + [Description("Wenn true, enthält jede Projektionszeile zusätzlich ein paths-Objekt mit den tatsächlich gewählten idShortPaths.")] bool withPaths = false) + { + using var _ = LogCallTimed($"aas_query {target} {DescribeConditions(conditions, combine)} limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} cursor={cursor ?? "-"} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); + + ResultType resultType; + string expression; + try + { + resultType = ParseTarget(target); + expression = BuildExpression(conditions, combine); + } + catch (ArgumentException ex) + { + // Lesbare Fehlermeldung an die KI zurückgeben (statt opakem MCP-"An error occurred"), + // damit sie den Aufruf selbst korrigieren kann. + return new { error = ex.Message }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var requestedLimit = NormalizeLimit(limit); + var pagination = new PaginationParameters(cursor, requestedLimit + 1); + + var (list, queryError) = await TryQuery(securityConfig, pagination, resultType, expression); + if (queryError != null) + { + return new { error = "Query fehlgeschlagen: " + queryError }; + } + + var fetchedIdentifiers = ExtractIdentifiers(list); + var hasMore = fetchedIdentifiers.Count > requestedLimit; + var identifiers = fetchedIdentifiers.Take(requestedLimit).ToList(); + + string? nextCursor = hasMore + ? (pagination.Cursor + identifiers.Count).ToString(CultureInfo.InvariantCulture) + : null; + + // Projektion: Wenn select angegeben ist, je Treffer eine Zeile mit den gewählten Feldwerten liefern + // (statt nur Identifier) — spart die vielen aas_get_submodel-Folgeaufrufe. Nur für Submodelle sinnvoll. + if (select is { Length: > 0 } && resultType == ResultType.Submodel) + { + var projectedRows = await BuildProjectionRows( + securityConfig, identifiers, select, lang, priority, deprioritize, withPaths, "aas_query"); + var rows = new JsonArray(); + foreach (var row in projectedRows) + { + rows.Add(row); + } + + return new { target, count = identifiers.Count, columns = select, rows, hasMore, nextCursor }; + } + + return new + { + target, + count = identifiers.Count, + identifiers, + hasMore, + nextCursor, + nextStep = identifiers.Count > 0 + ? "Dies sind nur Identifier. Inhalte lesen: aas_get_submodel(identifier), oder gleich Felder mitliefern via select=[...]. Für ALLE Daten eines Produkts (Technik+Hersteller+CO2): aas_get_product(identifier)." + : "Keine Treffer. Bedingung lockern (z.B. op=contains statt eq), Groß-/Kleinschreibung des idShort prüfen — oder mit aas_find_concepts die richtige semanticId ermitteln und über field=\"semanticId\" suchen.", + }; + } + + [McpServerTool(Name = "aas_query_export_csv", Title = "Export AAS Query as CSV", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Führt eine aas_query mit select aus und liefert das Ergebnis als echte CSV-Datei. " + + "Für Tabellen/Excel bevorzugt aas_query_export_xlsx verwenden; CSV nur, wenn explizit CSV gewünscht ist. " + + "Die Antwort enthält fileName, mimeType=text/csv, count, hasMore, downloadUrl (HTTP-Download) und resourceUri (MCP-Resource); " + + "content/contentBase64 sind nur bei kleinen Dateien (<200 KB) zusätzlich enthalten. " + + "select ist erforderlich; exportiert wird eine Zeile pro Treffer mit Spalte id plus den select-Spalten. " + + "Pfade ohne / beziehen sich auf das Treffer-Submodel. Für Felder aus einem anderen Submodel derselben AAS die Syntax /SubmodelIdShort/idShortPath nutzen, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber. " + + "Für mehr als 500 Treffer limit explizit setzen, z.B. limit=1000. Nicht vorher aas_count aufrufen — bei hasMore=true mit cursor weiterblättern.")] + public async Task AasQueryExportCsv( + [Description("Zielobjekt: aktuell \"submodels\".")] string target, + [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, + [Description("idShortPaths/Blattnamen als CSV-Spalten, z.B. [\"ProductCarbonFootprintCradleToGate00.PCFCO2eq\"]. Für andere Submodelle derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber.")] string[] select, + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and", + [Description("Maximale Trefferzahl (Default 500, Maximum 5000).")] int? limit = null, + [Description("Cursor (Offset) zum Weiterblättern; aus nextCursor einer vorigen Antwort.")] string? cursor = null, + [Description("Sprache für mehrsprachige Felder (Default \"en\").")] string lang = "en", + [Description("CSV-Trennzeichen; Default Semikolon für deutschsprachiges Excel.")] string delimiter = ";", + [Description("Dateiname der exportierten CSV.")] string fileName = "aas_query_export.csv", + [Description("Optionale Prioritätsreihenfolge für mehrdeutige Blattnamen.")] string[]? priority = null, + [Description("Optionale Pfadsegmente, die bei mehrdeutigen Blattnamen ans Ende gestellt werden.")] string[]? deprioritize = null) + { + using var _ = LogCallTimed($"aas_query_export_csv {target} {DescribeConditions(conditions, combine)} limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} cursor={cursor ?? "-"} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); + + var export = await RunExportQuery("aas_query_export_csv", target, conditions, select, combine, limit, cursor, lang, priority, deprioritize); + if (export.Error != null) + { + return export.Error; + } + + var exportWatch = Stopwatch.StartNew(); + var normalizedDelimiter = NormalizeCsvDelimiter(delimiter); + var csv = BuildCsv(export.Columns, export.Rows, normalizedDelimiter); + var bytes = Encoding.UTF8.GetBytes("\uFEFF" + csv); + + var normalizedFileName = NormalizeExportFileName(fileName, ".csv", "aas_query_export.csv"); + var token = McpExportFileStore.Add(bytes, normalizedFileName, "text/csv"); + LogMcpLine($"aas_query_export_csv file: {bytes.Length} bytes in {exportWatch.ElapsedMilliseconds} ms"); + + var inline = bytes.Length <= InlineExportContentMaxBytes; + return new + { + target, + count = export.Rows.Count, + hasMore = export.HasMore, + nextCursor = export.NextCursor, + fileName = normalizedFileName, + mimeType = "text/csv", + encoding = "utf-8-sig", + delimiter = normalizedDelimiter, + columns = export.Columns, + fileSizeBytes = bytes.Length, + downloadUrl = BuildExportDownloadUrl(token), + resourceUri = McpExportResources.BuildResourceUri(token), + contentBase64 = inline ? Convert.ToBase64String(bytes) : null, + content = inline ? csv : null, + note = inline + ? null + : "Datei zu groß für Inline-Inhalt; über downloadUrl herunterladen oder als MCP-Resource (resourceUri) lesen.", + }; + } + + [McpServerTool(Name = "aas_query_export_xlsx", Title = "Export AAS Query as Excel (XLSX)", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Führt eine aas_query mit select aus und liefert das Ergebnis als echte Excel-Datei (XLSX). " + + "BEVORZUGTES Tool, wenn der Benutzer eine Tabelle, Excel-Datei oder einen Datei-Download möchte — nicht viele aas_query-Zeilen im Chat weiterverarbeiten. " + + "Die Antwort enthält fileName, mimeType, count, hasMore, downloadUrl (HTTP-Download, dem Benutzer als Link geben) und resourceUri (MCP-Resource); " + + "contentBase64 ist nur bei kleinen Dateien (<200 KB) zusätzlich enthalten. " + + "select ist erforderlich; exportiert wird eine Zeile pro Treffer mit Spalte id plus den select-Spalten. " + + "Pfade ohne / beziehen sich auf das Treffer-Submodel. Für Felder aus einem anderen Submodel derselben AAS die Syntax /SubmodelIdShort/idShortPath nutzen, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber oder /TechnicalData/TechnicalProperties.Manufacturer.Product_type. " + + "Für mehr als 500 Treffer limit explizit setzen, z.B. limit=1000 (Maximum 5000). " + + "Nicht automatisch aas_count vorschalten — bei hasMore=true mit cursor weiterblättern.")] + public async Task AasQueryExportXlsx( + [Description("Zielobjekt: aktuell \"submodels\".")] string target, + [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, + [Description("idShortPaths/Blattnamen als Tabellenspalten, z.B. [\"GeneralInformation.ManufacturerArticleNumber\"]. Für andere Submodelle derselben AAS nutze /SubmodelIdShort/idShortPath, z.B. /TechnicalData/GeneralInformation.ManufacturerArticleNumber.")] string[] select, + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and", + [Description("Maximale Trefferzahl (Default 500, Maximum 5000).")] int? limit = null, + [Description("Cursor (Offset) zum Weiterblättern; aus nextCursor einer vorigen Antwort.")] string? cursor = null, + [Description("Sprache für mehrsprachige Felder (Default \"en\").")] string lang = "en", + [Description("Dateiname der exportierten Excel-Datei.")] string fileName = "aas_query_export.xlsx", + [Description("Optionale Prioritätsreihenfolge für mehrdeutige Blattnamen.")] string[]? priority = null, + [Description("Optionale Pfadsegmente, die bei mehrdeutigen Blattnamen ans Ende gestellt werden.")] string[]? deprioritize = null) + { + using var _ = LogCallTimed($"aas_query_export_xlsx {target} {DescribeConditions(conditions, combine)} limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} cursor={cursor ?? "-"} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); + + var export = await RunExportQuery("aas_query_export_xlsx", target, conditions, select, combine, limit, cursor, lang, priority, deprioritize); + if (export.Error != null) + { + return export.Error; + } + + var exportWatch = Stopwatch.StartNew(); + var bytes = XlsxBuilder.Build(export.Columns, export.Rows); + var normalizedFileName = NormalizeExportFileName(fileName, ".xlsx", "aas_query_export.xlsx"); + var token = McpExportFileStore.Add(bytes, normalizedFileName, XlsxBuilder.MimeType); + LogMcpLine($"aas_query_export_xlsx file: {bytes.Length} bytes in {exportWatch.ElapsedMilliseconds} ms"); + + var inline = bytes.Length <= InlineExportContentMaxBytes; + return new + { + target, + count = export.Rows.Count, + hasMore = export.HasMore, + nextCursor = export.NextCursor, + fileName = normalizedFileName, + mimeType = XlsxBuilder.MimeType, + columns = export.Columns, + fileSizeBytes = bytes.Length, + downloadUrl = BuildExportDownloadUrl(token), + resourceUri = McpExportResources.BuildResourceUri(token), + contentBase64 = inline ? Convert.ToBase64String(bytes) : null, + note = inline + ? null + : "Datei zu groß für Inline-Inhalt; über downloadUrl herunterladen oder als MCP-Resource (resourceUri) lesen.", + }; + } + + // Gemeinsamer Unterbau der Export-Tools: Query ausführen, Treffer paginieren, Projektions- + // zeilen bauen (Fast Path, wenn möglich) und die Spaltenliste (id + select) liefern. + private sealed class ExportQueryData + { + public object? Error { get; init; } + public bool HasMore { get; init; } + public string? NextCursor { get; init; } + public List Rows { get; init; } = new(); + public string[] Columns { get; init; } = []; + } + + private async Task RunExportQuery( + string toolName, string target, McpQueryCondition[] conditions, string[] select, string combine, + int? limit, string? cursor, string lang, string[]? priority, string[]? deprioritize) + { + if (select is null || select.Length == 0) + { + return new ExportQueryData { Error = new { error = $"select ist für {toolName} erforderlich, damit Tabellenspalten erzeugt werden können." } }; + } + + string expression; + try + { + var resultType = ParseTarget(target); + if (resultType != ResultType.Submodel) + { + throw new ArgumentException($"{toolName} unterstützt aktuell nur target=\"submodels\"."); + } + + expression = BuildExpression(conditions, combine); + } + catch (ArgumentException ex) + { + return new ExportQueryData { Error = new { error = ex.Message } }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var requestedLimit = NormalizeLimit(limit); + var pagination = new PaginationParameters(cursor, requestedLimit + 1); + + var (list, queryError) = await TryQuery(securityConfig, pagination, ResultType.Submodel, expression); + if (queryError != null) + { + return new ExportQueryData { Error = new { error = "Query fehlgeschlagen: " + queryError } }; + } + + var fetchedIdentifiers = ExtractIdentifiers(list); + var hasMore = fetchedIdentifiers.Count > requestedLimit; + var identifiers = fetchedIdentifiers.Take(requestedLimit).ToList(); + string? nextCursor = hasMore + ? (pagination.Cursor + identifiers.Count).ToString(CultureInfo.InvariantCulture) + : null; + + var rows = await BuildProjectionRows( + securityConfig, identifiers, select, lang, priority, deprioritize, withPaths: false, toolName); + var columns = new[] { "id" }.Concat(select.Where(s => !string.IsNullOrWhiteSpace(s)).Select(s => s.Trim())).ToArray(); + + return new ExportQueryData { HasMore = hasMore, NextCursor = nextCursor, Rows = rows, Columns = columns }; + } + + // Externer Download-Link für eine Exportdatei; nutzt die konfigurierte externe Server-URL. + private static string BuildExportDownloadUrl(string token) + { + var baseUrl = (Program.externalBlazor ?? string.Empty).TrimEnd('/'); + return baseUrl + "/mcp-exports/" + token; + } + + internal static string NormalizeExportFileName(string? fileName, string extension, string defaultName) + { + var name = string.IsNullOrWhiteSpace(fileName) ? defaultName : fileName.Trim(); + + // Nur der blanke Dateiname ist erlaubt (Token-URL und Content-Disposition). + name = name.Replace('\\', '_').Replace('/', '_').Replace("..", "_", StringComparison.Ordinal); + return name.EndsWith(extension, StringComparison.OrdinalIgnoreCase) ? name : name + extension; + } + + [McpServerTool(Name = "aas_count", Title = "Exact Count Only When Explicitly Requested", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Nicht für Exploration, Plausibilitätschecks oder vor normalen Suchen verwenden. Nutze stattdessen aas_query mit limit/cursor. " + + "Dieses Tool zählt nur, wenn der Benutzer ausdrücklich eine exakte Gesamtzahl verlangt; dann exactTotalRequested=true setzen. " + + "Gleiche Bedingungs-/combine-Logik wie aas_query. Nur für target=\"submodels\". " + + "Liefert die exakte Gesamtzahl (ungedeckelt), kann auf großen Datenbanken aber teuer sein.")] + public async Task AasCount( + [Description("Zielobjekt: aktuell nur \"submodels\".")] string target, + [Description("Liste von Suchbedingungen (mindestens eine).")] McpQueryCondition[] conditions, + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and", + [Description("Muss true sein, und nur setzen, wenn der Benutzer explizit nach der exakten Gesamtzahl fragt (z.B. \"wie viele insgesamt?\"). Für normale Suche/Listen false lassen und aas_query verwenden.")] bool exactTotalRequested = false) + { + using var _ = LogCallTimed($"aas_count {target} {DescribeConditions(conditions, combine)} exactTotalRequested={exactTotalRequested}"); + + if (!exactTotalRequested) + { + return new + { + skipped = true, + message = "aas_count wurde nicht ausgeführt. Nutze aas_query mit limit/cursor; count nur mit exactTotalRequested=true verwenden, wenn der Benutzer explizit die exakte Gesamtzahl verlangt.", + nextStep = "Rufe aas_query mit derselben Bedingung auf. Bei hasMore=true mit nextCursor weiterblättern." + }; + } + + string expression; + try + { + var resultType = ParseTarget(target); + if (resultType != ResultType.Submodel) + { + throw new ArgumentException("aas_count unterstützt in dieser Version nur target=\"submodels\". Für shells bitte aas_query nutzen."); + } + + expression = BuildExpression(conditions, combine); + } + catch (ArgumentException ex) + { + return new { error = ex.Message }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + + // Dedizierter COUNT-Pfad: zählt die Treffer-Ids im Query-Engine, OHNE die + // vollständigen Submodelle zu materialisieren ("Collect results" entfällt). + // Das skaliert auf große DBs und liefert die echte Gesamtzahl (ungedeckelt). + // pageSize = int.MaxValue => SQL-LIMIT umfasst alle Treffer (SQLite & Postgres). + var pagination = new PaginationParameters(null, int.MaxValue); + int totalCount; + try + { + totalCount = await _dbRequestHandlerService.QueryCountSMs( + securityConfig, string.Empty, string.Empty, string.Empty, pagination, ResultType.Submodel, expression); + } + catch (Exception ex) + { + return new { error = "Query fehlgeschlagen: " + ex.Message }; + } + + return new { target, totalCount, nextStep = "Treffer abrufen mit aas_query (gleiche Bedingung); danach aas_get_submodel oder aas_get_product für die Inhalte." }; + } + + [McpServerTool(Name = "aas_get_submodel", Title = "Get AAS Submodel", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Holt ein komplettes Submodel anhand seines Identifiers (z.B. einen Treffer aus aas_query), damit man mit den Inhalten weiterarbeiten kann. " + + "format=\"value\" (Default) liefert eine kompakte idShort->Wert-Darstellung — token-sparsam und ideal zum Auslesen von Werten. " + + "format=\"full\" liefert das vollständige, standardkonforme AAS-JSON (inkl. semanticId, valueType, Qualifier, Beschreibungen). " + + "Bei sehr großen Submodellen bevorzugt format=\"value\" verwenden. " + + "Brauchst du Daten aus einem ANDEREN Submodel desselben Produkts (z.B. Hersteller aus Nameplate, CO2 aus CarbonFootprint)? Dann über die Shell navigieren: aas_query target=\"shells\" -> aas_get_shell liefert alle Submodelle der Shell.")] + public async Task AasGetSubmodel( + [Description("Submodel-Identifier (die vollständige id, NICHT Base64-kodiert), typischerweise ein Treffer aus aas_query.")] string identifier, + [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (vollständiges AAS-JSON).")] string format = "value") + { + using var _ = LogCallTimed($"aas_get_submodel id={identifier} format={format}"); + if (string.IsNullOrWhiteSpace(identifier)) + { + throw new ArgumentException("identifier darf nicht leer sein."); + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + throw new ArgumentException($"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"."); + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + + IClass? submodel; + try + { + submodel = await _dbRequestHandlerService.ReadSubmodelById(securityConfig, aasIdentifier: null, identifier.Trim(), level: null, extent: null); + } + catch (NotFoundException ex) + { + return new { identifier, found = false, message = ex.Message }; + } + + if (submodel is null) + { + return new { identifier, found = false }; + } + + return new { identifier, format = fmt, submodel = SerializeValueOrFull(submodel, fmt) }; + } + + [McpServerTool(Name = "aas_get_submodels", Title = "Get Multiple AAS Submodels", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Holt MEHRERE Submodelle in EINEM Aufruf — eine Liste von Identifiern statt vieler Einzelabrufe. " + + "OHNE select: je Submodel der volle Inhalt (value/full). " + + "MIT select=[idShortPaths]: je Submodel nur eine kompakte Zeile mit diesen Feldern (Tabelle/Projektion, wie bei aas_query). " + + "Ideal, wenn man bereits eine Liste von Submodel-IDs hat (z.B. aus aas_query) und gezielt Felder oder Inhalte braucht.")] + public async Task AasGetSubmodels( + [Description("Liste von Submodel-Identifiern (vollständige ids, NICHT Base64-kodiert).")] string[] identifiers, + [Description("Ausgabeformat je Submodel, wenn KEIN select: \"value\" (kompakt, Default) oder \"full\".")] string format = "value", + [Description("Optional: idShortPaths zur Projektion (Tabellenspalten), z.B. [\"GeneralInformation.ManufacturerArticleNumber\", \"TechnicalProperties.Power_output\"]. Mit select wird je ID nur eine Zeile mit diesen Feldern geliefert.")] string[]? select = null, + [Description("Sprache für mehrsprachige Felder (MultiLanguageProperty) in der Projektion (Default \"en\"). Nur mit select relevant.")] string lang = "en", + [Description("Optionale Prioritätsreihenfolge für mehrdeutige Blattnamen. Default: Nameplate, GeneralInformation, TechnicalData, TechnicalProperties, HandoverDocumentation, ProductCarbonFootprint.")] string[]? priority = null, + [Description("Optionale Pfadsegmente, die bei mehrdeutigen Blattnamen ans Ende gestellt werden. Default: Associated_part, Part_relation.")] string[]? deprioritize = null, + [Description("Wenn true, enthält jede Projektionszeile zusätzlich ein paths-Objekt mit den tatsächlich gewählten idShortPaths.")] bool withPaths = false) + { + using var _ = LogCallTimed($"aas_get_submodels n={(identifiers?.Length ?? 0)} format={format} select={(select is { Length: > 0 } ? string.Join(",", select) : "-")}"); + + if (identifiers is null || identifiers.Length == 0) + { + return new { error = "identifiers darf nicht leer sein (Liste von Submodel-Identifiern)." }; + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + return new { error = $"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"." }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + + // Mit select: Tabelle (eine Zeile pro ID mit den gewählten Feldern). + if (select is { Length: > 0 }) + { + var rows = new JsonArray(); + foreach (var id in identifiers) + { + if (!string.IsNullOrWhiteSpace(id)) + { + rows.Add(await BuildProjectionRow(securityConfig, id.Trim(), select, lang, priority, deprioritize, withPaths)); + } + } + + return new { count = rows.Count, columns = select, rows }; + } + + // Ohne select: je Submodel der (value/full) Inhalt. + var submodels = new JsonArray(); + foreach (var id in identifiers) + { + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + + IClass? sm = null; + try + { + sm = await _dbRequestHandlerService.ReadSubmodelById(securityConfig, null, id.Trim(), level: null, extent: null); + } + catch (NotFoundException) + { + sm = null; + } + + var entry = new JsonObject { ["id"] = id.Trim() }; + if (sm is null) + { + entry["found"] = false; + } + else + { + if (sm is ISubmodel s) + { + entry["idShort"] = s.IdShort; + } + + entry["value"] = SerializeValueOrFull(sm, fmt); + } + + submodels.Add(entry); + } + + return new { count = submodels.Count, format = fmt, submodels }; + } + + [McpServerTool(Name = "aas_get_element", Title = "Get AAS Element", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Liest ein einzelnes SubmodelElement aus einem Submodel — gezielt, statt das ganze Submodel zu laden (spart Tokens). " + + "Ein idShortPath MIT Punkt (z.B. \"TechnicalProperties.flowMax\") adressiert exakt ein Element (normale AAS-API). " + + "Ein einzelner Name OHNE Punkt (z.B. \"flowMax\") wird als idShort im gesamten Submodel gesucht (beliebige Verschachtelungstiefe) und liefert ALLE Treffer mit ihrem vollen idShortPath — nützlich, wenn nur der Blattname bekannt ist und der Pfad nicht. " + + "format=\"value\" (Default) = kompakter Wert, \"full\" = vollständiges Element-JSON.")] + public async Task AasGetElement( + [Description("Submodel-Identifier (vollständige id, NICHT Base64-kodiert), typischerweise ein Treffer aus aas_query.")] string submodelIdentifier, + [Description("Voller idShortPath mit Punkt (exaktes Element) ODER einzelner idShort-Blattname ohne Punkt (Suche in beliebiger Tiefe, alle Treffer).")] string idShortPath, + [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (vollständiges Element-JSON).")] string format = "value") + { + using var _ = LogCallTimed($"aas_get_element sm={submodelIdentifier} path={idShortPath} format={format}"); + if (string.IsNullOrWhiteSpace(submodelIdentifier)) + { + throw new ArgumentException("submodelIdentifier darf nicht leer sein."); + } + + if (string.IsNullOrWhiteSpace(idShortPath)) + { + throw new ArgumentException("idShortPath darf nicht leer sein."); + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + throw new ArgumentException($"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"."); + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var smId = submodelIdentifier.Trim(); + var path = idShortPath.Trim(); + + // Mit Punkt: exakter Pfad über die normale AAS-API. + if (path.Contains('.', StringComparison.Ordinal)) + { + IClass? element; + try + { + element = await _dbRequestHandlerService.ReadSubmodelElementByPath(securityConfig, null, smId, path, level: null, extent: null); + } + catch (NotFoundException ex) + { + return new { submodelIdentifier = smId, idShortPath = path, found = false, message = ex.Message }; + } + + if (element is null) + { + return new { submodelIdentifier = smId, idShortPath = path, found = false }; + } + + return new { submodelIdentifier = smId, idShortPath = path, format = fmt, element = SerializeValueOrFull(element, fmt) }; + } + + // Nur Blattname: Submodel serverseitig laden und rekursiv nach idShort suchen (alle Treffer). + IClass? submodel; + try + { + submodel = await _dbRequestHandlerService.ReadSubmodelById(securityConfig, null, smId, level: null, extent: null); + } + catch (NotFoundException ex) + { + return new { submodelIdentifier = smId, idShort = path, found = false, message = ex.Message }; + } + + if (submodel is not ISubmodel sm) + { + return new { submodelIdentifier = smId, idShort = path, found = false }; + } + + var matches = new List<(string Path, ISubmodelElement Element)>(); + CollectByIdShort(sm.SubmodelElements, string.Empty, path, matches); + + var results = matches + .Select(m => new { idShortPath = m.Path, value = SerializeValueOrFull(m.Element, fmt) }) + .ToList(); + + return new { submodelIdentifier = smId, idShort = path, format = fmt, count = results.Count, matches = results }; + } + + [McpServerTool(Name = "aas_get_shell", Title = "Get AAS Shell", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Liest eine AAS (Asset Administration Shell) anhand ihres Identifiers — die Asset-Ebene ÜBER den Submodellen. " + + "WANN BENUTZEN: Immer wenn du zu einem Produkt weitere Daten aus einem ANDEREN Submodel DERSELBEN Shell brauchst — " + + "z.B. Hersteller/Seriennummer (Nameplate) oder CO2-Fußabdruck (CarbonFootprint), während du bisher nur z.B. das TechnicalData-Submodel hast. " + + "Der direkte Weg ist NICHT querfeldein nach Teilenummern zu suchen, sondern: erst die Shell ermitteln (aas_query target=\"shells\" mit derselben Bedingung), dann aas_get_shell aufrufen. " + + "Das liefert die Asset-Informationen (assetKind, globalAssetId, specificAssetIds wie Serien-/Herstellernummern) UND die Liste ALLER Submodel-Identifier der Shell; " + + "danach das passende Submodel gezielt mit aas_get_submodel / aas_get_element auslesen. " + + "format=\"value\" (Default) = kompakt, \"full\" = vollständiges AAS-JSON. " + + "Alternativ liefert auch aas_query target=\"submodels\" mit {scope:\"aas\", field:\"id\", op:\"eq\", value:} alle Submodelle einer AAS.")] + public async Task AasGetShell( + [Description("AAS-Identifier (vollständige id, NICHT Base64-kodiert), typischerweise ein Treffer aus aas_query target=\"shells\".")] string identifier, + [Description("Ausgabeformat: \"value\" (kompakt, Default) oder \"full\" (vollständiges AAS-JSON).")] string format = "value") + { + using var _ = LogCallTimed($"aas_get_shell id={identifier} format={format}"); + if (string.IsNullOrWhiteSpace(identifier)) + { + throw new ArgumentException("identifier darf nicht leer sein."); + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + throw new ArgumentException($"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"."); + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + + IAssetAdministrationShell? shell; + try + { + shell = await _dbRequestHandlerService.ReadAssetAdministrationShellById(securityConfig, identifier.Trim()); + } + catch (NotFoundException ex) + { + return new { identifier, found = false, message = ex.Message }; + } + + if (shell is null) + { + return new { identifier, found = false }; + } + + var result = BuildShellObject(shell, fmt); + result["identifier"] = identifier; + result["format"] = fmt; + return result; + } + + [McpServerTool(Name = "aas_get_shells", Title = "Get Multiple AAS Shells", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Holt MEHRERE AAS (Shells) in EINEM Aufruf — eine Liste von AAS-Identifiern statt vieler Einzelabrufe. " + + "Je Shell die AssetInformation + Submodel-Referenzen (format=\"value\", Default) oder das volle AAS-JSON (format=\"full\").")] + public async Task AasGetShells( + [Description("Liste von AAS-Identifiern (vollständige ids, NICHT Base64-kodiert).")] string[] identifiers, + [Description("Ausgabeformat je Shell: \"value\" (kompakt, Default) oder \"full\".")] string format = "value") + { + using var _ = LogCallTimed($"aas_get_shells n={(identifiers?.Length ?? 0)} format={format}"); + + if (identifiers is null || identifiers.Length == 0) + { + return new { error = "identifiers darf nicht leer sein (Liste von AAS-Identifiern)." }; + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + return new { error = $"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"." }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + + var shells = new JsonArray(); + foreach (var id in identifiers) + { + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + + IAssetAdministrationShell? shell = null; + try + { + shell = await _dbRequestHandlerService.ReadAssetAdministrationShellById(securityConfig, id.Trim()); + } + catch (NotFoundException) + { + shell = null; + } + + if (shell is null) + { + shells.Add(new JsonObject { ["identifier"] = id.Trim(), ["found"] = false }); + continue; + } + + var entry = BuildShellObject(shell, fmt); + entry["identifier"] = id.Trim(); + shells.Add(entry); + } + + return new { count = shells.Count, format = fmt, shells }; + } + + [McpServerTool(Name = "aas_get_product", Title = "Get Complete AAS Product", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Liefert in EINEM Aufruf ein komplettes Produkt: die AAS (assetInformation) PLUS die Werte ALLER ihrer Submodelle " + + "(z.B. TechnicalData + Nameplate + CarbonFootprint zusammen). " + + "BENUTZE DIES, wenn eine Frage Daten aus mehreren Submodellen braucht — etwa technische Daten UND Hersteller UND CO2-Fußabdruck — " + + "damit du NICHT einzeln navigieren musst. " + + "identifier kann eine AAS-id ODER eine Submodel-id sein (z.B. ein Treffer aus aas_query); die zugehörige Shell wird automatisch ermittelt. " + + "Typischer Ablauf: aas_query findet ein Submodel -> dessen id hier übergeben -> alles zum Produkt kommt zurück. " + + "Bei vielen/großen Submodellen über submodelLimit/submodelCursor weiterblättern; nextSubmodelCursor aus der Antwort verwenden.")] + public async Task AasGetProduct( + [Description("AAS-Identifier ODER Submodel-Identifier (vollständige id, NICHT Base64-kodiert). Ein Submodel-Treffer aus aas_query genügt — die Shell wird automatisch aufgelöst.")] string identifier, + [Description("Ausgabeformat je Submodel: \"value\" (kompakt, Default, nur idShort->Wert) oder \"full\" (vollständiges AAS-JSON inkl. semanticId, valueType, Qualifier). Nutze \"full\", wenn nach semanticId, Einheiten oder Datentyp gefragt wird.")] string format = "value", + [Description("Maximale Anzahl Submodelle dieses Produkts in dieser Antwort (Default und Maximum 50).")] int? submodelLimit = null, + [Description("Cursor zum Weiterblättern der Produkt-Submodelle; aus nextSubmodelCursor einer vorigen Antwort.")] string? submodelCursor = null) + { + using var _ = LogCallTimed($"aas_get_product id={identifier} format={format} submodelLimit={(submodelLimit?.ToString(CultureInfo.InvariantCulture) ?? "-")} submodelCursor={submodelCursor ?? "-"}"); + if (string.IsNullOrWhiteSpace(identifier)) + { + return new { error = "identifier darf nicht leer sein." }; + } + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + return new { error = $"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"." }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var shell = await ResolveShellForIdentifier(securityConfig, identifier.Trim()); + if (shell is null) + { + return new { identifier, found = false, message = "Identifier ist weder eine bekannte AAS noch ein bekanntes Submodel." }; + } + + return await BuildProductObject( + securityConfig, + shell, + fmt, + submodelCursor: NormalizeCursor(submodelCursor), + submodelLimit: NormalizeProductSubmodelLimit(submodelLimit)); + } + + [McpServerTool(Name = "aas_find_product", Title = "Find Complete AAS Product", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Sucht ein Produkt anhand von Bedingungen UND liefert es in EINEM Aufruf komplett zurück — ohne Verkettung. " + + "Das richtige Tool für Fragen wie 'finde das Ventil mit flowMax 80 und nenne Hersteller, technische Daten und CO2-Fußabdruck'. " + + "Bedingungslogik identisch zu aas_query (conditions/combine, Slots scope/field/idShortPath/op/value). " + + "Liefert zum ERSTEN Treffer die AAS (assetInformation) plus die Werte ALLER ihrer Submodelle (TechnicalData + Nameplate + CarbonFootprint usw.). " + + "format=\"value\" (Default, kompakt) oder \"full\" (vollständiges AAS-JSON je Submodel). " + + "totalMatches zeigt, wie viele Produkte insgesamt passen (geliefert wird das erste).")] + public async Task AasFindProduct( + [Description("Liste von Suchbedingungen (mindestens eine), gleiche Slots wie aas_query: scope/field/idShortPath/op/value.")] McpQueryCondition[] conditions, + [Description("Verknüpfung mehrerer Bedingungen: \"and\" oder \"or\".")] string combine = "and", + [Description("Ausgabeformat je Submodel: \"value\" (kompakt, Default, nur idShort->Wert) oder \"full\" (vollständiges AAS-JSON inkl. semanticId, valueType, Qualifier). Nutze \"full\", wenn nach semanticId, Einheiten oder Datentyp gefragt wird.")] string format = "value") + { + using var _ = LogCallTimed($"aas_find_product {DescribeConditions(conditions, combine)} format={format}"); + + var fmt = (format ?? "value").Trim().ToLowerInvariant(); + if (fmt != "value" && fmt != "full") + { + return new { error = $"Unbekanntes format \"{format}\". Erlaubt: \"value\" oder \"full\"." }; + } + + string expression; + try + { + expression = BuildExpression(conditions, combine); + } + catch (ArgumentException ex) + { + return new { error = ex.Message }; + } + + return await FindProductCore(expression, fmt); + } + + [McpServerTool(Name = "aas_find_product_simple", Title = "Find Product by Simple Property", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Findet ein Produkt anhand EINES Merkmals und liefert es komplett zurück — der einfachste Weg, ohne Bedingungs-Syntax. " + + "Gib einfach den Merkmalsnamen (idShort, z.B. \"flowMax\") und den gesuchten Wert (z.B. \"80\") an. " + + "Das Tool findet das Produkt, dessen Element mit diesem idShort den angegebenen Wert hat, und liefert die AAS plus die Werte ALLER Submodelle " + + "(Hersteller/Nameplate, technische Daten, CO2-Fußabdruck usw.) — alles in einem Aufruf. " + + "KEIN scope/op/conditions/format nötig: nur idShort und value. Das Ergebnis ist bewusst kompakt gehalten.")] + public async Task AasFindProductSimple( + [Description("Name des gesuchten Merkmals/Elements (idShort), z.B. \"flowMax\" oder \"ManufacturerName\".")] string idShort, + [Description("Gesuchter Wert dieses Merkmals, z.B. \"80\".")] string value) + { + using var _ = LogCallTimed($"aas_find_product_simple idShort={idShort} value={value}"); + + if (string.IsNullOrWhiteSpace(idShort)) + { + return new { error = "idShort darf nicht leer sein (z.B. \"flowMax\")." }; + } + + if (string.IsNullOrWhiteSpace(value)) + { + return new { error = "value darf nicht leer sein (z.B. \"80\")." }; + } + + var conditions = new[] + { + new McpQueryCondition { Scope = "sme", Field = "value", IdShortPath = idShort.Trim(), Op = "eq", Value = value }, + }; + + string expression; + try + { + expression = BuildExpression(conditions, "and"); + } + catch (ArgumentException ex) + { + return new { error = ex.Message }; + } + + // Bewusst immer "value" (kompakt) + coreOnly: full und Datei-/Doku-Submodelle überfordern sehr kleine Modelle (Halluzination). + return await FindProductCore(expression, "value", coreOnly: true); + } + + [McpServerTool(Name = "aas_find_concepts", Title = "Find Concept Definitions (Semantic IDs)", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Durchsucht den Merkmalskatalog (ConceptDescriptions, ECLASS/IEC 61360) nach Stichwörtern und liefert die semantischen Definitionen der Datenfelder: " + + "IRDI (= semanticId, z.B. 0173-1#02-AAM635#003), Namen (de/en), Einheit, Datentyp und Definition. " + + "ZUERST verwenden, wenn unklar ist, wie ein Merkmal in den Daten heißt (z.B. \"Ausgangsspannung\", \"output power\", \"Gewicht\") — " + + "danach mit aas_query exakt suchen, statt idShort-Schreibweisen zu raten: " + + "EIN Merkmal: field=\"semanticId\" op=\"eq\" value= plus optional eine value-Bedingung (combine=\"and\") — beide beziehen sich auf DASSELBE Element. " + + "MEHRERE Merkmale (z.B. Spannung UND Leistung): NICHT mehrere semanticId/value-Paare kombinieren (ergibt immer 0 Treffer) — stattdessen je Merkmal eine Bedingung mit idShortPath= und op/value; der idShort des Konzepts ist zugleich der idShort des Elements in den Daten. " + + "Die Suche ist case-insensitiv; mehrere Wörter sind UND-verknüpft und werden in idShort, preferredName, shortName und definition (de+en) gesucht. " + + "Bei 0 Treffern: Synonyme oder die jeweils andere Sprache probieren (z.B. \"voltage\" statt \"Spannung\"). " + + "usageCount gibt an, in wie vielen Submodellen die semanticId tatsächlich vorkommt (gedeckelt bei 1000; 1000 = 1000 oder mehr) — Konzepte mit usageCount=0 existieren nur im Katalog und sind für Suchen nutzlos.")] + public async Task AasFindConcepts( + [Description("Suchbegriffe, z.B. \"output voltage\" oder \"Ausgangsspannung\". Mehrere Wörter müssen alle vorkommen (UND). Gesucht wird in deutschen UND englischen Texten.")] string search, + [Description("Maximale Trefferzahl (Default 20, Maximum 100).")] int? limit = null, + [Description("Bevorzugte Sprache der definition im Ergebnis: \"en\" oder \"de\" (Default \"en\"; fehlt die Sprache, wird die andere genommen).")] string lang = "en", + [Description("Wenn true (Default), wird je Treffer gezählt, in wie vielen Submodellen die semanticId vorkommt (usageCount; wird für maximal die ersten 15 Treffer berechnet).")] bool withUsage = true) + { + using var _ = LogCallTimed($"aas_find_concepts search=\"{search}\" limit={(limit?.ToString(CultureInfo.InvariantCulture) ?? "-")} withUsage={withUsage}"); + + if (string.IsNullOrWhiteSpace(search)) + { + return new { error = "search darf nicht leer sein, z.B. \"output voltage\" oder \"Ausgangsspannung\"." }; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + + List catalog; + try + { + catalog = await GetConceptCatalog(securityConfig); + } + catch (Exception ex) + { + return new { error = "ConceptDescriptions konnten nicht geladen werden: " + ex.Message }; + } + + var terms = search.ToLowerInvariant().Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var requestedLimit = Math.Min(limit is > 0 ? limit.Value : DefaultConceptLimit, MaxConceptLimit); + + // Alle Terme müssen irgendwo vorkommen; Treffer im Namen (idShort/preferredName/shortName) + // zählen doppelt, damit "voltage" MaxOutputVoltage vor Konzepten mit "voltage" nur in der Definition listet. + // Mit withUsage wird ein größerer Kandidaten-Pool gezogen und erst NACH der Usage-Sortierung auf limit + // geschnitten — sonst verdrängen textlich gleichwertige Katalogleichen bei kleinem limit die real + // genutzten Konzepte (Counts sind dank 1000er-Deckel billig). + var ranked = catalog + .Select(c => new + { + Concept = c, + Score = terms.Sum(t => c.NameText.Contains(t, StringComparison.Ordinal) ? 2 + : c.SearchText.Contains(t, StringComparison.Ordinal) ? 1 : 0), + }) + .Where(x => terms.All(t => x.Concept.SearchText.Contains(t, StringComparison.Ordinal))) + .OrderByDescending(x => x.Score) + .ThenBy(x => x.Concept.IdShort, StringComparer.OrdinalIgnoreCase) + .Take(withUsage ? Math.Max(requestedLimit, ConceptRankingPoolSize) : requestedLimit) + .ToList(); + + // Usage nur für die vorderen Treffer zählen (je ein indizierter COUNT über die Query-Pipeline, gecacht). + var usage = new Dictionary(StringComparer.Ordinal); + if (withUsage) + { + foreach (var x in ranked.Take(MaxConceptUsageLookups)) + { + usage[x.Concept.Id] = await GetConceptUsageCount(securityConfig, x.Concept.Id); + } + } + + var preferGerman = string.Equals(lang?.Trim(), "de", StringComparison.OrdinalIgnoreCase); + var concepts = ranked + .OrderByDescending(x => usage.TryGetValue(x.Concept.Id, out var u) ? u ?? -1 : -1) + .ThenByDescending(x => x.Score) + .ThenBy(x => x.Concept.IdShort, StringComparer.OrdinalIgnoreCase) + .Take(requestedLimit) + .Select(x => new + { + id = x.Concept.Id, + idShort = x.Concept.IdShort, + nameEn = x.Concept.NameEn, + nameDe = x.Concept.NameDe, + unit = x.Concept.Unit, + dataType = x.Concept.DataType, + definition = preferGerman ? (x.Concept.DefDe ?? x.Concept.DefEn) : (x.Concept.DefEn ?? x.Concept.DefDe), + usageCount = usage.TryGetValue(x.Concept.Id, out var u) ? u : null, + }) + .ToList(); + + return new + { + search, + count = concepts.Count, + concepts, + nextStep = concepts.Count > 0 + ? "In den Daten suchen mit aas_query. EIN Merkmal: conditions=[{scope:\"sme\",field:\"semanticId\",op:\"eq\",value:\"\"},{scope:\"sme\",field:\"value\",op:\"gt\",value:\"...\"}] mit combine=\"and\" — " + + "beide Bedingungen beziehen sich auf DASSELBE Element. MEHRERE Merkmale: je Merkmal EINE Bedingung mit idShortPath= und op/value " + + "(z.B. {scope:\"sme\",idShortPath:\"Power_output\",op:\"gt\",value:\"500\"}); mehrere semanticId/value-Paare in einem and ergeben immer 0 Treffer. " + + "Konzepte mit usageCount=0 nicht verwenden; bei mehreren passenden Kandidaten op=\"in\" mit values=[,]." + : "Keine Konzepte gefunden. Synonyme oder die andere Sprache probieren (z.B. \"voltage\" statt \"Spannung\") oder weniger/kürzere Suchwörter verwenden.", + }; + } + + // --------------- ConceptDescription-Katalog (Cache für aas_find_concepts) --------------- + + private const int DefaultConceptLimit = 20; + private const int MaxConceptLimit = 100; + private const int MaxConceptUsageLookups = 40; + private const int ConceptRankingPoolSize = 30; + private const int UsageCountCap = 1000; + private static readonly TimeSpan ConceptCacheTtl = TimeSpan.FromMinutes(10); + + // Kompakte, durchsuchbare Sicht auf eine ConceptDescription (dedupliziert nach Id). + private sealed class ConceptInfo + { + public required string Id { get; init; } + public string? IdShort { get; init; } + public string? NameEn { get; init; } + public string? NameDe { get; init; } + public string? Unit { get; init; } + public string? DataType { get; init; } + public string? DefEn { get; init; } + public string? DefDe { get; init; } + public required string NameText { get; init; } // lowercase: idShort + Namen + public required string SearchText { get; init; } // lowercase: NameText + Definitionen + Id + } + + private static List? s_conceptCatalog; + private static DateTime s_conceptCatalogTime; + private static readonly System.Threading.SemaphoreSlim s_conceptCatalogLock = new(1, 1); + + // usageCount-Cache je IRDI: COUNT-Queries auf großen DBs nicht bei jeder Konzeptsuche wiederholen. + private static readonly System.Collections.Concurrent.ConcurrentDictionary s_conceptUsage = + new(StringComparer.Ordinal); + + private async Task> GetConceptCatalog(SecurityConfig securityConfig) + { + var cached = s_conceptCatalog; + if (cached != null && DateTime.UtcNow - s_conceptCatalogTime < ConceptCacheTtl) + { + return cached; + } + + await s_conceptCatalogLock.WaitAsync(); + try + { + cached = s_conceptCatalog; + if (cached != null && DateTime.UtcNow - s_conceptCatalogTime < ConceptCacheTtl) + { + return cached; + } + + var watch = Stopwatch.StartNew(); + var pagination = new PaginationParameters(null, int.MaxValue); + var cdList = await _dbRequestHandlerService.ReadPagedConceptDescriptions(pagination, securityConfig); + + // Die Liste enthält je Environment dieselbe CD erneut — nach Id deduplizieren. + var byId = new Dictionary(StringComparer.Ordinal); + foreach (var cd in (cdList ?? new List()).OfType()) + { + if (string.IsNullOrWhiteSpace(cd.Id) || byId.ContainsKey(cd.Id)) + { + continue; + } + + byId[cd.Id] = BuildConceptInfo(cd); + } + + var catalog = byId.Values.ToList(); + s_conceptCatalog = catalog; + s_conceptCatalogTime = DateTime.UtcNow; + LogMcpLine($"aas_find_concepts catalog: {catalog.Count} distinct concepts loaded in {watch.ElapsedMilliseconds} ms"); + return catalog; + } + finally + { + s_conceptCatalogLock.Release(); + } + } + + private static ConceptInfo BuildConceptInfo(IConceptDescription cd) + { + var iec = cd.EmbeddedDataSpecifications? + .Select(e => e?.DataSpecificationContent) + .OfType() + .FirstOrDefault(); + + var nameEn = PickLangText(iec?.PreferredName, "en") ?? PickLangText(iec?.ShortName, "en"); + var nameDe = PickLangText(iec?.PreferredName, "de") ?? PickLangText(iec?.ShortName, "de"); + var defEn = TruncateText(PickLangText(iec?.Definition, "en"), 300); + var defDe = TruncateText(PickLangText(iec?.Definition, "de"), 300); + + var nameText = JoinLower(cd.IdShort, nameEn, nameDe, PickLangText(iec?.ShortName, null)); + var searchText = JoinLower(nameText, defEn, defDe, cd.Id); + + return new ConceptInfo + { + Id = cd.Id, + IdShort = cd.IdShort, + NameEn = nameEn, + NameDe = nameDe, + Unit = string.IsNullOrWhiteSpace(iec?.Unit) ? null : iec.Unit, + DataType = iec?.DataType is { } dataType ? Stringification.ToString(dataType) : null, + DefEn = defEn, + DefDe = defDe, + NameText = nameText, + SearchText = searchText, + }; + } + + // Text einer LangString-Liste in der gewünschten Sprache; language=null nimmt den ersten Eintrag. + private static string? PickLangText(System.Collections.IEnumerable? langStrings, string? language) + { + if (langStrings is null) + { + return null; + } + + string? first = null; + foreach (var ls in langStrings.OfType()) + { + if (string.IsNullOrWhiteSpace(ls.Text)) + { + continue; + } + + first ??= ls.Text; + if (language != null && string.Equals(ls.Language?.Trim(), language, StringComparison.OrdinalIgnoreCase)) + { + return ls.Text; + } + } + + return language is null ? first : null; + } + + private static string JoinLower(params string?[] parts) + => string.Join('\n', parts.Where(p => !string.IsNullOrWhiteSpace(p))).ToLowerInvariant(); + + private static string? TruncateText(string? text, int maxLength) + { + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + var trimmed = text.Trim(); + return trimmed.Length <= maxLength ? trimmed : trimmed[..maxLength] + "…"; + } + + // Anzahl Submodelle, in denen die semanticId vorkommt — indizierter COUNT über die Query-Pipeline, gecacht. + private async Task GetConceptUsageCount(SecurityConfig securityConfig, string semanticId) + { + if (s_conceptUsage.TryGetValue(semanticId, out var entry) && DateTime.UtcNow - entry.Time < ConceptCacheTtl) + { + return entry.Count; + } + + try + { + var expression = BuildExpression( + new[] { new McpQueryCondition { Scope = "sme", Field = "semanticId", Op = "eq", Value = semanticId } }, "and"); + + // Gedeckelter Count (LIMIT im Subselect bricht früh ab): "1000" bedeutet "1000 oder mehr". + // Für die Frage "existiert das Konzept in den Daten und wie verbreitet ist es?" reicht das — + // exakte Zahlen über hunderttausende Treffer kosten Sekunden pro semanticId. + var count = await _dbRequestHandlerService.QueryCountSMs( + securityConfig, string.Empty, string.Empty, string.Empty, + new PaginationParameters(null, UsageCountCap), ResultType.Submodel, expression); + s_conceptUsage[semanticId] = (count, DateTime.UtcNow); + return count; + } + catch + { + // usageCount ist Zusatzinfo — ein Fehler (z.B. Engine-Limit) darf die Konzeptsuche nicht scheitern lassen. + return null; + } + } + + [McpServerTool(Name = "aas_describe_model", Title = "Describe Submodel Structures", Destructive = false, ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description( + "Liefert einen Überblick über die real vorhandenen Datenstrukturen: welche Submodel-Typen es gibt (idShort, semanticId, exakte Gesamtzahl — " + + "deterministisch und vollständig aus der Datenbank ermittelt, inkl. semanticId-Varianten je Typ) " + + "und welche Felder darin vorkommen — je Feld idShortPath, semanticId, Element-Typ, valueType, Einheit, Name und ein Beispielwert. " + + "Die Feldlisten stammen aus einer über den Bestand gestreuten Stichprobe echter Submodelle, angereichert mit den ConceptDescriptions. " + + "IDEALER ERSTER AUFRUF einer Session, BEVOR Suchanfragen formuliert werden: danach sind die korrekten idShortPaths und semanticIds " + + "für aas_query-Bedingungen, select-Projektionen und Exporte bekannt, statt Feldnamen zu raten. " + + "seenInSamples zeigt, in wie vielen der Stichproben-Submodelle das Feld vorkam — Felder, die nur bestimmte Produktklassen haben " + + "(z.B. Ausgangsspannung nur bei Netzteilen), können in der Stichprobe fehlen; solche Merkmale gezielt über aas_find_concepts suchen. " + + "Das Ergebnis wird serverseitig 10 Minuten gecacht.")] + public async Task AasDescribeModel( + [Description("Optional: nur diesen Submodel-Typ beschreiben (idShort, z.B. \"TechnicalData\"). Ohne Angabe werden alle gefundenen Submodel-Typen beschrieben.")] string? submodel = null, + [Description("Stichprobengröße je Submodel-Typ (Default 5, Maximum 20). Mehr Stichproben finden mehr produktklassen-spezifische Felder, dauern aber länger.")] int? samples = null, + [Description("Maximale Feldzahl je Submodel-Typ im Ergebnis (Default 200, Maximum 500). Häufige Felder zuerst; truncated=true zeigt Kürzung an.")] int? maxFields = null, + [Description("Bevorzugte Sprache für Namen aus den ConceptDescriptions und Beispielwerte mehrsprachiger Felder (Default \"en\").")] string lang = "en", + [Description("Wenn true, wird für SELTENE Felder (nicht in allen Stichproben) gezählt, in wie vielen Submodellen des Bestands ihre semanticId vorkommt (usageCount, gedeckelt bei 1000 = \"1000 oder mehr\"; gecacht, max. 30 frische Zählungen pro Aufruf). Allgegenwärtige Felder erhalten keinen usageCount — dort genügt seenInSamples. Default false.")] bool withUsage = false) + { + using var _ = LogCallTimed($"aas_describe_model submodel={submodel ?? "-"} samples={(samples?.ToString(CultureInfo.InvariantCulture) ?? "-")} withUsage={withUsage}"); + + var sampleCount = Math.Clamp(samples ?? DefaultDescribeSamples, 1, MaxDescribeSamples); + var fieldCap = Math.Clamp(maxFields ?? DefaultDescribeMaxFields, 10, MaxDescribeMaxFields); + var normalizedLang = string.IsNullOrWhiteSpace(lang) ? "en" : lang.Trim(); + var templateFilter = string.IsNullOrWhiteSpace(submodel) ? null : submodel.Trim(); + + var cacheKey = $"{templateFilter ?? "*"}|{sampleCount}|{fieldCap}|{normalizedLang}|{withUsage}"; + if (s_describeCache.TryGetValue(cacheKey, out var cached) && DateTime.UtcNow - cached.Time < ConceptCacheTtl) + { + return cached.Value; + } + + var securityConfig = new SecurityConfig(Program.noSecurity, null); + object result; + try + { + result = await BuildModelDescription(securityConfig, templateFilter, sampleCount, fieldCap, normalizedLang, withUsage); + } + catch (Exception ex) + { + return new { error = "Modellbeschreibung fehlgeschlagen: " + ex.Message }; + } + + s_describeCache[cacheKey] = (result, DateTime.UtcNow); + return result; + } + + // --------------- Modellbeschreibung (aas_describe_model) --------------- + + private const int DefaultDescribeSamples = 5; + private const int MaxDescribeSamples = 20; + private const int DefaultDescribeMaxFields = 200; + private const int MaxDescribeMaxFields = 500; + private const int MaxDescribeUsageLookups = 30; + private const int DescribeDiscoveryPageSize = 40; + private const int DescribeEmptyRetryPageSize = 10; + private const int MaxDescribeTemplates = 50; + private static readonly TimeSpan DescribeUsageTimeBudget = TimeSpan.FromSeconds(15); + + private static readonly System.Collections.Concurrent.ConcurrentDictionary s_describeCache = + new(StringComparer.Ordinal); + + // Aggregat eines Feldes über die Stichprobe: gleiche Struktur (Pfad mit neutralisierten Ziffern) + gleiche semanticId = ein Feld. + private sealed class FieldAggregate + { + public required string Path { get; init; } + public string? SemanticId { get; init; } + public required string Type { get; init; } + public string? ValueType { get; set; } + public string? Example { get; set; } + public int SeenIn { get; set; } + public HashSet Variants { get; } = new(StringComparer.Ordinal); + } + + private async Task BuildModelDescription( + SecurityConfig securityConfig, string? templateFilter, int sampleCount, int fieldCap, string lang, bool withUsage) + { + // 1) DETERMINISTISCH: vollständiges Template-Inventar der DB (GROUP BY IdShort+SemanticId über die + // SMSets — zwei Spalten, eine Zeile je Submodel; NICHT die großen SMESets). Erst danach Stichprobe. + List? inventory = null; + var inventoryWatch = Stopwatch.StartNew(); + try + { + inventory = await _dbRequestHandlerService.ReadSubmodelTemplates(securityConfig); + LogMcpLine($"aas_describe_model inventory: {inventory.Count} rows in {inventoryWatch.ElapsedMilliseconds} ms"); + } + catch (Exception ex) + { + // Fallback (z.B. Security aktiv): Templates aus der Discovery-Seite ableiten — dann stochastisch. + LogMcpLine($"aas_describe_model inventory unavailable ({ex.Message}); using page discovery"); + } + + // 2) Erste Stichproben aus einer Seite echter Submodelle; ohne Inventar zugleich Template-Entdeckung. + var discovery = await _dbRequestHandlerService.ReadPagedSubmodels( + new PaginationParameters(null, DescribeDiscoveryPageSize), securityConfig, null, null, null, null); + + var localSamples = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var sm in (discovery ?? new List()).OfType()) + { + if (string.IsNullOrWhiteSpace(sm.IdShort)) + { + continue; + } + + if (templateFilter != null && !string.Equals(sm.IdShort, templateFilter, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!localSamples.TryGetValue(sm.IdShort, out var list)) + { + localSamples[sm.IdShort] = list = new List(); + } + + // Höchstens 2 Muster aus dem Anfang des Bestands — der Rest wird gestreut nachgeladen, + // damit auch produktklassen-spezifische Felder (variable TechnicalData) auftauchen. + if (list.Count < 2) + { + list.Add(sm); + } + } + + // 3) Template-Liste: aus dem Inventar (vollständig, exakte Zahlen, semanticId-Varianten), + // im Fallback aus den Discovery-Stichproben. + List<(string Name, int? TotalCount, List? Variants)> templateInfos; + if (inventory != null) + { + templateInfos = inventory + .Where(row => !string.IsNullOrWhiteSpace(row.IdShort)) + .GroupBy(row => row.IdShort!.Trim(), StringComparer.OrdinalIgnoreCase) + .Where(g => templateFilter == null || string.Equals(g.Key, templateFilter, StringComparison.OrdinalIgnoreCase)) + .Select(g => ( + Name: g.Key, + TotalCount: (int?)g.Sum(row => row.Count), + Variants: (List?)g.OrderByDescending(row => row.Count).ToList())) + .OrderByDescending(t => t.TotalCount) + .ThenBy(t => t.Name, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + else + { + // Explizit angefragter Typ, der in der Discovery-Seite nicht vorkam: + // leer starten, die Stichprobe kommt dann komplett über die Query-Pipeline. + if (templateFilter != null && !localSamples.ContainsKey(templateFilter)) + { + localSamples[templateFilter] = new List(); + } + + templateInfos = localSamples.Keys + .OrderBy(name => name, StringComparer.OrdinalIgnoreCase) + .Select(name => (name, (int?)null, (List?)null)) + .ToList(); + } + + var catalog = await GetConceptCatalog(securityConfig); + var cdById = new Dictionary(StringComparer.Ordinal); + foreach (var concept in catalog) + { + cdById.TryAdd(concept.Id, concept); + } + + var templates = new List(); + foreach (var info in templateInfos.Take(MaxDescribeTemplates)) + { + localSamples.TryGetValue(info.Name, out var samplesForTemplate); + templates.Add(await DescribeTemplate( + securityConfig, info.Name, samplesForTemplate ?? new List(), + sampleCount, fieldCap, lang, withUsage, cdById, info.TotalCount, info.Variants)); + } + + return new + { + templateCount = templateInfos.Count, + truncatedTemplates = templateInfos.Count > MaxDescribeTemplates ? true : (bool?)null, + templates, + nextStep = templates.Count > 0 + ? "So damit suchen (aas_query): EIN Merkmal: conditions=[{scope:\"sme\",field:\"semanticId\",op:\"eq\",value:},{scope:\"sme\",field:\"value\",op:\"gt\",value:\"...\"}] mit combine=\"and\" (beide auf DASSELBE Element bezogen). " + + "MEHRERE Merkmale: je Merkmal EINE Bedingung {scope:\"sme\",idShortPath:,op,value}. " + + "Tabellen/Exporte: select=[], für Felder anderer Submodelle derselben AAS //. " + + "Einzelne Merkmale per Stichwort: aas_find_concepts." + : "Keine Submodelle gefunden. Ohne submodel-Filter aufrufen oder Schreibweise des idShort prüfen.", + }; + } + + private async Task DescribeTemplate( + SecurityConfig securityConfig, string templateName, List localSamples, + int sampleCount, int fieldCap, string lang, bool withUsage, Dictionary cdById, + int? knownTotalCount, List? semanticVariants) + { + var expression = BuildExpression( + new[] { new McpQueryCondition { Scope = "sm", Field = "idShort", Op = "eq", Value = templateName } }, "and"); + + var totalCount = knownTotalCount ?? 0; + if (knownTotalCount == null) + { + try + { + totalCount = await _dbRequestHandlerService.QueryCountSMs( + securityConfig, string.Empty, string.Empty, string.Empty, + new PaginationParameters(null, int.MaxValue), ResultType.Submodel, expression); + } + catch + { + // Gesamtzahl ist Zusatzinfo; Beschreibung geht mit totalCount=0 weiter. + } + } + + // Stichprobe auffüllen: gestreute Offsets über den Bestand statt nur Anfangsbereich. + var sampled = new List(localSamples); + var sampledIds = new HashSet(sampled.Select(s => s.Id ?? string.Empty), StringComparer.Ordinal); + var needed = Math.Min(sampleCount, totalCount > 0 ? totalCount : sampleCount) - sampled.Count; + for (var i = 1; i <= needed && totalCount > localSamples.Count; i++) + { + var offset = (long)totalCount * i / (needed + 1); + var (list, err) = await TryQuery( + securityConfig, new PaginationParameters(offset.ToString(CultureInfo.InvariantCulture), 1), ResultType.Submodel, expression); + if (err != null) + { + break; + } + + var id = ExtractIdentifiers(list).FirstOrDefault(); + if (id == null || !sampledIds.Add(id)) + { + continue; + } + + try + { + if (await _dbRequestHandlerService.ReadSubmodelById(securityConfig, null, id, null, null) is ISubmodel sm) + { + sampled.Add(sm); + } + } + catch (NotFoundException) + { + // Treffer zwischen Query und Read verschwunden — Stichprobe bleibt einfach kleiner. + } + } + + var fields = new Dictionary(StringComparer.Ordinal); + + void AggregateSample(ISubmodel sm) + { + var seenThisSample = new HashSet(StringComparer.Ordinal); + CollectFieldInfos(sm.SubmodelElements, string.Empty, (path, element) => + { + var semanticId = LastKeyValue(element.SemanticId); + var key = NormalizeStructurePath(path) + "|" + (semanticId ?? string.Empty); + if (!fields.TryGetValue(key, out var agg)) + { + fields[key] = agg = new FieldAggregate + { + Path = path, + SemanticId = semanticId, + Type = element.GetType().Name, + }; + } + + agg.Variants.Add(path); + if (seenThisSample.Add(key)) + { + agg.SeenIn++; + } + + agg.Example ??= ExampleValue(element, lang); + if (agg.ValueType == null && element is IProperty property) + { + agg.ValueType = Stringification.ToString(property.ValueType); + } + }); + } + + foreach (var sm in sampled) + { + AggregateSample(sm); + } + + // Alle Stichproben leer (z.B. CarbonFootprint: bei jedem Produkt vorhanden, aber überwiegend + // unbefüllt)? Dann gezielt eine Seite dieses Typs laden und die ersten befüllten Exemplare beschreiben. + if (fields.Count == 0 && totalCount > 0) + { + try + { + var retry = await _dbRequestHandlerService.ReadPagedSubmodels( + new PaginationParameters(null, DescribeEmptyRetryPageSize), securityConfig, null, templateName, null, null); + var added = 0; + foreach (var sm in (retry ?? new List()).OfType()) + { + if (sm.SubmodelElements is not { Count: > 0 } || !sampledIds.Add(sm.Id ?? string.Empty)) + { + continue; + } + + sampled.Add(sm); + AggregateSample(sm); + if (++added >= 3) + { + break; + } + } + } + catch + { + // Nachfass ist Best Effort; die Beschreibung bleibt sonst bei "alle Stichproben leer". + } + } + + var emptySamples = sampled.Count(s => s.SubmodelElements is not { Count: > 0 }); + string? note = null; + if (fields.Count == 0 && totalCount > 0) + { + note = $"Alle {sampled.Count} Stichproben sind leer — dieser Submodel-Typ existiert {totalCount}-mal, ist aber überwiegend unbefüllt. " + + "Befüllte Exemplare über aas_query finden (sm.idShort eq + eine sme-Bedingung, z.B. sme.idShort eq )."; + } + else if (emptySamples > 0) + { + note = $"{emptySamples} von {sampled.Count} Stichproben waren leer — dieser Submodel-Typ ist nicht bei allen Produkten befüllt."; + } + + var ordered = fields.Values + .OrderByDescending(f => f.SeenIn) + .ThenBy(f => f.Path, StringComparer.OrdinalIgnoreCase) + .ToList(); + + var usage = new Dictionary(StringComparer.Ordinal); + if (withUsage) + { + // Allgegenwärtige Felder nicht zählen: der COUNT läuft dann über hunderttausende Treffer + // (teuer, gemessen ~5 s/Stück) und sagt nichts Neues — seenInSamples == sampledCount genügt. + // WICHTIG: über die semanticId ausschließen, nicht nur über die Feldzeile — dieselbe IRDI + // taucht oft zusätzlich an seltenen Pfaden auf (z.B. Zubehör-Äste) und würde dort doch gezählt. + var ubiquitousIds = new HashSet( + fields.Values.Where(f => f.SeenIn >= sampled.Count && f.SemanticId != null).Select(f => f.SemanticId!), + StringComparer.Ordinal); + + // Seltenste zuerst (selektivste = billigste + informativste Counts); Stückzahl- UND Zeitbudget, + // denn auch sample-seltene Felder können hunderttausendfach vorkommen. + var budget = MaxDescribeUsageLookups; + var usageWatch = Stopwatch.StartNew(); + foreach (var f in ordered.Take(fieldCap).OrderBy(f => f.SeenIn)) + { + if (f.SemanticId == null || usage.ContainsKey(f.SemanticId) || ubiquitousIds.Contains(f.SemanticId)) + { + continue; + } + + if (s_conceptUsage.TryGetValue(f.SemanticId, out var entry) && DateTime.UtcNow - entry.Time < ConceptCacheTtl) + { + usage[f.SemanticId] = entry.Count; + continue; + } + + if (budget-- <= 0 || usageWatch.Elapsed > DescribeUsageTimeBudget) + { + continue; + } + + usage[f.SemanticId] = await GetConceptUsageCount(securityConfig, f.SemanticId); + } + } + + var preferGerman = string.Equals(lang, "de", StringComparison.OrdinalIgnoreCase); + var rows = ordered + .Take(fieldCap) + .Select(f => + { + ConceptInfo? cd = null; + if (f.SemanticId != null) + { + cdById.TryGetValue(f.SemanticId, out cd); + } + + return new + { + path = f.Path, + variants = f.Variants.Count > 1 ? (int?)f.Variants.Count : null, + type = f.Type, + valueType = f.ValueType, + semanticId = f.SemanticId, + name = cd == null ? null : (preferGerman ? cd.NameDe ?? cd.NameEn : cd.NameEn ?? cd.NameDe), + unit = cd?.Unit, + example = f.Example, + seenInSamples = f.SeenIn, + usageCount = f.SemanticId != null && usage.TryGetValue(f.SemanticId, out var u) ? u : null, + }; + }) + .ToList(); + + return new + { + submodel = templateName, + semanticId = semanticVariants is { Count: > 0 } + ? semanticVariants[0].SemanticId + : LastKeyValue(sampled.FirstOrDefault()?.SemanticId), + semanticIdVariants = semanticVariants is { Count: > 1 } + ? (object?)semanticVariants.Select(v => new { semanticId = v.SemanticId, count = v.Count }).ToList() + : null, + totalCount, + sampledCount = sampled.Count, + note, + fieldCount = fields.Count, + truncated = fields.Count > fieldCap ? true : (bool?)null, + fields = rows, + }; + } + + // Rekursiver Strukturlauf: ruft visit(idShortPath, element) für jedes Element inkl. Kindern von Collection/List/Entity. + private static void CollectFieldInfos(List? elements, string prefix, Action visit) + { + if (elements == null) + { + return; + } + + var index = 0; + foreach (var element in elements) + { + if (element == null) + { + continue; + } + + var name = string.IsNullOrEmpty(element.IdShort) ? $"[{index}]" : element.IdShort; + var path = prefix.Length == 0 ? name : prefix + "." + name; + visit(path, element); + + List? children = element switch + { + ISubmodelElementCollection collection => collection.Value, + ISubmodelElementList list => list.Value, + IEntity entity => entity.Statements, + _ => null, + }; + CollectFieldInfos(children, path, visit); + index++; + } + } + + // Nummerierte Wiederholungen (Application_standards00/01/02, [0]/[1]) auf eine Strukturzeile kollabieren: + // Ziffern am Segmentende werden für den Gruppierungsschlüssel neutralisiert. + private static string NormalizeStructurePath(string path) + => System.Text.RegularExpressions.Regex.Replace(path, @"\d+(?=\.|$)", "#"); + + private static string? LastKeyValue(IReference? reference) + => reference?.Keys is { Count: > 0 } keys ? keys[^1]?.Value : null; + + // Kompakter Beispielwert eines Elements für die Modellbeschreibung (Skalar bzw. eine Sprache, gekürzt). + private static string? ExampleValue(ISubmodelElement element, string lang) + => element switch + { + IProperty property => TruncateText(property.Value, 40), + IMultiLanguageProperty mlp => TruncateText(PickLangText(mlp.Value, lang) ?? PickLangText(mlp.Value, null), 40), + IRange range => TruncateText($"{range.Min}..{range.Max}", 40), + _ => null, + }; + + // --------------- Helfer --------------- + + // Führt eine Query aus und fängt Engine-Fehler ab (z.B. ungültiges SQL durch aas.* in Submodel-Queries), + // damit daraus ein lesbares {error} wird statt einer unhandled exception. + private async Task<(List? List, string? Error)> TryQuery(SecurityConfig securityConfig, PaginationParameters pagination, ResultType resultType, string expression) + { + try + { + var list = await _dbRequestHandlerService.QueryGetSMs(securityConfig, pagination, resultType, expression); + return (list, null); + } + catch (Exception ex) + { + return (null, ex.Message); + } + } + + // Gemeinsame Produkt-Suche: Expression ausführen, ersten Submodel-Treffer zur Shell auflösen, ganzes Produkt liefern. + private async Task FindProductCore(string expression, string fmt, bool coreOnly = false) + { + var securityConfig = new SecurityConfig(Program.noSecurity, null); + var pagination = new PaginationParameters(null, DefaultPageSize); + var (list, queryError) = await TryQuery(securityConfig, pagination, ResultType.Submodel, expression); + if (queryError != null) + { + return new { error = "Query fehlgeschlagen: " + queryError }; + } + + var ids = ExtractIdentifiers(list); + + if (ids.Count == 0) + { + return new { found = false, totalMatches = 0, message = "Keine Treffer. Anderen Wert/Schreibweise versuchen oder Groß-/Kleinschreibung des idShort prüfen." }; + } + + var firstId = ids[0]; + var shell = await ResolveShellForIdentifier(securityConfig, firstId); + if (shell is null) + { + return new { found = false, matchedSubmodel = firstId, totalMatches = ids.Count, message = "Treffer gefunden, aber keine zugehörige Shell auflösbar." }; + } + + var product = await BuildProductObject(securityConfig, shell, fmt, coreOnly); + product["matchedSubmodel"] = firstId; + product["totalMatches"] = ids.Count; + return product; + } + + // Kompaktes Console-Log jedes MCP-Aufrufs (eine Zeile), passend zum übrigen Server-Log. + private static void LogCall(string line) + { + DiagnosticsLog.WriteMcp(string.Empty); + DiagnosticsLog.WriteMcp(line, timestamp: true); + } + + private static void LogMcpLine(string line) + { + DiagnosticsLog.WriteMcp(line); + } + + // Loggt den Aufruf (Eingang) und beim Dispose die Dauer — für Performance-Analyse bei großer DB. + // Verwendung: using var _ = LogCallTimed($"aas_query ..."); -> beim Methodenende kommt "[MCP] done in X ms". + private static CallTimer LogCallTimed(string line) + { + var scope = DiagnosticsLog.BeginMcpScope(); + LogCall(line); + return new CallTimer(line.Split(' ', 2)[0], scope); + } + + private sealed class CallTimer : IDisposable + { + private readonly string _tool; + private readonly System.Diagnostics.Stopwatch _watch = System.Diagnostics.Stopwatch.StartNew(); + private readonly IDisposable _scope; + private bool _disposed; + + public CallTimer(string tool, IDisposable scope) + { + _tool = tool; + _scope = scope; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + LogMcpLine($"{_tool} done in {_watch.ElapsedMilliseconds} ms"); + _scope.Dispose(); + } + } + + // Kompakte Darstellung der Suchbedingungen fürs Log, z.B. [sm.idShort eq TechnicalData and sme.value ge 80]. + private static string DescribeConditions(McpQueryCondition[]? conditions, string? combine) + { + if (conditions == null || conditions.Length == 0) + { + return "[]"; + } + + var sep = " " + (string.IsNullOrWhiteSpace(combine) ? "and" : combine.Trim()) + " "; + var parts = conditions.Select(c => + { + var scope = string.IsNullOrWhiteSpace(c?.Scope) ? "sme" : c!.Scope!.Trim(); + var path = !string.IsNullOrWhiteSpace(c?.IdShortPath) ? "." + c!.IdShortPath!.Trim() : string.Empty; + var val = string.Equals(c?.Op?.Trim(), "in", StringComparison.OrdinalIgnoreCase) && c?.Values is { Length: > 0 } + ? "[" + string.Join("|", c.Values) + "]" + : c?.Value; + return $"{scope}{path}.{c?.Field} {c?.Op} {val}"; + }); + return "[" + string.Join(sep, parts) + "]"; + } + + private JsonNode? SerializeValueOrFull(IClass obj, string fmt) + { + if (fmt == "full") + { + return Jsonization.Serialize.ToJsonObject(obj); + } + + var dto = _mappingService.Map(obj, "value"); + return dto is IValueDTO valueDto ? ValueOnlyJsonSerializer.ToJsonObject(valueDto) : null; + } + + // Navigiert einen idShortPath im Submodel: mit Punkt exakt, ohne Punkt per konfigurierbarem Ranking. + private static (string Path, ISubmodelElement Element)? GetProjectionMatch( + ISubmodel submodel, string path, string[]? priority, string[]? deprioritize) + { + if (path.Contains('.', StringComparison.Ordinal)) + { + var exact = NavigatePath(submodel.SubmodelElements, path.Split('.'), 0); + return exact is null ? null : (path, exact); + } + + var matches = new List<(string Path, ISubmodelElement Element)>(); + CollectByIdShort(submodel.SubmodelElements, string.Empty, path, matches); + if (matches.Count == 0) + { + return null; + } + + var preferredPath = SelectPreferredProjectionPath( + matches.Select(match => match.Path).ToArray(), priority, deprioritize); + var preferred = matches.First(match => string.Equals(match.Path, preferredPath, StringComparison.Ordinal)); + return preferred; + } + + internal static string? SelectPreferredProjectionPath( + IReadOnlyList paths, string[]? priority = null, string[]? deprioritize = null) + { + if (paths.Count == 0) + { + return null; + } + + var priorities = NormalizeRankingList(priority, DefaultProjectionPriority); + var penalties = NormalizeRankingList(deprioritize, DefaultProjectionDeprioritize); + + return paths + .Select((path, index) => new + { + Path = path, + Index = index, + Penalty = PathContainsAnySegment(path, penalties) ? 1 : 0, + Priority = GetPathPriority(path, priorities), + Depth = path.Split('.', StringSplitOptions.RemoveEmptyEntries).Length, + }) + .OrderBy(x => x.Penalty) + .ThenBy(x => x.Priority) + .ThenBy(x => x.Depth) + .ThenBy(x => x.Index) + .Select(x => x.Path) + .FirstOrDefault(); + } + + private static string[] NormalizeRankingList(string[]? configured, string[] defaults) + => configured is null + ? defaults + : configured.Where(x => !string.IsNullOrWhiteSpace(x)).Select(x => x.Trim()).ToArray(); + + private static int GetPathPriority(string path, string[] priorities) + { + var segments = path.Split('.', StringSplitOptions.RemoveEmptyEntries); + for (var i = 0; i < priorities.Length; i++) + { + if (segments.Any(segment => string.Equals(segment, priorities[i], StringComparison.OrdinalIgnoreCase))) + { + return i; + } + } + + return priorities.Length; + } + + private static bool PathContainsAnySegment(string path, string[] candidates) + { + var segments = path.Split('.', StringSplitOptions.RemoveEmptyEntries); + return candidates.Any(candidate => + segments.Any(segment => string.Equals(segment, candidate, StringComparison.OrdinalIgnoreCase))); + } + + private static ISubmodelElement? NavigatePath(IReadOnlyList? elements, string[] segments, int index) + { + if (elements == null || index >= segments.Length) + { + return null; + } + + var match = elements.FirstOrDefault(e => string.Equals(e?.IdShort, segments[index], StringComparison.Ordinal)); + if (match == null) + { + return null; + } + + if (index == segments.Length - 1) + { + return match; + } + + IReadOnlyList? children = match switch + { + ISubmodelElementCollection collection => collection.Value, + ISubmodelElementList list => list.Value, + IEntity entity => entity.Statements, + _ => null, + }; + + return NavigatePath(children, segments, index + 1); + } + + // Projizierter Wert eines Elements: bei Property der Skalar, bei MultiLanguageProperty die bevorzugte Sprache, + // sonst die kompakte value-Darstellung. + private JsonNode? GetProjectedValue(ISubmodelElement? element, string lang) + { + if (element is null) + { + return null; + } + + if (element is IProperty property) + { + return property.Value; + } + + // MLP auf eine Sprache reduzieren (Default en), damit Tabellenzellen schlank bleiben statt aller Sprachen. + if (element is IMultiLanguageProperty mlp) + { + var langs = mlp.Value; + if (langs == null || langs.Count == 0) + { + return null; + } + + var pick = langs.FirstOrDefault(l => string.Equals(l?.Language, lang, StringComparison.OrdinalIgnoreCase)) + ?? langs[0]; + return pick?.Text; + } + + var node = SerializeValueOrFull(element, "value"); + + // ValueOnly wickelt als { idShort: } ein — für eine Tabellenzelle den inneren Wert auspacken. + if (node is JsonObject obj && obj.Count == 1) + { + return obj.First().Value?.DeepClone(); + } + + return node; + } + + // Projektions-Zeilen für alle Treffer: nutzt den SQL-Fast-Path, wenn alle select-Einträge + // explizite volle Pfade sind und keine Security aktiv ist; sonst den bisherigen Objektpfad. + private async Task> BuildProjectionRows( + SecurityConfig securityConfig, List identifiers, string[] select, string lang, + string[]? priority, string[]? deprioritize, bool withPaths, string toolName) + { + var fastRows = await TryBuildProjectionRowsFast(securityConfig, identifiers, select, lang, withPaths, toolName); + if (fastRows != null) + { + return fastRows; + } + + var rows = new List(identifiers.Count); + for (var index = 0; index < identifiers.Count; index++) + { + var id = identifiers[index]; + var projectionWatch = Stopwatch.StartNew(); + LogMcpLine($"{toolName} projection {index + 1}/{identifiers.Count}: {id}"); + rows.Add(await BuildProjectionRow(securityConfig, id, select, lang, priority, deprioritize, withPaths)); + LogMcpLine( + $"{toolName} projection {index + 1}/{identifiers.Count} done " + + $"in {projectionWatch.ElapsedMilliseconds} ms"); + } + + return rows; + } + + // SQL-Fast-Path: EINE Batch-Projektion (SMSets/SMRefSets/SMESets/ValueSets) für alle Treffer + // und alle select-Pfade statt vieler ReadSubmodel-/ReadSubmodelElement-Aufrufe pro Treffer. + // Liefert null, wenn der Fast Path nicht anwendbar ist (Security aktiv, Blattnamen im select, + // Indexpfade) — dann übernimmt der bisherige Objektpfad. + private async Task?> TryBuildProjectionRowsFast( + SecurityConfig securityConfig, List identifiers, string[] select, string lang, bool withPaths, string toolName) + { + if (!Program.noSecurity || !TryPlanFastProjection(select, out var plan)) + { + return null; + } + + var watch = Stopwatch.StartNew(); + List projection; + try + { + projection = await _dbRequestHandlerService.QueryProjectSMs(securityConfig, new DbProjectionRequest + { + SubmodelIdentifiers = identifiers, + Paths = plan, + }); + } + catch (Exception ex) + { + LogMcpLine($"{toolName} fast projection failed ({ex.Message}); falling back to object path"); + return null; + } + + var sqlMs = watch.ElapsedMilliseconds; + + var rows = new List(projection.Count); + var fallbackCells = 0; + foreach (var projRow in projection) + { + var row = new JsonObject { ["id"] = projRow.SubmodelIdentifier }; + JsonObject? selectedPaths = withPaths ? new JsonObject() : null; + foreach (var path in plan) + { + projRow.Cells.TryGetValue(path.RawPath, out var cell); + JsonNode? value = null; + string? resolvedPath = null; + if (cell is { Found: true }) + { + if (IsFastProjectableCell(cell)) + { + value = ProjectFastCellValue(cell, lang); + } + else + { + // Komplexes Element (SMC, File, Range, ...): nur diese Zelle über den + // Objektpfad lesen, damit der Wert exakt dem bisherigen Verhalten entspricht. + fallbackCells++; + var (element, _) = await ReadProjectedElement( + securityConfig, cell.SourceSubmodelIdentifier!, path.ElementIdShortPath, + priority: null, deprioritize: null); + value = GetProjectedValue(element, lang); + } + + resolvedPath = path.RawPath; + } + + row[path.RawPath] = value; + if (selectedPaths is not null) + { + selectedPaths[path.RawPath] = resolvedPath; + } + } + + if (selectedPaths is not null) + { + row["paths"] = selectedPaths; + } + + rows.Add(row); + } + + LogMcpLine( + $"{toolName} fast projection: {rows.Count} rows x {plan.Count} fields, sql {sqlMs} ms" + + (fallbackCells > 0 ? $", {fallbackCells} cells via object fallback" : string.Empty)); + return rows; + } + + // Fast-Path-Erkennung: JEDER select-Eintrag muss explizit adressierbar sein — + // "A.B.C" im Treffer-Submodel oder "/SubmodelIdShort/idShortPath" in einem Submodel derselben AAS. + // Blattnamen ohne Submodel-Präfix bleiben beim Objektpfad-Fallback, weil sie Ranking-Suche brauchen. + // Indexpfade mit "[" bleiben ebenfalls beim Fallback, weil SML-Kinder in SMESets.IdShortPath + // nicht bracket-adressierbar sind. + internal static bool TryPlanFastProjection(string[]? select, out List plan) + { + plan = new List(); + if (select is null || select.Length == 0) + { + return false; + } + + foreach (var rawPath in select) + { + if (string.IsNullOrWhiteSpace(rawPath)) + { + continue; + } + + var path = rawPath.Trim(); + if (path.Contains('[', StringComparison.Ordinal)) + { + return false; + } + + if (TryParseCrossSubmodelProjectionPath(path, out var targetSubmodelIdShort, out var elementPath)) + { + plan.Add(new DbProjectionPath + { + RawPath = path, + TargetSubmodelIdShort = targetSubmodelIdShort, + ElementIdShortPath = elementPath, + }); + continue; + } + + if (path.StartsWith("/", StringComparison.Ordinal) || !path.Contains('.', StringComparison.Ordinal)) + { + return false; + } + + plan.Add(new DbProjectionPath { RawPath = path, ElementIdShortPath = path }); + } + + return plan.Count > 0; + } + + // Nur Property und MultiLanguageProperty lassen sich direkt aus den ValueSets projizieren; + // alle anderen Typen brauchen die kompakte value-Serialisierung des Objektpfads. + private static readonly HashSet FastProjectableSmeTypes = new(StringComparer.Ordinal) { "Prop", "MLP" }; + + internal static bool IsFastProjectableCell(DbProjectionCell? cell) + => cell is { Found: true } && FastProjectableSmeTypes.Contains(NormalizeSmeType(cell.SmeType)); + + // SMEType kann einen Operation-Prefix tragen (z.B. "In-Prop"); nur der Basistyp zählt. + internal static string NormalizeSmeType(string? smeType) + { + if (string.IsNullOrEmpty(smeType)) + { + return string.Empty; + } + + var index = smeType.LastIndexOf('-'); + return index >= 0 ? smeType[(index + 1)..] : smeType; + } + + // Zellwert aus den ValueSets-Zeilen: Property als Skalar (numerisch bei TValue="D"), + // MultiLanguageProperty auf die gewünschte Sprache reduziert (sonst erste vorhandene). + internal static JsonNode? ProjectFastCellValue(DbProjectionCell cell, string lang) + { + if (NormalizeSmeType(cell.SmeType) == "MLP") + { + var pick = cell.Values.FirstOrDefault(v => string.Equals(v.Annotation, lang, StringComparison.OrdinalIgnoreCase)) + ?? cell.Values.FirstOrDefault(); + return pick?.SValue; + } + + var value = cell.Values.FirstOrDefault(); + if (value is null) + { + return null; + } + + if (value.NValue.HasValue) + { + return value.NValue.Value; + } + + return value.SValue; + } + + // Eine Projektions-Zeile für ein Submodel: liest es und extrahiert die select-Pfade als {id, :}. + private async Task BuildProjectionRow( + SecurityConfig securityConfig, string id, string[] select, string lang, + string[]? priority, string[]? deprioritize, bool withPaths) + { + var row = new JsonObject { ["id"] = id }; + JsonObject? selectedPaths = withPaths ? new JsonObject() : null; + ISubmodel? submodel = null; + var submodelLoadAttempted = false; + IAssetAdministrationShell? shell = null; + var shellResolveAttempted = false; + var crossSubmodelIdCache = new Dictionary(StringComparer.Ordinal); + + foreach (var rawPath in select) + { + if (string.IsNullOrWhiteSpace(rawPath)) + { + continue; + } + + var path = rawPath.Trim(); + + if (TryParseCrossSubmodelProjectionPath(path, out var targetSubmodelIdShort, out var targetPath)) + { + if (!shellResolveAttempted) + { + shellResolveAttempted = true; + shell = await ResolveShellForIdentifier(securityConfig, id); + } + + var targetSubmodelId = shell is null + ? null + : await ResolveSubmodelIdByIdShort( + securityConfig, shell, targetSubmodelIdShort, crossSubmodelIdCache); + + if (targetSubmodelId is null) + { + row[path] = null; + if (selectedPaths is not null) + { + selectedPaths[path] = null; + } + + continue; + } + + var (element, resolvedPath) = await ReadProjectedElement( + securityConfig, targetSubmodelId, targetPath, priority, deprioritize); + row[path] = GetProjectedValue(element, lang); + if (selectedPaths is not null) + { + selectedPaths[path] = element is null ? null : "/" + targetSubmodelIdShort + "/" + resolvedPath; + } + + continue; + } + + // A full idShortPath can be fetched directly. Loading the complete + // submodel for every projected cell is prohibitively expensive for + // very large TechnicalData submodels. + if (path.Contains('.', StringComparison.Ordinal)) + { + ISubmodelElement? element = null; + try + { + element = await _dbRequestHandlerService.ReadSubmodelElementByPath( + securityConfig, null, id, path, level: null, extent: null) as ISubmodelElement; + } + catch (NotFoundException) + { + element = null; + } + + row[path] = GetProjectedValue(element, lang); + if (selectedPaths is not null) + selectedPaths[path] = element is null ? null : path; + continue; + } + + // A leaf name can be ambiguous at any depth, so retain the ranked + // whole-submodel fallback only for this case and load it at most once. + if (!submodelLoadAttempted) + { + submodelLoadAttempted = true; + try + { + submodel = await _dbRequestHandlerService.ReadSubmodelById( + securityConfig, null, id, level: null, extent: null) as ISubmodel; + } + catch (NotFoundException) + { + submodel = null; + } + } + + var match = submodel is null ? null : GetProjectionMatch(submodel, path, priority, deprioritize); + row[path] = GetProjectedValue(match?.Element, lang); + if (selectedPaths is not null) + { + selectedPaths[path] = match?.Path; + } + } + + if (selectedPaths is not null) + { + row["paths"] = selectedPaths; + } + + return row; + } + + internal static bool TryParseCrossSubmodelProjectionPath( + string? rawPath, out string submodelIdShort, out string elementPath) + { + submodelIdShort = string.Empty; + elementPath = string.Empty; + + if (string.IsNullOrWhiteSpace(rawPath) || !rawPath.StartsWith("/", StringComparison.Ordinal)) + { + return false; + } + + var trimmed = rawPath.Trim(); + var separator = trimmed.IndexOf('/', 1); + if (separator <= 1 || separator == trimmed.Length - 1) + { + return false; + } + + submodelIdShort = trimmed[1..separator].Trim(); + elementPath = trimmed[(separator + 1)..].Trim(); + return submodelIdShort.Length > 0 && elementPath.Length > 0; + } + + private async Task ResolveSubmodelIdByIdShort( + SecurityConfig securityConfig, + IAssetAdministrationShell shell, + string submodelIdShort, + Dictionary cache) + { + if (cache.TryGetValue(submodelIdShort, out var cached)) + { + return cached; + } + + if (shell.Submodels != null) + { + foreach (var smRef in shell.Submodels) + { + var smId = smRef?.Keys?.LastOrDefault()?.Value; + if (string.IsNullOrEmpty(smId)) + { + continue; + } + + try + { + if (await _dbRequestHandlerService.ReadSubmodelById( + securityConfig, null, smId, level: null, extent: null) is ISubmodel submodel + && string.Equals(submodel.IdShort, submodelIdShort, StringComparison.Ordinal)) + { + cache[submodelIdShort] = smId; + return smId; + } + } + catch (NotFoundException) + { + // Ignore dangling references in the shell. + } + } + } + + cache[submodelIdShort] = null; + return null; + } + + private async Task<(ISubmodelElement? Element, string? ResolvedPath)> ReadProjectedElement( + SecurityConfig securityConfig, + string submodelId, + string path, + string[]? priority, + string[]? deprioritize) + { + if (path.Contains('.', StringComparison.Ordinal)) + { + try + { + return (await _dbRequestHandlerService.ReadSubmodelElementByPath( + securityConfig, null, submodelId, path, level: null, extent: null) as ISubmodelElement, path); + } + catch (NotFoundException) + { + return (null, null); + } + } + + try + { + if (await _dbRequestHandlerService.ReadSubmodelById( + securityConfig, null, submodelId, level: null, extent: null) is ISubmodel submodel) + { + var match = GetProjectionMatch(submodel, path, priority, deprioritize); + return (match?.Element, match?.Path); + } + } + catch (NotFoundException) + { + // Handled as empty projection cell. + } + + return (null, null); + } + + private async Task TryReadShell(SecurityConfig securityConfig, string aasId) + { + try + { + return await _dbRequestHandlerService.ReadAssetAdministrationShellById(securityConfig, aasId); + } + catch (NotFoundException) + { + return null; + } + } + + // Ermittelt die AAS-id, zu der ein Submodel gehört (Query: Shells, die ein Submodel mit dieser id haben). + private async Task ResolveShellOfSubmodel(SecurityConfig securityConfig, string submodelId) + { + var condition = new JsonObject + { + ["$eq"] = new JsonArray( + new JsonObject { ["$field"] = "$sm#id" }, + new JsonObject { ["$strVal"] = submodelId }), + }; + var expression = "$JSONGRAMMAR " + new JsonObject + { + ["Query"] = new JsonObject { ["$condition"] = condition }, + }.ToJsonString(); + + var pagination = new PaginationParameters(null, 1); + var (shells, _) = await TryQuery(securityConfig, pagination, ResultType.AssetAdministrationShell, expression); + + return ExtractIdentifiers(shells).FirstOrDefault(); + } + + private static List ExtractIdentifiers(List? items) + { + return (items ?? []) + .Select(item => item switch + { + string id => id, + IIdentifiable identifiable => identifiable.Id, + _ => null, + }) + .Where(id => !string.IsNullOrEmpty(id)) + .Select(id => id!) + .ToList(); + } + + // Löst eine id (AAS-Identifier ODER Submodel-Identifier) zur zugehörigen Shell auf. + private async Task ResolveShellForIdentifier(SecurityConfig securityConfig, string id) + { + var shell = await TryReadShell(securityConfig, id); + if (shell is null) + { + var aasId = await ResolveShellOfSubmodel(securityConfig, id); + if (aasId != null) + { + shell = await TryReadShell(securityConfig, aasId); + } + } + + return shell; + } + + // Baut das Shell-Objekt: bei value AssetInformation + Submodel-Referenzen, bei full das ganze AAS-JSON. + private static JsonObject BuildShellObject(IAssetAdministrationShell shell, string fmt) + { + var result = new JsonObject { ["idShort"] = shell.IdShort }; + + if (fmt == "full") + { + result["shell"] = Jsonization.Serialize.ToJsonObject(shell); + return result; + } + + var submodelIds = new JsonArray(); + if (shell.Submodels != null) + { + foreach (var smRef in shell.Submodels) + { + var key = smRef?.Keys?.LastOrDefault(); + if (key != null && !string.IsNullOrEmpty(key.Value)) + { + submodelIds.Add(key.Value); + } + } + } + + var ai = shell.AssetInformation; + var specificAssetIds = new JsonArray(); + if (ai?.SpecificAssetIds != null) + { + foreach (var said in ai.SpecificAssetIds) + { + specificAssetIds.Add(new JsonObject { ["name"] = said.Name, ["value"] = said.Value }); + } + } + + result["assetInformation"] = new JsonObject + { + ["assetKind"] = ai != null ? ai.AssetKind.ToString() : null, + ["globalAssetId"] = ai?.GlobalAssetId, + ["specificAssetIds"] = specificAssetIds, + }; + result["submodels"] = submodelIds; + return result; + } + + // Baut das Produkt-Objekt: AssetInformation + Werte ALLER Submodelle der Shell im gewählten Format. + // Jeder Submodel-Read ist indexiert (SMSet.Identifier, SMESet.SMId) und auf die Größe DIESES Submodels begrenzt. + private async Task BuildProductObject( + SecurityConfig securityConfig, + IAssetAdministrationShell shell, + string fmt, + bool coreOnly = false, + int submodelCursor = 0, + int submodelLimit = MaxProductSubmodels) + { + var ai = shell.AssetInformation; + var specificAssetIds = new JsonArray(); + if (ai?.SpecificAssetIds != null) + { + foreach (var said in ai.SpecificAssetIds) + { + specificAssetIds.Add(new JsonObject { ["name"] = said.Name, ["value"] = said.Value }); + } + } + + var assetInformation = new JsonObject + { + ["assetKind"] = ai != null ? ai.AssetKind.ToString() : null, + ["globalAssetId"] = ai?.GlobalAssetId, + ["specificAssetIds"] = specificAssetIds, + }; + + var submodels = new JsonArray(); + var totalSubmodelRefs = shell.Submodels?.Count ?? 0; + var hasMoreSubmodels = false; + string? nextSubmodelCursor = null; + if (shell.Submodels != null) + { + var endExclusive = Math.Min(totalSubmodelRefs, submodelCursor + submodelLimit); + hasMoreSubmodels = endExclusive < totalSubmodelRefs; + nextSubmodelCursor = hasMoreSubmodels + ? endExclusive.ToString(CultureInfo.InvariantCulture) + : null; + + foreach (var smRef in shell.Submodels.Skip(submodelCursor).Take(submodelLimit)) + { + var smId = smRef?.Keys?.LastOrDefault()?.Value; + if (string.IsNullOrEmpty(smId)) + { + continue; + } + + IClass? sm = null; + try + { + sm = await _dbRequestHandlerService.ReadSubmodelById(securityConfig, null, smId, level: null, extent: null); + } + catch (NotFoundException) + { + sm = null; + } + + // coreOnly (simple-Stufe): sperrige Datei-/Doku-Submodelle (CAD, BoM, HandoverDocumentation) weglassen, + // damit kleine Modelle die relevanten Daten (Nameplate/TechnicalData/CarbonFootprint) nicht im Rauschen verlieren. + var smIdShort = (sm as ISubmodel)?.IdShort; + if (coreOnly && smIdShort != null + && NoiseSubmodelKeywords.Any(k => smIdShort.Contains(k, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + var entry = new JsonObject { ["id"] = smId }; + if (sm is ISubmodel s) + { + entry["idShort"] = s.IdShort; + } + + entry["value"] = sm != null ? SerializeValueOrFull(sm, fmt) : null; + submodels.Add(entry); + } + } + + return new JsonObject + { + ["aasId"] = shell.Id, + ["idShort"] = shell.IdShort, + ["format"] = fmt, + ["assetInformation"] = assetInformation, + ["submodels"] = submodels, + ["submodelCursor"] = submodelCursor, + ["submodelLimit"] = submodelLimit, + ["returnedSubmodels"] = submodels.Count, + ["totalSubmodelRefs"] = totalSubmodelRefs, + ["hasMoreSubmodels"] = hasMoreSubmodels, + ["nextSubmodelCursor"] = nextSubmodelCursor, + ["truncated"] = hasMoreSubmodels, + }; + } + + // Rekursive Suche nach allen SubmodelElementen mit passendem idShort; baut dabei den vollen idShortPath. + private static void CollectByIdShort( + IReadOnlyList? elements, string prefix, string targetIdShort, List<(string Path, ISubmodelElement Element)> matches) + { + if (elements == null) + { + return; + } + + for (var i = 0; i < elements.Count; i++) + { + var sme = elements[i]; + if (sme is null) + { + continue; + } + + // Listen-Kinder haben oft keinen idShort -> Index-Notation [i]. + var segment = sme.IdShort ?? "[" + i.ToString(CultureInfo.InvariantCulture) + "]"; + string path; + if (prefix.Length == 0) + { + path = segment; + } + else + { + path = segment.StartsWith("[", StringComparison.Ordinal) ? prefix + segment : prefix + "." + segment; + } + + if (string.Equals(sme.IdShort, targetIdShort, StringComparison.Ordinal)) + { + matches.Add((path, sme)); + } + + switch (sme) + { + case ISubmodelElementCollection collection: + CollectByIdShort(collection.Value, path, targetIdShort, matches); + break; + case ISubmodelElementList list: + CollectByIdShort(list.Value, path, targetIdShort, matches); + break; + case IEntity entity: + CollectByIdShort(entity.Statements, path, targetIdShort, matches); + break; + case IAnnotatedRelationshipElement annotated: + CollectByIdShort(annotated.Annotations?.Cast().ToList(), path, targetIdShort, matches); + break; + } + } + } + + internal static string BuildCsv(string[] columns, IReadOnlyList rows, string delimiter) + { + var sb = new StringBuilder(); + sb.AppendLine(string.Join(delimiter, columns.Select(column => CsvEscape(column, delimiter)))); + + foreach (var row in rows) + { + sb.AppendLine(string.Join(delimiter, columns.Select(column => + CsvEscape(row.TryGetPropertyValue(column, out var value) ? value : null, delimiter)))); + } + + return sb.ToString(); + } + + internal static string CsvEscape(JsonNode? value, string delimiter) + { + if (value is null) + { + return string.Empty; + } + + if (value is JsonValue jsonValue) + { + if (jsonValue.TryGetValue(out var s)) + return CsvEscape(s, delimiter); + if (jsonValue.TryGetValue(out var i)) + return i.ToString(CultureInfo.InvariantCulture); + if (jsonValue.TryGetValue(out var l)) + return l.ToString(CultureInfo.InvariantCulture); + if (jsonValue.TryGetValue(out var d)) + return d.ToString(CultureInfo.InvariantCulture); + if (jsonValue.TryGetValue(out var m)) + return m.ToString(CultureInfo.InvariantCulture); + if (jsonValue.TryGetValue(out var b)) + return b ? "true" : "false"; + } + + return CsvEscape(value.ToJsonString(), delimiter); + } + + internal static string CsvEscape(string? value, string delimiter) + { + if (string.IsNullOrEmpty(value)) + { + return string.Empty; + } + + var mustQuote = value.Contains('"', StringComparison.Ordinal) + || value.Contains('\r', StringComparison.Ordinal) + || value.Contains('\n', StringComparison.Ordinal) + || value.Contains(delimiter, StringComparison.Ordinal); + if (!mustQuote) + { + return value; + } + + return "\"" + value.Replace("\"", "\"\"", StringComparison.Ordinal) + "\""; + } + + private static string NormalizeCsvDelimiter(string? delimiter) + { + if (string.IsNullOrEmpty(delimiter)) + { + return ";"; + } + + return delimiter is "," or ";" or "\t" ? delimiter : ";"; + } + + private static ResultType ParseTarget(string target) => target?.Trim().ToLowerInvariant() switch + { + "submodels" or "submodel" or "sm" => ResultType.Submodel, + "shells" or "shell" or "aas" => ResultType.AssetAdministrationShell, + _ => throw new ArgumentException($"Unbekanntes target \"{target}\". Erlaubt: \"submodels\" oder \"shells\"."), + }; + + internal static int NormalizeLimit(int? limit) + { + if (limit is null || limit <= 0) + { + return DefaultPageSize; + } + + return Math.Min(limit.Value, MaxPageSize); + } + + internal static int NormalizeProductSubmodelLimit(int? limit) + { + if (limit is null || limit <= 0) + { + return MaxProductSubmodels; + } + + return Math.Min(limit.Value, MaxProductSubmodels); + } + + internal static int NormalizeCursor(string? cursor) + => string.IsNullOrWhiteSpace(cursor) || !int.TryParse(cursor, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) || parsed < 0 + ? 0 + : parsed; + + /// + /// Baut aus den Slots eine AASQL-JSON-Query (Modus-Präfix "$JSONGRAMMAR"). + /// Eine Bedingung -> direkte Vergleichsoperation; mehrere -> $and/$or. + /// + private static string BuildExpression(McpQueryCondition[] conditions, string combine) + { + if (conditions is null || conditions.Length == 0) + { + throw new ArgumentException("Es muss mindestens eine Bedingung (conditions) angegeben werden."); + } + + JsonNode condition; + if (conditions.Length == 1) + { + condition = BuildComparison(conditions[0]); + } + else + { + var combineKey = combine?.Trim().ToLowerInvariant() switch + { + "or" => "$or", + "and" or null or "" => "$and", + _ => throw new ArgumentException($"Unbekanntes combine \"{combine}\". Erlaubt: \"and\" oder \"or\"."), + }; + + var array = new JsonArray(); + foreach (var c in conditions) + { + array.Add(BuildComparison(c)); + } + + condition = new JsonObject { [combineKey] = array }; + } + + var query = new JsonObject + { + ["Query"] = new JsonObject + { + // aas_query needs identifiers first. Without this, Query.GetQueryData + // materializes every complete submodel before MCP projects the selected + // fields, causing the same large submodels to be loaded twice. + ["$select"] = "id", + ["$condition"] = condition, + }, + }; + + return "$JSONGRAMMAR " + query.ToJsonString(); + } + + private static JsonObject BuildComparison(McpQueryCondition c) + { + if (c is null) + { + throw new ArgumentException("Leere Bedingung."); + } + + var scope = (c.Scope ?? "sme").Trim().ToLowerInvariant(); + if (!AllowedScopes.Contains(scope)) + { + throw new ArgumentException($"Unbekannter scope \"{c.Scope}\". Erlaubt: \"aas\", \"sm\", \"sme\"."); + } + + var op = (c.Op ?? string.Empty).Trim().ToLowerInvariant(); + + // op="in": ODER-Verknüpfung von eq-Vergleichen über die Werteliste — nutzt die komplette Feld-/Pfad-/Numerik-Logik wieder. + if (op == "in") + { + var values = c.Values ?? System.Array.Empty(); + if (values.Length == 0) + { + throw new ArgumentException("op=\"in\" benötigt eine nicht-leere Werteliste (values)."); + } + + var orArray = new JsonArray(); + foreach (var v in values) + { + orArray.Add(BuildComparison(new McpQueryCondition + { + Scope = c.Scope, + Field = c.Field, + IdShortPath = c.IdShortPath, + Op = "eq", + Value = v, + })); + } + + return new JsonObject { ["$or"] = orArray }; + } + + if (!OpMap.TryGetValue(op, out var opKey)) + { + throw new ArgumentException($"Unbekannter Operator \"{c.Op}\". Erlaubt: {string.Join(", ", OpMap.Keys)}, in."); + } + + var validationField = string.IsNullOrWhiteSpace(c.Field) && scope == "sme" ? "value" : (c.Field ?? string.Empty).Trim(); + ValidateField(scope, validationField); + var validationPath = scope == "sme" && !string.IsNullOrWhiteSpace(c.IdShortPath) + ? c.IdShortPath!.Trim() + : null; + ValidateWildcardOperatorForField(op, scope, validationField, validationPath); + + if (op == "regex") + { + throw new ArgumentException( + "Der Operator regex wird vom SQL-Backend derzeit nicht unterstützt. " + + "Verwende eq, starts-with, ends-with oder contains."); + } + + if (op == "contains" && (c.Value ?? string.Empty).Length < 3) + { + throw new ArgumentException( + $"contains mit \"{c.Value}\" ist zu kurz: Der SQLite-Trigrammindex benötigt mindestens 3 Zeichen. " + + "Verwende eq, starts-with oder einen spezifischeren Teilstring mit mindestens 3 Zeichen " + + "(z.B. 24V, 24D oder /24)."); + } + + // Für scope=sme ist "value" der sinnvolle Default, wenn field leer ist — LLMs lassen field + // bei idShortPath-/Wert-Suchen oft weg (z.B. {scope:sme, idShortPath:"flowMax", op:eq, value:"80"}). + var field = string.IsNullOrWhiteSpace(c.Field) && scope == "sme" ? "value" : (c.Field ?? string.Empty).Trim(); + ValidateField(scope, field); + + var rawPath = scope == "sme" && !string.IsNullOrWhiteSpace(c.IdShortPath) + ? c.IdShortPath!.Trim() + : null; + + // idShortPath nur als positionalen Pfad behandeln, wenn er einen "." enthält + // (z.B. "TechnicalProperties.flowMax"). Ein einzelner Name ohne "." ist ein idShort-Blattname + // in beliebiger Tiefe. WICHTIG (BUG A): Das muss am SELBEN Element korreliert sein. Früher wurde + // $and[$sme#idShort==X, $sme# op Y] gebaut — das matcht aber "irgendein idShort=X UND + // irgendein Wert op Y" (verschiedene Elemente!) -> Falsch-Treffer. Korrekt ist der korrelierte + // Pfad-Mechanismus der Engine: top-level ($sme.#field) ODER verschachtelt ($sme.%.#field, + // %-Wildcard = beliebiger Pfad-Präfix). Beides ist je ein korreliertes Pfad-Subquery (idShortPath + Wert in einer Zeile). + if (rawPath != null && !rawPath.Contains('.', StringComparison.Ordinal) && field != "idShort") + { + var allowNumeric = NumericCapableOps.Contains(op); + return BuildFieldComparison( + opKey, + "$" + scope + ".%." + rawPath + "#" + field, + c.Value, + allowNumeric); + } + + var pathSegment = rawPath != null ? "." + rawPath : string.Empty; + var fieldRef = "$" + scope + pathSegment + "#" + field; + return BuildFieldComparison(opKey, fieldRef, c.Value, allowNumeric: NumericCapableOps.Contains(op)); + } + + private static JsonObject BuildFieldComparison(string opKey, string fieldRef, string? value, bool allowNumeric) + { + JsonObject StrRhs() => new JsonObject { ["$strVal"] = value ?? string.Empty }; + JsonObject Leaf(string op, JsonObject rhs) => new JsonObject + { + [op] = new JsonArray(new JsonObject { ["$field"] = fieldRef }, rhs), + }; + + // Nicht numerikfähig oder keine Zahl -> reiner String-Vergleich. + // NumberStyles.Float (NICHT .Any!): keine Tausendertrennung, sonst würde "1,32" als 132 fehlinterpretiert. + if (!allowNumeric || !double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var num)) + { + return Leaf(opKey, StrRhs()); + } + + JsonObject NumRhs() => new JsonObject { ["$numVal"] = num }; + + // eq/ne: gegen String ODER Zahl prüfen — der valueType (xs:string vs xs:double) ist bei der Query unbekannt. + // So findet eq "80" einen Double-Wert 80 UND eq "2908936" eine ziffrige String-Artikelnummer. + if (opKey == "$eq") + { + return new JsonObject { ["$or"] = new JsonArray(Leaf("$eq", StrRhs()), Leaf("$eq", NumRhs())) }; + } + + if (opKey == "$ne") + { + return new JsonObject { ["$and"] = new JsonArray(Leaf("$ne", StrRhs()), Leaf("$ne", NumRhs())) }; + } + + // gt/ge/lt/le: numerischer Vergleich. + return Leaf(opKey, NumRhs()); + } + + internal static void ValidateWildcardOperatorForField(string op, string scope, string field, string? idShortPath = null) + { + if (!WildcardOps.Contains(op)) + { + return; + } + + var normalizedField = NormalizeWildcardFieldName(scope, field, idShortPath); + if (WildcardFields.Contains(normalizedField)) + { + return; + } + + throw new ArgumentException(FormatWildcardFieldError(normalizedField)); + } + + private static string NormalizeWildcardFieldName(string scope, string field, string? idShortPath) + { + var f = (field ?? string.Empty).Trim(); + if (scope == "sme" && !string.IsNullOrWhiteSpace(idShortPath) && f == "idShort") + { + return "idShortPath"; + } + + return f is "id" or "identifier" ? "identifier" : f; + } + + private static string DisplayWildcardFieldName(string field) + => field == "identifier" ? "id" : field; + + private static string FormatWildcardFieldError(string field) + { + var display = DisplayWildcardFieldName(field); + return $"Wildcard search is not supported for field '{display}'.{System.Environment.NewLine}" + + $"Allowed wildcard fields are:{System.Environment.NewLine}" + + $"- value{System.Environment.NewLine}" + + $"- idShort{System.Environment.NewLine}" + + $"- idShortPath{System.Environment.NewLine}" + + $"Use 'eq' or 'in' for {display}."; + } + + private static void ValidateField(string scope, string field) + { + if (string.IsNullOrWhiteSpace(field)) + { + throw new ArgumentException("field darf nicht leer sein."); + } + + var f = field.Trim(); + bool ok = scope switch + { + "sme" => f is "idShort" or "idShortPath" or "value" or "valueType" or "language" || f.StartsWith("semanticId", StringComparison.Ordinal), + "sm" => f is "idShort" or "id" || f.StartsWith("semanticId", StringComparison.Ordinal), + "aas" => f is "idShort" or "id" + || f.StartsWith("assetInformation", StringComparison.Ordinal) + || f.StartsWith("submodels", StringComparison.Ordinal), + _ => false, + }; + + if (!ok) + { + throw new ArgumentException($"Feld \"{field}\" ist für scope \"{scope}\" nicht erlaubt."); + } + } +} diff --git a/src/IO.Swagger.Lib.V3/MCP/XlsxBuilder.cs b/src/IO.Swagger.Lib.V3/MCP/XlsxBuilder.cs new file mode 100644 index 000000000..da987ccbe --- /dev/null +++ b/src/IO.Swagger.Lib.V3/MCP/XlsxBuilder.cs @@ -0,0 +1,198 @@ +/******************************************************************************** +* Copyright (c) {2019 - 2025} Contributors to the Eclipse Foundation +* +* See the NOTICE file(s) distributed with this work for additional +* information regarding copyright ownership. +* +* This program and the accompanying materials are made available under the +* terms of the Apache License Version 2.0 which is available at +* https://www.apache.org/licenses/LICENSE-2.0 +* +* SPDX-License-Identifier: Apache-2.0 +********************************************************************************/ + +namespace IO.Swagger.Lib.V3.MCP; + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Text; +using System.Text.Json.Nodes; + +/// +/// Minimaler XLSX-Writer ohne externe Abhängigkeit: ein Sheet, Inline-Strings, +/// Zahlen als numerische Zellen. Reicht für tabellarische Query-Exporte und wird +/// von Excel/LibreOffice/Google Sheets gelesen. +/// +internal static class XlsxBuilder +{ + public const string MimeType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + + public static byte[] Build(string[] columns, IReadOnlyList rows, string sheetName = "Export") + { + var sheet = BuildSheetXml(columns, rows); + + using var stream = new MemoryStream(); + using (var zip = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + AddEntry(zip, "[Content_Types].xml", + "" + + "" + + "" + + "" + + "" + + "" + + ""); + AddEntry(zip, "_rels/.rels", + "" + + "" + + "" + + ""); + AddEntry(zip, "xl/workbook.xml", + "" + + "" + + $"" + + ""); + AddEntry(zip, "xl/_rels/workbook.xml.rels", + "" + + "" + + "" + + ""); + AddEntry(zip, "xl/worksheets/sheet1.xml", sheet); + } + + return stream.ToArray(); + } + + private static string BuildSheetXml(string[] columns, IReadOnlyList rows) + { + var sb = new StringBuilder(); + sb.Append(""); + sb.Append(""); + sb.Append(""); + + AppendRow(sb, 1, columns.Length, col => ((string?)columns[col], null)); + for (var r = 0; r < rows.Count; r++) + { + var row = rows[r]; + AppendRow(sb, r + 2, columns.Length, col => + GetCellValue(row.TryGetPropertyValue(columns[col], out var value) ? value : null)); + } + + sb.Append(""); + sb.Append(""); + return sb.ToString(); + } + + private static void AppendRow(StringBuilder sb, int rowNumber, int columnCount, Func cell) + { + sb.Append(""); + for (var col = 0; col < columnCount; col++) + { + var (text, number) = cell(col); + if (number.HasValue) + { + sb.Append("") + .Append(number.Value.ToString("R", CultureInfo.InvariantCulture)) + .Append(""); + } + else if (text is not null) + { + sb.Append("") + .Append(XmlEscape(text)) + .Append(""); + } + } + + sb.Append(""); + } + + // Zellwert aus dem Projektions-JSON: Zahlen als numerische Zelle, alles andere als Text. + internal static (string? Text, double? Number) GetCellValue(JsonNode? value) + { + if (value is null) + { + return (null, null); + } + + if (value is JsonValue jsonValue) + { + if (jsonValue.TryGetValue(out var i)) + return (null, i); + if (jsonValue.TryGetValue(out var l)) + return (null, l); + if (jsonValue.TryGetValue(out var d)) + return double.IsFinite(d) ? (null, d) : (d.ToString(CultureInfo.InvariantCulture), null); + if (jsonValue.TryGetValue(out var m)) + return (null, (double)m); + if (jsonValue.TryGetValue(out var b)) + return (b ? "true" : "false", null); + if (jsonValue.TryGetValue(out var s)) + return (s, null); + } + + return (value.ToJsonString(), null); + } + + internal static string CellRef(int columnIndex, int rowNumber) + { + var column = string.Empty; + var remaining = columnIndex; + while (remaining >= 0) + { + column = (char)('A' + (remaining % 26)) + column; + remaining = remaining / 26 - 1; + } + + return column + rowNumber.ToString(CultureInfo.InvariantCulture); + } + + internal static string XmlEscape(string value) + { + var sb = new StringBuilder(value.Length); + foreach (var c in value) + { + switch (c) + { + case '<': sb.Append("<"); break; + case '>': sb.Append(">"); break; + case '&': sb.Append("&"); break; + case '"': sb.Append("""); break; + default: + // Steuerzeichen sind in XML 1.0 nicht erlaubt (Excel verweigert die Datei sonst). + if (c < 0x20 && c != '\t' && c != '\n' && c != '\r') + { + sb.Append(' '); + } + else + { + sb.Append(c); + } + + break; + } + } + + return sb.ToString(); + } + + private static string SanitizeSheetName(string? sheetName) + { + var name = string.IsNullOrWhiteSpace(sheetName) ? "Export" : sheetName.Trim(); + foreach (var invalid in new[] { ':', '\\', '/', '?', '*', '[', ']' }) + { + name = name.Replace(invalid, '_'); + } + + return name.Length > 31 ? name[..31] : name; + } + + private static void AddEntry(ZipArchive zip, string path, string content) + { + var entry = zip.CreateEntry(path, CompressionLevel.Fastest); + using var writer = new StreamWriter(entry.Open(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + writer.Write(content); + } +} diff --git a/src/IO.Swagger.Lib.V3/Middleware/ExceptionMiddleware.cs b/src/IO.Swagger.Lib.V3/Middleware/ExceptionMiddleware.cs index 116abf2a5..431ab5da0 100644 --- a/src/IO.Swagger.Lib.V3/Middleware/ExceptionMiddleware.cs +++ b/src/IO.Swagger.Lib.V3/Middleware/ExceptionMiddleware.cs @@ -54,6 +54,13 @@ private async Task HandleExceptionAsync(HttpContext context, Exception exception { logger.LogError(exception.Message); logger.LogDebug(exception.StackTrace ?? $"No Stacktrace for {exception}"); + if (context.Response.HasStarted) + { + // The client may cancel an MCP request after response streaming has + // begun. Headers are immutable at that point, so no JSON error + // response can safely be written anymore. + return; + } context.Response.ContentType = "application/json"; var result = new Result(); var message = new Message();