Skip to content

Commit b231ac3

Browse files
committed
Fix fusion summary LLM trace payloads
Refs #973
1 parent 6a81ce7 commit b231ac3

5 files changed

Lines changed: 315 additions & 11 deletions

File tree

api/apps/api-server/src/routes/applications/application_runtime/runtime_debug_artifacts/visible_internal_llm_route_traces.rs

Lines changed: 144 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,7 @@ fn branch_trace_from_event(
397397
status: status.to_string(),
398398
route_model,
399399
provider_route,
400-
input_payload: value_field(event_object, &["input_payload"]),
400+
input_payload: branch_input_payload(event_object),
401401
output_payload,
402402
output,
403403
output_summary,
@@ -412,6 +412,66 @@ fn branch_trace_from_event(
412412
Some(branch_trace)
413413
}
414414

415+
fn branch_input_payload(event_object: &Map<String, Value>) -> Option<Value> {
416+
value_field(event_object, &["input_payload"])
417+
.or_else(|| historical_llm_input_payload_from_debug_context(event_object))
418+
}
419+
420+
fn historical_llm_input_payload_from_debug_context(
421+
event_object: &Map<String, Value>,
422+
) -> Option<Value> {
423+
if string_field(event_object, &["node_type"]).as_deref() != Some("llm") {
424+
return None;
425+
}
426+
427+
let llm_context = event_object
428+
.get("debug_payload")?
429+
.get("llm_context")?
430+
.as_object()?;
431+
let mut prompt_messages = Vec::new();
432+
433+
prompt_messages.extend(
434+
llm_context
435+
.get("provider_messages")?
436+
.as_array()?
437+
.iter()
438+
.filter_map(|message| {
439+
message
440+
.as_object()
441+
.map(|object| Value::Object(object.clone()))
442+
}),
443+
);
444+
445+
if let Some(effective_system) = llm_context
446+
.get("effective_system")
447+
.and_then(Value::as_str)
448+
.map(str::trim)
449+
.filter(|value| !value.is_empty())
450+
.filter(|_| {
451+
!prompt_messages
452+
.iter()
453+
.filter_map(Value::as_object)
454+
.any(|message| string_field(message, &["role"]).as_deref() == Some("system"))
455+
})
456+
{
457+
prompt_messages.insert(
458+
0,
459+
json!({
460+
"role": "system",
461+
"content": effective_system,
462+
}),
463+
);
464+
}
465+
466+
if prompt_messages.is_empty() {
467+
return None;
468+
}
469+
470+
Some(json!({
471+
"prompt_messages": prompt_messages,
472+
}))
473+
}
474+
415475
fn apply_branch_node_run_payload(
416476
branch_trace: &mut VisibleInternalLlmToolBranchTraceFacts,
417477
node_run: &VisibleInternalLlmToolBranchNodeRunPayload,
@@ -1347,4 +1407,87 @@ mod tests {
13471407
json!("artifact-panel-b")
13481408
);
13491409
}
1410+
1411+
#[test]
1412+
fn fusion_trace_projects_historical_summary_llm_detail_from_debug_context() {
1413+
let debug_payload = json!({
1414+
"visible_internal_llm_tool_events": [
1415+
{
1416+
"event_type": "visible_internal_llm_tool_started",
1417+
"main_node_id": "node-main-llm",
1418+
"target_node_id": "node-panel-a",
1419+
"tool_name": "fusion_review",
1420+
"tool_call_id": "call_fusion",
1421+
"tool_mode": "fusion",
1422+
"execution_mode": "bounded_parallel_panel"
1423+
},
1424+
{
1425+
"event_type": "visible_internal_llm_tool_completed",
1426+
"main_node_id": "node-main-llm",
1427+
"target_node_id": "node-panel-a",
1428+
"tool_name": "fusion_review",
1429+
"tool_call_id": "call_fusion",
1430+
"tool_mode": "fusion",
1431+
"execution_mode": "bounded_parallel_panel",
1432+
"node_id": "node-judge",
1433+
"node_alias": "LLM5",
1434+
"node_type": "llm",
1435+
"provider_route": {
1436+
"model": "gpt-5.4-mini",
1437+
"provider_code": "fixture_provider"
1438+
},
1439+
"metrics_payload": {
1440+
"usage": {
1441+
"input_tokens": 5513,
1442+
"output_tokens": 2455,
1443+
"total_tokens": 7968
1444+
}
1445+
},
1446+
"debug_payload": {
1447+
"llm_context": {
1448+
"effective_system": "You are the fusion judge.",
1449+
"provider_messages": [
1450+
{
1451+
"role": "user",
1452+
"content": "Merge panel answers."
1453+
}
1454+
]
1455+
},
1456+
"assistant_message": {
1457+
"role": "assistant",
1458+
"content": "judge merged answer"
1459+
}
1460+
},
1461+
"content": "judge merged answer"
1462+
}
1463+
]
1464+
});
1465+
1466+
let traces = collect_visible_internal_llm_tool_route_traces(&debug_payload);
1467+
1468+
assert_eq!(traces.len(), 1);
1469+
let detail = traces[0].detail_payload();
1470+
let branch_trace = &detail["branch_traces"][0];
1471+
assert_eq!(branch_trace["node_alias"], json!("LLM5"));
1472+
assert_eq!(
1473+
branch_trace["input_payload"]["prompt_messages"][0]["role"],
1474+
json!("system")
1475+
);
1476+
assert_eq!(
1477+
branch_trace["input_payload"]["prompt_messages"][0]["content"],
1478+
json!("You are the fusion judge.")
1479+
);
1480+
assert_eq!(
1481+
branch_trace["input_payload"]["prompt_messages"][1]["content"],
1482+
json!("Merge panel answers.")
1483+
);
1484+
assert_eq!(
1485+
branch_trace["output_payload"]["text"],
1486+
json!("judge merged answer")
1487+
);
1488+
assert_eq!(
1489+
branch_trace["metrics_payload"]["usage"]["total_tokens"],
1490+
json!(7968)
1491+
);
1492+
}
13501493
}

api/crates/orchestration-runtime/src/_tests/execution_engine_tests/human_and_tool_resume/visible_internal_llm_tools.rs

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -524,6 +524,26 @@ async fn fusion_visible_internal_llm_tool_executes_panel_llms_in_bounded_paralle
524524
"panel LLM event should preserve node-run-like output payload"
525525
);
526526
}
527+
let judge_event = route_events
528+
.iter()
529+
.find(|event| {
530+
event["event_type"] == json!("visible_internal_llm_tool_completed")
531+
&& event["node_id"] == json!("node-judge")
532+
})
533+
.expect("fusion summary LLM completed event should exist");
534+
assert_eq!(judge_event["node_type"], json!("llm"));
535+
assert!(
536+
judge_event["input_payload"]["prompt_messages"].is_array(),
537+
"fusion summary LLM event should preserve node-run-like input payload"
538+
);
539+
assert_eq!(
540+
judge_event["output_payload"]["text"],
541+
json!("judge-result ")
542+
);
543+
assert_eq!(
544+
judge_event["metrics_payload"]["usage"]["total_tokens"],
545+
json!(24)
546+
);
527547

528548
let captured = invoker
529549
.captured_inputs

api/crates/orchestration-runtime/src/execution_engine/visible_internal_llm_tools.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -740,6 +740,8 @@ where
740740
"node_id": node.node_id,
741741
"node_alias": node.alias,
742742
"node_type": node.node_type,
743+
"input_payload": node_output.input_payload.clone(),
744+
"output_payload": node_output.output_payload.clone(),
743745
"provider_route": node_output.output_payload
744746
.get("provider_route")
745747
.cloned()

web/app/src/features/agent-flow/_tests/debug-console/debug-conversation-log-panel.test.tsx

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,88 @@ const fusionSummaryOnlyAssistantMessage: AgentFlowDebugMessage = {
434434
]
435435
};
436436

437+
const fusionHistoricalBranchDetailAssistantMessage: AgentFlowDebugMessage = {
438+
...assistantMessage,
439+
traceSummary: [
440+
{
441+
...assistantMessage.traceSummary[1],
442+
nodeId: 'node-main-llm',
443+
nodeRunId: 'node-run-main-llm',
444+
nodeAlias: 'LLM',
445+
debugPayload: {
446+
llm_rounds: [
447+
{
448+
round_index: 0,
449+
assistant: {
450+
role: 'assistant',
451+
tool_calls: [
452+
{
453+
id: 'call_fusion',
454+
name: 'fusion_review'
455+
}
456+
]
457+
}
458+
}
459+
],
460+
visible_internal_llm_tool_trace: [
461+
{
462+
kind: 'visible_internal_llm_tool_trace',
463+
preview_kind: 'visible_internal_llm_tool_trace',
464+
route_kind: 'fusion',
465+
tool_call_id: 'call_fusion',
466+
tool_name: 'fusion_review',
467+
status: 'succeeded',
468+
branch_count: 1,
469+
branch_traces: [
470+
{
471+
node_id: 'node-judge',
472+
node_alias: 'LLM5',
473+
node_type: 'llm',
474+
status: 'succeeded',
475+
route_model: 'gpt-5.4-mini',
476+
input_payload: {
477+
prompt_messages: [
478+
{
479+
role: 'system',
480+
content: 'You are the fusion judge.'
481+
},
482+
{
483+
role: 'user',
484+
content: 'Merge panel answers.'
485+
}
486+
]
487+
},
488+
output_payload: {
489+
text: 'judge merged answer'
490+
},
491+
metrics_payload: {
492+
usage: {
493+
input_tokens: 5513,
494+
output_tokens: 2455,
495+
total_tokens: 7968
496+
}
497+
},
498+
debug_payload: {
499+
assistant_message: {
500+
role: 'assistant',
501+
content: 'judge merged answer'
502+
}
503+
},
504+
output_summary: {
505+
kind: 'text',
506+
preview: 'judge merged answer',
507+
char_count: 19,
508+
truncated: false
509+
}
510+
}
511+
]
512+
}
513+
]
514+
}
515+
}
516+
]
517+
};
518+
437519
const answerSnapshotAssistantMessage: AgentFlowDebugMessage = {
438520
...assistantMessage,
439521
status: 'waiting_callback',
@@ -716,6 +798,53 @@ describe('debug conversation log panel', () => {
716798
).not.toBeInTheDocument();
717799
}, 10_000);
718800

801+
test('shows fusion branch LLM tokens from metrics payload and reuses node detail sections', () => {
802+
renderConsole({
803+
messages: [
804+
{
805+
id: 'user-1',
806+
role: 'user',
807+
status: 'completed',
808+
runId: 'run-1',
809+
content: '做 fusion 评审',
810+
rawOutput: null,
811+
traceSummary: []
812+
},
813+
fusionHistoricalBranchDetailAssistantMessage
814+
]
815+
});
816+
817+
fireEvent.click(screen.getByRole('button', { name: '查看对话日志' }));
818+
const panel = screen.getByRole('complementary', { name: '对话日志' });
819+
fireEvent.click(within(panel).getByRole('tab', { name: '追踪' }));
820+
fireEvent.click(within(panel).getByRole('button', { name: /LLM/ }));
821+
fireEvent.click(
822+
within(panel).getByRole('button', { name: /.*1 / })
823+
);
824+
fireEvent.click(
825+
within(panel).getByRole('button', { name: /fusion_review/ })
826+
);
827+
828+
const branchNode = within(panel).getByTestId('debug-llm-route-branch-node');
829+
const branchButton = within(branchNode).getByRole('button', {
830+
name: /LLM5/
831+
});
832+
expect(branchButton).toHaveTextContent('7.96 K tokens');
833+
expect(branchButton).not.toHaveTextContent('执行成功');
834+
835+
fireEvent.click(branchButton);
836+
837+
expect(within(branchNode).getByLabelText('输入 JSON')).toHaveTextContent(
838+
'Merge panel answers.'
839+
);
840+
expect(
841+
within(branchNode).getByLabelText('数据处理 JSON')
842+
).toHaveTextContent('assistant_message');
843+
expect(within(branchNode).getByLabelText('输出 JSON')).toHaveTextContent(
844+
'judge merged answer'
845+
);
846+
}, 10_000);
847+
719848
test('collapses repeated LLM node runs into one trace row', () => {
720849
renderConsole({
721850
messages: [

web/app/src/features/agent-flow/components/debug-console/conversation/DebugWorkflowNodeRow.tsx

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,27 +30,37 @@ function statusTone(status: string) {
3030
}
3131
}
3232

33-
function readOutputTotalTokens(outputPayload: unknown) {
34-
if (
35-
!outputPayload ||
36-
typeof outputPayload !== 'object' ||
37-
Array.isArray(outputPayload)
38-
) {
33+
function readRecord(value: unknown): Record<string, unknown> | null {
34+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
3935
return null;
4036
}
4137

42-
const usage = (outputPayload as Record<string, unknown>).usage;
38+
return value as Record<string, unknown>;
39+
}
40+
41+
function readTotalTokens(payload: unknown) {
42+
const record = readRecord(payload);
43+
if (!record) {
44+
return null;
45+
}
46+
47+
const directTotalTokens = record.total_tokens;
48+
if (typeof directTotalTokens === 'number') {
49+
return directTotalTokens;
50+
}
4351

44-
if (!usage || typeof usage !== 'object' || Array.isArray(usage)) {
52+
const usage = readRecord(record.usage);
53+
if (!usage) {
4554
return null;
4655
}
4756

48-
const totalTokens = (usage as Record<string, unknown>).total_tokens;
57+
const totalTokens = usage.total_tokens;
4958
return typeof totalTokens === 'number' ? totalTokens : null;
5059
}
5160

5261
function metricText(item: AgentFlowTraceItem) {
53-
const tokens = readOutputTotalTokens(item.outputPayload);
62+
const tokens =
63+
readTotalTokens(item.metricsPayload) ?? readTotalTokens(item.outputPayload);
5464
const durationMs = item.durationMs;
5565
const toolCount =
5666
item.nodeType === 'llm'

0 commit comments

Comments
 (0)