diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp index 41c01d6e..b9752d48 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp @@ -178,7 +178,7 @@ bool Graph::queueHeadNodes(bool start) queueHeadNodesForExecution(); } - return (nodes_in_flight_ > 0); + return (executing_nodes > 0); } inline uint32_t Graph::countExecutableNodes(bool start) @@ -223,7 +223,15 @@ inline void Graph::tryQueueNode(const std::shared_ptr& node) } else { - LM_LOG_WARN() << "Failed to queue node for execution " << push_res.error(); + // This means the job will never be queued so we'll never get the nodeExecuted() call, we need to call it + // here + LM_LOG_ERROR() << "Failed to queue node for execution " << push_res.error(); + + abort(getLastExecutionError(), ControlClientCode::kSetStateFailed); + markNodeInFlight(); // This will be decremented below, avoid it going negative + // Also, we need to be careful not to recurse or deadlock here. The below function does not lock any mutex + // nor call this function + handleNonTransitionExecution(GraphState::kAborting); break; } } diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp index f1a8f2fc..0e15fcc6 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp @@ -320,7 +320,7 @@ class Graph final { /// @brief Queue head nodes to start a graph. Return false if there were no head nodes /// @param start true if this run of the graph is to start processes, false it it is to stop them - /// @return true if one or more head nodes were queued + /// @return true if one or more head nodes were found bool queueHeadNodes(bool start); /// @brief Queue the jobs to kick-off a stop process graph diff --git a/tests/integration/process_wrong_binary_failure/BUILD b/tests/integration/process_wrong_binary_failure/BUILD new file mode 100644 index 00000000..3a36ccd4 --- /dev/null +++ b/tests/integration/process_wrong_binary_failure/BUILD @@ -0,0 +1,66 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files") +load("@rules_pkg//pkg:tar.bzl", "pkg_tar") +load("//:defs.bzl", "launch_manager_config") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +launch_manager_config( + name = "lm_process_wrong_binary_failure_config", + config = "//tests/integration/process_wrong_binary_failure:process_wrong_binary_failure.json", + flatbuffer_out_dir = "etc", +) + +cc_binary( + name = "control_client_mock", + srcs = ["control_client_mock.cpp"], + deps = [ + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +pkg_files( + name = "process_wrong_binary_failure_main_files", + srcs = [ + ":control_client_mock", + "//score/launch_manager", + "//tests/utils/test_helper:verification_process", + ], + attributes = pkg_attributes(mode = "0755"), + prefix = "tests/process_wrong_binary_failure", +) + +pkg_files( + name = "process_wrong_binary_failure_etc_files", + srcs = [":lm_process_wrong_binary_failure_config"], + prefix = "tests/process_wrong_binary_failure", +) + +pkg_tar( + name = "process_wrong_binary_failure_binaries", + srcs = [ + ":process_wrong_binary_failure_etc_files", + ":process_wrong_binary_failure_main_files", + ], +) + +integration_test( + name = "process_wrong_binary_failure", + srcs = ["process_wrong_binary_failure.py"], + tags = ["integration"], + test_binaries = ":process_wrong_binary_failure_binaries", + deps = ["//tests/utils/testing_utils"], +) diff --git a/tests/integration/process_wrong_binary_failure/control_client_mock.cpp b/tests/integration/process_wrong_binary_failure/control_client_mock.cpp new file mode 100644 index 00000000..5dfbb353 --- /dev/null +++ b/tests/integration/process_wrong_binary_failure/control_client_mock.cpp @@ -0,0 +1,52 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include + +TEST(MissingBinaryFailure, ControlClientMock) +{ + score::mw::lifecycle::ControlClient client; + + ASSERT_TRUE(check_clean({test_end_location, fallback_file})); + + TEST_STEP("Report kRunning from ControlClientMock") + { + score::mw::lifecycle::report_running(); + } + + TEST_STEP("Activate RunTarget containing a component with a missing binary") + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget("run_target_with_missing_binary").Get(stop_token); + EXPECT_FALSE(result.has_value()) << "Activating a run target with a missing binary should fail."; + } + // Limitation: we cannot wait for the transition to fallback to complete + sleep(1); + TEST_STEP("Verify fallback run target was activated") + { + EXPECT_TRUE(std::filesystem::exists(fallback_file)) << "Fallback run target should have been activated"; + } + + TEST_STEP("Activate RunTarget Off") + { + client.ActivateRunTarget("Off"); + } +} + +int main(int argc, char** argv) +{ + return TestRunner(__FILE__, TerminationBehavior::kContinue, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/process_wrong_binary_failure/process_wrong_binary_failure.json b/tests/integration/process_wrong_binary_failure/process_wrong_binary_failure.json new file mode 100644 index 00000000..1a0c1386 --- /dev/null +++ b/tests/integration/process_wrong_binary_failure/process_wrong_binary_failure.json @@ -0,0 +1,127 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/process_wrong_binary_failure", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": true, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "control_client_mock": { + "component_properties": { + "binary_name": "control_client_mock", + "application_profile": { + "application_type": "State_Manager", + "alive_supervision": { + "min_indications": 0 + } + } + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "control_client_mock" + } + } + }, + "component_with_missing_binary": { + "component_properties": { + "binary_name": "abc_complex_reporting_process", + "process_arguments": [ + "1500.0" + ] + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "component_with_missing_binary" + } + } + }, + "verification_component": { + "component_properties": { + "binary_name": "verification_process", + "application_profile": { + "application_type": "Native", + "is_self_terminating": true + }, + "ready_condition": { + "process_state": "Terminated" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "control_client_mock" + ], + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + } + }, + "run_target_with_missing_binary": { + "depends_on": [ + "control_client_mock", + "component_with_missing_binary" + ], + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + } + }, + "Off": { + "depends_on": [] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [ + "control_client_mock", + "verification_component" + ] + } +} diff --git a/tests/integration/process_wrong_binary_failure/process_wrong_binary_failure.py b/tests/integration/process_wrong_binary_failure/process_wrong_binary_failure.py new file mode 100644 index 00000000..8c84d436 --- /dev/null +++ b/tests/integration/process_wrong_binary_failure/process_wrong_binary_failure.py @@ -0,0 +1,40 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + + +@add_test_properties( + partially_verifies=[], + test_type="interface-test", + derivation_technique="error-guessing", +) +def test_process_wrong_binary_failure( + target, setup_test, assert_test_results, remote_test_dir +): + """ + Objective: Verifies that activating a run target containing a component whose binary does not exist + results in a failure and triggers the configured recovery action (switch to fallback run target). + """ + + run_until_file_deployed( + target=target, + binary_path=str(remote_test_dir / "launch_manager"), + file_path=remote_test_dir.parent / "test_end", + cwd=str(remote_test_dir), + timeout_s=6, + ) + + assert_test_results({"control_client_mock.xml"})