From 13a990590c6faf35ea3710170fe240c07a92c53b Mon Sep 17 00:00:00 2001 From: svcnemo-autobot Date: Mon, 27 Jul 2026 00:40:08 +0000 Subject: [PATCH 1/3] fix(docker): persist cancellation status Signed-off-by: svcnemo-autobot --- .../run/torchx_backend/schedulers/docker.py | 4 +++- .../torchx_backend/schedulers/test_docker.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/nemo_run/run/torchx_backend/schedulers/docker.py b/nemo_run/run/torchx_backend/schedulers/docker.py index 4f68920c..50a3a014 100644 --- a/nemo_run/run/torchx_backend/schedulers/docker.py +++ b/nemo_run/run/torchx_backend/schedulers/docker.py @@ -276,8 +276,10 @@ def local_logs(container: DockerContainer): def _cancel_existing(self, app_id: str) -> None: req = DockerJobRequest.load(app_id=app_id) if not req: - return None + return for container in req.containers: + status_file = Path(req.executor.job_dir) / f"status_{container.name}.out" + status_file.write_text(json.dumps({"id": app_id, "exit_code": "1"}) + "\n") container.delete(client=self._docker_client, id=req.id) def close(self) -> None: diff --git a/test/run/torchx_backend/schedulers/test_docker.py b/test/run/torchx_backend/schedulers/test_docker.py index 551d8a60..33a2983c 100644 --- a/test/run/torchx_backend/schedulers/test_docker.py +++ b/test/run/torchx_backend/schedulers/test_docker.py @@ -84,6 +84,25 @@ def test_docker_scheduler_methods(docker_scheduler): assert hasattr(docker_scheduler, "close") +def test_cancel_existing_persists_terminal_status(tmp_path): + docker_scheduler = object.__new__(PersistentDockerScheduler) + docker_scheduler._DockerWorkspaceMixin__docker_client = mock.Mock() + docker_scheduler._scheduled_reqs = [] + container = mock.Mock(name="container") + container.name = "task-1-0" + request = mock.Mock() + request.id = "app-id" + request.executor.job_dir = str(tmp_path) + request.containers = [container] + + with mock.patch.object(DockerJobRequest, "load", return_value=request): + docker_scheduler._cancel_existing("app-id") + + container.delete.assert_called_once_with(client=docker_scheduler._docker_client, id="app-id") + status = json.loads((tmp_path / "status_task-1-0.out").read_text()) + assert status == {"id": "app-id", "exit_code": "1"} + + def test_schedule(docker_scheduler, mock_app_def, docker_executor): with ( mock.patch.object(DockerExecutor, "package") as mock_package, From 1ea2e1795926f491424a24fec1802f46a9b32ab5 Mon Sep 17 00:00:00 2001 From: svcnemo-autobot Date: Mon, 27 Jul 2026 01:05:15 +0000 Subject: [PATCH 2/3] test(docker): cover missing cancellation request Signed-off-by: svcnemo-autobot --- nemo_run/run/torchx_backend/schedulers/docker.py | 3 ++- test/run/torchx_backend/schedulers/test_docker.py | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nemo_run/run/torchx_backend/schedulers/docker.py b/nemo_run/run/torchx_backend/schedulers/docker.py index 50a3a014..b162cfe1 100644 --- a/nemo_run/run/torchx_backend/schedulers/docker.py +++ b/nemo_run/run/torchx_backend/schedulers/docker.py @@ -277,9 +277,10 @@ def _cancel_existing(self, app_id: str) -> None: req = DockerJobRequest.load(app_id=app_id) if not req: return + status = json.dumps({"id": app_id, "exit_code": "1"}) + "\n" for container in req.containers: status_file = Path(req.executor.job_dir) / f"status_{container.name}.out" - status_file.write_text(json.dumps({"id": app_id, "exit_code": "1"}) + "\n") + status_file.write_text(status) container.delete(client=self._docker_client, id=req.id) def close(self) -> None: diff --git a/test/run/torchx_backend/schedulers/test_docker.py b/test/run/torchx_backend/schedulers/test_docker.py index 33a2983c..e555d4ae 100644 --- a/test/run/torchx_backend/schedulers/test_docker.py +++ b/test/run/torchx_backend/schedulers/test_docker.py @@ -84,6 +84,15 @@ def test_docker_scheduler_methods(docker_scheduler): assert hasattr(docker_scheduler, "close") +def test_cancel_existing_without_request_is_noop(): + docker_scheduler = object.__new__(PersistentDockerScheduler) + docker_scheduler._DockerWorkspaceMixin__docker_client = mock.Mock() + docker_scheduler._scheduled_reqs = [] + + with mock.patch.object(DockerJobRequest, "load", return_value=None): + docker_scheduler._cancel_existing("app-id") + + def test_cancel_existing_persists_terminal_status(tmp_path): docker_scheduler = object.__new__(PersistentDockerScheduler) docker_scheduler._DockerWorkspaceMixin__docker_client = mock.Mock() From ddd40a8f24847f5c919f911d0240bd622653612f Mon Sep 17 00:00:00 2001 From: svcnemo-autobot Date: Mon, 27 Jul 2026 01:19:14 +0000 Subject: [PATCH 3/3] fix(ci): skip release rehearsal for forks Signed-off-by: svcnemo-autobot --- .github/workflows/release.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2305ca21..2d9b2329 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,12 @@ concurrency: jobs: release: - if: '!cancelled()' + if: | + !cancelled() + && ( + github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository + ) uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_release_library.yml@v1.4.3 with: release-ref: ${{ inputs.release-ref || github.sha }} @@ -82,7 +87,7 @@ jobs: release-summary: needs: [release] - if: ${{ !cancelled() }} + if: ${{ !cancelled() && needs.release.result != 'skipped' }} runs-on: ubuntu-latest steps: - name: Result