diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2305ca21..2d9b2329 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,12 @@ concurrency: jobs: release: - if: '!cancelled()' + if: | + !cancelled() + && ( + github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository + ) uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_release_library.yml@v1.4.3 with: release-ref: ${{ inputs.release-ref || github.sha }} @@ -82,7 +87,7 @@ jobs: release-summary: needs: [release] - if: ${{ !cancelled() }} + if: ${{ !cancelled() && needs.release.result != 'skipped' }} runs-on: ubuntu-latest steps: - name: Result diff --git a/nemo_run/run/torchx_backend/schedulers/docker.py b/nemo_run/run/torchx_backend/schedulers/docker.py index 4f68920c..b162cfe1 100644 --- a/nemo_run/run/torchx_backend/schedulers/docker.py +++ b/nemo_run/run/torchx_backend/schedulers/docker.py @@ -276,8 +276,11 @@ def local_logs(container: DockerContainer): def _cancel_existing(self, app_id: str) -> None: req = DockerJobRequest.load(app_id=app_id) if not req: - return None + return + status = json.dumps({"id": app_id, "exit_code": "1"}) + "\n" for container in req.containers: + status_file = Path(req.executor.job_dir) / f"status_{container.name}.out" + status_file.write_text(status) container.delete(client=self._docker_client, id=req.id) def close(self) -> None: diff --git a/test/run/torchx_backend/schedulers/test_docker.py b/test/run/torchx_backend/schedulers/test_docker.py index 551d8a60..e555d4ae 100644 --- a/test/run/torchx_backend/schedulers/test_docker.py +++ b/test/run/torchx_backend/schedulers/test_docker.py @@ -84,6 +84,34 @@ def test_docker_scheduler_methods(docker_scheduler): assert hasattr(docker_scheduler, "close") +def test_cancel_existing_without_request_is_noop(): + docker_scheduler = object.__new__(PersistentDockerScheduler) + docker_scheduler._DockerWorkspaceMixin__docker_client = mock.Mock() + docker_scheduler._scheduled_reqs = [] + + with mock.patch.object(DockerJobRequest, "load", return_value=None): + docker_scheduler._cancel_existing("app-id") + + +def test_cancel_existing_persists_terminal_status(tmp_path): + docker_scheduler = object.__new__(PersistentDockerScheduler) + docker_scheduler._DockerWorkspaceMixin__docker_client = mock.Mock() + docker_scheduler._scheduled_reqs = [] + container = mock.Mock(name="container") + container.name = "task-1-0" + request = mock.Mock() + request.id = "app-id" + request.executor.job_dir = str(tmp_path) + request.containers = [container] + + with mock.patch.object(DockerJobRequest, "load", return_value=request): + docker_scheduler._cancel_existing("app-id") + + container.delete.assert_called_once_with(client=docker_scheduler._docker_client, id="app-id") + status = json.loads((tmp_path / "status_task-1-0.out").read_text()) + assert status == {"id": "app-id", "exit_code": "1"} + + def test_schedule(docker_scheduler, mock_app_def, docker_executor): with ( mock.patch.object(DockerExecutor, "package") as mock_package,