diff --git a/packages/k8s/src/hooks/run-script-step.ts b/packages/k8s/src/hooks/run-script-step.ts index 1db2bb2e..3c70022a 100644 --- a/packages/k8s/src/hooks/run-script-step.ts +++ b/packages/k8s/src/hooks/run-script-step.ts @@ -75,13 +75,35 @@ export async function runScriptStep( args.entryPoint = 'sh' args.entryPointArgs = ['-e', containerPath] try { - await execPodStep( - [args.entryPoint, ...args.entryPointArgs], - state.jobPod, - JOB_CONTAINER_NAME - ) + await (async function () { + const retries = 3 + const delayMs = 5000 + + function attempt(n: number): Promise { + return execPodStep( + [args.entryPoint, ...args.entryPointArgs], + state.jobPod, + JOB_CONTAINER_NAME + ).catch((e: any) => { + const msg = String(e?.message || e) + const is404 = + msg.indexOf('404') !== -1 || + msg.indexOf('Unexpected server response') !== -1 + + if (is404 && n > 1) { + core.debug('execPodStep got 404, retrying...') + return new Promise(r => setTimeout(r, delayMs)).then(() => + attempt(n - 1) + ) + } + throw e + }) + } + + return attempt(retries) + })() } catch (err) { - core.debug(`execPodStep failed: ${JSON.stringify(err)}`) + core.debug(`execPodStep failed: ${(err as any)?.message || String(err)}`) const message = (err as any)?.response?.body?.message || err throw new Error(`failed to run script step: ${message}`) } finally {