Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@ node_modules/
lib/
dist/
**/tests/_temp/**
packages/k8s/tests/test-kind.yaml
packages/k8s/tests/test-kind.yaml
.idea
60 changes: 48 additions & 12 deletions packages/k8s/src/k8s/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,29 @@ export async function pruneSecrets(): Promise<void> {
)
}

const UNRECOVERABLE_WAITING_REASONS = new Set([
'ImagePullBackOff',
'ErrImagePull',
'InvalidImageName',
'CreateContainerConfigError',
'CreateContainerError'
])

function getContainerErrors(pod: k8s.V1Pod): string[] {
const errors: string[] = []
const allStatuses = [
...(pod.status?.initContainerStatuses ?? []),
...(pod.status?.containerStatuses ?? [])
]
for (const cs of allStatuses) {
const waiting = cs.state?.waiting
if (waiting?.reason && UNRECOVERABLE_WAITING_REASONS.has(waiting.reason)) {
errors.push(`container "${cs.name}": ${waiting.reason}${waiting.message ? ` - ${waiting.message}` : ''}`)
}
}
return errors
}

export async function waitForPodPhases(
podName: string,
awaitingPhases: Set<PodPhase>,
Expand All @@ -683,7 +706,8 @@ export async function waitForPodPhases(
let phase: PodPhase = PodPhase.UNKNOWN
try {
while (true) {
phase = await getPodPhase(podName)
const pod = await readPod(podName)
phase = parsePodPhase(pod)
if (awaitingPhases.has(phase)) {
return
}
Expand All @@ -693,11 +717,20 @@ export async function waitForPodPhases(
`Pod ${podName} is unhealthy with phase status ${phase}`
)
}

const containerErrors = getContainerErrors(pod)
if (containerErrors.length > 0) {
throw new Error(
`Pod ${podName} has unrecoverable container errors: ${containerErrors.join('; ')}`
)
}

await backOffManager.backOff()
}
} catch (error) {
const additionalPodErrors = error instanceof Error ? error.message : JSON.stringify(error)
throw new Error(
`Pod ${podName} is unhealthy with phase status ${phase}: ${JSON.stringify(error)}`
`Pod ${podName} is unhealthy with phase status ${phase}: ${additionalPodErrors}`
)
}
}
Expand All @@ -721,23 +754,26 @@ export function getPrepareJobTimeoutSeconds(): number {
return timeoutSeconds
}

async function getPodPhase(name: string): Promise<PodPhase> {
const podPhaseLookup = new Set<string>([
PodPhase.PENDING,
PodPhase.RUNNING,
PodPhase.SUCCEEDED,
PodPhase.FAILED,
PodPhase.UNKNOWN
])
const pod = await k8sApi.readNamespacedPod({
async function readPod(name: string): Promise<k8s.V1Pod> {
return k8sApi.readNamespacedPod({
name,
namespace: namespace()
})
}

const podPhaseLookup = new Set<string>([
PodPhase.PENDING,
PodPhase.RUNNING,
PodPhase.SUCCEEDED,
PodPhase.FAILED,
PodPhase.UNKNOWN
])

function parsePodPhase(pod: k8s.V1Pod): PodPhase {
if (!pod.status?.phase || !podPhaseLookup.has(pod.status.phase)) {
return PodPhase.UNKNOWN
}
return pod.status?.phase as PodPhase
return pod.status.phase as PodPhase
}

async function isJobSucceeded(name: string): Promise<boolean> {
Expand Down