diff --git a/packages/k8s/src/k8s/index.ts b/packages/k8s/src/k8s/index.ts index ae773da3..802721fc 100644 --- a/packages/k8s/src/k8s/index.ts +++ b/packages/k8s/src/k8s/index.ts @@ -16,6 +16,8 @@ import { PodPhase, mergePodSpecWithOptions, mergeObjectMeta, + injectSameNodePreference, + useSameNodePreference, fixArgs, listDirAllCommand, sleep, @@ -35,6 +37,33 @@ const k8sAuthorizationV1Api = kc.makeApiClient(k8s.AuthorizationV1Api) const DEFAULT_WAIT_FOR_POD_TIME_SECONDS = 10 * 60 // 10 min +async function applySameNodePreference(spec: k8s.V1PodSpec): Promise { + if (!useSameNodePreference()) { + return + } + try { + const runnerPodName = process.env.ACTIONS_RUNNER_POD_NAME + if (!runnerPodName) { + core.warning( + 'Same-node scheduling preference is enabled but ACTIONS_RUNNER_POD_NAME is not set' + ) + return + } + const runnerPod = await k8sApi.readNamespacedPod({ + name: runnerPodName, + namespace: namespace() + }) + const runnerNodeName = runnerPod.spec?.nodeName + if (runnerNodeName) { + injectSameNodePreference(spec, runnerNodeName) + } + } catch (err) { + core.warning( + `Could not look up runner pod node for same-node scheduling: ${err}` + ) + } +} + export const requiredPermissions = [ { group: '', @@ -176,6 +205,8 @@ export async function createJobPod( mergePodSpecWithOptions(appPod.spec, extension.spec) } + await applySameNodePreference(appPod.spec) + return await k8sApi.createNamespacedPod({ namespace: namespace(), body: appPod @@ -229,6 +260,8 @@ export async function createContainerStepPod( mergePodSpecWithOptions(appPod.spec, extension.spec) } + await applySameNodePreference(appPod.spec) + return await k8sApi.createNamespacedPod({ namespace: namespace(), body: appPod diff --git a/packages/k8s/src/k8s/utils.ts b/packages/k8s/src/k8s/utils.ts index 9e744004..45307c0b 100644 --- a/packages/k8s/src/k8s/utils.ts +++ b/packages/k8s/src/k8s/utils.ts @@ -12,6 +12,7 @@ export const DEFAULT_CONTAINER_ENTRY_POINT = 'tail' export const ENV_HOOK_TEMPLATE_PATH = 'ACTIONS_RUNNER_CONTAINER_HOOK_TEMPLATE' export const ENV_USE_KUBE_SCHEDULER = 'ACTIONS_RUNNER_USE_KUBE_SCHEDULER' +export const ENV_SAME_NODE_PREFERENCE = 'ACTIONS_RUNNER_SAME_NODE_PREFERENCE' export const EXTERNALS_VOLUME_NAME = 'externals' export const GITHUB_VOLUME_NAME = 'github' @@ -269,6 +270,42 @@ export function useKubeScheduler(): boolean { return process.env[ENV_USE_KUBE_SCHEDULER] === 'true' } +export function useSameNodePreference(): boolean { + return process.env[ENV_SAME_NODE_PREFERENCE] === 'true' +} + +export function injectSameNodePreference( + spec: k8s.V1PodSpec, + nodeName: string +): void { + if (!spec.affinity) { + spec.affinity = {} + } + if (!spec.affinity.nodeAffinity) { + spec.affinity.nodeAffinity = {} + } + if ( + !spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution + ) { + spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution = + [] + } + spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.push( + { + weight: 100, + preference: { + matchExpressions: [ + { + key: 'kubernetes.io/hostname', + operator: 'In', + values: [nodeName] + } + ] + } + } + ) +} + export enum PodPhase { PENDING = 'Pending', RUNNING = 'Running', diff --git a/packages/k8s/tests/k8s-utils-test.ts b/packages/k8s/tests/k8s-utils-test.ts index bfb6c453..d37d95a3 100644 --- a/packages/k8s/tests/k8s-utils-test.ts +++ b/packages/k8s/tests/k8s-utils-test.ts @@ -6,6 +6,9 @@ import { mergePodSpecWithOptions, mergeContainerWithOptions, readExtensionFromFile, + injectSameNodePreference, + useSameNodePreference, + ENV_SAME_NODE_PREFERENCE, ENV_HOOK_TEMPLATE_PATH } from '../src/k8s/utils' import * as k8s from '@kubernetes/client-node' @@ -339,6 +342,131 @@ spec: expect(base).toStrictEqual(expectContainer) }) + describe('useSameNodePreference', () => { + let savedValue: string | undefined + + beforeEach(() => { + savedValue = process.env[ENV_SAME_NODE_PREFERENCE] + }) + + afterEach(() => { + if (savedValue === undefined) { + delete process.env[ENV_SAME_NODE_PREFERENCE] + } else { + process.env[ENV_SAME_NODE_PREFERENCE] = savedValue + } + }) + + it('should return false when env var is not set', () => { + delete process.env[ENV_SAME_NODE_PREFERENCE] + expect(useSameNodePreference()).toBe(false) + }) + + it('should return false when env var is empty', () => { + process.env[ENV_SAME_NODE_PREFERENCE] = '' + expect(useSameNodePreference()).toBe(false) + }) + + it('should return false when env var is not "true"', () => { + process.env[ENV_SAME_NODE_PREFERENCE] = 'false' + expect(useSameNodePreference()).toBe(false) + + process.env[ENV_SAME_NODE_PREFERENCE] = 'TRUE' + expect(useSameNodePreference()).toBe(false) + + process.env[ENV_SAME_NODE_PREFERENCE] = '1' + expect(useSameNodePreference()).toBe(false) + }) + + it('should return true when env var is "true"', () => { + process.env[ENV_SAME_NODE_PREFERENCE] = 'true' + expect(useSameNodePreference()).toBe(true) + }) + }) + + describe('injectSameNodePreference', () => { + it('should add nodeAffinity to empty spec', () => { + const spec = { containers: [] } as k8s.V1PodSpec + injectSameNodePreference(spec, 'node-1') + + const preferred = + spec.affinity?.nodeAffinity + ?.preferredDuringSchedulingIgnoredDuringExecution + expect(preferred).toHaveLength(1) + expect(preferred![0].weight).toBe(100) + expect( + preferred![0].preference.matchExpressions![0] + ).toStrictEqual({ + key: 'kubernetes.io/hostname', + operator: 'In', + values: ['node-1'] + }) + }) + + it('should append to existing preferred scheduling entries', () => { + const spec = { + containers: [], + affinity: { + nodeAffinity: { + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: 50, + preference: { + matchExpressions: [ + { + key: 'node.kubernetes.io/instance-type', + operator: 'In', + values: ['m5.xlarge'] + } + ] + } + } + ] + } + } + } as k8s.V1PodSpec + + injectSameNodePreference(spec, 'node-2') + + const preferred = + spec.affinity!.nodeAffinity! + .preferredDuringSchedulingIgnoredDuringExecution! + expect(preferred).toHaveLength(2) + expect(preferred[0].weight).toBe(50) + expect(preferred[1].weight).toBe(100) + expect( + preferred[1].preference.matchExpressions![0].values + ).toStrictEqual(['node-2']) + }) + + it('should not touch requiredDuringScheduling', () => { + const required = { + nodeSelectorTerms: [ + { + matchExpressions: [ + { key: 'zone', operator: 'In', values: ['us-east-1a'] } + ] + } + ] + } + const spec = { + containers: [], + affinity: { + nodeAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: required + } + } + } as k8s.V1PodSpec + + injectSameNodePreference(spec, 'node-3') + + expect( + spec.affinity!.nodeAffinity! + .requiredDuringSchedulingIgnoredDuringExecution + ).toStrictEqual(required) + }) + }) + it('should merge pod spec', () => { const base = { containers: [ diff --git a/packages/k8s/tests/prepare-job-test.ts b/packages/k8s/tests/prepare-job-test.ts index f73ee93b..1ca49d59 100644 --- a/packages/k8s/tests/prepare-job-test.ts +++ b/packages/k8s/tests/prepare-job-test.ts @@ -3,7 +3,11 @@ import * as path from 'path' import { cleanupJob } from '../src/hooks' import { createContainerSpec, prepareJob } from '../src/hooks/prepare-job' import { TestHelper } from './test-setup' -import { ENV_HOOK_TEMPLATE_PATH, generateContainerName } from '../src/k8s/utils' +import { + ENV_HOOK_TEMPLATE_PATH, + ENV_SAME_NODE_PREFERENCE, + generateContainerName +} from '../src/k8s/utils' import { execPodStep, getPodByName } from '../src/k8s' import { V1Container } from '@kubernetes/client-node' import { JOB_CONTAINER_NAME } from '../src/hooks/constants' @@ -243,4 +247,48 @@ describe('Prepare job', () => { 'ghcr.io/actions/actions-runner:latest' ) }) + + it('should apply same-node scheduling preference when enabled', async () => { + process.env[ENV_SAME_NODE_PREFERENCE] = 'true' + + const runnerPodName = process.env.ACTIONS_RUNNER_POD_NAME! + let runnerNodeName: string | undefined + for (let i = 0; i < 30; i++) { + const runnerPod = await getPodByName(runnerPodName) + runnerNodeName = runnerPod.spec?.nodeName + if (runnerNodeName) break + await new Promise(r => setTimeout(r, 1000)) + } + expect(runnerNodeName).toBeTruthy() + + await prepareJob(prepareJobData.args, prepareJobOutputFilePath) + + delete process.env[ENV_SAME_NODE_PREFERENCE] + + const content = JSON.parse( + fs.readFileSync(prepareJobOutputFilePath).toString() + ) + + const got = await getPodByName(content.state.jobPod) + const preferred = + got.spec?.affinity?.nodeAffinity + ?.preferredDuringSchedulingIgnoredDuringExecution + expect(preferred).toBeDefined() + expect(preferred).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + weight: 100, + preference: { + matchExpressions: [ + { + key: 'kubernetes.io/hostname', + operator: 'In', + values: [runnerNodeName] + } + ] + } + }) + ]) + ) + }) })