Skip to content

fix: make environment service-account annotations cloud-neutral #217

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vmelikyan wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: make environment service-account annotations cloud-neutral #217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/server/db/migrations/001_seed.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,7 +456,7 @@ export async function up(knex: Knex): Promise<any> {
INSERT INTO global_config (key, config, "createdAt", "updatedAt", "deletedAt", description) VALUES ('rdsRestoreSettings', '{"vpcId":"","accountId":"","region":"us-west-2","securityGroupIds":[],"subnetGroupName":"","engine":"mysql","engineVersion":"8.0.33","tagMatch":{"key":"restore-for"},"instanceSize":"db.t3.small","restoreSize":"db.t3.small"}', now(), now(), null, 'Default RDS database settings to use for restore');
INSERT INTO global_config (key, config, "createdAt", "updatedAt", "deletedAt", description) VALUES ('minio', '{"version":"3.7.2","args":"--force --timeout 60m0s --wait","action":"install","chart":{"name":"minio","repoUrl":"https://charts.bitnami.com/bitnami","version":"15.0.7","values":[],"valueFiles":[]},"label":"labels","tolerations":"tolerations","affinity":"affinity","nodeSelector":"nodeSelector"}', now(), now(), null, 'Default minio s3 compatible bucket');
INSERT INTO global_config (key, config, "createdAt", "updatedAt", "deletedAt", description) VALUES ('features', '{"webhooks":true,"reconcileDeletedServices":false}', now(), now(), null, 'Configuration for feature flags controlled from database');
INSERT INTO global_config (key, config, "createdAt", "updatedAt", "deletedAt", description) VALUES ('serviceAccount', '{"name": "default","role":"replace_me"}', now(), now(), null, 'Default IAM role name to be used to annotate service account');
INSERT INTO global_config (key, config, "createdAt", "updatedAt", "deletedAt", description) VALUES ('serviceAccount', '{"name": "default"}', now(), now(), null, 'Service account for environment namespaces. Optional annotations map is applied to the generated service account (e.g. cloud workload identity keys).');
INSERT INTO global_config (key, config, "createdAt", "updatedAt", "deletedAt", description) VALUES ('app_setup', '{"state":"","created":false,"installed":false,"restarted":false,"org":"","url":"","name":""}', now(), now(), null, 'Application setup state');
INSERT INTO global_config (key, config, "createdAt", "updatedAt", "deletedAt", description) VALUES ('labels', '{"deploy":["lifecycle-deploy!"],"disabled":["lifecycle-disabled!"],"statusComments":["lifecycle-status-comments!"],"defaultStatusComments":true,"defaultControlComments":true}', now(), now(), null, 'Configurable PR labels for deploy, disabled, and status comments');
INSERT INTO global_config (key, config, "createdAt", "updatedAt", "deletedAt", description) VALUES ('logArchival', '{"enabled": false}', now(), now(), null, 'Log archival feature flag. Enable to archive build/deploy logs to object storage.');
Expand Down
14 changes: 7 additions & 7 deletions src/server/lib/agentSession/__tests__/serviceAccountFactory.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@
* limitations under the License.
*/

const mockSetupReadOnlyServiceAccountInNamespace = jest.fn();
const mockEnsureServiceAccount = jest.fn();

jest.mock('server/lib/kubernetes/rbac', () => ({
setupReadOnlyServiceAccountInNamespace: mockSetupReadOnlyServiceAccountInNamespace,
jest.mock('server/lib/kubernetes/common/serviceAccount', () => ({
ensureServiceAccount: mockEnsureServiceAccount,
}));

function loadModule() {
Expand All @@ -36,7 +36,7 @@ describe('serviceAccountFactory', () => {

it('reuses the in-flight setup promise for the same namespace', async () => {
let resolveSetup!: () => void;
mockSetupReadOnlyServiceAccountInNamespace.mockImplementationOnce(
mockEnsureServiceAccount.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
resolveSetup = resolve;
Expand All @@ -47,7 +47,7 @@ describe('serviceAccountFactory', () => {
const firstCall = ensureAgentSessionServiceAccount('test-ns');
const secondCall = ensureAgentSessionServiceAccount('test-ns');

expect(mockSetupReadOnlyServiceAccountInNamespace).toHaveBeenCalledTimes(1);
expect(mockEnsureServiceAccount).toHaveBeenCalledTimes(1);

resolveSetup();

Expand All @@ -57,12 +57,12 @@ describe('serviceAccountFactory', () => {

it('clears the namespace cache after a failed setup', async () => {
const setupError = new Error('setup failed');
mockSetupReadOnlyServiceAccountInNamespace.mockRejectedValueOnce(setupError).mockResolvedValueOnce(undefined);
mockEnsureServiceAccount.mockRejectedValueOnce(setupError).mockResolvedValueOnce(undefined);

const { ensureAgentSessionServiceAccount } = loadModule();

await expect(ensureAgentSessionServiceAccount('test-ns')).rejects.toThrow('setup failed');
await expect(ensureAgentSessionServiceAccount('test-ns')).resolves.toBe('agent-sa');
expect(mockSetupReadOnlyServiceAccountInNamespace).toHaveBeenCalledTimes(2);
expect(mockEnsureServiceAccount).toHaveBeenCalledTimes(2);
});
});
4 changes: 2 additions & 2 deletions src/server/lib/agentSession/serviceAccountFactory.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* limitations under the License.
*/

import { setupReadOnlyServiceAccountInNamespace } from 'server/lib/kubernetes/rbac';
import { ensureServiceAccount } from 'server/lib/kubernetes/common/serviceAccount';

export const AGENT_SESSION_SERVICE_ACCOUNT_NAME = 'agent-sa';
const serviceAccountSetupByNamespace = new Map<string, Promise<string>>();
Expand All @@ -23,7 +23,7 @@ export async function ensureAgentSessionServiceAccount(namespace: string): Promi
let setupPromise = serviceAccountSetupByNamespace.get(namespace);
if (!setupPromise) {
setupPromise = (async () => {
await setupReadOnlyServiceAccountInNamespace(namespace, AGENT_SESSION_SERVICE_ACCOUNT_NAME);
await ensureServiceAccount({ namespace, name: AGENT_SESSION_SERVICE_ACCOUNT_NAME, permissions: 'read' });
return AGENT_SESSION_SERVICE_ACCOUNT_NAME;
})();
serviceAccountSetupByNamespace.set(namespace, setupPromise);
Expand Down
97 changes: 1 addition & 96 deletions src/server/lib/kubernetes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,14 @@ import _ from 'lodash';
import { Build, Deploy, Deployable, Service } from 'server/models';
import { CLIDeployTypes, KubernetesDeployTypes, MEDIUM_TYPE, DEFAULT_TTL_INACTIVITY_DAYS } from 'shared/constants';
import { shellPromise } from './shell';
import { flattenObject, getKeepLabel, parsePullRequestLabels, waitUntil } from 'server/lib/utils';
import { flattenObject, getKeepLabel, parsePullRequestLabels } from 'server/lib/utils';
import { ServiceDiskConfig } from 'server/models/yaml';
import * as k8s from '@kubernetes/client-node';
import { HttpError, V1Status, CoreV1Api, KubeConfig } from '@kubernetes/client-node';
import { IncomingMessage } from 'http';
import { APP_ENV, TMP_PATH } from 'shared/config';
import fs from 'fs';
import GlobalConfigService from 'server/services/globalConfig';
import { setupServiceAccountWithRBAC } from './kubernetes/rbac';
import { staticEnvTolerations } from './helm/constants';
import { parseSecretRefsFromEnv, SecretRefWithEnvKey } from './secretRefs';
import { generateSecretName } from './kubernetes/externalSecret';
Expand Down Expand Up @@ -384,100 +383,6 @@ export async function createOrUpdateNamespace({
}
}

export async function createOrUpdateServiceAccount({ namespace, role }: { namespace: string; role: string }) {
const kc = new k8s.KubeConfig();
kc.loadFromDefault();
const client = kc.makeApiClient(k8s.CoreV1Api);

// Get the service account name from global config
const { serviceAccount } = await GlobalConfigService.getInstance().getAllConfigs();
const serviceAccountName = serviceAccount?.name || 'default';

const serviceAccountExists = async () => {
try {
const saResponse = await client.readNamespacedServiceAccount(serviceAccountName, namespace);
return Boolean(saResponse?.body);
} catch (error) {
return false;
}
};

// If it's not the default service account, create it first
if (serviceAccountName !== 'default') {
const serviceAccountManifest = {
apiVersion: 'v1',
kind: 'ServiceAccount',
metadata: {
name: serviceAccountName,
},
};

try {
if (!(await serviceAccountExists())) {
getLogger({ namespace, serviceAccountName }).debug('ServiceAccount: creating');
await client.createNamespacedServiceAccount(namespace, serviceAccountManifest);
getLogger({ namespace, serviceAccountName }).debug('Created service account');
} else {
getLogger({ namespace, serviceAccountName }).debug('Service account already exists');
}
} catch (err) {
getLogger({
namespace,
serviceAccountName,
error: err,
statusCode: err?.response?.statusCode,
statusMessage: err?.response?.statusMessage,
}).error('ServiceAccount: create failed');
throw err;
}
} else {
try {
await waitUntil(serviceAccountExists, {
timeoutMs: 120000,
intervalMs: 2000,
});
} catch (error) {
getLogger({ namespace, serviceAccountName, error }).error('ServiceAccount: wait timeout');
throw error;
}
}

// patch the service account with the role
const patch = {
metadata: {
annotations: {
'eks.amazonaws.com/role-arn': role,
},
},
};

try {
await client.patchNamespacedServiceAccount(
serviceAccountName,
namespace,
patch,
undefined,
undefined,
undefined,
undefined,
undefined,
{ headers: { 'Content-Type': 'application/merge-patch+json' } }
);
getLogger({ namespace, serviceAccountName }).debug('Annotated service account');

await setupServiceAccountWithRBAC({
namespace,
serviceAccountName,
awsRoleArn: role,
permissions: 'deploy',
});
getLogger({ namespace, serviceAccountName }).debug('RBAC: configured');
} catch (err) {
getLogger({ namespace, serviceAccountName, error: err }).error('ServiceAccount: setup failed');
throw err;
}
}

/**
*
* @param build
Expand Down
Loading
Loading