Skip to content

Commit ad2312d

Browse files
crazy-maxcrazy-max
authored
Merge pull request #1016 from crazy-max/imagetools-inspect-retry
buildx(imagetools): make manifest retries configurable
2 parents 4d926d8 + 21e2b75 commit ad2312d

6 files changed

Lines changed: 213 additions & 43 deletions

File tree

‎__tests__/buildx/imagetools.test.itg.ts‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,51 +30,51 @@ const maybe = !process.env.GITHUB_ACTIONS || (process.env.GITHUB_ACTIONS === 'tr
3030

3131
maybe('inspectImage', () => {
3232
it('inspect single platform', async () => {
33-
const image = await new ImageTools().inspectImage('moby/buildkit:latest@sha256:5769c54b98840147b74128f38fb0b0a049e24b11a75bd81664131edd2854593f');
33+
const image = await new ImageTools().inspectImage({name: 'moby/buildkit:latest@sha256:5769c54b98840147b74128f38fb0b0a049e24b11a75bd81664131edd2854593f'});
3434
const expectedImage = <Image>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-01.json'), {encoding: 'utf-8'}).trim());
3535
expect(image).toEqual(expectedImage);
3636
});
3737
it('inspect multi platform', async () => {
38-
const image = await new ImageTools().inspectImage('moby/buildkit:latest@sha256:86c0ad9d1137c186e9d455912167df20e530bdf7f7c19de802e892bb8ca16552');
38+
const image = await new ImageTools().inspectImage({name: 'moby/buildkit:latest@sha256:86c0ad9d1137c186e9d455912167df20e530bdf7f7c19de802e892bb8ca16552'});
3939
const expectedImage = <Record<string, Image>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-02.json'), {encoding: 'utf-8'}).trim());
4040
expect(image).toEqual(expectedImage);
4141
});
4242
});
4343

4444
maybe('inspectManifest', () => {
4545
it('inspect descriptor', async () => {
46-
const manifest = await new ImageTools().inspectManifest('moby/buildkit:latest@sha256:dccc69dd895968c4f21aa9e43e715f25f0cedfce4b17f1014c88c307928e22fc');
46+
const manifest = await new ImageTools().inspectManifest({name: 'moby/buildkit:latest@sha256:dccc69dd895968c4f21aa9e43e715f25f0cedfce4b17f1014c88c307928e22fc'});
4747
const expectedManifest = <Descriptor>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-03.json'), {encoding: 'utf-8'}).trim());
4848
expect(manifest).toEqual(expectedManifest);
4949
});
5050
it('inspect index', async () => {
51-
const manifest = await new ImageTools().inspectManifest('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6');
51+
const manifest = await new ImageTools().inspectManifest({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'});
5252
const expectedManifest = <ImageToolsManifest>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-04.json'), {encoding: 'utf-8'}).trim());
5353
expect(manifest).toEqual(expectedManifest);
5454
});
5555
});
5656

5757
maybe('attestationDescriptors', () => {
5858
it('returns buildkit attestations descriptors', async () => {
59-
const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6');
59+
const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'});
6060
const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-05.json'), {encoding: 'utf-8'}).trim());
6161
expect(attestations).toEqual(expectedAttestations);
6262
});
6363
it('returns buildkit attestations descriptors for linux/amd64', async () => {
64-
const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'amd64'});
64+
const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'amd64'}});
6565
const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-06.json'), {encoding: 'utf-8'}).trim());
6666
expect(attestations).toEqual(expectedAttestations);
6767
});
6868
it('returns buildkit attestations descriptors for linux/arm/v7', async () => {
69-
const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'arm', variant: 'v7'});
69+
const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'arm', variant: 'v7'}});
7070
const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-07.json'), {encoding: 'utf-8'}).trim());
7171
expect(attestations).toEqual(expectedAttestations);
7272
});
7373
});
7474

7575
maybe('attestationDigests', () => {
7676
it('returns buildkit attestations digests', async () => {
77-
const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6');
77+
const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'});
7878
// prettier-ignore
7979
expect(digests).toEqual([
8080
'sha256:2ba4ad6eae1efcafee73a971953093c7c32b6938f2f9fd4998c8bf4d0fbe76f2',
@@ -86,11 +86,11 @@ maybe('attestationDigests', () => {
8686
]);
8787
});
8888
it('returns buildkit attestations digests for linux/amd64', async () => {
89-
const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'amd64'});
89+
const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'amd64'}});
9090
expect(digests).toEqual(['sha256:2ba4ad6eae1efcafee73a971953093c7c32b6938f2f9fd4998c8bf4d0fbe76f2']);
9191
});
9292
it('returns buildkit attestations digests for linux/arm/v7', async () => {
93-
const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'arm', variant: 'v7'});
93+
const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'arm', variant: 'v7'}});
9494
expect(digests).toEqual(['sha256:0709528fae1747ce17638ad2978ee7936b38a294136eaadaf692e415f64b1e03']);
9595
});
9696
});

‎__tests__/buildx/imagetools.test.ts‎

Lines changed: 124 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17-
import {afterEach, describe, expect, it, vi} from 'vitest';
17+
import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
1818
import fs from 'fs';
1919
import os from 'os';
2020
import path from 'path';
@@ -38,10 +38,133 @@ vi.spyOn(Context, 'tmpName').mockImplementation((): string => {
3838
});
3939

4040
afterEach(() => {
41+
vi.useRealTimers();
4142
vi.clearAllMocks();
4243
rimraf.sync(tmpDir);
4344
});
4445

46+
beforeEach(() => {
47+
vi.useRealTimers();
48+
fs.mkdirSync(tmpDir, {recursive: true});
49+
});
50+
51+
describe('inspectManifest', () => {
52+
it('retries transient manifest unknown errors when requested', async () => {
53+
vi.useFakeTimers();
54+
55+
const getCommand = vi.fn().mockResolvedValue({
56+
command: 'docker',
57+
args: ['buildx', 'imagetools', 'inspect']
58+
});
59+
const buildx = {getCommand} as unknown as Buildx;
60+
const execSpy = vi
61+
.spyOn(Exec, 'getExecOutput')
62+
.mockResolvedValueOnce({
63+
exitCode: 1,
64+
stdout: '',
65+
stderr: 'ERROR: MANIFEST_UNKNOWN: manifest unknown'
66+
})
67+
.mockResolvedValueOnce({
68+
exitCode: 0,
69+
stdout: JSON.stringify({
70+
schemaVersion: 2,
71+
mediaType: 'application/vnd.oci.image.index.v1+json',
72+
manifests: []
73+
}),
74+
stderr: ''
75+
});
76+
77+
const inspectPromise = new ImageTools({buildx}).inspectManifest({
78+
name: 'docker.io/library/alpine:latest',
79+
retryOnManifestUnknown: true,
80+
retryLimit: 2
81+
});
82+
83+
await vi.runAllTimersAsync();
84+
85+
expect(await inspectPromise).toEqual({
86+
schemaVersion: 2,
87+
mediaType: 'application/vnd.oci.image.index.v1+json',
88+
manifests: []
89+
});
90+
expect(getCommand).toHaveBeenCalledWith(['imagetools', 'inspect', 'docker.io/library/alpine:latest', '--format', '{{json .Manifest}}']);
91+
expect(execSpy).toHaveBeenCalledTimes(2);
92+
});
93+
94+
it('does not retry non-manifest errors', async () => {
95+
const getCommand = vi.fn().mockResolvedValue({
96+
command: 'docker',
97+
args: ['buildx', 'imagetools', 'inspect']
98+
});
99+
const buildx = {getCommand} as unknown as Buildx;
100+
const execSpy = vi.spyOn(Exec, 'getExecOutput').mockResolvedValue({
101+
exitCode: 1,
102+
stdout: '',
103+
stderr: 'ERROR: unauthorized'
104+
});
105+
106+
const result = await new ImageTools({buildx})
107+
.inspectManifest({
108+
name: 'docker.io/library/alpine:latest',
109+
retryOnManifestUnknown: true
110+
})
111+
.then(
112+
value => ({value, error: undefined}),
113+
error => ({value: undefined, error: error as Error})
114+
);
115+
116+
expect(result.value).toBeUndefined();
117+
expect(result.error).toBeInstanceOf(Error);
118+
expect(result.error?.message).toContain('ERROR: unauthorized');
119+
120+
expect(execSpy).toHaveBeenCalledTimes(1);
121+
});
122+
});
123+
124+
describe('inspectImage', () => {
125+
it('retries transient manifest unknown errors when requested', async () => {
126+
vi.useFakeTimers();
127+
128+
const getCommand = vi.fn().mockResolvedValue({
129+
command: 'docker',
130+
args: ['buildx', 'imagetools', 'inspect']
131+
});
132+
const buildx = {getCommand} as unknown as Buildx;
133+
const execSpy = vi
134+
.spyOn(Exec, 'getExecOutput')
135+
.mockResolvedValueOnce({
136+
exitCode: 1,
137+
stdout: '',
138+
stderr: 'ERROR: MANIFEST_UNKNOWN: manifest unknown'
139+
})
140+
.mockResolvedValueOnce({
141+
exitCode: 0,
142+
stdout: JSON.stringify({
143+
config: {
144+
digest: 'sha256:test'
145+
}
146+
}),
147+
stderr: ''
148+
});
149+
150+
const inspectPromise = new ImageTools({buildx}).inspectImage({
151+
name: 'docker.io/library/alpine:latest',
152+
retryOnManifestUnknown: true,
153+
retryLimit: 2
154+
});
155+
156+
await vi.runAllTimersAsync();
157+
158+
expect(await inspectPromise).toEqual({
159+
config: {
160+
digest: 'sha256:test'
161+
}
162+
});
163+
expect(getCommand).toHaveBeenCalledWith(['imagetools', 'inspect', 'docker.io/library/alpine:latest', '--format', '{{json .Image}}']);
164+
expect(execSpy).toHaveBeenCalledTimes(2);
165+
});
166+
});
167+
45168
describe('create', () => {
46169
it('parses metadata and supports cwd sources', async () => {
47170
const getCommand = vi.fn().mockResolvedValue({

‎src/buildx/imagetools.ts‎

Lines changed: 53 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ import {Buildx} from './buildx.js';
2121
import {Context} from '../context.js';
2222
import {Exec} from '../exec.js';
2323

24-
import {CreateOpts, CreateResponse, CreateResult, Manifest as ImageToolsManifest} from '../types/buildx/imagetools.js';
24+
import {AttestationInspectOpts, CreateOpts, CreateResponse, CreateResult, InspectOpts, Manifest as ImageToolsManifest} from '../types/buildx/imagetools.js';
2525
import {Image} from '../types/oci/config.js';
26-
import {Descriptor, Platform} from '../types/oci/descriptor.js';
26+
import {Descriptor} from '../types/oci/descriptor.js';
2727
import {Digest} from '../types/oci/digest.js';
2828

2929
export interface ImageToolsOpts {
@@ -49,16 +49,8 @@ export class ImageTools {
4949
return await this.getCommand(['create', ...args]);
5050
}
5151

52-
public async inspectImage(name: string): Promise<Record<string, Image> | Image> {
53-
const cmd = await this.getInspectCommand([name, '--format', '{{json .Image}}']);
54-
return await Exec.getExecOutput(cmd.command, cmd.args, {
55-
ignoreReturnCode: true,
56-
silent: true
57-
}).then(res => {
58-
if (res.stderr.length > 0 && res.exitCode != 0) {
59-
throw new Error(res.stderr.trim());
60-
}
61-
const parsedOutput = JSON.parse(res.stdout);
52+
public async inspectImage(opts: InspectOpts): Promise<Record<string, Image> | Image> {
53+
return await this.inspect(opts, '{{json .Image}}', parsedOutput => {
6254
if (typeof parsedOutput === 'object' && !Array.isArray(parsedOutput) && parsedOutput !== null) {
6355
if (Object.prototype.hasOwnProperty.call(parsedOutput, 'config')) {
6456
return <Image>parsedOutput;
@@ -70,16 +62,8 @@ export class ImageTools {
7062
});
7163
}
7264

73-
public async inspectManifest(name: string): Promise<ImageToolsManifest | Descriptor> {
74-
const cmd = await this.getInspectCommand([name, '--format', '{{json .Manifest}}']);
75-
return await Exec.getExecOutput(cmd.command, cmd.args, {
76-
ignoreReturnCode: true,
77-
silent: true
78-
}).then(res => {
79-
if (res.stderr.length > 0 && res.exitCode != 0) {
80-
throw new Error(res.stderr.trim());
81-
}
82-
const parsedOutput = JSON.parse(res.stdout);
65+
public async inspectManifest(opts: InspectOpts): Promise<ImageToolsManifest | Descriptor> {
66+
return await this.inspect(opts, '{{json .Manifest}}', parsedOutput => {
8367
if (typeof parsedOutput === 'object' && !Array.isArray(parsedOutput) && parsedOutput !== null) {
8468
if (Object.prototype.hasOwnProperty.call(parsedOutput, 'manifests')) {
8569
return <ImageToolsManifest>parsedOutput;
@@ -91,17 +75,18 @@ export class ImageTools {
9175
});
9276
}
9377

94-
public async attestationDescriptors(name: string, platform?: Platform): Promise<Array<Descriptor>> {
95-
const manifest = await this.inspectManifest(name);
78+
public async attestationDescriptors(opts: AttestationInspectOpts): Promise<Array<Descriptor>> {
79+
const manifest = await this.inspectManifest(opts);
9680

9781
if (typeof manifest !== 'object' || manifest === null || !('manifests' in manifest) || !Array.isArray(manifest.manifests)) {
98-
throw new Error(`No descriptor found for ${name}`);
82+
throw new Error(`No descriptor found for ${opts.name}`);
9983
}
10084

10185
const attestations = manifest.manifests.filter(m => m.annotations?.['vnd.docker.reference.type'] === 'attestation-manifest');
102-
if (!platform) {
86+
if (!opts.platform) {
10387
return attestations;
10488
}
89+
const platform = opts.platform;
10590

10691
const manifestByDigest = new Map<string, Descriptor>();
10792
for (const m of manifest.manifests) {
@@ -123,8 +108,8 @@ export class ImageTools {
123108
});
124109
}
125110

126-
public async attestationDigests(name: string, platform?: Platform): Promise<Array<Digest>> {
127-
return (await this.attestationDescriptors(name, platform)).map(attestation => attestation.digest);
111+
public async attestationDigests(opts: AttestationInspectOpts): Promise<Array<Digest>> {
112+
return (await this.attestationDescriptors(opts)).map(attestation => attestation.digest);
128113
}
129114

130115
public async create(opts: CreateOpts): Promise<CreateResult | undefined> {
@@ -205,4 +190,44 @@ export class ImageTools {
205190
}
206191
});
207192
}
193+
194+
private async inspect<T>(opts: InspectOpts, format: string, parser: (parsedOutput: unknown) => T): Promise<T> {
195+
const cmd = await this.getInspectCommand([opts.name, '--format', format]);
196+
if (!opts.retryOnManifestUnknown) {
197+
return await this.execInspect(cmd.command, cmd.args, parser);
198+
}
199+
200+
const retries = opts.retryLimit ?? 15;
201+
let lastError: Error | undefined;
202+
for (let attempt = 0; attempt < retries; attempt++) {
203+
try {
204+
return await this.execInspect(cmd.command, cmd.args, parser);
205+
} catch (err) {
206+
lastError = err as Error;
207+
if (!ImageTools.isManifestUnknownError(lastError.message) || attempt === retries - 1) {
208+
throw lastError;
209+
}
210+
core.info(`buildx imagetools inspect command failed with MANIFEST_UNKNOWN, retrying attempt ${attempt + 1}/${retries}...\n${lastError.message}`);
211+
await new Promise(res => setTimeout(res, Math.pow(2, attempt) * 100));
212+
}
213+
}
214+
215+
throw lastError ?? new Error(`ImageTools inspect command failed for ${opts.name}`);
216+
}
217+
218+
private async execInspect<T>(command: string, args: Array<string>, parser: (parsedOutput: unknown) => T): Promise<T> {
219+
return await Exec.getExecOutput(command, args, {
220+
ignoreReturnCode: true,
221+
silent: true
222+
}).then(res => {
223+
if (res.stderr.length > 0 && res.exitCode != 0) {
224+
throw new Error(res.stderr.trim());
225+
}
226+
return parser(JSON.parse(res.stdout));
227+
});
228+
}
229+
230+
private static isManifestUnknownError(message: string): boolean {
231+
return /(MANIFEST_UNKNOWN|manifest unknown|not found: not found)/i.test(message);
232+
}
208233
}

‎src/sigstore/sigstore.ts‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,11 @@ export class Sigstore {
113113
}
114114

115115
for (const imageName of opts.imageNames) {
116-
const attestationDigests = await this.imageTools.attestationDigests(`${imageName}@${opts.imageDigest}`);
116+
const attestationDigests = await this.imageTools.attestationDigests({
117+
name: `${imageName}@${opts.imageDigest}`,
118+
retryOnManifestUnknown: opts.retryOnManifestUnknown,
119+
retryLimit: opts.retryLimit
120+
});
117121
for (const attestationDigest of attestationDigests) {
118122
const attestationRef = `${imageName}@${attestationDigest}`;
119123
await core.group(`Signing attestation manifest ${attestationRef}`, async () => {
@@ -183,7 +187,12 @@ export class Sigstore {
183187
public async verifyImageAttestations(image: string, opts: VerifySignedManifestsOpts): Promise<Record<string, VerifySignedManifestsResult>> {
184188
const result: Record<string, VerifySignedManifestsResult> = {};
185189

186-
const attestationDigests = await this.imageTools.attestationDigests(image, opts.platform);
190+
const attestationDigests = await this.imageTools.attestationDigests({
191+
name: image,
192+
platform: opts.platform,
193+
retryOnManifestUnknown: opts.retryOnManifestUnknown,
194+
retryLimit: opts.retryLimit
195+
});
187196
if (attestationDigests.length === 0) {
188197
throw new Error(`No attestation manifests found for ${image}`);
189198
}
@@ -237,7 +246,7 @@ export class Sigstore {
237246
};
238247
}
239248

240-
const retries = 15;
249+
const retries = opts.retryLimit ?? 15;
241250
let lastError: Error | undefined;
242251
core.info(`[command]cosign ${[...cosignArgs, attestationRef].join(' ')}`);
243252
for (let attempt = 0; attempt < retries; attempt++) {

0 commit comments

Comments
 (0)