Skip to content

Commit 0e2da13

Browse files
nbbeekennbbeeken
committed
NODE-7534: Add prose tests for retry behavior with mixed overload/non-overload errors
- Add 4 new prose tests per spec commit 7039e69 - Test 4 (reads/writes): Verify MAX_RETRIES applies to all errors after overload - Test 5 (reads/writes): Verify backoff only applies to overload errors - Uses mocking pattern consistent with existing prose tests
1 parent 16a899d commit 0e2da13

2 files changed

Lines changed: 287 additions & 2 deletions

File tree

‎test/integration/retryable-reads/retryable_reads.spec.prose.test.ts‎

Lines changed: 153 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,18 @@
11
/* eslint-disable @typescript-eslint/no-non-null-assertion */
22
import { expect } from 'chai';
3+
import * as sinon from 'sinon';
34

45
import {
56
type Collection,
67
type CommandFailedEvent,
78
type CommandSucceededEvent,
8-
type MongoClient
9+
MAX_RETRIES,
10+
type MongoClient,
11+
MongoErrorLabel,
12+
MongoServerError,
13+
Server
914
} from '../../mongodb';
15+
import { measureDuration } from '../../tools/utils';
1016
import { filterForCommands } from '../shared';
1117

1218
describe('Retryable Reads Spec Prose', () => {
@@ -279,4 +285,150 @@ describe('Retryable Reads Spec Prose', () => {
279285
});
280286
});
281287
});
288+
289+
describe('4: Test that drivers set the maximum number of retries for all retryable read errors when an overload error is encountered', () => {
290+
// This test MUST be executed against a MongoDB 4.4+ server that supports `retryReads=true` and has enabled the
291+
// `configureFailPoint` command with the `errorLabels` option.
292+
293+
const TEST_METADATA: MongoDBMetadataUI = {
294+
requires: { mongodb: '>=4.4' }
295+
};
296+
297+
let client: MongoClient;
298+
299+
beforeEach(async function () {
300+
// 1. Create a client.
301+
client = this.configuration.newClient({
302+
monitorCommands: true
303+
});
304+
await client.connect();
305+
});
306+
307+
afterEach(async function () {
308+
sinon.restore();
309+
await client?.close();
310+
});
311+
312+
it(
313+
'should retry MAX_RETRIES times for all retryable errors after encountering an overload error',
314+
TEST_METADATA,
315+
async () => {
316+
// 2. Configure a fail point with error code `91` (ShutdownInProgress) with the `RetryableError` and
317+
// `SystemOverloadedError` error labels:
318+
319+
// 3. Via the command monitoring CommandFailedEvent, configure a fail point with error code `91` (ShutdownInProgress) and
320+
// the `RetryableError` label:
321+
322+
// We use mocking to simulate the failpoint sequence:
323+
// - First call: error WITH SystemOverloadedError
324+
// - Subsequent calls: error WITHOUT SystemOverloadedError (but still retryable)
325+
const serverCommandStub = sinon
326+
.stub(Server.prototype, 'command')
327+
.callsFake(async function () {
328+
const errorLabels =
329+
serverCommandStub.callCount === 1
330+
? [MongoErrorLabel.RetryableError, MongoErrorLabel.SystemOverloadedError]
331+
: [MongoErrorLabel.RetryableError];
332+
333+
throw new MongoServerError({
334+
message: 'Server Error',
335+
errorLabels,
336+
code: 91,
337+
ok: 0
338+
});
339+
});
340+
341+
// 4. Attempt a `findOne` operation on any record for any database and collection. Expect the `findOne` to fail with a
342+
// server error. Assert that `MAX_RETRIES + 1` attempts were made.
343+
const error = await client
344+
.db('test')
345+
.collection('test')
346+
.findOne({})
347+
.catch(e => e);
348+
349+
expect(error).to.exist;
350+
expect(serverCommandStub.callCount).to.equal(MAX_RETRIES + 1);
351+
}
352+
);
353+
});
354+
355+
describe('5: Test that drivers do not apply backoff to non-overload errors', () => {
356+
// This test MUST be executed against a MongoDB 4.4+ server that supports `retryReads=true` and has enabled the
357+
// `configureFailPoint` command with the `errorLabels` option.
358+
359+
const TEST_METADATA: MongoDBMetadataUI = {
360+
requires: { mongodb: '>=4.4' }
361+
};
362+
363+
let client: MongoClient;
364+
365+
beforeEach(async function () {
366+
// 1. Create a client.
367+
client = this.configuration.newClient({
368+
monitorCommands: true
369+
});
370+
await client.connect();
371+
});
372+
373+
afterEach(async function () {
374+
sinon.restore();
375+
await client?.close();
376+
});
377+
378+
it(
379+
'should apply backoff only once for the initial overload error and not for subsequent non-overload retryable errors',
380+
TEST_METADATA,
381+
async function () {
382+
// Configure the random number generator used for jitter to always return a number as close as possible to `1`.
383+
const randomStub = sinon.stub(Math, 'random');
384+
randomStub.returns(0.99);
385+
386+
// 2. Configure a fail point with error code `91` (ShutdownInProgress) with the `RetryableError` and
387+
// `SystemOverloadedError` error labels:
388+
389+
// 3. Via the command monitoring CommandFailedEvent, configure a fail point with error code `91` (ShutdownInProgress) and
390+
// the `RetryableError` label:
391+
392+
// We use mocking to simulate the failpoint sequence:
393+
// - First call: error WITH SystemOverloadedError
394+
// - Subsequent calls: error WITHOUT SystemOverloadedError (but still retryable)
395+
const serverCommandStub = sinon
396+
.stub(Server.prototype, 'command')
397+
.callsFake(async function () {
398+
const errorLabels =
399+
serverCommandStub.callCount === 1
400+
? [MongoErrorLabel.RetryableError, MongoErrorLabel.SystemOverloadedError]
401+
: [MongoErrorLabel.RetryableError];
402+
403+
throw new MongoServerError({
404+
message: 'Server Error',
405+
errorLabels,
406+
code: 91,
407+
ok: 0
408+
});
409+
});
410+
411+
// 4. Attempt a `findOne` operation on any record for any database and collection. Expect the `findOne` to fail with a
412+
// server error. Assert that backoff was applied only once for the initial overload error and not for the subsequent
413+
// non-overload retryable errors.
414+
const { duration } = await measureDuration(async () => {
415+
const error = await client
416+
.db('test')
417+
.collection('test')
418+
.findOne({})
419+
.catch(e => e);
420+
expect(error).to.exist;
421+
});
422+
423+
// The expected backoff for the first (overload) error is: Math.random() * Math.min(10000, 100 * 2^0)
424+
// With Math.random() = 0.99, this gives us: 0.99 * 100 = 99ms
425+
// Subsequent errors are non-overload, so they should have NO backoff applied.
426+
// We add a margin for test execution overhead.
427+
const expectedMinBackoff = 99; // First backoff
428+
const expectedMaxBackoff = expectedMinBackoff + 1000; // Allow 1 second margin for test overhead
429+
430+
expect(duration).to.be.within(expectedMinBackoff, expectedMaxBackoff);
431+
}
432+
);
433+
});
282434
});

‎test/integration/retryable-writes/retryable_writes.spec.prose.test.ts‎

Lines changed: 134 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@ import * as sinon from 'sinon';
66

77
import {
88
type Collection,
9+
MAX_RETRIES,
910
type MongoClient,
1011
MongoError,
1112
MongoErrorLabel,
1213
MongoServerError,
1314
MongoWriteConcernError,
1415
Server
1516
} from '../../mongodb';
16-
import { sleep } from '../../tools/utils';
17+
import { measureDuration, sleep } from '../../tools/utils';
1718

1819
describe('Retryable Writes Spec Prose', () => {
1920
describe('1. Test that retryable writes raise an exception when using the MMAPv1 storage engine.', () => {
@@ -551,5 +552,137 @@ describe('Retryable Writes Spec Prose', () => {
551552
expect(insertResult.errorLabels).to.not.include(MongoErrorLabel.NoWritesPerformed);
552553
}
553554
);
555+
556+
it(
557+
'Case 4: Test that drivers set the maximum number of retries for all retryable write errors when an overload error is encountered',
558+
{ requires: { topology: 'replicaset', mongodb: '>=6.0' } },
559+
async () => {
560+
// 2. Configure a fail point with error code `91` (ShutdownInProgress) with the `RetryableError` and
561+
// `SystemOverloadedError` error labels:
562+
// ```javascript
563+
// {
564+
// configureFailPoint: "failCommand",
565+
// mode: {times: 1},
566+
// data: {
567+
// failCommands: ["insert"],
568+
// errorLabels: ["RetryableError", "SystemOverloadedError"],
569+
// errorCode: 91
570+
// }
571+
// }
572+
// ```
573+
574+
// 3. Via the command monitoring CommandFailedEvent, configure a fail point with error code `91` (ShutdownInProgress) and
575+
// the `RetryableWriteError` and `RetryableError` labels:
576+
// ```javascript
577+
// {
578+
// configureFailPoint: "failCommand",
579+
// mode: "alwaysOn",
580+
// data: {
581+
// failCommands: ["insert"],
582+
// errorLabels: ["RetryableError", "RetryableWriteError"],
583+
// errorCode: 91
584+
// }
585+
// }
586+
// ```
587+
// Configure the second fail point command only if the failed event is for the first error configured in step 2.
588+
const serverCommandStub = sinon
589+
.stub(Server.prototype, 'command')
590+
.callsFake(async function () {
591+
// First call: error WITH SystemOverloadedError
592+
// Subsequent calls: error WITHOUT SystemOverloadedError (but still retryable)
593+
const errorLabels =
594+
serverCommandStub.callCount === 1
595+
? [MongoErrorLabel.RetryableError, MongoErrorLabel.SystemOverloadedError]
596+
: [MongoErrorLabel.RetryableError, MongoErrorLabel.RetryableWriteError];
597+
598+
throw new MongoServerError({
599+
message: 'Server Error',
600+
errorLabels,
601+
code: 91,
602+
ok: 0
603+
});
604+
});
605+
606+
// 4. Attempt an `insertOne` operation on any record for any database and collection. Expect the `insertOne` to fail with a
607+
// server error. Assert that `MAX_RETRIES + 1` attempts were made.
608+
const insertResult = await collection.insertOne({ _id: 1 }).catch(error => error);
609+
610+
expect(insertResult).to.be.instanceOf(MongoServerError);
611+
expect(serverCommandStub.callCount).to.equal(MAX_RETRIES + 1);
612+
}
613+
);
614+
615+
it(
616+
'Case 5: Test that drivers do not apply backoff to non-overload errors',
617+
{ requires: { topology: 'replicaset', mongodb: '>=6.0' } },
618+
async function () {
619+
// Configure the random number generator used for jitter to always return a number as close as possible to `1`.
620+
const stub = sinon.stub(Math, 'random');
621+
stub.returns(0.99);
622+
623+
// 2. Configure a fail point with error code `91` (ShutdownInProgress) with the `RetryableError` and
624+
// `SystemOverloadedError` error labels:
625+
// ```javascript
626+
// {
627+
// configureFailPoint: "failCommand",
628+
// mode: {times: 1},
629+
// data: {
630+
// failCommands: ["insert"],
631+
// errorLabels: ["RetryableError", "SystemOverloadedError"],
632+
// errorCode: 91
633+
// }
634+
// }
635+
// ```
636+
637+
// 3. Via the command monitoring CommandFailedEvent, configure a fail point with error code `91` (ShutdownInProgress) and
638+
// the `RetryableWriteError` and `RetryableError` labels:
639+
// ```javascript
640+
// {
641+
// configureFailPoint: "failCommand",
642+
// mode: "alwaysOn",
643+
// data: {
644+
// failCommands: ["insert"],
645+
// errorLabels: ["RetryableError", "RetryableWriteError"],
646+
// errorCode: 91
647+
// }
648+
// }
649+
// ```
650+
// Configure the second fail point command only if the failed event is for the first error configured in step 2.
651+
const serverCommandStub = sinon
652+
.stub(Server.prototype, 'command')
653+
.callsFake(async function () {
654+
// First call: error WITH SystemOverloadedError
655+
// Subsequent calls: error WITHOUT SystemOverloadedError (but still retryable)
656+
const errorLabels =
657+
serverCommandStub.callCount === 1
658+
? [MongoErrorLabel.RetryableError, MongoErrorLabel.SystemOverloadedError]
659+
: [MongoErrorLabel.RetryableError, MongoErrorLabel.RetryableWriteError];
660+
661+
throw new MongoServerError({
662+
message: 'Server Error',
663+
errorLabels,
664+
code: 91,
665+
ok: 0
666+
});
667+
});
668+
669+
// 4. Attempt an `insertOne` operation on any record for any database and collection. Expect the `insertOne` to fail with a
670+
// server error. Assert that backoff was applied only once for the initial overload error and not for the subsequent
671+
// non-overload retryable errors.
672+
const { duration } = await measureDuration(async () => {
673+
const insertResult = await collection.insertOne({ _id: 1 }).catch(error => error);
674+
expect(insertResult).to.be.instanceOf(MongoServerError);
675+
});
676+
677+
// The expected backoff for the first (overload) error is: Math.random() * Math.min(10000, 100 * 2^0)
678+
// With Math.random() = 0.99, this gives us: 0.99 * 100 = 99ms
679+
// Subsequent errors are non-overload, so they should have NO backoff applied.
680+
// We add a margin for test execution overhead.
681+
const expectedMinBackoff = 99; // First backoff
682+
const expectedMaxBackoff = expectedMinBackoff + 1000; // Allow 1 second margin for test overhead
683+
684+
expect(duration).to.be.within(expectedMinBackoff, expectedMaxBackoff);
685+
}
686+
);
554687
});
555688
});

0 commit comments

Comments
 (0)