From 8e1da84aa86692345775274c3ae2898a3f23bd21 Mon Sep 17 00:00:00 2001 From: bailey Date: Fri, 21 Nov 2025 10:59:35 -0700 Subject: [PATCH 1/7] add token bucket --- src/cmap/connect.ts | 2 + src/cmap/connection.ts | 3 + src/index.ts | 1 + src/operations/execute_operation.ts | 102 +- src/operations/operation.ts | 6 + src/sdam/topology.ts | 8 +- src/sessions.ts | 55 +- src/token_bucket.ts | 23 + src/utils.ts | 16 + sync.sh | 4 + .../client-backpressure.prose.test.ts | 60 + .../client-backpressure.spec.test.ts | 16 + .../mongodb-handshake.prose.test.ts | 303 +- test/spec/client-backpressure/README.md | 57 + .../backpressure-retry-loop.json | 2759 +++++++++++++ .../backpressure-retry-loop.yml | 1444 +++++++ .../backpressure-retry-loop.yml.template | 116 + .../backpressure-retry-max-attempts.json | 3448 +++++++++++++++++ .../backpressure-retry-max-attempts.yml | 1844 +++++++++ ...ckpressure-retry-max-attempts.yml.template | 111 + .../client-backpressure/getMore-retried.json | 291 ++ .../client-backpressure/getMore-retried.yml | 149 + .../unified/backpressure-retryable-abort.json | 357 ++ .../unified/backpressure-retryable-abort.yml | 213 + .../backpressure-retryable-commit.json | 374 ++ .../unified/backpressure-retryable-commit.yml | 220 ++ .../unified/backpressure-retryable-reads.json | 328 ++ .../unified/backpressure-retryable-reads.yml | 192 + .../backpressure-retryable-writes.json | 440 +++ .../unified/backpressure-retryable-writes.yml | 250 ++ test/unit/index.test.ts | 1 + 31 files changed, 13019 insertions(+), 174 deletions(-) create mode 100644 src/token_bucket.ts create mode 100644 sync.sh create mode 100644 test/integration/client-backpressure/client-backpressure.prose.test.ts create mode 100644 test/integration/client-backpressure/client-backpressure.spec.test.ts create mode 100644 test/spec/client-backpressure/README.md create mode 100644 test/spec/client-backpressure/backpressure-retry-loop.json create mode 100644 test/spec/client-backpressure/backpressure-retry-loop.yml create mode 100644 test/spec/client-backpressure/backpressure-retry-loop.yml.template create mode 100644 test/spec/client-backpressure/backpressure-retry-max-attempts.json create mode 100644 test/spec/client-backpressure/backpressure-retry-max-attempts.yml create mode 100644 test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template create mode 100644 test/spec/client-backpressure/getMore-retried.json create mode 100644 test/spec/client-backpressure/getMore-retried.yml create mode 100644 test/spec/transactions/unified/backpressure-retryable-abort.json create mode 100644 test/spec/transactions/unified/backpressure-retryable-abort.yml create mode 100644 test/spec/transactions/unified/backpressure-retryable-commit.json create mode 100644 test/spec/transactions/unified/backpressure-retryable-commit.yml create mode 100644 test/spec/transactions/unified/backpressure-retryable-reads.json create mode 100644 test/spec/transactions/unified/backpressure-retryable-reads.yml create mode 100644 test/spec/transactions/unified/backpressure-retryable-writes.json create mode 100644 test/spec/transactions/unified/backpressure-retryable-writes.yml diff --git a/src/cmap/connect.ts b/src/cmap/connect.ts index 544ec471a99..711335f3d1d 100644 --- a/src/cmap/connect.ts +++ b/src/cmap/connect.ts @@ -224,6 +224,7 @@ export interface HandshakeDocument extends Document { compression: string[]; saslSupportedMechs?: string; loadBalanced?: boolean; + backpressure: true; } /** @@ -241,6 +242,7 @@ export async function prepareHandshakeDocument( const handshakeDoc: HandshakeDocument = { [serverApi?.version || options.loadBalanced === true ? 'hello' : LEGACY_HELLO_COMMAND]: 1, + backpressure: true, helloOk: true, client: clientMetadata, compression: compressors diff --git a/src/cmap/connection.ts b/src/cmap/connection.ts index 9652e3a5e4f..2a3e5a3b9a0 100644 --- a/src/cmap/connection.ts +++ b/src/cmap/connection.ts @@ -582,6 +582,9 @@ export class Connection extends TypedEventEmitter { this.throwIfAborted(); } } catch (error) { + if (options.session != null && !(error instanceof MongoServerError)) { + updateSessionFromResponse(options.session, MongoDBResponse.empty); + } if (this.shouldEmitAndLogCommand) { this.emitAndLogCommand( this.monitorCommands, diff --git a/src/index.ts b/src/index.ts index 8f5c4cfa60e..ca66d31e201 100644 --- a/src/index.ts +++ b/src/index.ts @@ -87,6 +87,7 @@ export { MongoWriteConcernError, WriteConcernErrorResult } from './error'; +export { TokenBucket } from './token_bucket'; export { AbstractCursor, // Actual driver classes exported diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index b6ddbba2b66..f68ce5b1d9f 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -1,3 +1,5 @@ +import { setTimeout } from 'node:timers/promises'; + import { MIN_SUPPORTED_SNAPSHOT_READS_WIRE_VERSION } from '../cmap/wire_protocol/constants'; import { isRetryableReadError, @@ -10,6 +12,7 @@ import { MongoInvalidArgumentError, MongoNetworkError, MongoNotConnectedError, + MongoOperationTimeoutError, MongoRuntimeError, MongoServerError, MongoTransactionError, @@ -26,9 +29,15 @@ import { import type { Topology } from '../sdam/topology'; import type { ClientSession } from '../sessions'; import { TimeoutContext } from '../timeout'; -import { abortable, maxWireVersion, supportsRetryableWrites } from '../utils'; +import { RETRY_COST, TOKEN_REFRESH_RATE } from '../token_bucket'; +import { + abortable, + ExponentialBackoffProvider, + maxWireVersion, + supportsRetryableWrites +} from '../utils'; import { AggregateOperation } from './aggregate'; -import { AbstractOperation, Aspect } from './operation'; +import { AbstractOperation, Aspect, RetryContext } from './operation'; const MMAPv1_RETRY_WRITES_ERROR_CODE = MONGODB_ERROR_CODES.IllegalOperation; const MMAPv1_RETRY_WRITES_ERROR_MESSAGE = @@ -50,7 +59,7 @@ type ResultTypeFromOperation = ReturnType< * The expectation is that this function: * - Connects the MongoClient if it has not already been connected, see {@link autoConnect} * - Creates a session if none is provided and cleans up the session it creates - * - Tries an operation and retries under certain conditions, see {@link tryOperation} + * - Tries an operation and retries under certain conditions, see {@link executeOperationWithRetries} * * @typeParam T - The operation's type * @typeParam TResult - The type of the operation's result, calculated from T @@ -120,7 +129,7 @@ export async function executeOperation< }); try { - return await tryOperation(operation, { + return await executeOperationWithRetries(operation, { topology, timeoutContext, session, @@ -184,7 +193,10 @@ type RetryOptions = { * * @param operation - The operation to execute * */ -async function tryOperation>( +async function executeOperationWithRetries< + T extends AbstractOperation, + TResult = ResultTypeFromOperation +>( operation: T, { topology, timeoutContext, session, readPreference }: RetryOptions ): Promise { @@ -233,11 +245,27 @@ async function tryOperation timeoutContext.remainingTimeMS) { + // TODO: is this the right error to throw? + throw new MongoOperationTimeoutError( + `MongoDB SystemOverload exponential backoff would exceed timeoutMS deadline: remaining CSOT deadline=${timeoutContext.remainingTimeMS}, backoff delayMS=${delayMS}`, + { + cause: previousOperationError + } + ); + } + + if (!topology.tokenBucket.consume(RETRY_COST)) { + throw previousOperationError; + } + + await setTimeout(delayMS); } if ( @@ -285,19 +337,34 @@ async function tryOperation 0 and we are command batching we need to reset the batch. - if (tries > 0 && operation.hasAspect(Aspect.COMMAND_BATCHING)) { + const isRetry = attempt > 0; + + // If attempt > 0 and we are command batching we need to reset the batch. + if (isRetry && operation.hasAspect(Aspect.COMMAND_BATCHING)) { operation.resetBatch(); } try { const result = await server.command(operation, timeoutContext); + topology.tokenBucket.deposit( + isRetry + ? // on successful retry, deposit the retry cost + the refresh rate. + TOKEN_REFRESH_RATE + RETRY_COST + : // otherwise, just deposit the refresh rate. + TOKEN_REFRESH_RATE + ); return operation.handleOk(result); } catch (error) { return operation.handleError(error); } } catch (operationError) { if (!(operationError instanceof MongoError)) throw operationError; + + if (!operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { + // if an operation fails with an error that does not contain the SystemOverloadError, deposit 1 token. + topology.tokenBucket.deposit(RETRY_COST); + } + if ( previousOperationError != null && operationError.hasErrorLabel(MongoErrorLabel.NoWritesPerformed) @@ -312,8 +379,5 @@ async function tryOperation { /** Specifies the time an operation will run until it throws a timeout error. */ timeoutMS?: number; + retryContext?: RetryContext; + private _session: ClientSession | undefined; static aspects?: Set; diff --git a/src/sdam/topology.ts b/src/sdam/topology.ts index 2a158525fa0..16288261ebc 100644 --- a/src/sdam/topology.ts +++ b/src/sdam/topology.ts @@ -35,6 +35,7 @@ import { type Abortable, TypedEventEmitter } from '../mongo_types'; import { ReadPreference, type ReadPreferenceLike } from '../read_preference'; import type { ClientSession } from '../sessions'; import { Timeout, TimeoutContext, TimeoutError } from '../timeout'; +import { TokenBucket } from '../token_bucket'; import type { Transaction } from '../transactions'; import { addAbortListener, @@ -207,18 +208,15 @@ export type TopologyEvents = { * @internal */ export class Topology extends TypedEventEmitter { - /** @internal */ s: TopologyPrivate; - /** @internal */ waitQueue: List; - /** @internal */ hello?: Document; - /** @internal */ _type?: string; + tokenBucket = new TokenBucket(1000); + client!: MongoClient; - /** @internal */ private connectionLock?: Promise; /** @event */ diff --git a/src/sessions.ts b/src/sessions.ts index c1d9ab70b03..ea5e34265fa 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -25,6 +25,7 @@ import { import type { MongoClient, MongoOptions } from './mongo_client'; import { TypedEventEmitter } from './mongo_types'; import { executeOperation } from './operations/execute_operation'; +import { RetryContext } from './operations/operation'; import { RunCommandOperation } from './operations/run_command'; import { ReadConcernLevel } from './read_concern'; import { ReadPreference } from './read_preference'; @@ -468,7 +469,11 @@ export class ClientSession } else { const wcKeys = Object.keys(wc); if (wcKeys.length > 2 || (!wcKeys.includes('wtimeoutMS') && !wcKeys.includes('wTimeoutMS'))) - // if the write concern was specified with wTimeoutMS, then we set both wtimeoutMS and wTimeoutMS, guaranteeing at least two keys, so if we have more than two keys, then we can automatically assume that we should add the write concern to the command. If it has 2 or fewer keys, we need to check that those keys aren't the wtimeoutMS or wTimeoutMS options before we add the write concern to the command + // if the write concern was specified with wTimeoutMS, then we set both wtimeoutMS + // and wTimeoutMS, guaranteeing at least two keys, so if we have more than two keys, + // then we can automatically assume that we should add the write concern to the command. + // If it has 2 or fewer keys, we need to check that those keys aren't the wtimeoutMS + // or wTimeoutMS options before we add the write concern to the command WriteConcern.apply(command, { ...wc, wtimeoutMS: undefined }); } } @@ -489,11 +494,14 @@ export class ClientSession command.recoveryToken = this.transaction.recoveryToken; } + const retryContext = new RetryContext(5); + const operation = new RunCommandOperation(new MongoDBNamespace('admin'), command, { session: this, readPreference: ReadPreference.primary, bypassPinningCheck: true }); + operation.retryContext = retryContext; const timeoutContext = this.timeoutContext ?? @@ -518,15 +526,13 @@ export class ClientSession this.unpin({ force: true }); try { - await executeOperation( - this.client, - new RunCommandOperation(new MongoDBNamespace('admin'), command, { - session: this, - readPreference: ReadPreference.primary, - bypassPinningCheck: true - }), - timeoutContext - ); + const op = new RunCommandOperation(new MongoDBNamespace('admin'), command, { + session: this, + readPreference: ReadPreference.primary, + bypassPinningCheck: true + }); + op.retryContext = retryContext; + await executeOperation(this.client, op, timeoutContext); return; } catch (retryCommitError) { // If the retry failed, we process that error instead of the original @@ -1013,6 +1019,11 @@ export class ServerSession { id: ServerSessionId; lastUse: number; txnNumber: number; + + /* + * Indicates that a network error has been encountered while using this session. + * Once a session is marked as dirty, it is always dirty. + */ isDirty: boolean; /** @internal */ @@ -1106,16 +1117,15 @@ export class ServerSessionPool { * @param session - The session to release to the pool */ release(session: ServerSession): void { - const sessionTimeoutMinutes = this.client.topology?.logicalSessionTimeoutMinutes ?? 10; + if (this.client.topology?.loadBalanced) { + if (session.isDirty) return; - if (this.client.topology?.loadBalanced && !sessionTimeoutMinutes) { this.sessions.unshift(session); - } - - if (!sessionTimeoutMinutes) { return; } + const sessionTimeoutMinutes = this.client.topology?.logicalSessionTimeoutMinutes ?? 10; + this.sessions.prune(session => session.hasTimedOut(sessionTimeoutMinutes)); if (!session.hasTimedOut(sessionTimeoutMinutes)) { @@ -1203,9 +1213,9 @@ export function applySession( command.autocommit = false; if (session.transaction.state === TxnState.STARTING_TRANSACTION) { - session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); command.startTransaction = true; + // TODO: read concern only applied if it is not the same as the server's default const readConcern = session.transaction.options.readConcern || session?.clientOptions?.readConcern; if (readConcern) { @@ -1241,4 +1251,17 @@ export function updateSessionFromResponse(session: ClientSession, document: Mong session.snapshotTime = atClusterTime; } } + + if (session.transaction.state === TxnState.STARTING_TRANSACTION) { + if (document.ok === 1) { + session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); + } else { + const error = new MongoServerError(document.toObject()); + const isBackpressureError = error.hasErrorLabel(MongoErrorLabel.RetryableError); + + if (!isBackpressureError) { + session.transaction.transition(TxnState.TRANSACTION_IN_PROGRESS); + } + } + } } diff --git a/src/token_bucket.ts b/src/token_bucket.ts new file mode 100644 index 00000000000..9d542bf719d --- /dev/null +++ b/src/token_bucket.ts @@ -0,0 +1,23 @@ +/** + * @internal + */ +export class TokenBucket { + private budget: number; + constructor(allowance: number) { + this.budget = allowance; + } + deposit(tokens: number) { + this.budget += tokens; + } + + consume(tokens: number): boolean { + if (tokens > this.budget) return false; + + this.budget -= tokens; + return true; + } +} + +export const TOKEN_REFRESH_RATE = 0.1; +export const INITIAL_SIZE = 1000; +export const RETRY_COST = 1; diff --git a/src/utils.ts b/src/utils.ts index 397850649d5..7908add0290 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1432,3 +1432,19 @@ export async function abortable( abortListener?.[kDispose](); } } + +export class ExponentialBackoffProvider { + constructor( + public readonly maxBackoff: number, + public readonly baseBackoff: number, + public readonly backoffIncreaseRate: number, + public iteration = 0 + ) {} + + getNextBackoffDuration(): number { + return ( + Math.random() * + Math.min(this.maxBackoff, this.baseBackoff * this.backoffIncreaseRate ** this.iteration) + ); + } +} diff --git a/sync.sh b/sync.sh new file mode 100644 index 00000000000..b99d9ed1086 --- /dev/null +++ b/sync.sh @@ -0,0 +1,4 @@ + + +cp ~/dev/specifications/source/client-backpressure/tests/* ~/dev/node-mongodb-native/test/spec/client-backpressure +cp ~/dev/specifications/source/transactions/tests/unified/backpressure* ~/dev/node-mongodb-native/test/spec/transactions/unified/ \ No newline at end of file diff --git a/test/integration/client-backpressure/client-backpressure.prose.test.ts b/test/integration/client-backpressure/client-backpressure.prose.test.ts new file mode 100644 index 00000000000..56d274fffe1 --- /dev/null +++ b/test/integration/client-backpressure/client-backpressure.prose.test.ts @@ -0,0 +1,60 @@ +import { expect } from 'chai'; +import * as sinon from 'sinon'; + +import { type Collection, type MongoClient, MongoServerError } from '../../../src'; +import { clearFailPoint, configureFailPoint, measureDuration } from '../../tools/utils'; + +describe('Client Backpressure (Prose)', function () { + let client: MongoClient; + let collection: Collection; + + beforeEach(async function () { + client = this.configuration.newClient(); + await client.connect(); + + collection = client.db('foo').collection('bar'); + }); + + afterEach(async function () { + await client.close(); + await clearFailPoint(this.configuration); + }); + + it( + 'Test 1: Operation Retry Uses Exponential Backoff', + { + requires: { + mongodb: '4.4' + } + }, + async function () { + await configureFailPoint(this.configuration, { + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] + } + }); + + const stub = sinon.stub(Math, 'random'); + + stub.returns(0); + + const { duration: durationNoBackoff } = await measureDuration(async () => { + const error = await collection.insertOne({ a: 1 }).catch(e => e); + expect(error).to.be.instanceof(MongoServerError); + }); + + stub.returns(1); + + const { duration: durationBackoff } = await measureDuration(async () => { + const error = await collection.insertOne({ a: 1 }).catch(e => e); + expect(error).to.be.instanceof(MongoServerError); + }); + + expect(durationBackoff - durationNoBackoff).to.be.within(3100 - 1000, 3100 + 1000); + } + ); +}); diff --git a/test/integration/client-backpressure/client-backpressure.spec.test.ts b/test/integration/client-backpressure/client-backpressure.spec.test.ts new file mode 100644 index 00000000000..9485c46a96c --- /dev/null +++ b/test/integration/client-backpressure/client-backpressure.spec.test.ts @@ -0,0 +1,16 @@ +import { loadSpecTests } from '../../spec'; +import { runUnifiedSuite } from '../../tools/unified-spec-runner/runner'; +import { type Test } from '../../tools/unified-spec-runner/schema'; + +const skippedTests = { + 'collection.dropIndexes retries at most maxAttempts=5 times': + 'TODO(NODE-6517): dropIndexes squashes all errors other than ns not found' +}; + +function shouldSkip({ description }: Test) { + return skippedTests[description] ?? false; +} + +describe('Client Backpressure (spec)', function () { + runUnifiedSuite(loadSpecTests('client-backpressure'), shouldSkip); +}); diff --git a/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts b/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts index 219ef6f70e7..e5f4f82dde5 100644 --- a/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts +++ b/test/integration/mongodb-handshake/mongodb-handshake.prose.test.ts @@ -37,68 +37,68 @@ describe('Handshake Prose Tests', function () { expectedProvider: string | undefined; env: EnvironmentVariables; }> = [ - { - context: '1. Valid AWS', - expectedProvider: 'aws.lambda', - env: [ - ['AWS_EXECUTION_ENV', 'AWS_Lambda_java8'], - ['AWS_REGION', 'us-east-2'], - ['AWS_LAMBDA_FUNCTION_MEMORY_SIZE', '1024'] - ] - }, - { - context: '2. Valid Azure', - expectedProvider: 'azure.func', - env: [['FUNCTIONS_WORKER_RUNTIME', 'node']] - }, - { - context: '3. Valid GCP', - expectedProvider: 'gcp.func', - env: [ - ['K_SERVICE', 'servicename'], - ['FUNCTION_MEMORY_MB', '1024'], - ['FUNCTION_TIMEOUT_SEC', '60'], - ['FUNCTION_REGION', 'us-central1'] - ] - }, - { - context: '4. Valid Vercel', - expectedProvider: 'vercel', - env: [ - ['VERCEL', '1'], - ['VERCEL_REGION', 'cdg1'] - ] - }, - { - expectedProvider: undefined, - context: '5. Invalid - multiple providers', - env: [ - ['AWS_EXECUTION_ENV', 'AWS_Lambda_java8'], - ['FUNCTIONS_WORKER_RUNTIME', 'node'] - ] - }, - { - expectedProvider: 'aws.lambda', - context: '6. Invalid - long string', - env: [ - ['AWS_EXECUTION_ENV', 'AWS_Lambda_java8'], - ['AWS_REGION', 'a'.repeat(1024)] - ] - }, - { - expectedProvider: 'aws.lambda', - context: '7. Invalid - wrong types', - env: [ - ['AWS_EXECUTION_ENV', 'AWS_Lambda_java8'], - ['AWS_LAMBDA_FUNCTION_MEMORY_SIZE', 'big'] - ] - }, - { - expectedProvider: undefined, - context: '8. Invalid - AWS_EXECUTION_ENV does not start with "AWS_Lambda_"', - env: [['AWS_EXECUTION_ENV', 'EC2']] - } - ]; + { + context: '1. Valid AWS', + expectedProvider: 'aws.lambda', + env: [ + ['AWS_EXECUTION_ENV', 'AWS_Lambda_java8'], + ['AWS_REGION', 'us-east-2'], + ['AWS_LAMBDA_FUNCTION_MEMORY_SIZE', '1024'] + ] + }, + { + context: '2. Valid Azure', + expectedProvider: 'azure.func', + env: [['FUNCTIONS_WORKER_RUNTIME', 'node']] + }, + { + context: '3. Valid GCP', + expectedProvider: 'gcp.func', + env: [ + ['K_SERVICE', 'servicename'], + ['FUNCTION_MEMORY_MB', '1024'], + ['FUNCTION_TIMEOUT_SEC', '60'], + ['FUNCTION_REGION', 'us-central1'] + ] + }, + { + context: '4. Valid Vercel', + expectedProvider: 'vercel', + env: [ + ['VERCEL', '1'], + ['VERCEL_REGION', 'cdg1'] + ] + }, + { + expectedProvider: undefined, + context: '5. Invalid - multiple providers', + env: [ + ['AWS_EXECUTION_ENV', 'AWS_Lambda_java8'], + ['FUNCTIONS_WORKER_RUNTIME', 'node'] + ] + }, + { + expectedProvider: 'aws.lambda', + context: '6. Invalid - long string', + env: [ + ['AWS_EXECUTION_ENV', 'AWS_Lambda_java8'], + ['AWS_REGION', 'a'.repeat(1024)] + ] + }, + { + expectedProvider: 'aws.lambda', + context: '7. Invalid - wrong types', + env: [ + ['AWS_EXECUTION_ENV', 'AWS_Lambda_java8'], + ['AWS_LAMBDA_FUNCTION_MEMORY_SIZE', 'big'] + ] + }, + { + expectedProvider: undefined, + context: '8. Invalid - AWS_EXECUTION_ENV does not start with "AWS_Lambda_"', + env: [['AWS_EXECUTION_ENV', 'EC2']] + } + ]; for (const { context: name, env, expectedProvider } of tests) { context(name, function () { @@ -461,10 +461,10 @@ describe('Client Metadata Update Prose Tests', function () { const expected = isDriverInfoEqual(driverInfo, originalDriverInfo) ? originalDriverInfo : { - name: `library|${driverInfo.name}`, - platform: `Library Platform|${driverInfo.platform}`, - version: `1.2|${driverInfo.version}` - }; + name: `library|${driverInfo.name}`, + platform: `Library Platform|${driverInfo.platform}`, + version: `1.2|${driverInfo.version}` + }; expect(actual).to.deep.equal(expected); @@ -753,43 +753,43 @@ describe('Client Metadata Update Prose Tests', function () { initial: DriverInfo; appended: DriverInfo; }> = [ - { - initial: { - name: undefined, - version: '1.2', - platform: 'Library Platform' - }, - appended: { - name: '', - version: '1.2', - platform: 'Library Platform' - } - }, - { - initial: { - name: 'library', - version: undefined, - platform: 'Library Platform' + { + initial: { + name: undefined, + version: '1.2', + platform: 'Library Platform' + }, + appended: { + name: '', + version: '1.2', + platform: 'Library Platform' + } }, - appended: { - name: 'library', - version: '', - platform: 'Library Platform' - } - }, - { - initial: { - name: 'library', - version: '1.2', - platform: undefined + { + initial: { + name: 'library', + version: undefined, + platform: 'Library Platform' + }, + appended: { + name: 'library', + version: '', + platform: 'Library Platform' + } }, - appended: { - name: 'library', - version: '1.2', - platform: '' + { + initial: { + name: 'library', + version: '1.2', + platform: undefined + }, + appended: { + name: 'library', + version: '1.2', + platform: '' + } } - } - ]; + ]; for (const [metadata, index] of driverInfos.map((infos, i) => [infos, i] as const)) { describe(`Test ${index + 1}`, function () { @@ -856,43 +856,43 @@ describe('Client Metadata Update Prose Tests', function () { initial: DriverInfo; appended: DriverInfo; }> = [ - { - initial: { - name: undefined, - version: '1.2', - platform: 'Library Platform' - }, - appended: { - name: '', - version: '1.2', - platform: 'Library Platform' - } - }, - { - initial: { - name: 'library', - version: undefined, - platform: 'Library Platform' + { + initial: { + name: undefined, + version: '1.2', + platform: 'Library Platform' + }, + appended: { + name: '', + version: '1.2', + platform: 'Library Platform' + } }, - appended: { - name: 'library', - version: '', - platform: 'Library Platform' - } - }, - { - initial: { - name: 'library', - version: '1.2', - platform: undefined + { + initial: { + name: 'library', + version: undefined, + platform: 'Library Platform' + }, + appended: { + name: 'library', + version: '', + platform: 'Library Platform' + } }, - appended: { - name: 'library', - version: '1.2', - platform: '' + { + initial: { + name: 'library', + version: '1.2', + platform: undefined + }, + appended: { + name: 'library', + version: '1.2', + platform: '' + } } - } - ]; + ]; for (const [metadata, index] of driverInfos.map((infos, i) => [infos, i] as const)) { describe(`Test ${index + 1}`, function () { @@ -943,3 +943,38 @@ describe('Client Metadata Update Prose Tests', function () { } }); }); + +// TODO: add prose test descriptions here to align the test with the spec. +describe('Backpressure Metadata', function () { + let client: MongoClient; + let spy: sinon.SinonSpy>; + + beforeEach(async function () { + client = this.configuration.newClient(); + spy = sinon.spy(Connection.prototype, 'command'); + await client.connect(); + + // run an operation to force a connection establishment, + // if we're testing noauth load balanced mode. + await client.db('foo').collection('bar').insertOne({ name: 'bumpy' }); + }); + + afterEach(async function () { + sinon.restore(); + await client?.close(); + }); + + it('includes backpressure in the handshake document', function () { + const isHello = (cmd: Document): cmd is HandshakeDocument => + `hello` in cmd || LEGACY_HELLO_COMMAND in cmd; + + const hellos = spy.args.map(([_ns, command, _options]) => command).filter(isHello); + + expect(hellos.length).to.be.greaterThan(0); + + expect( + hellos.every(hello => hello.backpressure === true), + `some handshake documents did not specify backpressure: true` + ); + }); +}); diff --git a/test/spec/client-backpressure/README.md b/test/spec/client-backpressure/README.md new file mode 100644 index 00000000000..9f49b1ed203 --- /dev/null +++ b/test/spec/client-backpressure/README.md @@ -0,0 +1,57 @@ +# Client Backpressure Tests + +______________________________________________________________________ + +## Introduction + +The YAML and JSON files in this directory are platform-independent tests meant to exercise a driver's implementation of +retryable reads. These tests utilize the [Unified Test Format](../../unified-test-format/unified-test-format.md). + +Several prose tests, which are not easily expressed in YAML, are also presented in this file. Those tests will need to +be manually implemented by each driver. + +### Prose Tests + +#### Test 1: Operation Retry Uses Exponential Backoff + +Drivers should test that retries do not occur immediately when a SystemOverloadedError is encountered. + +1. Let `client` be a `MongoClient` +2. Let `collection` be a collection +3. Now, run transactions without backoff: + 1. Configure the random number generator used for jitter to always return `0` -- this effectively disables backoff. + + 2. Configure the following failPoint: + + ```javascript + { + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { + failCommands: ['insert'], + errorCode: 2, + errorLabels: ['SystemOverloadedError', 'RetryableError'] + } + } + ``` + + 3. Insert the document `{ a: 1 }`. Expect that the command errors. Measure the duration of the command execution. + + ```javascript + const start = performance.now(); + expect( + await coll.insertOne({ a: 1 }).catch(e => e) + ).to.be.an.instanceof(MongoServerError); + const end = performance.now(); + ``` + + 4. Configure the random number generator used for jitter to always return `1`. + + 5. Execute step 3 again. + + 6. Compare the two time between the two runs. + ```python + assertTrue(with_backoff_time - no_backoff_time >= 2.1) + ``` + The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + runs. diff --git a/test/spec/client-backpressure/backpressure-retry-loop.json b/test/spec/client-backpressure/backpressure-retry-loop.json new file mode 100644 index 00000000000..2542344b382 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.json @@ -0,0 +1,2759 @@ +{ + "description": "tests that operations respect overload backoff retry loop", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "internal_client", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "internal_db", + "client": "internal_client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "retryable-writes-tests", + "database": "internal_db", + "collectionName": "coll" + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [] + } + ], + "_yamlAnchors": { + "bulWriteInsertNamespace": "retryable-writes-tests.coll" + }, + "tests": [ + { + "description": "client.listDatabases retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandSucceededEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries using operation loop", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandSucceededEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandSucceededEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandSucceededEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandSucceededEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandSucceededEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandSucceededEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandSucceededEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandSucceededEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandSucceededEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandSucceededEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries using operation loop", + "operations": [ + { + "object": "retryable-writes-tests", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + } + }, + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries using operation loop", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "internal_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes" + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandSucceededEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml b/test/spec/client-backpressure/backpressure-retry-loop.yml new file mode 100644 index 00000000000..6a3033989b4 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml @@ -0,0 +1,1444 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.3' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - + client: + id: &internal_client internal_client + useMultipleMongoses: false + + - + database: + id: &internal_db internal_db + client: *internal_client + databaseName: &database_name retryable-writes-tests + + - + collection: + id: &internal_collection retryable-writes-tests + database: *internal_db + collectionName: &collection_name coll + + - + database: + id: &database database + client: *client + databaseName: *database_name + + - + collection: + id: &collection collection + database: *database + collectionName: *collection_name + +initialData: +- collectionName: *collection_name + databaseName: *database_name + documents: [] + +_yamlAnchors: + bulWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +tests: + + - + description: 'client.listDatabases retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: listDatabases + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - + description: 'client.listDatabaseNames retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: listDatabaseNames + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandSucceededEvent: + commandName: listDatabases + + - + description: 'client.createChangeStream retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: createChangeStream + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'client.clientBulkWrite retries using operation loop' + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [bulkWrite] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *client_bulk_write_ns + document: { _id: 8, x: 88 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandSucceededEvent: + commandName: bulkWrite + + - + description: 'database.aggregate retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: aggregate + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'database.listCollections retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: listCollections + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - + description: 'database.listCollectionNames retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: listCollectionNames + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandSucceededEvent: + commandName: listCollections + + - + description: 'database.runCommand retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: runCommand + arguments: + command: { ping: 1 } + commandName: ping + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandSucceededEvent: + commandName: ping + + - + description: 'database.createChangeStream retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: createChangeStream + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.aggregate retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: aggregate + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.countDocuments retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: countDocuments + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.estimatedDocumentCount retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [count] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: estimatedDocumentCount + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandSucceededEvent: + commandName: count + + - + description: 'collection.distinct retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [distinct] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: distinct + arguments: + fieldName: x + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandSucceededEvent: + commandName: distinct + + - + description: 'collection.find retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: find + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - + description: 'collection.findOne retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOne + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + + - + description: 'collection.listIndexes retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: listIndexes + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - + description: 'collection.listIndexNames retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: listIndexNames + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandSucceededEvent: + commandName: listIndexes + + - + description: 'collection.createChangeStream retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: createChangeStream + arguments: + pipeline: [] + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandSucceededEvent: + commandName: aggregate + + - + description: 'collection.insertOne retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: insertOne + arguments: + document: { _id: 2, x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.insertMany retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: insertMany + arguments: + documents: + - { _id: 2, x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.deleteOne retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: deleteOne + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - + description: 'collection.deleteMany retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: deleteMany + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandSucceededEvent: + commandName: delete + + - + description: 'collection.replaceOne retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: replaceOne + arguments: + filter: {} + replacement: { x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.updateOne retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: updateOne + arguments: + filter: {} + update: { $set: { x: 22 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.updateMany retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: updateMany + arguments: + filter: {} + update: { $set: { x: 22 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandSucceededEvent: + commandName: update + + - + description: 'collection.findOneAndDelete retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndDelete + arguments: + filter: {} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.findOneAndReplace retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndReplace + arguments: + filter: {} + replacement: { x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.findOneAndUpdate retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndUpdate + arguments: + filter: {} + update: { $set: { x: 22 } } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandSucceededEvent: + commandName: findAndModify + + - + description: 'collection.bulkWrite retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: bulkWrite + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandSucceededEvent: + commandName: insert + + - + description: 'collection.createIndex retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [createIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandSucceededEvent: + commandName: createIndexes + + - + description: 'collection.dropIndex retries using operation loop' + operations: + - + object: *internal_collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: dropIndex + arguments: + name: "x_11" + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes + + - + description: 'collection.dropIndexes retries using operation loop' + operations: + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: dropIndexes + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandSucceededEvent: + commandName: dropIndexes diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml.template b/test/spec/client-backpressure/backpressure-retry-loop.yml.template new file mode 100644 index 00000000000..df1afe95cf7 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml.template @@ -0,0 +1,116 @@ +# Tests in this file are generated from backpressure-retry-loop.yml.template. + +description: tests that operations respect overload backoff retry loop + +schemaVersion: '1.3' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - + client: + id: &internal_client internal_client + useMultipleMongoses: false + + - + database: + id: &internal_db internal_db + client: *internal_client + databaseName: &database_name retryable-writes-tests + + - + collection: + id: &internal_collection retryable-writes-tests + database: *internal_db + collectionName: &collection_name coll + + - + database: + id: &database database + client: *client + databaseName: *database_name + + - + collection: + id: &collection collection + database: *database + collectionName: *collection_name + +initialData: +- collectionName: *collection_name + databaseName: *database_name + documents: [] + +_yamlAnchors: + bulWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +tests: +{% for operation in operations %} + - + description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + {%- endif %} + operations: + {%- if operation.operation_name == "dropIndex" %} + - + object: *internal_collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + {%- endif %} + + + - name: failPoint + object: testRunner + arguments: + client: *internal_client + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [{{operation.command_name}}] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *{{operation.object}} + name: {{operation.operation_name}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandSucceededEvent: + commandName: {{operation.command_name}} +{% endfor -%} diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.json b/test/spec/client-backpressure/backpressure-retry-max-attempts.json new file mode 100644 index 00000000000..1de8cb38d4e --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.json @@ -0,0 +1,3448 @@ +{ + "description": "tests that operations retry at most maxAttempts=5 times", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "client": { + "id": "fail_point_client", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database", + "client": "client", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection", + "database": "database", + "collectionName": "coll" + } + } + ], + "_yamlAnchors": { + "bulkWriteInsertNamespace": "retryable-writes-tests.coll" + }, + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "client.listDatabases retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabases", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.listDatabaseNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listDatabases" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "listDatabaseNames", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandStartedEvent": { + "commandName": "listDatabases" + } + }, + { + "commandFailedEvent": { + "commandName": "listDatabases" + } + } + ] + } + ] + }, + { + "description": "client.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "client.clientBulkWrite retries at most maxAttempts=5 times", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "bulkWrite" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "client", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "insertOne": { + "namespace": "retryable-writes-tests.coll", + "document": { + "_id": 8, + "x": 88 + } + } + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandStartedEvent": { + "commandName": "bulkWrite" + } + }, + { + "commandFailedEvent": { + "commandName": "bulkWrite" + } + } + ] + } + ] + }, + { + "description": "database.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "database.listCollections retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollections", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.listCollectionNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listCollections" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "listCollectionNames", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + }, + { + "commandStartedEvent": { + "commandName": "listCollections" + } + }, + { + "commandFailedEvent": { + "commandName": "listCollections" + } + } + ] + } + ] + }, + { + "description": "database.runCommand retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "ping" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "runCommand", + "arguments": { + "command": { + "ping": 1 + }, + "commandName": "ping" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + }, + { + "commandStartedEvent": { + "commandName": "ping" + } + }, + { + "commandFailedEvent": { + "commandName": "ping" + } + } + ] + } + ] + }, + { + "description": "database.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "database", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.aggregate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "aggregate", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.countDocuments retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "countDocuments", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.estimatedDocumentCount retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "count" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "estimatedDocumentCount", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + }, + { + "commandStartedEvent": { + "commandName": "count" + } + }, + { + "commandFailedEvent": { + "commandName": "count" + } + } + ] + } + ] + }, + { + "description": "collection.distinct retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "distinct" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "distinct", + "arguments": { + "fieldName": "x", + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + }, + { + "commandStartedEvent": { + "commandName": "distinct" + } + }, + { + "commandFailedEvent": { + "commandName": "distinct" + } + } + ] + } + ] + }, + { + "description": "collection.find retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "find", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.findOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOne", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexes retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexes", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.listIndexNames retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "listIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "listIndexNames", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "listIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "listIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.createChangeStream retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "aggregate" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createChangeStream", + "arguments": { + "pipeline": [] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + }, + { + "commandStartedEvent": { + "commandName": "aggregate" + } + }, + { + "commandFailedEvent": { + "commandName": "aggregate" + } + } + ] + } + ] + }, + { + "description": "collection.insertOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2, + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.insertMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.deleteOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteOne", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.deleteMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "delete" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "deleteMany", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + }, + { + "commandStartedEvent": { + "commandName": "delete" + } + }, + { + "commandFailedEvent": { + "commandName": "delete" + } + } + ] + } + ] + }, + { + "description": "collection.replaceOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "replaceOne", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateOne retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateOne", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.updateMany retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "update" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "updateMany", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + }, + { + "commandStartedEvent": { + "commandName": "update" + } + }, + { + "commandFailedEvent": { + "commandName": "update" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndDelete retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndDelete", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndReplace retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndReplace", + "arguments": { + "filter": {}, + "replacement": { + "x": 22 + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.findOneAndUpdate retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "findAndModify" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "findOneAndUpdate", + "arguments": { + "filter": {}, + "update": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandStartedEvent": { + "commandName": "findAndModify" + } + }, + { + "commandFailedEvent": { + "commandName": "findAndModify" + } + } + ] + } + ] + }, + { + "description": "collection.bulkWrite retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "bulkWrite", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 2, + "x": 22 + } + } + } + ] + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandFailedEvent": { + "commandName": "insert" + } + } + ] + } + ] + }, + { + "description": "collection.createIndex retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "createIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "createIndex", + "arguments": { + "keys": { + "x": 11 + }, + "name": "x_11" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "createIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "createIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndex retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndex", + "arguments": { + "name": "x_11" + }, + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + }, + { + "description": "collection.dropIndexes retries at most maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "fail_point_client", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "dropIndexes" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "object": "collection", + "name": "dropIndexes", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client", + "events": [ + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandStartedEvent": { + "commandName": "dropIndexes" + } + }, + { + "commandFailedEvent": { + "commandName": "dropIndexes" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml new file mode 100644 index 00000000000..3800b20a333 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml @@ -0,0 +1,1844 @@ +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.3' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - + client: + id: &fail_point_client fail_point_client + useMultipleMongoses: false + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +_yamlAnchors: + bulkWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + + - + description: 'client.listDatabases retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: listDatabases + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + + - + description: 'client.listDatabaseNames retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listDatabases] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: listDatabaseNames + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + - commandStartedEvent: + commandName: listDatabases + - commandFailedEvent: + commandName: listDatabases + + + - + description: 'client.createChangeStream retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: createChangeStream + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'client.clientBulkWrite retries at most maxAttempts=5 times' + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [bulkWrite] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *client + name: clientBulkWrite + arguments: + models: + - insertOne: + namespace: *client_bulk_write_ns + document: { _id: 8, x: 88 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + - commandStartedEvent: + commandName: bulkWrite + - commandFailedEvent: + commandName: bulkWrite + + + - + description: 'database.aggregate retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: aggregate + arguments: + pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'database.listCollections retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: listCollections + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + + - + description: 'database.listCollectionNames retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listCollections] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: listCollectionNames + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + - commandStartedEvent: + commandName: listCollections + - commandFailedEvent: + commandName: listCollections + + + - + description: 'database.runCommand retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [ping] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: runCommand + arguments: + command: { ping: 1 } + commandName: ping + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + - commandStartedEvent: + commandName: ping + - commandFailedEvent: + commandName: ping + + + - + description: 'database.createChangeStream retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *database + name: createChangeStream + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'collection.aggregate retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: aggregate + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'collection.countDocuments retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: countDocuments + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'collection.estimatedDocumentCount retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [count] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: estimatedDocumentCount + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + - commandStartedEvent: + commandName: count + - commandFailedEvent: + commandName: count + + + - + description: 'collection.distinct retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [distinct] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: distinct + arguments: + fieldName: x + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + - commandStartedEvent: + commandName: distinct + - commandFailedEvent: + commandName: distinct + + + - + description: 'collection.find retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: find + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + + - + description: 'collection.findOne retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [find] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOne + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandFailedEvent: + commandName: find + + + - + description: 'collection.listIndexes retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: listIndexes + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + + - + description: 'collection.listIndexNames retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [listIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: listIndexNames + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + - commandStartedEvent: + commandName: listIndexes + - commandFailedEvent: + commandName: listIndexes + + + - + description: 'collection.createChangeStream retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [aggregate] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: createChangeStream + arguments: + pipeline: [] + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + - commandStartedEvent: + commandName: aggregate + - commandFailedEvent: + commandName: aggregate + + + - + description: 'collection.insertOne retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: insertOne + arguments: + document: { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + + - + description: 'collection.insertMany retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: insertMany + arguments: + documents: + - { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + + - + description: 'collection.deleteOne retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: deleteOne + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + + - + description: 'collection.deleteMany retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [delete] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: deleteMany + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + - commandStartedEvent: + commandName: delete + - commandFailedEvent: + commandName: delete + + + - + description: 'collection.replaceOne retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: replaceOne + arguments: + filter: {} + replacement: { x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + + - + description: 'collection.updateOne retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: updateOne + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + + - + description: 'collection.updateMany retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [update] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: updateMany + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + - commandStartedEvent: + commandName: update + - commandFailedEvent: + commandName: update + + + - + description: 'collection.findOneAndDelete retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndDelete + arguments: + filter: {} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + + - + description: 'collection.findOneAndReplace retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndReplace + arguments: + filter: {} + replacement: { x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + + - + description: 'collection.findOneAndUpdate retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [findAndModify] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: findOneAndUpdate + arguments: + filter: {} + update: { $set: { x: 22 } } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + - commandStartedEvent: + commandName: findAndModify + - commandFailedEvent: + commandName: findAndModify + + + - + description: 'collection.bulkWrite retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [insert] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: bulkWrite + arguments: + requests: + - insertOne: + document: { _id: 2, x: 22 } + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandFailedEvent: + commandName: insert + + + - + description: 'collection.createIndex retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [createIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: createIndex + arguments: + keys: { x: 11 } + name: "x_11" + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + - commandStartedEvent: + commandName: createIndexes + - commandFailedEvent: + commandName: createIndexes + + + - + description: 'collection.dropIndex retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: dropIndex + arguments: + name: "x_11" + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + + + - + description: 'collection.dropIndexes retries at most maxAttempts=5 times' + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [dropIndexes] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *collection + name: dropIndexes + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + - commandStartedEvent: + commandName: dropIndexes + - commandFailedEvent: + commandName: dropIndexes + diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template new file mode 100644 index 00000000000..3117d44b890 --- /dev/null +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template @@ -0,0 +1,111 @@ +# Tests in this file are generated from backpressure-retry-max-attempts.yml.template. + +description: tests that operations retry at most maxAttempts=5 times + +schemaVersion: '1.3' + +runOnRequirements: + - + minServerVersion: '4.4' # failCommand + topologies: [replicaset, sharded, load-balanced] + +createEntities: + - + client: + id: &client client + useMultipleMongoses: false + observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] + + - + client: + id: &fail_point_client fail_point_client + useMultipleMongoses: false + + - + database: + id: &database database + client: *client + databaseName: &database_name retryable-writes-tests + - + collection: + id: &collection collection + database: *database + collectionName: &collection_name coll + +_yamlAnchors: + bulkWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: +{% for operation in operations %} + - + description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' + {%- if ((operation.operation_name == 'clientBulkWrite')) %} + runOnRequirements: + - minServerVersion: '8.0' # client bulk write added to server in 8.0 + {%- endif %} + + operations: + - name: failPoint + object: testRunner + arguments: + client: *fail_point_client + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [{{operation.command_name}}] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - + object: *{{operation.object}} + name: {{operation.operation_name}} + {%- if operation.arguments|length > 0 %} + arguments: + {%- for arg in operation.arguments %} + {{arg}} + {%- endfor -%} + {%- endif %} + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + # we expect 6 pairs of command started and succeeded events: + # 1 initial attempt and 5 retries. + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + - commandStartedEvent: + commandName: {{operation.command_name}} + - commandFailedEvent: + commandName: {{operation.command_name}} + +{% endfor -%} diff --git a/test/spec/client-backpressure/getMore-retried.json b/test/spec/client-backpressure/getMore-retried.json new file mode 100644 index 00000000000..70eff846128 --- /dev/null +++ b/test/spec/client-backpressure/getMore-retried.json @@ -0,0 +1,291 @@ +{ + "description": "getMore-retries-backpressure", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandFailedEvent", + "commandSucceededEvent" + ] + } + }, + { + "client": { + "id": "failPointClient" + } + }, + { + "database": { + "id": "db", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "coll", + "database": "db", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "default", + "collectionName": "default", + "documents": [ + { + "a": 1 + }, + { + "a": 2 + }, + { + "a": 3 + } + ] + } + ], + "tests": [ + { + "description": "getMores are retried", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 3 + }, + "data": { + "failCommands": [ + "getMore" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "find", + "arguments": { + "batchSize": 2, + "filter": {}, + "sort": { + "a": 1 + } + }, + "object": "coll", + "expectResult": [ + { + "a": 1 + }, + { + "a": 2 + }, + { + "a": 3 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandSucceededEvent": { + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "getMores are retried maxAttempts=5 times", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "failPointClient", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "getMore" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 2 + } + } + } + }, + { + "name": "find", + "arguments": { + "batchSize": 2, + "filter": {} + }, + "object": "coll", + "expectError": { + "isError": true, + "isClientError": false + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandSucceededEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "getMore" + } + }, + { + "commandFailedEvent": { + "commandName": "getMore" + } + }, + { + "commandStartedEvent": { + "commandName": "killCursors" + } + }, + { + "commandSucceededEvent": { + "commandName": "killCursors" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/client-backpressure/getMore-retried.yml b/test/spec/client-backpressure/getMore-retried.yml new file mode 100644 index 00000000000..3a5d180aa54 --- /dev/null +++ b/test/spec/client-backpressure/getMore-retried.yml @@ -0,0 +1,149 @@ +description: getMore-retries-backpressure +schemaVersion: "1.3" +runOnRequirements: + - + minServerVersion: '4.4' # failCommand +createEntities: + - client: + id: &client client0 + observeEvents: + - commandStartedEvent + - commandFailedEvent + - commandSucceededEvent + - client: + id: &failPointClient failPointClient + - database: + id: db + client: *client + databaseName: &dbName default + - collection: + id: &collection coll + database: db + collectionName: &collectionName default +initialData: + - databaseName: *dbName + collectionName: *collectionName + documents: + - { a: 1 } + - { a: 2 } + - { a: 3 } + +tests: + - description: "getMores are retried" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: { times: 3 } + data: + failCommands: [getMore] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: find + arguments: + # batch size of 2 with 3 docs in the collection ensures exactly one find + one getMore exhaust the cursor + batchSize: 2 + filter: {} + # ensure stable ordering of result documents + sort: { a: 1 } + object: *collection + expectResult: + - { a: 1 } + - { a: 2 } + - { a: 3 } + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + # first attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # second attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # third attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # success + - commandStartedEvent: + commandName: getMore + - commandSucceededEvent: + commandName: getMore + + - description: "getMores are retried maxAttempts=5 times" + operations: + - name: failPoint + object: testRunner + arguments: + client: *failPointClient + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: [getMore] + errorLabels: [RetryableError, SystemOverloadedError] + errorCode: 2 + + - name: find + arguments: + batchSize: 2 + filter: {} + object: *collection + expectError: + isError: true + isClientError: false + + expectEvents: + - client: *client + events: + - commandStartedEvent: + commandName: find + - commandSucceededEvent: + commandName: find + # first attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # second attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # third attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # fourth attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # fifth attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + # final attempt + - commandStartedEvent: + commandName: getMore + - commandFailedEvent: + commandName: getMore + - commandStartedEvent: + commandName: killCursors + - commandSucceededEvent: + commandName: killCursors \ No newline at end of file diff --git a/test/spec/transactions/unified/backpressure-retryable-abort.json b/test/spec/transactions/unified/backpressure-retryable-abort.json new file mode 100644 index 00000000000..53fc9c6f090 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-abort.json @@ -0,0 +1,357 @@ +{ + "description": "backpressure-retryable-abort", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "abortTransaction retries if backpressure labels are added", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "abortTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + }, + { + "description": "abortTransaction is retried maxAttempts=5 times if backpressure labels are added", + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "abortTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-abort.yml b/test/spec/transactions/unified/backpressure-retryable-abort.yml new file mode 100644 index 00000000000..85532e1f605 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-abort.yml @@ -0,0 +1,213 @@ +description: backpressure-retryable-abort +schemaVersion: "1.3" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - replicaset + - sharded + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: abortTransaction retries if backpressure labels are added + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 2 + data: + failCommands: + - abortTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] + - description: abortTransaction is retried maxAttempts=5 times if backpressure labels are added + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - abortTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: abortTransaction + databaseName: admin + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + - commandStartedEvent: + commandName: abortTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] diff --git a/test/spec/transactions/unified/backpressure-retryable-commit.json b/test/spec/transactions/unified/backpressure-retryable-commit.json new file mode 100644 index 00000000000..ae873561a99 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-commit.json @@ -0,0 +1,374 @@ +{ + "description": "backpressure-retryable-commit", + "schemaVersion": "1.4", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "sharded", + "replicaset", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "commitTransaction retries if backpressure labels are added", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "commitTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + }, + { + "description": "commitTransaction is retried maxAttempts=5 times if backpressure labels are added", + "runOnRequirements": [ + { + "serverless": "forbid" + } + ], + "operations": [ + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "commitTransaction" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "session0", + "name": "commitTransaction", + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + }, + { + "commandStartedEvent": { + "commandName": "commitTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-commit.yml b/test/spec/transactions/unified/backpressure-retryable-commit.yml new file mode 100644 index 00000000000..8099e1c1eba --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-commit.yml @@ -0,0 +1,220 @@ +description: backpressure-retryable-commit +schemaVersion: "1.4" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - sharded + - replicaset + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: commitTransaction retries if backpressure labels are added + runOnRequirements: + - serverless: forbid + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 2 + data: + failCommands: + - commitTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: commitTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: + - _id: 1 + - description: commitTransaction is retried maxAttempts=5 times if backpressure labels are added + runOnRequirements: + - serverless: forbid + operations: + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - commitTransaction + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: *session0 + name: commitTransaction + expectError: + isError: true + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + - commandStartedEvent: + commandName: commitTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] diff --git a/test/spec/transactions/unified/backpressure-retryable-reads.json b/test/spec/transactions/unified/backpressure-retryable-reads.json new file mode 100644 index 00000000000..731762830e3 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-reads.json @@ -0,0 +1,328 @@ +{ + "description": "backpressure-retryable-reads", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "reads are retried if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": {}, + "session": "session0" + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "find", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "find", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ] + }, + { + "description": "reads are retried maxAttempts=5 times if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "find" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": {}, + "session": "session0" + }, + "expectError": { + "isError": true + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-reads.yml b/test/spec/transactions/unified/backpressure-retryable-reads.yml new file mode 100644 index 00000000000..18bbdaadbfa --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-reads.yml @@ -0,0 +1,192 @@ +description: backpressure-retryable-reads +schemaVersion: "1.3" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - replicaset + - sharded + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: reads are retried if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - find + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: find + arguments: + filter: {} + session: *session0 + - object: *session0 + name: commitTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + find: test + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: find + databaseName: *database_name + - commandStartedEvent: + command: + find: test + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: find + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + - description: reads are retried maxAttempts=5 times if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - find + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: find + arguments: + filter: {} + session: *session0 + expectError: + isError: true + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: find + - commandStartedEvent: + commandName: abortTransaction diff --git a/test/spec/transactions/unified/backpressure-retryable-writes.json b/test/spec/transactions/unified/backpressure-retryable-writes.json new file mode 100644 index 00000000000..0817e03f2f7 --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-writes.json @@ -0,0 +1,440 @@ +{ + "description": "backpressure-retryable-writes", + "schemaVersion": "1.3", + "runOnRequirements": [ + { + "minServerVersion": "4.4", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "writes are retried if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + } + }, + { + "object": "session0", + "name": "commitTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": { + "$numberLong": "1" + }, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + }, + { + "description": "writes are retried maxAttempts=5 times if backpressure labels are added", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": "alwaysOn", + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + }, + "expectError": { + "isError": true + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + }, + { + "description": "retry succeeds if backpressure labels are added to the first operation in a transaction", + "operations": [ + { + "object": "session0", + "name": "startTransaction" + }, + { + "object": "testRunner", + "name": "failPoint", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorLabels": [ + "RetryableError", + "SystemOverloadedError" + ], + "errorCode": 112 + } + } + } + }, + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + } + }, + { + "object": "session0", + "name": "abortTransaction" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "insert" + } + }, + { + "commandStartedEvent": { + "commandName": "abortTransaction" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ] + } + ] +} diff --git a/test/spec/transactions/unified/backpressure-retryable-writes.yml b/test/spec/transactions/unified/backpressure-retryable-writes.yml new file mode 100644 index 00000000000..630c9d9694b --- /dev/null +++ b/test/spec/transactions/unified/backpressure-retryable-writes.yml @@ -0,0 +1,250 @@ +description: backpressure-retryable-writes +schemaVersion: "1.3" +runOnRequirements: + - minServerVersion: "4.4" + topologies: + - replicaset + - sharded + - load-balanced +createEntities: + - + client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: + - commandStartedEvent + - + database: + id: &database0 database0 + client: *client0 + databaseName: &database_name transaction-tests + - + collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection_name test + - + session: + id: &session0 session0 + client: *client0 + +initialData: + - + collectionName: *collection_name + databaseName: *database_name + documents: [] +tests: + - description: writes are retried if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - insert + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 2 + - object: *session0 + name: commitTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: test + documents: + - _id: 1 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: true + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + insert: test + documents: + - _id: 2 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + insert: test + documents: + - _id: 2 + ordered: true + readConcern: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + autocommit: false + writeConcern: + $$exists: false + commandName: insert + databaseName: *database_name + - commandStartedEvent: + command: + abortTransaction: + $$exists: false + lsid: + $$sessionLsid: *session0 + txnNumber: + $numberLong: "1" + startTransaction: + $$exists: false + autocommit: false + writeConcern: + $$exists: false + commandName: commitTransaction + databaseName: admin + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: + - { _id: 1 } + - { _id: 2 } + - description: writes are retried maxAttempts=5 times if backpressure labels are added + operations: + - object: *session0 + name: startTransaction + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 1 + expectResult: + $$unsetOrMatches: + insertedId: + $$unsetOrMatches: 1 + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: alwaysOn + data: + failCommands: + - insert + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 2 + expectError: + isError: true + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: abortTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] + - description: retry succeeds if backpressure labels are added to the first operation in a transaction + operations: + - object: *session0 + name: startTransaction + - object: testRunner + name: failPoint + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: + times: 1 + data: + failCommands: + - insert + errorLabels: + - RetryableError + - SystemOverloadedError + errorCode: 112 + - object: *collection0 + name: insertOne + arguments: + session: *session0 + document: + _id: 2 + - object: *session0 + name: abortTransaction + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: insert + - commandStartedEvent: + commandName: abortTransaction + outcome: + - collectionName: *collection_name + databaseName: *database_name + documents: [] diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 7b73c22b82b..9e74808e29c 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -132,6 +132,7 @@ const EXPECTED_EXPORTS = [ 'SeverityLevel', 'SrvPollingEvent', 'Timestamp', + 'TokenBucket', 'TopologyClosedEvent', 'TopologyDescriptionChangedEvent', 'TopologyOpeningEvent', From d960f8acfaffaac4cd020871555afd2bcb292042 Mon Sep 17 00:00:00 2001 From: bailey Date: Sat, 20 Dec 2025 11:00:07 -0700 Subject: [PATCH 2/7] token bucket comments --- src/token_bucket.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/token_bucket.ts b/src/token_bucket.ts index 9d542bf719d..74bec7f83d3 100644 --- a/src/token_bucket.ts +++ b/src/token_bucket.ts @@ -18,6 +18,18 @@ export class TokenBucket { } } +/** + * @internal + * The amount to deposit on successful operations, as defined in the backpressure specification. + */ export const TOKEN_REFRESH_RATE = 0.1; -export const INITIAL_SIZE = 1000; +/** + * @internal + * The initial size of the token bucket, as defined in the backpressure specification. + */ +export const INITIAL_TOKEN_BUCKET_SIZE = 1000; +/** + * @internal + * The cost of a retry, as defined in the backpressure specification. + */ export const RETRY_COST = 1; From 5ac5dbd37984d1fd9eaf1f92a757275acdd4f1d4 Mon Sep 17 00:00:00 2001 From: bailey Date: Sat, 20 Dec 2025 11:02:13 -0700 Subject: [PATCH 3/7] hack --- src/operations/execute_operation.ts | 14 +++++++------- src/operations/operation.ts | 6 +----- src/sessions.ts | 8 ++++---- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index f68ce5b1d9f..6cb279b6bc5 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -37,7 +37,7 @@ import { supportsRetryableWrites } from '../utils'; import { AggregateOperation } from './aggregate'; -import { AbstractOperation, Aspect, RetryContext } from './operation'; +import { AbstractOperation, Aspect } from './operation'; const MMAPv1_RETRY_WRITES_ERROR_CODE = MONGODB_ERROR_CODES.IllegalOperation; const MMAPv1_RETRY_WRITES_ERROR_MESSAGE = @@ -254,17 +254,17 @@ async function executeOperationWithRetries< 2 // backoff rate ); - const retryContext = - operation.retryContext ?? - new RetryContext(willRetry ? (timeoutContext.csotEnabled() ? Infinity : 2) : 1); + let maxAttempts = + (operation.maxAttempts ?? willRetry) ? (timeoutContext.csotEnabled() ? Infinity : 2) : 1; + for ( let attempt = 0; - attempt < retryContext.maxAttempts; + attempt < maxAttempts; attempt++, - retryContext.maxAttempts = + maxAttempts = willRetry && previousOperationError?.hasErrorLabel(MongoErrorLabel.SystemOverloadedError) ? 6 - : retryContext.maxAttempts + : maxAttempts ) { if (previousOperationError) { if (hasWriteAspect && previousOperationError.code === MMAPv1_RETRY_WRITES_ERROR_CODE) { diff --git a/src/operations/operation.ts b/src/operations/operation.ts index 8edf6b05090..3ab68c6596b 100644 --- a/src/operations/operation.ts +++ b/src/operations/operation.ts @@ -45,10 +45,6 @@ export interface OperationOptions extends BSONSerializeOptions { timeoutMS?: number; } -export class RetryContext { - constructor(public maxAttempts: number) {} -} - /** * This class acts as a parent class for any operation and is responsible for setting this.options, * as well as setting and getting a session. @@ -70,7 +66,7 @@ export abstract class AbstractOperation { /** Specifies the time an operation will run until it throws a timeout error. */ timeoutMS?: number; - retryContext?: RetryContext; + maxAttempts?: number; private _session: ClientSession | undefined; diff --git a/src/sessions.ts b/src/sessions.ts index ea5e34265fa..0e6fa0949d2 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -25,7 +25,7 @@ import { import type { MongoClient, MongoOptions } from './mongo_client'; import { TypedEventEmitter } from './mongo_types'; import { executeOperation } from './operations/execute_operation'; -import { RetryContext } from './operations/operation'; +import { RetryAttemptContext } from './operations/operation'; import { RunCommandOperation } from './operations/run_command'; import { ReadConcernLevel } from './read_concern'; import { ReadPreference } from './read_preference'; @@ -494,14 +494,14 @@ export class ClientSession command.recoveryToken = this.transaction.recoveryToken; } - const retryContext = new RetryContext(5); + const retryContext = new RetryAttemptContext(5); const operation = new RunCommandOperation(new MongoDBNamespace('admin'), command, { session: this, readPreference: ReadPreference.primary, bypassPinningCheck: true }); - operation.retryContext = retryContext; + operation.attempts = retryContext; const timeoutContext = this.timeoutContext ?? @@ -531,7 +531,7 @@ export class ClientSession readPreference: ReadPreference.primary, bypassPinningCheck: true }); - op.retryContext = retryContext; + op.attempts = retryContext; await executeOperation(this.client, op, timeoutContext); return; } catch (retryCommitError) { From 4327b3d0d45d135e8e00af3300dcdf1c7d91bb22 Mon Sep 17 00:00:00 2001 From: bailey Date: Fri, 23 Jan 2026 11:31:18 -0700 Subject: [PATCH 4/7] sync yml tests --- .../backpressure-retry-loop.json | 76 ++-- .../backpressure-retry-loop.yml | 347 ++++++------------ .../backpressure-retry-loop.yml.template | 41 +-- .../backpressure-retry-max-attempts.json | 64 ++-- .../backpressure-retry-max-attempts.yml | 275 ++++---------- ...ckpressure-retry-max-attempts.yml.template | 30 +- .../client-backpressure/getMore-retried.json | 4 +- .../client-backpressure/getMore-retried.yml | 8 +- 8 files changed, 275 insertions(+), 570 deletions(-) diff --git a/test/spec/client-backpressure/backpressure-retry-loop.json b/test/spec/client-backpressure/backpressure-retry-loop.json index 2542344b382..8c01020ca52 100644 --- a/test/spec/client-backpressure/backpressure-retry-loop.json +++ b/test/spec/client-backpressure/backpressure-retry-loop.json @@ -96,8 +96,8 @@ } }, { - "object": "client", "name": "listDatabases", + "object": "client", "arguments": { "filter": {} } @@ -178,8 +178,8 @@ } }, { - "object": "client", - "name": "listDatabaseNames" + "name": "listDatabaseNames", + "object": "client" } ], "expectEvents": [ @@ -257,8 +257,8 @@ } }, { - "object": "client", "name": "createChangeStream", + "object": "client", "arguments": { "pipeline": [] } @@ -344,8 +344,8 @@ } }, { - "object": "client", "name": "clientBulkWrite", + "object": "client", "arguments": { "models": [ { @@ -436,8 +436,8 @@ } }, { - "object": "database", "name": "aggregate", + "object": "database", "arguments": { "pipeline": [ { @@ -525,8 +525,8 @@ } }, { - "object": "database", "name": "listCollections", + "object": "database", "arguments": { "filter": {} } @@ -607,8 +607,8 @@ } }, { - "object": "database", "name": "listCollectionNames", + "object": "database", "arguments": { "filter": {} } @@ -689,8 +689,8 @@ } }, { - "object": "database", "name": "runCommand", + "object": "database", "arguments": { "command": { "ping": 1 @@ -774,8 +774,8 @@ } }, { - "object": "database", "name": "createChangeStream", + "object": "database", "arguments": { "pipeline": [] } @@ -856,8 +856,8 @@ } }, { - "object": "collection", "name": "aggregate", + "object": "collection", "arguments": { "pipeline": [] } @@ -938,8 +938,8 @@ } }, { - "object": "collection", "name": "countDocuments", + "object": "collection", "arguments": { "filter": {} } @@ -1020,8 +1020,8 @@ } }, { - "object": "collection", - "name": "estimatedDocumentCount" + "name": "estimatedDocumentCount", + "object": "collection" } ], "expectEvents": [ @@ -1099,8 +1099,8 @@ } }, { - "object": "collection", "name": "distinct", + "object": "collection", "arguments": { "fieldName": "x", "filter": {} @@ -1182,8 +1182,8 @@ } }, { - "object": "collection", "name": "find", + "object": "collection", "arguments": { "filter": {} } @@ -1264,8 +1264,8 @@ } }, { - "object": "collection", "name": "findOne", + "object": "collection", "arguments": { "filter": {} } @@ -1346,8 +1346,8 @@ } }, { - "object": "collection", - "name": "listIndexes" + "name": "listIndexes", + "object": "collection" } ], "expectEvents": [ @@ -1425,8 +1425,8 @@ } }, { - "object": "collection", - "name": "listIndexNames" + "name": "listIndexNames", + "object": "collection" } ], "expectEvents": [ @@ -1504,8 +1504,8 @@ } }, { - "object": "collection", "name": "createChangeStream", + "object": "collection", "arguments": { "pipeline": [] } @@ -1586,8 +1586,8 @@ } }, { - "object": "collection", "name": "insertOne", + "object": "collection", "arguments": { "document": { "_id": 2, @@ -1671,8 +1671,8 @@ } }, { - "object": "collection", "name": "insertMany", + "object": "collection", "arguments": { "documents": [ { @@ -1758,8 +1758,8 @@ } }, { - "object": "collection", "name": "deleteOne", + "object": "collection", "arguments": { "filter": {} } @@ -1840,8 +1840,8 @@ } }, { - "object": "collection", "name": "deleteMany", + "object": "collection", "arguments": { "filter": {} } @@ -1922,8 +1922,8 @@ } }, { - "object": "collection", "name": "replaceOne", + "object": "collection", "arguments": { "filter": {}, "replacement": { @@ -2007,8 +2007,8 @@ } }, { - "object": "collection", "name": "updateOne", + "object": "collection", "arguments": { "filter": {}, "update": { @@ -2094,8 +2094,8 @@ } }, { - "object": "collection", "name": "updateMany", + "object": "collection", "arguments": { "filter": {}, "update": { @@ -2181,8 +2181,8 @@ } }, { - "object": "collection", "name": "findOneAndDelete", + "object": "collection", "arguments": { "filter": {} } @@ -2263,8 +2263,8 @@ } }, { - "object": "collection", "name": "findOneAndReplace", + "object": "collection", "arguments": { "filter": {}, "replacement": { @@ -2348,8 +2348,8 @@ } }, { - "object": "collection", "name": "findOneAndUpdate", + "object": "collection", "arguments": { "filter": {}, "update": { @@ -2435,8 +2435,8 @@ } }, { - "object": "collection", "name": "bulkWrite", + "object": "collection", "arguments": { "requests": [ { @@ -2526,8 +2526,8 @@ } }, { - "object": "collection", "name": "createIndex", + "object": "collection", "arguments": { "keys": { "x": 11 @@ -2588,8 +2588,8 @@ "description": "collection.dropIndex retries using operation loop", "operations": [ { - "object": "retryable-writes-tests", "name": "createIndex", + "object": "retryable-writes-tests", "arguments": { "keys": { "x": 11 @@ -2621,8 +2621,8 @@ } }, { - "object": "collection", "name": "dropIndex", + "object": "collection", "arguments": { "name": "x_11" } @@ -2703,8 +2703,8 @@ } }, { - "object": "collection", - "name": "dropIndexes" + "name": "dropIndexes", + "object": "collection" } ], "expectEvents": [ diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml b/test/spec/client-backpressure/backpressure-retry-loop.yml index 6a3033989b4..eda57d4979d 100644 --- a/test/spec/client-backpressure/backpressure-retry-loop.yml +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml @@ -5,42 +5,35 @@ description: tests that operations respect overload backoff retry loop schemaVersion: '1.3' runOnRequirements: - - - minServerVersion: '4.4' # failCommand + - minServerVersion: '4.4' # failCommand topologies: [replicaset, sharded, load-balanced] createEntities: - - - client: + - client: id: &client client useMultipleMongoses: false observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - - - client: + - client: id: &internal_client internal_client useMultipleMongoses: false - - - database: + - database: id: &internal_db internal_db client: *internal_client databaseName: &database_name retryable-writes-tests - - - collection: + - collection: id: &internal_collection retryable-writes-tests database: *internal_db collectionName: &collection_name coll - - - database: + - database: id: &database database client: *client databaseName: *database_name - - - collection: + - collection: id: &collection collection database: *database collectionName: *collection_name @@ -53,13 +46,9 @@ initialData: _yamlAnchors: bulWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll -tests: - - - - description: 'client.listDatabases retries using operation loop' - operations: - - +tests: + - description: 'client.listDatabases retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -72,9 +61,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listDatabases object: *client - name: listDatabases arguments: filter: {} @@ -98,11 +86,8 @@ tests: - commandSucceededEvent: commandName: listDatabases - - - description: 'client.listDatabaseNames retries using operation loop' - operations: - - + - description: 'client.listDatabaseNames retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -115,9 +100,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listDatabaseNames object: *client - name: listDatabaseNames expectEvents: - client: *client @@ -139,11 +123,8 @@ tests: - commandSucceededEvent: commandName: listDatabases - - - description: 'client.createChangeStream retries using operation loop' - operations: - - + - description: 'client.createChangeStream retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -156,9 +137,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: createChangeStream object: *client - name: createChangeStream arguments: pipeline: [] @@ -182,13 +162,10 @@ tests: - commandSucceededEvent: commandName: aggregate - - - description: 'client.clientBulkWrite retries using operation loop' + - description: 'client.clientBulkWrite retries using operation loop' runOnRequirements: - minServerVersion: '8.0' # client bulk write added to server in 8.0 - operations: - - + operations: - name: failPoint object: testRunner arguments: @@ -201,9 +178,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: clientBulkWrite object: *client - name: clientBulkWrite arguments: models: - insertOne: @@ -230,11 +206,8 @@ tests: - commandSucceededEvent: commandName: bulkWrite - - - description: 'database.aggregate retries using operation loop' - operations: - - + - description: 'database.aggregate retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -247,9 +220,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: aggregate object: *database - name: aggregate arguments: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] @@ -273,11 +245,8 @@ tests: - commandSucceededEvent: commandName: aggregate - - - description: 'database.listCollections retries using operation loop' - operations: - - + - description: 'database.listCollections retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -290,9 +259,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listCollections object: *database - name: listCollections arguments: filter: {} @@ -316,11 +284,8 @@ tests: - commandSucceededEvent: commandName: listCollections - - - description: 'database.listCollectionNames retries using operation loop' - operations: - - + - description: 'database.listCollectionNames retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -333,9 +298,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listCollectionNames object: *database - name: listCollectionNames arguments: filter: {} @@ -359,11 +323,8 @@ tests: - commandSucceededEvent: commandName: listCollections - - - description: 'database.runCommand retries using operation loop' - operations: - - + - description: 'database.runCommand retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -376,9 +337,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: runCommand object: *database - name: runCommand arguments: command: { ping: 1 } commandName: ping @@ -403,11 +363,8 @@ tests: - commandSucceededEvent: commandName: ping - - - description: 'database.createChangeStream retries using operation loop' - operations: - - + - description: 'database.createChangeStream retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -420,9 +377,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: createChangeStream object: *database - name: createChangeStream arguments: pipeline: [] @@ -446,11 +402,8 @@ tests: - commandSucceededEvent: commandName: aggregate - - - description: 'collection.aggregate retries using operation loop' - operations: - - + - description: 'collection.aggregate retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -463,9 +416,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: aggregate object: *collection - name: aggregate arguments: pipeline: [] @@ -489,11 +441,8 @@ tests: - commandSucceededEvent: commandName: aggregate - - - description: 'collection.countDocuments retries using operation loop' - operations: - - + - description: 'collection.countDocuments retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -506,9 +455,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: countDocuments object: *collection - name: countDocuments arguments: filter: {} @@ -532,11 +480,8 @@ tests: - commandSucceededEvent: commandName: aggregate - - - description: 'collection.estimatedDocumentCount retries using operation loop' - operations: - - + - description: 'collection.estimatedDocumentCount retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -549,9 +494,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: estimatedDocumentCount object: *collection - name: estimatedDocumentCount expectEvents: - client: *client @@ -573,11 +517,8 @@ tests: - commandSucceededEvent: commandName: count - - - description: 'collection.distinct retries using operation loop' - operations: - - + - description: 'collection.distinct retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -590,9 +531,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: distinct object: *collection - name: distinct arguments: fieldName: x filter: {} @@ -617,11 +557,8 @@ tests: - commandSucceededEvent: commandName: distinct - - - description: 'collection.find retries using operation loop' - operations: - - + - description: 'collection.find retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -634,9 +571,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: find object: *collection - name: find arguments: filter: {} @@ -660,11 +596,8 @@ tests: - commandSucceededEvent: commandName: find - - - description: 'collection.findOne retries using operation loop' - operations: - - + - description: 'collection.findOne retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -677,9 +610,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: findOne object: *collection - name: findOne arguments: filter: {} @@ -703,11 +635,8 @@ tests: - commandSucceededEvent: commandName: find - - - description: 'collection.listIndexes retries using operation loop' - operations: - - + - description: 'collection.listIndexes retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -720,9 +649,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listIndexes object: *collection - name: listIndexes expectEvents: - client: *client @@ -744,11 +672,8 @@ tests: - commandSucceededEvent: commandName: listIndexes - - - description: 'collection.listIndexNames retries using operation loop' - operations: - - + - description: 'collection.listIndexNames retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -761,9 +686,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listIndexNames object: *collection - name: listIndexNames expectEvents: - client: *client @@ -785,11 +709,8 @@ tests: - commandSucceededEvent: commandName: listIndexes - - - description: 'collection.createChangeStream retries using operation loop' - operations: - - + - description: 'collection.createChangeStream retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -802,9 +723,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: createChangeStream object: *collection - name: createChangeStream arguments: pipeline: [] @@ -828,11 +748,8 @@ tests: - commandSucceededEvent: commandName: aggregate - - - description: 'collection.insertOne retries using operation loop' - operations: - - + - description: 'collection.insertOne retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -845,9 +762,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: insertOne object: *collection - name: insertOne arguments: document: { _id: 2, x: 22 } @@ -871,11 +787,8 @@ tests: - commandSucceededEvent: commandName: insert - - - description: 'collection.insertMany retries using operation loop' - operations: - - + - description: 'collection.insertMany retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -888,9 +801,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: insertMany object: *collection - name: insertMany arguments: documents: - { _id: 2, x: 22 } @@ -915,11 +827,8 @@ tests: - commandSucceededEvent: commandName: insert - - - description: 'collection.deleteOne retries using operation loop' - operations: - - + - description: 'collection.deleteOne retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -932,9 +841,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: deleteOne object: *collection - name: deleteOne arguments: filter: {} @@ -958,11 +866,8 @@ tests: - commandSucceededEvent: commandName: delete - - - description: 'collection.deleteMany retries using operation loop' - operations: - - + - description: 'collection.deleteMany retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -975,9 +880,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: deleteMany object: *collection - name: deleteMany arguments: filter: {} @@ -1001,11 +905,8 @@ tests: - commandSucceededEvent: commandName: delete - - - description: 'collection.replaceOne retries using operation loop' - operations: - - + - description: 'collection.replaceOne retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -1018,9 +919,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: replaceOne object: *collection - name: replaceOne arguments: filter: {} replacement: { x: 22 } @@ -1045,11 +945,8 @@ tests: - commandSucceededEvent: commandName: update - - - description: 'collection.updateOne retries using operation loop' - operations: - - + - description: 'collection.updateOne retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -1062,9 +959,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: updateOne object: *collection - name: updateOne arguments: filter: {} update: { $set: { x: 22 } } @@ -1089,11 +985,8 @@ tests: - commandSucceededEvent: commandName: update - - - description: 'collection.updateMany retries using operation loop' - operations: - - + - description: 'collection.updateMany retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -1106,9 +999,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: updateMany object: *collection - name: updateMany arguments: filter: {} update: { $set: { x: 22 } } @@ -1133,11 +1025,8 @@ tests: - commandSucceededEvent: commandName: update - - - description: 'collection.findOneAndDelete retries using operation loop' - operations: - - + - description: 'collection.findOneAndDelete retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -1150,9 +1039,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: findOneAndDelete object: *collection - name: findOneAndDelete arguments: filter: {} @@ -1176,11 +1064,8 @@ tests: - commandSucceededEvent: commandName: findAndModify - - - description: 'collection.findOneAndReplace retries using operation loop' - operations: - - + - description: 'collection.findOneAndReplace retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -1193,9 +1078,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: findOneAndReplace object: *collection - name: findOneAndReplace arguments: filter: {} replacement: { x: 22 } @@ -1220,11 +1104,8 @@ tests: - commandSucceededEvent: commandName: findAndModify - - - description: 'collection.findOneAndUpdate retries using operation loop' - operations: - - + - description: 'collection.findOneAndUpdate retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -1237,9 +1118,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: findOneAndUpdate object: *collection - name: findOneAndUpdate arguments: filter: {} update: { $set: { x: 22 } } @@ -1264,11 +1144,8 @@ tests: - commandSucceededEvent: commandName: findAndModify - - - description: 'collection.bulkWrite retries using operation loop' - operations: - - + - description: 'collection.bulkWrite retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -1281,9 +1158,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: bulkWrite object: *collection - name: bulkWrite arguments: requests: - insertOne: @@ -1309,11 +1185,8 @@ tests: - commandSucceededEvent: commandName: insert - - - description: 'collection.createIndex retries using operation loop' - operations: - - + - description: 'collection.createIndex retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -1326,9 +1199,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: createIndex object: *collection - name: createIndex arguments: keys: { x: 11 } name: "x_11" @@ -1353,17 +1225,13 @@ tests: - commandSucceededEvent: commandName: createIndexes - - - description: 'collection.dropIndex retries using operation loop' + - description: 'collection.dropIndex retries using operation loop' operations: - - + - name: createIndex object: *internal_collection - name: createIndex arguments: keys: { x: 11 } - name: "x_11" - - + name: "x_11" - name: failPoint object: testRunner arguments: @@ -1376,9 +1244,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: dropIndex object: *collection - name: dropIndex arguments: name: "x_11" @@ -1402,11 +1269,8 @@ tests: - commandSucceededEvent: commandName: dropIndexes - - - description: 'collection.dropIndexes retries using operation loop' - operations: - - + - description: 'collection.dropIndexes retries using operation loop' + operations: - name: failPoint object: testRunner arguments: @@ -1419,9 +1283,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: dropIndexes object: *collection - name: dropIndexes expectEvents: - client: *client diff --git a/test/spec/client-backpressure/backpressure-retry-loop.yml.template b/test/spec/client-backpressure/backpressure-retry-loop.yml.template index df1afe95cf7..bd64827d4e6 100644 --- a/test/spec/client-backpressure/backpressure-retry-loop.yml.template +++ b/test/spec/client-backpressure/backpressure-retry-loop.yml.template @@ -5,42 +5,35 @@ description: tests that operations respect overload backoff retry loop schemaVersion: '1.3' runOnRequirements: - - - minServerVersion: '4.4' # failCommand + - minServerVersion: '4.4' # failCommand topologies: [replicaset, sharded, load-balanced] createEntities: - - - client: + - client: id: &client client useMultipleMongoses: false observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - - - client: + - client: id: &internal_client internal_client useMultipleMongoses: false - - - database: + - database: id: &internal_db internal_db client: *internal_client databaseName: &database_name retryable-writes-tests - - - collection: + - collection: id: &internal_collection retryable-writes-tests database: *internal_db collectionName: &collection_name coll - - - database: + - database: id: &database database client: *client databaseName: *database_name - - - collection: + - collection: id: &collection collection database: *database collectionName: *collection_name @@ -53,25 +46,18 @@ initialData: _yamlAnchors: bulWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll -tests: -{% for operation in operations %} - - - description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' - {%- if ((operation.operation_name == 'clientBulkWrite')) %} +tests: {% for operation in operations %} + - description: '{{operation.object}}.{{operation.operation_name}} retries using operation loop' {%- if ((operation.operation_name == 'clientBulkWrite')) %} runOnRequirements: - minServerVersion: '8.0' # client bulk write added to server in 8.0 {%- endif %} - operations: - {%- if operation.operation_name == "dropIndex" %} - - + operations: {%- if operation.operation_name == "dropIndex" %} + - name: createIndex object: *internal_collection - name: createIndex arguments: keys: { x: 11 } name: "x_11" - {%- endif %} - - + {%- endif %} - name: failPoint object: testRunner arguments: @@ -84,9 +70,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: {{operation.operation_name}} object: *{{operation.object}} - name: {{operation.operation_name}} {%- if operation.arguments|length > 0 %} arguments: {%- for arg in operation.arguments %} diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.json b/test/spec/client-backpressure/backpressure-retry-max-attempts.json index 1de8cb38d4e..efde5426212 100644 --- a/test/spec/client-backpressure/backpressure-retry-max-attempts.json +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.json @@ -89,8 +89,8 @@ } }, { - "object": "client", "name": "listDatabases", + "object": "client", "arguments": { "filter": {} }, @@ -193,8 +193,8 @@ } }, { - "object": "client", "name": "listDatabaseNames", + "object": "client", "expectError": { "isError": true, "isClientError": false @@ -294,8 +294,8 @@ } }, { - "object": "client", "name": "createChangeStream", + "object": "client", "arguments": { "pipeline": [] }, @@ -403,8 +403,8 @@ } }, { - "object": "client", "name": "clientBulkWrite", + "object": "client", "arguments": { "models": [ { @@ -517,8 +517,8 @@ } }, { - "object": "database", "name": "aggregate", + "object": "database", "arguments": { "pipeline": [ { @@ -628,8 +628,8 @@ } }, { - "object": "database", "name": "listCollections", + "object": "database", "arguments": { "filter": {} }, @@ -732,8 +732,8 @@ } }, { - "object": "database", "name": "listCollectionNames", + "object": "database", "arguments": { "filter": {} }, @@ -836,8 +836,8 @@ } }, { - "object": "database", "name": "runCommand", + "object": "database", "arguments": { "command": { "ping": 1 @@ -943,8 +943,8 @@ } }, { - "object": "database", "name": "createChangeStream", + "object": "database", "arguments": { "pipeline": [] }, @@ -1047,8 +1047,8 @@ } }, { - "object": "collection", "name": "aggregate", + "object": "collection", "arguments": { "pipeline": [] }, @@ -1151,8 +1151,8 @@ } }, { - "object": "collection", "name": "countDocuments", + "object": "collection", "arguments": { "filter": {} }, @@ -1255,8 +1255,8 @@ } }, { - "object": "collection", "name": "estimatedDocumentCount", + "object": "collection", "expectError": { "isError": true, "isClientError": false @@ -1356,8 +1356,8 @@ } }, { - "object": "collection", "name": "distinct", + "object": "collection", "arguments": { "fieldName": "x", "filter": {} @@ -1461,8 +1461,8 @@ } }, { - "object": "collection", "name": "find", + "object": "collection", "arguments": { "filter": {} }, @@ -1565,8 +1565,8 @@ } }, { - "object": "collection", "name": "findOne", + "object": "collection", "arguments": { "filter": {} }, @@ -1669,8 +1669,8 @@ } }, { - "object": "collection", "name": "listIndexes", + "object": "collection", "expectError": { "isError": true, "isClientError": false @@ -1770,8 +1770,8 @@ } }, { - "object": "collection", "name": "listIndexNames", + "object": "collection", "expectError": { "isError": true, "isClientError": false @@ -1871,8 +1871,8 @@ } }, { - "object": "collection", "name": "createChangeStream", + "object": "collection", "arguments": { "pipeline": [] }, @@ -1975,8 +1975,8 @@ } }, { - "object": "collection", "name": "insertOne", + "object": "collection", "arguments": { "document": { "_id": 2, @@ -2082,8 +2082,8 @@ } }, { - "object": "collection", "name": "insertMany", + "object": "collection", "arguments": { "documents": [ { @@ -2191,8 +2191,8 @@ } }, { - "object": "collection", "name": "deleteOne", + "object": "collection", "arguments": { "filter": {} }, @@ -2295,8 +2295,8 @@ } }, { - "object": "collection", "name": "deleteMany", + "object": "collection", "arguments": { "filter": {} }, @@ -2399,8 +2399,8 @@ } }, { - "object": "collection", "name": "replaceOne", + "object": "collection", "arguments": { "filter": {}, "replacement": { @@ -2506,8 +2506,8 @@ } }, { - "object": "collection", "name": "updateOne", + "object": "collection", "arguments": { "filter": {}, "update": { @@ -2615,8 +2615,8 @@ } }, { - "object": "collection", "name": "updateMany", + "object": "collection", "arguments": { "filter": {}, "update": { @@ -2724,8 +2724,8 @@ } }, { - "object": "collection", "name": "findOneAndDelete", + "object": "collection", "arguments": { "filter": {} }, @@ -2828,8 +2828,8 @@ } }, { - "object": "collection", "name": "findOneAndReplace", + "object": "collection", "arguments": { "filter": {}, "replacement": { @@ -2935,8 +2935,8 @@ } }, { - "object": "collection", "name": "findOneAndUpdate", + "object": "collection", "arguments": { "filter": {}, "update": { @@ -3044,8 +3044,8 @@ } }, { - "object": "collection", "name": "bulkWrite", + "object": "collection", "arguments": { "requests": [ { @@ -3157,8 +3157,8 @@ } }, { - "object": "collection", "name": "createIndex", + "object": "collection", "arguments": { "keys": { "x": 11 @@ -3264,8 +3264,8 @@ } }, { - "object": "collection", "name": "dropIndex", + "object": "collection", "arguments": { "name": "x_11" }, @@ -3368,8 +3368,8 @@ } }, { - "object": "collection", "name": "dropIndexes", + "object": "collection", "expectError": { "isError": true, "isClientError": false diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml index 3800b20a333..39e4859368a 100644 --- a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml @@ -10,24 +10,21 @@ runOnRequirements: topologies: [replicaset, sharded, load-balanced] createEntities: - - - client: + - client: id: &client client useMultipleMongoses: false observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - - - client: + - client: id: &fail_point_client fail_point_client useMultipleMongoses: false - - - database: + - database: id: &database database client: *client databaseName: &database_name retryable-writes-tests - - - collection: + + - collection: id: &collection collection database: *database collectionName: &collection_name coll @@ -36,18 +33,14 @@ _yamlAnchors: bulkWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll initialData: - - - collectionName: *collection_name + - collectionName: *collection_name databaseName: *database_name documents: - { _id: 1, x: 11 } - { _id: 2, x: 22 } -tests: - - - - description: 'client.listDatabases retries at most maxAttempts=5 times' - +tests: + - description: 'client.listDatabases retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -61,9 +54,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listDatabases object: *client - name: listDatabases arguments: filter: {} expectError: @@ -100,10 +92,7 @@ tests: - commandFailedEvent: commandName: listDatabases - - - - description: 'client.listDatabaseNames retries at most maxAttempts=5 times' - + - description: 'client.listDatabaseNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -117,9 +106,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listDatabaseNames object: *client - name: listDatabaseNames expectError: isError: true isClientError: false @@ -154,10 +142,7 @@ tests: - commandFailedEvent: commandName: listDatabases - - - - description: 'client.createChangeStream retries at most maxAttempts=5 times' - + - description: 'client.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -171,9 +156,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: createChangeStream object: *client - name: createChangeStream arguments: pipeline: [] expectError: @@ -210,12 +194,9 @@ tests: - commandFailedEvent: commandName: aggregate - - - - description: 'client.clientBulkWrite retries at most maxAttempts=5 times' + - description: 'client.clientBulkWrite retries at most maxAttempts=5 times' runOnRequirements: - minServerVersion: '8.0' # client bulk write added to server in 8.0 - operations: - name: failPoint object: testRunner @@ -229,9 +210,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: clientBulkWrite object: *client - name: clientBulkWrite arguments: models: - insertOne: @@ -271,10 +251,7 @@ tests: - commandFailedEvent: commandName: bulkWrite - - - - description: 'database.aggregate retries at most maxAttempts=5 times' - + - description: 'database.aggregate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -288,9 +265,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: aggregate object: *database - name: aggregate arguments: pipeline: [ { $listLocalSessions: {} }, { $limit: 1 } ] expectError: @@ -327,10 +303,7 @@ tests: - commandFailedEvent: commandName: aggregate - - - - description: 'database.listCollections retries at most maxAttempts=5 times' - + - description: 'database.listCollections retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -344,9 +317,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listCollections object: *database - name: listCollections arguments: filter: {} expectError: @@ -383,10 +355,7 @@ tests: - commandFailedEvent: commandName: listCollections - - - - description: 'database.listCollectionNames retries at most maxAttempts=5 times' - + - description: 'database.listCollectionNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -400,9 +369,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listCollectionNames object: *database - name: listCollectionNames arguments: filter: {} expectError: @@ -439,10 +407,7 @@ tests: - commandFailedEvent: commandName: listCollections - - - - description: 'database.runCommand retries at most maxAttempts=5 times' - + - description: 'database.runCommand retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -456,9 +421,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: runCommand object: *database - name: runCommand arguments: command: { ping: 1 } commandName: ping @@ -496,10 +460,7 @@ tests: - commandFailedEvent: commandName: ping - - - - description: 'database.createChangeStream retries at most maxAttempts=5 times' - + - description: 'database.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -513,9 +474,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: createChangeStream object: *database - name: createChangeStream arguments: pipeline: [] expectError: @@ -552,10 +512,7 @@ tests: - commandFailedEvent: commandName: aggregate - - - - description: 'collection.aggregate retries at most maxAttempts=5 times' - + - description: 'collection.aggregate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -569,9 +526,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: aggregate object: *collection - name: aggregate arguments: pipeline: [] expectError: @@ -608,10 +564,7 @@ tests: - commandFailedEvent: commandName: aggregate - - - - description: 'collection.countDocuments retries at most maxAttempts=5 times' - + - description: 'collection.countDocuments retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -625,9 +578,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: countDocuments object: *collection - name: countDocuments arguments: filter: {} expectError: @@ -664,10 +616,7 @@ tests: - commandFailedEvent: commandName: aggregate - - - - description: 'collection.estimatedDocumentCount retries at most maxAttempts=5 times' - + - description: 'collection.estimatedDocumentCount retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -681,9 +630,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: estimatedDocumentCount object: *collection - name: estimatedDocumentCount expectError: isError: true isClientError: false @@ -718,10 +666,7 @@ tests: - commandFailedEvent: commandName: count - - - - description: 'collection.distinct retries at most maxAttempts=5 times' - + - description: 'collection.distinct retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -735,9 +680,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: distinct object: *collection - name: distinct arguments: fieldName: x filter: {} @@ -775,10 +719,7 @@ tests: - commandFailedEvent: commandName: distinct - - - - description: 'collection.find retries at most maxAttempts=5 times' - + - description: 'collection.find retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -792,9 +733,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: find object: *collection - name: find arguments: filter: {} expectError: @@ -831,10 +771,7 @@ tests: - commandFailedEvent: commandName: find - - - - description: 'collection.findOne retries at most maxAttempts=5 times' - + - description: 'collection.findOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -848,9 +785,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: findOne object: *collection - name: findOne arguments: filter: {} expectError: @@ -887,10 +823,7 @@ tests: - commandFailedEvent: commandName: find - - - - description: 'collection.listIndexes retries at most maxAttempts=5 times' - + - description: 'collection.listIndexes retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -904,9 +837,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listIndexes object: *collection - name: listIndexes expectError: isError: true isClientError: false @@ -941,10 +873,7 @@ tests: - commandFailedEvent: commandName: listIndexes - - - - description: 'collection.listIndexNames retries at most maxAttempts=5 times' - + - description: 'collection.listIndexNames retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -958,9 +887,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: listIndexNames object: *collection - name: listIndexNames expectError: isError: true isClientError: false @@ -995,10 +923,7 @@ tests: - commandFailedEvent: commandName: listIndexes - - - - description: 'collection.createChangeStream retries at most maxAttempts=5 times' - + - description: 'collection.createChangeStream retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1012,9 +937,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: createChangeStream object: *collection - name: createChangeStream arguments: pipeline: [] expectError: @@ -1051,10 +975,7 @@ tests: - commandFailedEvent: commandName: aggregate - - - - description: 'collection.insertOne retries at most maxAttempts=5 times' - + - description: 'collection.insertOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1068,9 +989,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: insertOne object: *collection - name: insertOne arguments: document: { _id: 2, x: 22 } expectError: @@ -1107,10 +1027,7 @@ tests: - commandFailedEvent: commandName: insert - - - - description: 'collection.insertMany retries at most maxAttempts=5 times' - + - description: 'collection.insertMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1124,9 +1041,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: insertMany object: *collection - name: insertMany arguments: documents: - { _id: 2, x: 22 } @@ -1164,10 +1080,7 @@ tests: - commandFailedEvent: commandName: insert - - - - description: 'collection.deleteOne retries at most maxAttempts=5 times' - + - description: 'collection.deleteOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1181,9 +1094,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: deleteOne object: *collection - name: deleteOne arguments: filter: {} expectError: @@ -1220,10 +1132,7 @@ tests: - commandFailedEvent: commandName: delete - - - - description: 'collection.deleteMany retries at most maxAttempts=5 times' - + - description: 'collection.deleteMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1237,9 +1146,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: deleteMany object: *collection - name: deleteMany arguments: filter: {} expectError: @@ -1276,10 +1184,7 @@ tests: - commandFailedEvent: commandName: delete - - - - description: 'collection.replaceOne retries at most maxAttempts=5 times' - + - description: 'collection.replaceOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1293,9 +1198,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: replaceOne object: *collection - name: replaceOne arguments: filter: {} replacement: { x: 22 } @@ -1333,10 +1237,7 @@ tests: - commandFailedEvent: commandName: update - - - - description: 'collection.updateOne retries at most maxAttempts=5 times' - + - description: 'collection.updateOne retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1350,9 +1251,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: updateOne object: *collection - name: updateOne arguments: filter: {} update: { $set: { x: 22 } } @@ -1390,10 +1290,7 @@ tests: - commandFailedEvent: commandName: update - - - - description: 'collection.updateMany retries at most maxAttempts=5 times' - + - description: 'collection.updateMany retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1407,9 +1304,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: updateMany object: *collection - name: updateMany arguments: filter: {} update: { $set: { x: 22 } } @@ -1447,10 +1343,7 @@ tests: - commandFailedEvent: commandName: update - - - - description: 'collection.findOneAndDelete retries at most maxAttempts=5 times' - + - description: 'collection.findOneAndDelete retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1464,9 +1357,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: findOneAndDelete object: *collection - name: findOneAndDelete arguments: filter: {} expectError: @@ -1503,10 +1395,7 @@ tests: - commandFailedEvent: commandName: findAndModify - - - - description: 'collection.findOneAndReplace retries at most maxAttempts=5 times' - + - description: 'collection.findOneAndReplace retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1520,9 +1409,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: findOneAndReplace object: *collection - name: findOneAndReplace arguments: filter: {} replacement: { x: 22 } @@ -1560,10 +1448,7 @@ tests: - commandFailedEvent: commandName: findAndModify - - - - description: 'collection.findOneAndUpdate retries at most maxAttempts=5 times' - + - description: 'collection.findOneAndUpdate retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1577,9 +1462,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: findOneAndUpdate object: *collection - name: findOneAndUpdate arguments: filter: {} update: { $set: { x: 22 } } @@ -1617,10 +1501,7 @@ tests: - commandFailedEvent: commandName: findAndModify - - - - description: 'collection.bulkWrite retries at most maxAttempts=5 times' - + - description: 'collection.bulkWrite retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1634,9 +1515,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: bulkWrite object: *collection - name: bulkWrite arguments: requests: - insertOne: @@ -1675,10 +1555,7 @@ tests: - commandFailedEvent: commandName: insert - - - - description: 'collection.createIndex retries at most maxAttempts=5 times' - + - description: 'collection.createIndex retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1692,9 +1569,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: createIndex object: *collection - name: createIndex arguments: keys: { x: 11 } name: "x_11" @@ -1732,10 +1608,7 @@ tests: - commandFailedEvent: commandName: createIndexes - - - - description: 'collection.dropIndex retries at most maxAttempts=5 times' - + - description: 'collection.dropIndex retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1749,9 +1622,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: dropIndex object: *collection - name: dropIndex arguments: name: "x_11" expectError: @@ -1788,10 +1660,7 @@ tests: - commandFailedEvent: commandName: dropIndexes - - - - description: 'collection.dropIndexes retries at most maxAttempts=5 times' - + - description: 'collection.dropIndexes retries at most maxAttempts=5 times' operations: - name: failPoint object: testRunner @@ -1805,9 +1674,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - + - name: dropIndexes object: *collection - name: dropIndexes expectError: isError: true isClientError: false @@ -1841,4 +1709,3 @@ tests: commandName: dropIndexes - commandFailedEvent: commandName: dropIndexes - diff --git a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template index 3117d44b890..3a9cb27ab53 100644 --- a/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template +++ b/test/spec/client-backpressure/backpressure-retry-max-attempts.yml.template @@ -10,24 +10,21 @@ runOnRequirements: topologies: [replicaset, sharded, load-balanced] createEntities: - - - client: + - client: id: &client client useMultipleMongoses: false observeEvents: [commandStartedEvent, commandSucceededEvent, commandFailedEvent] - - - client: + - client: id: &fail_point_client fail_point_client useMultipleMongoses: false - - - database: + - database: id: &database database client: *client databaseName: &database_name retryable-writes-tests - - - collection: + + - collection: id: &collection collection database: *database collectionName: &collection_name coll @@ -36,22 +33,18 @@ _yamlAnchors: bulkWriteInsertNamespace: &client_bulk_write_ns retryable-writes-tests.coll initialData: - - - collectionName: *collection_name + - collectionName: *collection_name databaseName: *database_name documents: - { _id: 1, x: 11 } - { _id: 2, x: 22 } -tests: -{% for operation in operations %} - - - description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' +tests: {% for operation in operations %} + - description: '{{operation.object}}.{{operation.operation_name}} retries at most maxAttempts=5 times' {%- if ((operation.operation_name == 'clientBulkWrite')) %} runOnRequirements: - minServerVersion: '8.0' # client bulk write added to server in 8.0 {%- endif %} - operations: - name: failPoint object: testRunner @@ -65,10 +58,8 @@ tests: errorLabels: [RetryableError, SystemOverloadedError] errorCode: 2 - - - object: *{{operation.object}} - name: {{operation.operation_name}} - {%- if operation.arguments|length > 0 %} + - name: {{operation.operation_name}} + object: *{{operation.object}} {%- if operation.arguments|length > 0 %} arguments: {%- for arg in operation.arguments %} {{arg}} @@ -107,5 +98,4 @@ tests: commandName: {{operation.command_name}} - commandFailedEvent: commandName: {{operation.command_name}} - {% endfor -%} diff --git a/test/spec/client-backpressure/getMore-retried.json b/test/spec/client-backpressure/getMore-retried.json index 70eff846128..fed6ab8cb95 100644 --- a/test/spec/client-backpressure/getMore-retried.json +++ b/test/spec/client-backpressure/getMore-retried.json @@ -1,5 +1,5 @@ { - "description": "getMore-retries-backpressure", + "description": "getMore-retried-backpressure", "schemaVersion": "1.3", "runOnRequirements": [ { @@ -83,6 +83,7 @@ }, { "name": "find", + "object": "coll", "arguments": { "batchSize": 2, "filter": {}, @@ -90,7 +91,6 @@ "a": 1 } }, - "object": "coll", "expectResult": [ { "a": 1 diff --git a/test/spec/client-backpressure/getMore-retried.yml b/test/spec/client-backpressure/getMore-retried.yml index 3a5d180aa54..45f5ad9deb6 100644 --- a/test/spec/client-backpressure/getMore-retried.yml +++ b/test/spec/client-backpressure/getMore-retried.yml @@ -1,8 +1,7 @@ -description: getMore-retries-backpressure +description: getMore-retried-backpressure schemaVersion: "1.3" runOnRequirements: - - - minServerVersion: '4.4' # failCommand + - minServerVersion: '4.4' # failCommand createEntities: - client: id: &client client0 @@ -44,17 +43,18 @@ tests: errorCode: 2 - name: find + object: *collection arguments: # batch size of 2 with 3 docs in the collection ensures exactly one find + one getMore exhaust the cursor batchSize: 2 filter: {} # ensure stable ordering of result documents sort: { a: 1 } - object: *collection expectResult: - { a: 1 } - { a: 2 } - { a: 3 } + expectEvents: - client: *client events: From 410fb45c7b5c973b9a58e7967e3ace27495e382e Mon Sep 17 00:00:00 2001 From: bailey Date: Fri, 23 Jan 2026 11:31:46 -0700 Subject: [PATCH 5/7] squash --- test/spec/client-backpressure/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/spec/client-backpressure/README.md b/test/spec/client-backpressure/README.md index 9f49b1ed203..e9048e1b4a3 100644 --- a/test/spec/client-backpressure/README.md +++ b/test/spec/client-backpressure/README.md @@ -50,8 +50,8 @@ Drivers should test that retries do not occur immediately when a SystemOverloade 5. Execute step 3 again. 6. Compare the two time between the two runs. - ```python - assertTrue(with_backoff_time - no_backoff_time >= 2.1) - ``` - The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two - runs. + ```python + assertTrue(with_backoff_time - no_backoff_time >= 2.1) + ``` + The sum of 5 backoffs is 3.1 seconds. There is a 1-second window to account for potential variance between the two + runs. From 0363694b4bce04a8cb79ff7f37988ee41a1ca67f Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 27 Jan 2026 11:55:57 -0700 Subject: [PATCH 6/7] WIP --- src/operations/execute_operation.ts | 13 ++++++------ src/sessions.ts | 33 +++++++++++++---------------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index 6cb279b6bc5..ab6ef1ea5f9 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -38,6 +38,7 @@ import { } from '../utils'; import { AggregateOperation } from './aggregate'; import { AbstractOperation, Aspect } from './operation'; +import { RunCommandOperation } from './run_command'; const MMAPv1_RETRY_WRITES_ERROR_CODE = MONGODB_ERROR_CODES.IllegalOperation; const MMAPv1_RETRY_WRITES_ERROR_MESSAGE = @@ -256,15 +257,15 @@ async function executeOperationWithRetries< let maxAttempts = (operation.maxAttempts ?? willRetry) ? (timeoutContext.csotEnabled() ? Infinity : 2) : 1; + const shouldRetry = operation.hasAspect(Aspect.READ_OPERATION) && topology.s.options.retryReads || (operation.hasAspect(Aspect.WRITE_OPERATION) || operation instanceof RunCommandOperation) && topology.s.options.retryWrites; for ( let attempt = 0; attempt < maxAttempts; attempt++, - maxAttempts = - willRetry && previousOperationError?.hasErrorLabel(MongoErrorLabel.SystemOverloadedError) - ? 6 - : maxAttempts + maxAttempts = shouldRetry && previousOperationError?.hasErrorLabel(MongoErrorLabel.SystemOverloadedError) + ? 6 + : maxAttempts ) { if (previousOperationError) { if (hasWriteAspect && previousOperationError.code === MMAPv1_RETRY_WRITES_ERROR_CODE) { @@ -349,9 +350,9 @@ async function executeOperationWithRetries< topology.tokenBucket.deposit( isRetry ? // on successful retry, deposit the retry cost + the refresh rate. - TOKEN_REFRESH_RATE + RETRY_COST + TOKEN_REFRESH_RATE + RETRY_COST : // otherwise, just deposit the refresh rate. - TOKEN_REFRESH_RATE + TOKEN_REFRESH_RATE ); return operation.handleOk(result); } catch (error) { diff --git a/src/sessions.ts b/src/sessions.ts index 0e6fa0949d2..a3259c265da 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -25,7 +25,6 @@ import { import type { MongoClient, MongoOptions } from './mongo_client'; import { TypedEventEmitter } from './mongo_types'; import { executeOperation } from './operations/execute_operation'; -import { RetryAttemptContext } from './operations/operation'; import { RunCommandOperation } from './operations/run_command'; import { ReadConcernLevel } from './read_concern'; import { ReadPreference } from './read_preference'; @@ -105,8 +104,7 @@ export interface EndSessionOptions { */ export class ClientSession extends TypedEventEmitter - implements AsyncDisposable -{ + implements AsyncDisposable { /** @internal */ client: MongoClient; /** @internal */ @@ -494,23 +492,22 @@ export class ClientSession command.recoveryToken = this.transaction.recoveryToken; } - const retryContext = new RetryAttemptContext(5); const operation = new RunCommandOperation(new MongoDBNamespace('admin'), command, { session: this, readPreference: ReadPreference.primary, bypassPinningCheck: true }); - operation.attempts = retryContext; + operation.maxAttempts = 5; const timeoutContext = this.timeoutContext ?? (typeof timeoutMS === 'number' ? TimeoutContext.create({ - serverSelectionTimeoutMS: this.clientOptions.serverSelectionTimeoutMS, - socketTimeoutMS: this.clientOptions.socketTimeoutMS, - timeoutMS - }) + serverSelectionTimeoutMS: this.clientOptions.serverSelectionTimeoutMS, + socketTimeoutMS: this.clientOptions.socketTimeoutMS, + timeoutMS + }) : null); try { @@ -531,7 +528,7 @@ export class ClientSession readPreference: ReadPreference.primary, bypassPinningCheck: true }); - op.attempts = retryContext; + op.maxAttempts = operation.maxAttempts; await executeOperation(this.client, op, timeoutContext); return; } catch (retryCommitError) { @@ -612,10 +609,10 @@ export class ClientSession const timeoutContext = timeoutMS != null ? TimeoutContext.create({ - timeoutMS, - serverSelectionTimeoutMS: this.clientOptions.serverSelectionTimeoutMS, - socketTimeoutMS: this.clientOptions.socketTimeoutMS - }) + timeoutMS, + serverSelectionTimeoutMS: this.clientOptions.serverSelectionTimeoutMS, + socketTimeoutMS: this.clientOptions.socketTimeoutMS + }) : null; const wc = this.transaction.options.writeConcern ?? this.clientOptions?.writeConcern; @@ -728,10 +725,10 @@ export class ClientSession this.timeoutContext = timeoutMS != null ? TimeoutContext.create({ - timeoutMS, - serverSelectionTimeoutMS: this.clientOptions.serverSelectionTimeoutMS, - socketTimeoutMS: this.clientOptions.socketTimeoutMS - }) + timeoutMS, + serverSelectionTimeoutMS: this.clientOptions.serverSelectionTimeoutMS, + socketTimeoutMS: this.clientOptions.socketTimeoutMS + }) : null; // 1. Record the current monotonic time, which will be used to enforce the 120-second timeout before later retry attempts. From 7ba74261a7d271190cb4b5780e34882e5d610c0c Mon Sep 17 00:00:00 2001 From: bailey Date: Tue, 27 Jan 2026 15:54:58 -0700 Subject: [PATCH 7/7] refactor + retry writes error logic --- src/operations/execute_operation.ts | 127 +++++++++--------- .../retryable_writes.spec.prose.test.ts | 99 +++++++++++++- 2 files changed, 165 insertions(+), 61 deletions(-) diff --git a/src/operations/execute_operation.ts b/src/operations/execute_operation.ts index ab6ef1ea5f9..6a7d464dc5c 100644 --- a/src/operations/execute_operation.ts +++ b/src/operations/execute_operation.ts @@ -246,9 +246,7 @@ async function executeOperationWithRetries< session.incrementTransactionNumber(); } - let previousOperationError: MongoError | undefined; const deprioritizedServers = new DeprioritizedServers(); - const backoffDelayProvider = new ExponentialBackoffProvider( 10_000, // MAX_BACKOFF 100, // base backoff @@ -257,40 +255,86 @@ async function executeOperationWithRetries< let maxAttempts = (operation.maxAttempts ?? willRetry) ? (timeoutContext.csotEnabled() ? Infinity : 2) : 1; + const shouldRetry = operation.hasAspect(Aspect.READ_OPERATION) && topology.s.options.retryReads || (operation.hasAspect(Aspect.WRITE_OPERATION) || operation instanceof RunCommandOperation) && topology.s.options.retryWrites; + let error: MongoError | null = null; + for ( let attempt = 0; attempt < maxAttempts; - attempt++, - maxAttempts = shouldRetry && previousOperationError?.hasErrorLabel(MongoErrorLabel.SystemOverloadedError) - ? 6 - : maxAttempts + attempt++ ) { - if (previousOperationError) { - if (hasWriteAspect && previousOperationError.code === MMAPv1_RETRY_WRITES_ERROR_CODE) { + + operation.server = server; + + try { + const isRetry = attempt > 0; + + try { + const result = await server.command(operation, timeoutContext); + topology.tokenBucket.deposit( + isRetry + ? // on successful retry, deposit the retry cost + the refresh rate. + TOKEN_REFRESH_RATE + RETRY_COST + : // otherwise, just deposit the refresh rate. + TOKEN_REFRESH_RATE + ); + return operation.handleOk(result); + } catch (error) { + return operation.handleError(error); + } + } catch (operationError) { + // Should never happen but if it does - propragate the error. + if (!(operationError instanceof MongoError)) throw operationError; + + if (!operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { + // if an operation fails with an error that does not contain the SystemOverloadError, deposit 1 token. + topology.tokenBucket.deposit(RETRY_COST); + } + + if (error == null) { + error = operationError; + } else { + if (!operationError.hasErrorLabel(MongoErrorLabel.NoWritesPerformed)) { + error = operationError; + } + } + + if (hasWriteAspect && operationError.code === MMAPv1_RETRY_WRITES_ERROR_CODE) { throw new MongoServerError({ message: MMAPv1_RETRY_WRITES_ERROR_MESSAGE, errmsg: MMAPv1_RETRY_WRITES_ERROR_MESSAGE, - originalError: previousOperationError + originalError: operationError }); } + // prepare for retry const isRetryable = // bulk write commands are retryable if all operations in the batch are retryable (operation.hasAspect(Aspect.COMMAND_BATCHING) && operation.canRetryWrite) || // if we have a retryable read or write operation, we can retry - (hasWriteAspect && willRetryWrite && isRetryableWriteError(previousOperationError)) || - (hasReadAspect && willRetryRead && isRetryableReadError(previousOperationError)) || + (!operation.hasAspect(Aspect.COMMAND_BATCHING) && hasWriteAspect && willRetryWrite && isRetryableWriteError(operationError)) || + (hasReadAspect && willRetryRead && isRetryableReadError(operationError)) || // if we have a retryable, system overloaded error, we can retry - (previousOperationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError) && - previousOperationError.hasErrorLabel(MongoErrorLabel.RetryableError)); + (operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError) && + operationError.hasErrorLabel(MongoErrorLabel.RetryableError)); + + if (!isRetryable) throw error; - if (!isRetryable) { - throw previousOperationError; + maxAttempts = shouldRetry && operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError) + ? 6 + : maxAttempts + if (attempt >= maxAttempts) { + throw error; } - if (previousOperationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { + // safe to retry - reset timeout context, apply backoff if necessary and re-run server selection + + // Reset timeouts + timeoutContext.clear(); + + if (operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { const delayMS = backoffDelayProvider.getNextBackoffDuration(); // if the delay would exhaust the CSOT timeout, short-circuit. @@ -299,20 +343,20 @@ async function executeOperationWithRetries< throw new MongoOperationTimeoutError( `MongoDB SystemOverload exponential backoff would exceed timeoutMS deadline: remaining CSOT deadline=${timeoutContext.remainingTimeMS}, backoff delayMS=${delayMS}`, { - cause: previousOperationError + cause: error } ); } if (!topology.tokenBucket.consume(RETRY_COST)) { - throw previousOperationError; + throw error; } await setTimeout(delayMS); } if ( - previousOperationError instanceof MongoNetworkError && + operationError instanceof MongoNetworkError && operation.hasAspect(Aspect.CURSOR_CREATING) && session != null && session.isPinned && @@ -321,6 +365,8 @@ async function executeOperationWithRetries< session.unpin({ force: true, forceClear: true }); } + deprioritizedServers.add(server.description); + server = await topology.selectServer(selector, { session, operationName: operation.commandName, @@ -333,52 +379,13 @@ async function executeOperationWithRetries< 'Selected server does not support retryable writes' ); } - } - - operation.server = server; - - try { - const isRetry = attempt > 0; // If attempt > 0 and we are command batching we need to reset the batch. - if (isRetry && operation.hasAspect(Aspect.COMMAND_BATCHING)) { + if (operation.hasAspect(Aspect.COMMAND_BATCHING)) { operation.resetBatch(); } - - try { - const result = await server.command(operation, timeoutContext); - topology.tokenBucket.deposit( - isRetry - ? // on successful retry, deposit the retry cost + the refresh rate. - TOKEN_REFRESH_RATE + RETRY_COST - : // otherwise, just deposit the refresh rate. - TOKEN_REFRESH_RATE - ); - return operation.handleOk(result); - } catch (error) { - return operation.handleError(error); - } - } catch (operationError) { - if (!(operationError instanceof MongoError)) throw operationError; - - if (!operationError.hasErrorLabel(MongoErrorLabel.SystemOverloadedError)) { - // if an operation fails with an error that does not contain the SystemOverloadError, deposit 1 token. - topology.tokenBucket.deposit(RETRY_COST); - } - - if ( - previousOperationError != null && - operationError.hasErrorLabel(MongoErrorLabel.NoWritesPerformed) - ) { - throw previousOperationError; - } - deprioritizedServers.add(server.description); - previousOperationError = operationError; - - // Reset timeouts - timeoutContext.clear(); } } - throw previousOperationError ?? new MongoRuntimeError('ahh'); + throw error ?? new MongoRuntimeError('ahh'); } diff --git a/test/integration/retryable-writes/retryable_writes.spec.prose.test.ts b/test/integration/retryable-writes/retryable_writes.spec.prose.test.ts index 58fe83ca97a..d3a89f219cf 100644 --- a/test/integration/retryable-writes/retryable_writes.spec.prose.test.ts +++ b/test/integration/retryable-writes/retryable_writes.spec.prose.test.ts @@ -8,8 +8,9 @@ import { type Collection, type MongoClient, MongoError, + MongoErrorLabel, MongoServerError, - MongoWriteConcernError + MongoWriteConcernError, } from '../../../src'; import { Server } from '../../../src/sdam/server'; import { sleep } from '../../tools/utils'; @@ -344,4 +345,100 @@ describe('Retryable Writes Spec Prose', () => { } ); }); + + describe('6. Test that drivers return the original error after encountering multiple WriteConcernErrors with a RetryableWriteError label', () => { + let client: MongoClient; + let collection: Collection<{ _id: 1 }>; + + beforeEach(async function () { + // This test MUST: + // - be implemented by any driver that implements the Command Monitoring specification, + // - only run against replica sets as mongos does not propagate the NoWritesPerformed label to the drivers. + // - be run against server versions 6.0 and above. + // - be implemented by any driver that has implemented the Client Backpressure specification. + + // Additionally, this test requires drivers to set a fail point after an `insertOne` operation but before the subsequent + // retry. Drivers that are unable to set a failCommand after the CommandFailedEvent SHOULD use mocking or write a unit test + // to cover the same sequence of events. + + // 1. Create a client with `retryWrites=true`. + client = this.configuration.newClient({ monitorCommands: true, retryWrites: true }); + await client + .db() + .collection('retryReturnsOriginal') + .drop() + .catch(() => null); + collection = client.db().collection('retryReturnsOriginal'); + }); + + afterEach(async function () { + // 5. Disable the fail point: + // ```javascript + // { + // configureFailPoint: "failCommand", + // mode: "off" + // } + // ``` + + // (we don't use a failPoint, so we use sinon.restore instead) + sinon.restore(); + await client.close(); + }); + + it( + 'when a retry attempt fails with an error labeled NoWritesPerformed, drivers MUST return the original error', + { requires: { topology: 'replicaset', mongodb: '>=4.2.9' } }, + async () => { + // 2. Configure a fail point with error code `91` (ShutdownInProgress) with the `RetryableError` and + // `SystemOverloadedError` error labels: + + // ```javascript + // { + // configureFailPoint: "failCommand", + // mode: {times: 1}, + // data: { + // failCommands: ["insert"], + // errorLabels: ["RetryableError", "SystemOverloadedError", "NoWritesPerformed"], + // errorCode: 91 + // } + // } + // ``` + + // 3. Via the command monitoring CommandFailedEvent, configure a fail point with error code `10107` (NotWritablePrimary) + // and a NoWritesPerformed label: + + // ```javascript + // { + // configureFailPoint: "failCommand", + // mode: "alwaysOn", + // data: { + // failCommands: ["insert"], + // errorCode: 10107, + // errorLabels: ["RetryableError", "SystemOverloadedError", , "NoWritesPerformed"] + // } + // } + // ``` + + // Drivers SHOULD only configure the `10107` fail point command if the the failed event is for the `91` error configured + // in step 2. + const serverCommandStub = sinon.stub(Server.prototype, 'command').callsFake(async function () { + throw new MongoServerError({ + message: 'Server Error', + errorLabels: [MongoErrorLabel.RetryableError, MongoErrorLabel.SystemOverloadedError, MongoErrorLabel.NoWritesPerformed], + code: serverCommandStub.callCount === 1 ? 91 : 10107, + ok: 0 + }); + }); + + const insertResult = await collection.insertOne({ _id: 1 }).catch(error => error); + + expect(serverCommandStub.callCount).to.equal(6); + + // 4. Attempt an `insertOne` operation on any record for any database and collection. Expect the `insertOne` to fail with a + // server error. Assert that the error code of the server error is 91. + expect(insertResult).to.be.instanceOf(MongoServerError); + expect(insertResult).to.have.property('code', 91); + } + ); + }) });