Merge pull request #19850 from mozilla/normalize-email-bounce-addrs-2

task(auth): Normalize email bounce lookup

Author: dschom

packages/db-migrations/databases/fxa/patches/patch-180-181.sql

@@ -0,0 +1,25 @@
+SET NAMES utf8mb4 COLLATE utf8mb4_bin;
+
+CALL assertPatchLevel('180');
+
+CREATE PROCEDURE `fetchEmailBounces_4` (
+  IN `inEmail` VARCHAR(255)
+)
+BEGIN
+  SELECT
+      e.email,
+      t.emailType,
+      e.bounceType,
+      e.bounceSubType,
+      e.createdAt,
+      e.diagnosticCode
+  FROM emailBounces e
+  JOIN emailTypes t
+      ON t.id = e.emailTypeId
+  WHERE
+    e.email LIKE inEmail
+  ORDER BY createdAt DESC
+  LIMIT 20;
+END;
+
+UPDATE dbMetadata SET value = '181' WHERE name = 'schema-patch-level';

packages/db-migrations/databases/fxa/patches/patch-181-180.sql

@@ -0,0 +1,7 @@
+-- -- Drop new column and new constraint
+
+-- -- Drop new stored procedures
+-- DROP PROCEDURE `fetchEmailBounces_4`;
+
+-- -- Decrement the schema version
+-- UPDATE dbMetadata SET value = '180' WHERE name = 'schema-patch-level';

packages/db-migrations/databases/fxa/target-patch.json

@@ -1,3 +1,3 @@
 {
-  "level": 180
+  "level": 181
 }

packages/functional-tests/lib/email.ts

@@ -3,6 +3,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 import got from 'got';
+import mysql from 'mysql2/promise';
 
 function wait() {
   return new Promise((r) => setTimeout(r, 50));
@@ -253,4 +254,40 @@ export class EmailClient {
     await this.clear(email);
     return code;
   }
+
+  /** Creates a bounce record. Note, this only works on localhost. For stage / prod, we expect we can generate a real bounce. */
+  async createBounce(
+    email: string,
+    bounceType = 1,
+    bounceSubType = 1,
+    emailTypeId = 39,
+    diagnosticCode = ''
+  ): Promise<void> {
+    try {
+      // create the connection to database
+      const connection = await mysql.createConnection({
+        host: 'localhost',
+        user: 'root',
+        database: 'fxa',
+        // Since this is local, wait up to a second and fail other wise.
+        connectTimeout: 1000,
+      });
+
+      // execute will internally call prepare and query
+      connection.execute(
+        `INSERT INTO emailBounces (email, bounceType, bounceSubType, createdAt, emailTypeId, diagnosticCode)
+         VALUES (?, ?, ?, ?, ?, ?);`,
+        [
+          email,
+          bounceType,
+          bounceSubType,
+          Date.now(),
+          emailTypeId,
+          diagnosticCode,
+        ]
+      );
+    } catch (err) {
+      console.log('Could not create bounce record in local db!');
+    }
+  }
 }

packages/functional-tests/lib/testAccountTracker.ts

@@ -17,6 +17,7 @@ import { SessionStatus } from 'fxa-auth-client/lib/client';
 enum EmailPrefix {
   BLOCKED = 'blocked',
   BOUNCED = 'bounced',
+  BOUNCED_ALIAS = 'bounced+',
   FORCED_PWD_CHANGE = 'forcepwdchange',
   SIGNIN = 'signin',
   SIGNUP = 'signup',
@@ -85,9 +86,12 @@ export class TestAccountTracker {
    * @param prefix email prefix to use when generating email
    * @returns email
    */
-  generateEmail(prefix: EmailPrefix = EmailPrefix.SIGNIN): string {
+  generateEmail(
+    prefix: EmailPrefix = EmailPrefix.SIGNIN,
+    domain = 'restmail.net'
+  ): string {
     const id = crypto.randomBytes(8).toString('hex');
-    return `${prefix}${id}@restmail.net`;
+    return `${prefix}${id}@${domain}`;
   }
 
   /**
@@ -107,6 +111,15 @@ export class TestAccountTracker {
     return this.generateAccountDetails(EmailPrefix.BOUNCED);
   }
 
+  /**
+   * Creates a new email address with the 'bounced' prefix and a new randomized
+   * password
+   * @returns AccountDetails
+   */
+  generateBouncedAliasAccountDetails(domain: string): AccountDetails {
+    return this.generateAccountDetails(EmailPrefix.BOUNCED_ALIAS, domain);
+  }
+
   /**
    * Creates a new email address with the 'signup' prefix and a new
    * randomized password
@@ -141,10 +154,11 @@ export class TestAccountTracker {
    * @returns AccountDetails
    */
   generateAccountDetails(
-    prefix: EmailPrefix = EmailPrefix.SIGNIN
+    prefix: EmailPrefix = EmailPrefix.SIGNIN,
+    domain = 'restmail.net'
   ): AccountDetails {
     const account = {
-      email: this.generateEmail(prefix),
+      email: this.generateEmail(prefix, domain),
       password: this.generatePassword(),
     };
     this.accounts.push(account);

packages/functional-tests/pages/signin.ts

@@ -86,6 +86,12 @@ export class SigninPage extends BaseLayout {
     });
   }
 
+  get emailReturnedWarning() {
+    return this.page.getByText(
+      'Your confirmation email was just returned. Mistyped email?'
+    );
+  }
+
   // for backwards compatibility with Backbone
   // not currently implemented in React, see FXA-8827
   get permissionsHeading() {
@@ -152,10 +158,7 @@ export class SigninPage extends BaseLayout {
    * assertions up to the caller.
    * @param Credentials
    */
-  async emailFirstSignin({
-    email,
-    password,
-  }: Credentials) {
+  async emailFirstSignin({ email, password }: Credentials) {
     if (this.page.url() !== this.url) {
       // avoid navigating if we don't need to.
       // Checking url is faster than an extra navigation

packages/functional-tests/tests/signin/signinBlocked.spec.ts

@@ -40,6 +40,54 @@ test.describe('severity-2 #smoke', () => {
       await removeAccount(settings, deleteAccount, page, credentials.password);
     });
 
+    test('email bounced', async ({
+      target,
+      page,
+      pages: {
+        deleteAccount,
+        secondaryEmail,
+        settings,
+        signin,
+        signup,
+        signinUnblock,
+      },
+      testAccountTracker,
+    }) => {
+      // We might have to wait a bit for the bounce to get picked up. So
+      // we will flag this as a slow test. If on local, we manually create
+      // a hard bounce record. On local, this will insert a new bounce
+      // record. On stage prod, this will silently fail, and we will rely
+      // on the email to actually bounce.
+      if (target.name === 'local') {
+        await target.emailClient.createBounce('[email protected]');
+      } else {
+        test.slow();
+      }
+
+      // Here, we check that an email alias ie `bounced+123` will trip a bounce
+      // check when the bounce record is based on the non-aliased email form.
+      const credentials =
+        testAccountTracker.generateBouncedAliasAccountDetails('mozilla.com');
+
+      // Start the sign up process
+      await page.goto(target.contentServerUrl);
+      await signin.fillOutEmailFirstForm(credentials.email);
+      await signup.fillOutSignupForm(credentials.password);
+
+      // The service that polls for bounces does so on approximately a 5 second interval.
+      // Add a little timeout to make sure that catches the bounce, and the polling
+      // mechanism can have a chance to redirect.
+      if (target.name === 'local') {
+        await page.waitForTimeout(1000);
+      } else {
+        await page.waitForTimeout(9000);
+      }
+
+      // This indicates the email bounced. The front end starts polling for bounces,
+      // it should etecte one, and then kick the user back to the sign in page.
+      await expect(signin.emailReturnedWarning).toBeVisible();
+    });
+
     test('incorrect code entered', async ({
       target,
       page,

packages/fxa-auth-server/config/index.ts

@@ -612,6 +612,14 @@ const convictConf = convict({
         default: true,
         env: 'BOUNCES_DELETE_ACCOUNT',
       },
+      emailAliasNormalization: {
+        default: JSON.stringify([
+          { domain: 'mozilla.com', regex: '\\+.*', replace: '%' },
+        ]),
+        doc: 'List of email domain configurations for alias normalization. Each entry should have domain, regex, and replace properties. Example: [{domain: "mozilla.com", regex: "\\+[^@]+" }].',
+        env: 'BOUNCES_EMAIL_ALIAS_NORMALIZATION',
+        format: String,
+      },
     },
     connectionTimeout: {
       doc: 'Milliseconds to wait for the connection to establish (default is 2 minutes)',
@@ -2318,7 +2326,9 @@ const convictConf = convict({
       format: Array,
     },
     emailAliasNormalization: {
-      default: '',
+      default: JSON.stringify([
+        { domain: 'mozilla.com', regex: '\\+.*', replace: '' },
+      ]),
       doc: 'List of email domain configurations for alias normalization. Each entry should have domain, regex, and replace properties. Example: [{domain: "mozilla.com", regex: "\\+[^@]+", replace: ""}]',
       env: 'RATE_LIMIT__EMAIL_ALIAS_NORMALIZATION',
       format: String,

packages/fxa-auth-server/lib/bounces.js

@@ -5,6 +5,7 @@
 'use strict';
 
 const { AppError: error } = require('@fxa/accounts/errors');
+const { EmailNormalization } = require('fxa-shared/email/email-normalization');
 
 module.exports = (config, db) => {
   const configBounces = (config.smtp && config.smtp.bounces) || {};
@@ -28,12 +29,24 @@ module.exports = (config, db) => {
     [BOUNCE_TYPE_COMPLAINT]: error.emailComplaint,
   };
 
-  function checkBounces(email, template) {
+  const emailNormalization = new EmailNormalization(
+    configBounces.emailAliasNormalization
+  );
+
+  async function checkBounces(email, template) {
     if (ignoreTemplates.includes(template)) {
       return;
     }
 
-    return db.emailBounces(email).then(applyRules);
+    // This strips out 'alias' stuff from an email and replace
+    // them with wildcards, allowing us to turn up bounce records
+    // on email aliases.
+    const normalizedEmail = emailNormalization.normalizeEmailAliases(
+      email,
+      '%'
+    );
+    const bounces = await db.emailBounces(normalizedEmail);
+    return applyRules(bounces);
   }
 
   // Relies on the order of the bounces array to be sorted by date,