[𝘀𝗽𝗿] initial version

Created using spr 1.3.7

Author: jroelofs

lnt/server/db/migrations/new_suite.py

@@ -2,6 +2,7 @@
 from . import upgrade_2_to_3
 from . import upgrade_7_to_8
 from . import upgrade_8_to_9
+from . import upgrade_18_to_19
 
 
 def init_new_testsuite(engine, session, name):
@@ -17,3 +18,6 @@ def init_new_testsuite(engine, session, name):
     session.commit()
     upgrade_8_to_9.upgrade_testsuite(engine, session, name)
     session.commit()
+    ts_defn = session.query(upgrade_0_to_1.TestSuite).filter_by(name=name).first()
+    upgrade_18_to_19.upgrade_testsuite(engine, ts_defn.db_key_name)
+    session.commit()

lnt/server/db/migrations/upgrade_18_to_19.py

@@ -0,0 +1,107 @@
+"""Add ABRun, ABSample, and ABExperiment tables for A/B performance testing.
+
+ABRun intentionally omits order_id so that A/B test runs never participate
+in trend analysis or FieldChange/Regression detection.
+
+This migration creates the AB tables with their dynamic run/sample field
+columns following the same column-type rules as upgrade_0_to_1.
+"""
+
+import sqlalchemy
+from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer,
+                        LargeBinary, String, select)
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.declarative import declarative_base
+
+import lnt.server.db.migrations.upgrade_0_to_1 as upgrade_0_to_1
+from lnt.server.db.migrations.util import introspect_table
+
+
+def _add_ab_tables(test_suite):
+    """Return a Base with ABRun, ABSample, and ABExperiment for test_suite.
+
+    Machine and Test stubs are included in the same Base so that FK
+    references resolve during create_all."""
+    db_key_name = test_suite.db_key_name
+    Base = declarative_base()
+
+    class Machine(Base):
+        __tablename__ = db_key_name + '_Machine'
+        __table_args__ = {'extend_existing': True}
+        id = Column("ID", Integer, primary_key=True)
+
+    class Test(Base):
+        __tablename__ = db_key_name + '_Test'
+        __table_args__ = {'extend_existing': True}
+        id = Column("ID", Integer, primary_key=True)
+
+    class ABRun(Base):
+        __tablename__ = db_key_name + '_ABRun'
+        id = Column("ID", Integer, primary_key=True)
+        machine_id = Column("MachineID", Integer, ForeignKey(Machine.id),
+                            index=True)
+        start_time = Column("StartTime", DateTime)
+        end_time = Column("EndTime", DateTime)
+        parameters_data = Column("Parameters", LargeBinary)
+
+        class_dict = locals()
+        for item in test_suite.run_fields:
+            class_dict[item.name] = Column(item.name, String(256))
+
+    class ABSample(Base):
+        __tablename__ = db_key_name + '_ABSample'
+        id = Column("ID", Integer, primary_key=True)
+        run_id = Column("RunID", Integer, ForeignKey(ABRun.id), index=True)
+        test_id = Column("TestID", Integer, ForeignKey(Test.id), index=True)
+
+        class_dict = locals()
+        for item in test_suite.sample_fields:
+            if item.type.name == 'Real':
+                class_dict[item.name] = Column(item.name, Float)
+            elif item.type.name == 'Status':
+                class_dict[item.name] = Column(
+                    item.name, Integer,
+                    ForeignKey(upgrade_0_to_1.StatusKind.id))
+            elif item.type.name == 'Hash':
+                class_dict[item.name] = Column(item.name, String)
+
+    class ABExperiment(Base):
+        __tablename__ = db_key_name + '_ABExperiment'
+        id = Column("ID", Integer, primary_key=True)
+        name = Column("Name", String(256))
+        created_time = Column("CreatedTime", DateTime)
+        extra = Column("Extra", String)
+        pinned = Column("Pinned", Boolean, default=False)
+        control_run_id = Column("ControlRunID", Integer,
+                                ForeignKey(ABRun.id))
+        variant_run_id = Column("VariantRunID", Integer,
+                                ForeignKey(ABRun.id))
+
+    return Base
+
+
+def upgrade_testsuite(engine, db_key_name):
+    """Create the AB tables for a single test suite identified by db_key_name."""
+    session = sessionmaker(engine)()
+    try:
+        test_suite = session.query(upgrade_0_to_1.TestSuite).filter_by(
+            db_key_name=db_key_name).first()
+        if test_suite is None:
+            return
+        Base = _add_ab_tables(test_suite)
+        # Only create new tables; use checkfirst=True (the default) so
+        # existing tables are left untouched.
+        Base.metadata.create_all(engine, checkfirst=True)
+    finally:
+        session.close()
+
+
+def upgrade(engine):
+    """Create AB tables for every existing test suite."""
+    test_suite_table = introspect_table(engine, 'TestSuite')
+
+    with engine.begin() as trans:
+        suites = list(trans.execute(select([test_suite_table.c.DBKeyName])))
+
+    for (db_key_name,) in suites:
+        upgrade_testsuite(engine, db_key_name)

lnt/server/db/testsuitedb.py

@@ -13,7 +13,7 @@
 import aniso8601
 import sqlalchemy
 import flask
-from sqlalchemy import Float, String, Integer, Column, ForeignKey, Binary, DateTime
+from sqlalchemy import Float, String, Integer, Column, ForeignKey, Binary, Boolean, DateTime
 from sqlalchemy.orm import relation
 from sqlalchemy.orm.exc import ObjectDeletedError
 from lnt.util import logger
@@ -756,6 +756,81 @@ class Baseline(self.base, ParameterizedMixin):
             def __str__(self):
                 return "Baseline({})".format(self.name)
 
+        # A/B testing tables.  ABRun intentionally omits order_id so that A/B
+        # runs never participate in trend analysis or FieldChange/Regression
+        # detection.
+
+        class ABRun(self.base, ParameterizedMixin):
+            __tablename__ = db_key_name + '_ABRun'
+
+            fields = self.run_fields
+            id = Column("ID", Integer, primary_key=True)
+            machine_id = Column("MachineID", Integer, ForeignKey(Machine.id),
+                                index=True)
+            start_time = Column("StartTime", DateTime)
+            end_time = Column("EndTime", DateTime)
+            parameters_data = Column("Parameters", Binary, index=False,
+                                     unique=False)
+
+            machine = relation(Machine)
+
+            # Dynamic run-field columns.  Create fresh Column objects rather
+            # than reusing item.column, which points to the Run table.
+            class_dict = locals()
+            for item in fields:
+                class_dict[item.name] = testsuite.make_run_column(item.name)
+
+            @property
+            def parameters(self):
+                return dict(json.loads(self.parameters_data))
+
+            @parameters.setter
+            def parameters(self, data):
+                self.parameters_data = json.dumps(
+                    sorted(data.items())).encode("utf-8")
+
+        class ABSample(self.base, ParameterizedMixin):
+            __tablename__ = db_key_name + '_ABSample'
+
+            fields = self.sample_fields
+            id = Column("ID", Integer, primary_key=True)
+            run_id = Column("RunID", Integer, ForeignKey(ABRun.id), index=True)
+            test_id = Column("TestID", Integer, ForeignKey(Test.id),
+                             index=True)
+
+            run = relation(ABRun)
+            test = relation(Test)
+
+            # Dynamic sample-field columns.  Create fresh Column objects rather
+            # than reusing item.column, which points to the Sample table.
+            class_dict = locals()
+            for item in fields:
+                class_dict[item.name] = testsuite.make_sample_column(
+                    item.name, item.type.name)
+
+        ABRun.ab_samples = relation(ABSample, back_populates='run',
+                                    cascade="all, delete-orphan")
+
+        class ABExperiment(self.base, ParameterizedMixin):
+            __tablename__ = db_key_name + '_ABExperiment'
+
+            fields = []
+            id = Column("ID", Integer, primary_key=True)
+            name = Column("Name", String(256))
+            created_time = Column("CreatedTime", DateTime)
+            extra = Column("Extra", String)
+            pinned = Column("Pinned", Boolean, default=False)
+
+            # Two FK references to ABRun; foreign_keys disambiguates them.
+            control_run_id = Column("ControlRunID", Integer,
+                                    ForeignKey(ABRun.id))
+            variant_run_id = Column("VariantRunID", Integer,
+                                    ForeignKey(ABRun.id))
+            control_run = relation(ABRun,
+                                   foreign_keys=[control_run_id])
+            variant_run = relation(ABRun,
+                                   foreign_keys=[variant_run_id])
+
         self.Machine = Machine
         self.Run = Run
         self.Test = Test
@@ -767,10 +842,15 @@ def __str__(self):
         self.RegressionIndicator = RegressionIndicator
         self.ChangeIgnore = ChangeIgnore
         self.Baseline = Baseline
+        self.ABRun = ABRun
+        self.ABSample = ABSample
+        self.ABExperiment = ABExperiment
 
         # Create the compound index we cannot declare inline.
         sqlalchemy.schema.Index("ix_%s_Sample_RunID_TestID" % db_key_name,
                                 Sample.run_id, Sample.test_id)
+        sqlalchemy.schema.Index("ix_%s_ABSample_RunID_TestID" % db_key_name,
+                                ABSample.run_id, ABSample.test_id)
 
     def create_tables(self, engine):
         self.base.metadata.create_all(engine)
@@ -1125,6 +1205,99 @@ def importDataFromDict(self, session, data, config, select_machine,
         self._importSampleValues(session, data['tests'], run, config)
         return run
 
+    def _importABRun(self, session, run_data, machine):
+        """Create and insert an ABRun for the given machine.
+
+        No order tracking is performed; A/B runs are isolated from trend
+        analysis and regression detection.
+        """
+        run_parameters = run_data.copy()
+        run_parameters.pop('id', None)
+        run_parameters.pop('order_by', None)
+        run_parameters.pop('order_id', None)
+        run_parameters.pop('machine_id', None)
+
+        start_time_str = run_parameters.pop('start_time', None)
+        if start_time_str:
+            try:
+                start_time = aniso8601.parse_datetime(start_time_str)
+            except ValueError:
+                start_time = datetime.datetime.strptime(start_time_str,
+                                                        "%Y-%m-%d %H:%M:%S")
+        else:
+            start_time = None
+
+        end_time_str = run_parameters.pop('end_time', None)
+        if end_time_str:
+            try:
+                end_time = aniso8601.parse_datetime(end_time_str)
+            except ValueError:
+                end_time = datetime.datetime.strptime(end_time_str,
+                                                      "%Y-%m-%d %H:%M:%S")
+        else:
+            end_time = None
+
+        run = self.ABRun()
+        run.machine = machine
+        run.start_time = start_time
+        run.end_time = end_time
+        for item in self.run_fields:
+            value = run_parameters.pop(item.name, None)
+            run.set_field(item, value)
+        run.parameters = run_parameters
+        session.add(run)
+        return run
+
+    def _importABSampleValues(self, session, tests_data, run):
+        """Create ABSample rows for the given tests data and ABRun.
+
+        Mirrors _importSampleValues but creates ABSample objects and skips
+        profile handling.
+        """
+        test_cache = dict((test.name, test)
+                          for test in session.query(self.Test))
+        field_dict = dict([(f.name, f) for f in self.sample_fields])
+        all_samples = []
+
+        for test_data in tests_data:
+            name = test_data['name']
+            test = test_cache.get(name)
+            if test is None:
+                test = self.Test(test_data['name'])
+                test_cache[name] = test
+                session.add(test)
+
+            samples = []
+            for key, values in test_data.items():
+                if key == 'name' or key == 'id' or key.endswith('_id'):
+                    continue
+                field = field_dict.get(key)
+                if field is None:
+                    raise ValueError("test %s: Metric %r unknown in suite" %
+                                     (name, key))
+                if not isinstance(values, list):
+                    values = [values]
+                while len(samples) < len(values):
+                    sample = self.ABSample()
+                    sample.run = run
+                    sample.test = test
+                    samples.append(sample)
+                    all_samples.append(sample)
+                for sample, value in zip(samples, values):
+                    sample.set_field(field, value)
+
+        session.add_all(all_samples)
+
+    def importABDataFromDict(self, session, data):
+        """Import a report into ABRun/ABSample tables.
+
+        No order or regression tracking is performed.
+        """
+        machine = self._getOrCreateMachine(session, data['machine'], 'match')
+        run = self._importABRun(session, data['run'], machine)
+        self._importABSampleValues(session, data['tests'], run)
+        return run
+
     # Simple query support (mostly used by templates)
 
     def machines(self, session, name=None):

lnt/server/reporting/analysis.py

@@ -419,3 +419,29 @@ def _load_samples_for_runs(self, session, run_ids, only_tests):
                 self.profile_map[(run_id, test_id)] = profile_id
 
         self.loaded_run_ids |= to_load
+
+
+class ABRunInfo(RunInfo):
+    """Like RunInfo but queries ABSample instead of Sample.
+
+    Uses getattr(ts.ABSample, f.name) to reference columns rather than
+    f.column, which points to the Sample table.
+    """
+
+    def _load_samples_for_runs(self, session, run_ids, only_tests):
+        to_load = set(run_ids) - self.loaded_run_ids
+        if not to_load:
+            return
+
+        ABSample = self.testsuite.ABSample
+        columns = [ABSample.run_id, ABSample.test_id]
+        columns.extend(
+            getattr(ABSample, f.name) for f in self.testsuite.sample_fields)
+        q = session.query(*columns)
+        if only_tests:
+            q = q.filter(ABSample.test_id.in_(only_tests))
+        q = q.filter(ABSample.run_id.in_(to_load))
+        for (run_id, test_id, *sample_values) in q:
+            self.sample_map[(run_id, test_id)] = sample_values
+
+        self.loaded_run_ids |= to_load