[𝘀𝗽𝗿] initial version

Created using spr 1.3.7

Author: jroelofs

lnt/lnttool/__init__.py

@@ -13,6 +13,8 @@
 from .runtest import group_runtest
 from .send_daily_report import action_send_daily_report
 from .send_run_comparison import action_send_run_comparison
+from .abtest import group_abtest
+from .expire_abtests import action_expire_abtests
 from .showtests import action_showtests
 from .submit import action_submit
 from .updatedb import action_updatedb
@@ -45,6 +47,8 @@ def main():
 main.add_command(action_checkformat)
 main.add_command(action_convert)
 main.add_command(action_create)
+main.add_command(group_abtest)
+main.add_command(action_expire_abtests)
 main.add_command(action_import)
 main.add_command(action_importreport)
 main.add_command(action_profile)

lnt/lnttool/abtest.py

@@ -0,0 +1,141 @@
+"""lnt abtest — manage A/B performance experiments on a remote LNT server."""
+import json
+import ssl
+import urllib.error
+import urllib.request
+
+import certifi
+import click
+
+
+def _api_url(server_url, database, testsuite, *path_parts):
+    base = '%s/api/db_%s/v4/%s' % (server_url.rstrip('/'), database, testsuite)
+    if path_parts:
+        return '%s/%s' % (base, '/'.join(str(p) for p in path_parts))
+    return base
+
+
+def _api_request(method, url, body=None, auth_token=None):
+    headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
+    if auth_token:
+        headers['AuthToken'] = auth_token
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(url, data=data, headers=headers, method=method)
+    context = ssl.create_default_context(cafile=certifi.where())
+    try:
+        resp = urllib.request.urlopen(req, context=context)
+        return json.loads(resp.read())
+    except urllib.error.HTTPError as e:
+        body_text = e.read().decode(errors='replace')
+        raise click.ClickException('HTTP %d: %s' % (e.code, body_text))
+    except urllib.error.URLError as e:
+        raise click.ClickException('Could not connect to %s: %s' % (url, e))
+
+
+@click.group("abtest")
+def group_abtest():
+    """manage A/B performance experiments on a remote LNT server"""
+
+
+@group_abtest.command("create")
+@click.argument("server_url")
+@click.option("--database", default="default", show_default=True,
+              help="LNT database name")
+@click.option("--testsuite", "-s", default="nts", show_default=True,
+              help="testsuite name")
+@click.option("--name", default="",
+              help="human-readable experiment name")
+@click.option("--control", "control_file",
+              type=click.Path(exists=True), default=None,
+              help="control run report JSON")
+@click.option("--variant", "variant_file",
+              type=click.Path(exists=True), default=None,
+              help="variant run report JSON")
+@click.option("--auth-token", envvar="LNT_AUTH_TOKEN",
+              help="API auth token (or set LNT_AUTH_TOKEN)")
+def action_abtest_create(server_url, database, testsuite, name,
+                         control_file, variant_file, auth_token):
+    """Create an A/B experiment on a remote LNT server.
+
+\b
+Two workflows are supported:
+
+  Atomic — both runs available at the same time:
+
+    lnt abtest create SERVER --name "pr-42" \\
+        --control control.json --variant variant.json
+
+  Two-phase — control and variant submitted by independent CI jobs:
+
+    # Orchestrator: create the experiment and capture the ID
+    ID=$(lnt abtest create SERVER --name "pr-42")
+
+    # Control CI job
+    lnt abtest submit SERVER $ID --control control.json
+
+    # Variant CI job
+    lnt abtest submit SERVER $ID --variant variant.json
+    """
+    if bool(control_file) != bool(variant_file):
+        raise click.UsageError(
+            "Provide both --control and --variant for atomic creation, "
+            "or neither to create a pending experiment.")
+
+    body = {'name': name}
+    if control_file:
+        with open(control_file) as f:
+            body['control'] = json.load(f)
+        with open(variant_file) as f:
+            body['variant'] = json.load(f)
+
+    url = _api_url(server_url, database, testsuite, 'abtest')
+    result = _api_request('POST', url, body=body, auth_token=auth_token)
+
+    # Print just the ID to stdout so scripts can capture it with $(...).
+    click.echo(result['id'])
+    exp_url = result.get('url')
+    if exp_url:
+        click.echo('Experiment: %s' % exp_url, err=True)
+
+
+@group_abtest.command("submit")
+@click.argument("server_url")
+@click.argument("experiment_id", type=int)
+@click.option("--database", default="default", show_default=True,
+              help="LNT database name")
+@click.option("--testsuite", "-s", default="nts", show_default=True,
+              help="testsuite name")
+@click.option("--control", "control_file",
+              type=click.Path(exists=True), default=None,
+              help="submit this JSON as the control run")
+@click.option("--variant", "variant_file",
+              type=click.Path(exists=True), default=None,
+              help="submit this JSON as the variant run")
+@click.option("--auth-token", envvar="LNT_AUTH_TOKEN",
+              help="API auth token (or set LNT_AUTH_TOKEN)")
+def action_abtest_submit(server_url, experiment_id, database, testsuite,
+                         control_file, variant_file, auth_token):
+    """Submit a control or variant run to an existing A/B experiment.
+
+\b
+Used in the two-phase workflow after 'lnt abtest create' has returned an ID:
+
+    lnt abtest submit SERVER ID --control control.json
+    lnt abtest submit SERVER ID --variant variant.json
+    """
+    if not control_file and not variant_file:
+        raise click.UsageError("Provide --control or --variant.")
+    if control_file and variant_file:
+        raise click.UsageError(
+            "Provide --control or --variant, not both. "
+            "To submit both at once use 'lnt abtest create'.")
+
+    role = 'control' if control_file else 'variant'
+    report_file = control_file or variant_file
+
+    with open(report_file) as f:
+        body = json.load(f)
+
+    url = _api_url(server_url, database, testsuite, 'abtest', experiment_id, role)
+    _api_request('POST', url, body=body, auth_token=auth_token)
+    click.echo('Submitted %s run for experiment %d.' % (role, experiment_id))

lnt/lnttool/expire_abtests.py

@@ -0,0 +1,93 @@
+import contextlib
+import datetime
+import re
+
+import click
+
+
+def _parse_age(value):
+    """Parse an age string like '90d', '4w', '6m', '1y' into a cutoff datetime."""
+    m = re.fullmatch(r'(\d+)([dwmy])', value)
+    if not m:
+        raise click.BadParameter(
+            "expected a positive integer followed by d/w/m/y "
+            "(e.g. 90d, 4w, 6m, 1y)",
+            param_hint="'--older-than'")
+    n, unit = int(m.group(1)), m.group(2)
+    days = {'d': n, 'w': n * 7, 'm': n * 30, 'y': n * 365}[unit]
+    return datetime.datetime.utcnow() - datetime.timedelta(days=days)
+
+
+@click.command("expire-abtests")
+@click.argument("instance_path", type=click.UNPROCESSED)
+@click.option("--database", default="default", show_default=True,
+              help="database to expire experiments from")
+@click.option("--testsuite", "-s", default="nts", show_default=True,
+              help="testsuite to expire experiments from")
+@click.option("--older-than", "older_than", required=True,
+              help="delete experiments older than this age (e.g. 90d, 4w, 6m, 1y)")
+@click.option("--dry-run", is_flag=True,
+              help="print what would be deleted without making any changes")
+def action_expire_abtests(instance_path, database, testsuite,
+                          older_than, dry_run):
+    """Delete unpinned A/B experiments older than a given age.
+
+\b
+Removes unpinned ABExperiment records (and their associated ABRun and
+ABSample rows) whose creation time predates the specified age threshold.
+Pinned ('Keep Forever') experiments are never deleted.
+
+\b
+Age format: a positive integer followed by a unit:
+  d  days       (e.g. 90d)
+  w  weeks      (e.g. 4w)
+  m  months     (approx 30 days each, e.g. 6m)
+  y  years      (approx 365 days each, e.g. 1y)
+    """
+    import lnt.server.instance
+
+    cutoff = _parse_age(older_than)
+
+    instance = lnt.server.instance.Instance.frompath(instance_path)
+    with contextlib.closing(instance.get_database(database)) as db:
+        session = db.make_session()
+        ts = db.testsuite[testsuite]
+
+        to_delete = (
+            session.query(ts.ABExperiment)
+            .filter(ts.ABExperiment.created_time < cutoff,
+                    ts.ABExperiment.pinned == False)  # noqa: E712
+            .all())
+
+        if not to_delete:
+            click.echo("No experiments to delete.")
+            return
+
+        for exp in to_delete:
+            click.echo("%s experiment #%d: %s" % (
+                "Would delete" if dry_run else "Deleting",
+                exp.id, exp.name or "(unnamed)"))
+
+        if dry_run:
+            return
+
+        # Delete ABSample and ABRun children before the ABExperiment rows to
+        # respect foreign-key constraints.
+        run_ids = [rid for exp in to_delete
+                   for rid in (exp.control_run_id, exp.variant_run_id)
+                   if rid is not None]
+
+        session.query(ts.ABSample) \
+            .filter(ts.ABSample.run_id.in_(run_ids)) \
+            .delete(synchronize_session=False)
+
+        session.query(ts.ABRun) \
+            .filter(ts.ABRun.id.in_(run_ids)) \
+            .delete(synchronize_session=False)
+
+        for exp in to_delete:
+            session.delete(exp)
+
+        session.commit()
+        click.echo("Deleted %d experiment%s." %
+                   (len(to_delete), "s" if len(to_delete) != 1 else ""))

lnt/server/db/migrations/new_suite.py

@@ -2,6 +2,7 @@
 from . import upgrade_2_to_3
 from . import upgrade_7_to_8
 from . import upgrade_8_to_9
+from . import upgrade_18_to_19
 
 
 def init_new_testsuite(engine, session, name):
@@ -17,3 +18,6 @@ def init_new_testsuite(engine, session, name):
     session.commit()
     upgrade_8_to_9.upgrade_testsuite(engine, session, name)
     session.commit()
+    ts_defn = session.query(upgrade_0_to_1.TestSuite).filter_by(name=name).first()
+    upgrade_18_to_19.upgrade_testsuite(engine, ts_defn.db_key_name)
+    session.commit()

lnt/server/db/migrations/upgrade_18_to_19.py

@@ -0,0 +1,107 @@
+"""Add ABRun, ABSample, and ABExperiment tables for A/B performance testing.
+
+ABRun intentionally omits order_id so that A/B test runs never participate
+in trend analysis or FieldChange/Regression detection.
+
+This migration creates the AB tables with their dynamic run/sample field
+columns following the same column-type rules as upgrade_0_to_1.
+"""
+
+import sqlalchemy
+from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer,
+                        LargeBinary, String, select)
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.declarative import declarative_base
+
+import lnt.server.db.migrations.upgrade_0_to_1 as upgrade_0_to_1
+from lnt.server.db.migrations.util import introspect_table
+
+
+def _add_ab_tables(test_suite):
+    """Return a Base with ABRun, ABSample, and ABExperiment for test_suite.
+
+    Machine and Test stubs are included in the same Base so that FK
+    references resolve during create_all."""
+    db_key_name = test_suite.db_key_name
+    Base = declarative_base()
+
+    class Machine(Base):
+        __tablename__ = db_key_name + '_Machine'
+        __table_args__ = {'extend_existing': True}
+        id = Column("ID", Integer, primary_key=True)
+
+    class Test(Base):
+        __tablename__ = db_key_name + '_Test'
+        __table_args__ = {'extend_existing': True}
+        id = Column("ID", Integer, primary_key=True)
+
+    class ABRun(Base):
+        __tablename__ = db_key_name + '_ABRun'
+        id = Column("ID", Integer, primary_key=True)
+        machine_id = Column("MachineID", Integer, ForeignKey(Machine.id),
+                            index=True)
+        start_time = Column("StartTime", DateTime)
+        end_time = Column("EndTime", DateTime)
+        parameters_data = Column("Parameters", LargeBinary)
+
+        class_dict = locals()
+        for item in test_suite.run_fields:
+            class_dict[item.name] = Column(item.name, String(256))
+
+    class ABSample(Base):
+        __tablename__ = db_key_name + '_ABSample'
+        id = Column("ID", Integer, primary_key=True)
+        run_id = Column("RunID", Integer, ForeignKey(ABRun.id), index=True)
+        test_id = Column("TestID", Integer, ForeignKey(Test.id), index=True)
+
+        class_dict = locals()
+        for item in test_suite.sample_fields:
+            if item.type.name == 'Real':
+                class_dict[item.name] = Column(item.name, Float)
+            elif item.type.name == 'Status':
+                class_dict[item.name] = Column(
+                    item.name, Integer,
+                    ForeignKey(upgrade_0_to_1.StatusKind.id))
+            elif item.type.name == 'Hash':
+                class_dict[item.name] = Column(item.name, String)
+
+    class ABExperiment(Base):
+        __tablename__ = db_key_name + '_ABExperiment'
+        id = Column("ID", Integer, primary_key=True)
+        name = Column("Name", String(256))
+        created_time = Column("CreatedTime", DateTime)
+        extra = Column("Extra", String)
+        pinned = Column("Pinned", Boolean, default=False)
+        control_run_id = Column("ControlRunID", Integer,
+                                ForeignKey(ABRun.id))
+        variant_run_id = Column("VariantRunID", Integer,
+                                ForeignKey(ABRun.id))
+
+    return Base
+
+
+def upgrade_testsuite(engine, db_key_name):
+    """Create the AB tables for a single test suite identified by db_key_name."""
+    session = sessionmaker(engine)()
+    try:
+        test_suite = session.query(upgrade_0_to_1.TestSuite).filter_by(
+            db_key_name=db_key_name).first()
+        if test_suite is None:
+            return
+        Base = _add_ab_tables(test_suite)
+        # Only create new tables; use checkfirst=True (the default) so
+        # existing tables are left untouched.
+        Base.metadata.create_all(engine, checkfirst=True)
+    finally:
+        session.close()
+
+
+def upgrade(engine):
+    """Create AB tables for every existing test suite."""
+    test_suite_table = introspect_table(engine, 'TestSuite')
+
+    with engine.begin() as trans:
+        suites = list(trans.execute(select([test_suite_table.c.DBKeyName])))
+
+    for (db_key_name,) in suites:
+        upgrade_testsuite(engine, db_key_name)