improve aggregated metrics view

kitsune/dashboards/jinja2/dashboards/aggregated_metrics.html

@@ -6,8 +6,6 @@
 {% set crumbs = [(None, title)] %}
 {% set classes = 'aggregated-metrics' %}
 
-{% set product_slug = product.slug if product else 'null' %}
-
 
 {% block content %}
     <article class="dashboards sumo-page-section">
@@ -19,23 +17,23 @@ <h1 class="sumo-page-heading">{{ title }}</h1>
         {{ _('The dashboard data is being loaded. This may take some time.') }}
       </div>
 
-      <div id="dashboard-readouts" class="sumo-page-section">
-        <section id="percent-localized-top100" data-url="{{ url('api.wikimetric_list')|urlparams(product=product_slug, page_size=1000) }}">
+      <div id="dashboard-readouts" class="sumo-page-section" data-url="{{ url('dashboards.wiki_metrics_data')|urlparams(product=(product.slug if product else None)) }}">
+        <section id="percent-localized-top100">
           {# Double % required below to escape % for gettext #}
           <h2 class="sumo-page-subheading">{{ _('Top 100 Articles: %% Localized') }}</h2>
         </section>
 
-        <section id="percent-localized-top20" class="sumo-page-section" data-url="{{ url('api.wikimetric_list')|urlparams(product=product_slug, page_size=1000) }}">
+        <section id="percent-localized-top20" class="sumo-page-section">
           {# Double % required below to escape % for gettext #}
           <h2 class="sumo-page-subheading">{{ _('Top 20 Articles: %% Localized') }}</h2>
         </section>
 
-        <section id="percent-localized-all" class="sumo-page-section" data-url="{{ url('api.wikimetric_list')|urlparams(product=product_slug, page_size=1000) }}">
+        <section id="percent-localized-all" class="sumo-page-section">
           {# Double % required below to escape % for gettext #}
           <h2 class="sumo-page-subheading">{{ _('All Articles: %% Localized') }}</h2>
         </section>
 
-        <section id="active-contributors" class="sumo-page-section" data-url="{{ url('api.wikimetric_list')|urlparams(product=product_slug, page_size=1000) }}">
+        <section id="active-contributors" class="sumo-page-section">
           <h2 class="sumo-page-subheading">{{ _('Active Contributors') }}</h2>
         </section>
       </div>

kitsune/dashboards/jinja2/dashboards/locale_metrics.html

@@ -6,26 +6,24 @@
 {% set crumbs = [(None, title)] %}
 {% set classes = 'locale-metrics' %}
 
-{% set product_slug = product.slug if product else 'null' %}
-
-
 {% block content %}
 <article id="localize" class="dashboards sumo-page-section">
       <h1 class="sumo-page-heading">{{ title }}</h1>
 
       {{ product_choice_list(products, product, url('dashboards.locale_metrics', locale_code=current_locale)) }}
 
       {% if current_locale != settings.WIKI_DEFAULT_LANGUAGE %}
-        <section class="sumo-page-section" id="localization-metrics" data-url="{{ url('api.wikimetric_list')|urlparams(locale=current_locale, page_size=10000, product=product_slug) }}">
+        <section class="sumo-page-section" id="localization-metrics" data-url="{{ url('dashboards.wiki_metrics_data')|urlparams(locale=current_locale, product=(product.slug if product else None)) }}">
           <h2 class="sumo-page-subheading">{{ _('Localization Percentage') }}</h2>
         </section>
       {% endif %}
 
-      <section class="sumo-page-section" id="active-contributors" data-url="{{ url('api.wikimetric_list')|urlparams(locale=current_locale, page_size=10000, product=product_slug) }}">
+      <section class="sumo-page-section" id="active-contributors" data-url="{{ url('dashboards.wiki_metrics_data')|urlparams(locale=current_locale, product=(product.slug if product else None)) }}">
         <h2 class="sumo-page-subheading">{{ _('Active Contributors') }}</h2>
       </section>
 
-      <section class="sumo-page-section" id="kpi-vote" data-url="{{ url('api.kpi.kb-votes')|urlparams(locale=current_locale, product=product_slug) }}">
+      {# The kb-votes KPI endpoint uses the literal string "null" as its "all products" sentinel. #}
+      <section class="sumo-page-section" id="kpi-vote" data-url="{{ url('api.kpi.kb-votes')|urlparams(locale=current_locale, product=(product.slug if product else 'null')) }}">
         <h2 class="sumo-page-subheading">{{ _('Helpful Votes') }}</h2>
       </section>
 </article>

kitsune/dashboards/metrics.py

@@ -0,0 +1,93 @@
+from datetime import date, timedelta
+
+from django.core.cache import cache
+from django.db.models.functions import TruncWeek
+
+from kitsune.dashboards.models import WikiMetric
+from kitsune.products.models import Product
+
+# The metrics dashboards always show a fixed one-year window.
+WINDOW_DAYS = 365
+
+# The coverage cron re-warms these payloads once a day. Keep the TTL comfortably
+# longer than that 24-hour interval so a successful warm always overwrites a
+# still-live entry; if the TTL matched the interval, expiry would race the next
+# warm and leave a cold gap (forcing an on-request rebuild) on roughly half the
+# days. The extra two hours absorb cron jitter and Celery queue latency. The
+# cache only goes cold when a warm is actually missed.
+WIKI_METRICS_CACHE_TIMEOUT = 26 * 60 * 60  # 26 hours
+
+
+def _build_payload(product, locale):
+    """Build the shaped metrics payload from the database.
+
+    The daily coverage codes are downsampled to one point per ISO week (the
+    most recent row in each week); the monthly active-contributors code already
+    has at most one row per month, so it passes through unchanged.
+
+    Returns {code: {locale: [{"date": iso, "value": float}, ...ascending]}}.
+    """
+    cutoff = date.today() - timedelta(days=WINDOW_DAYS)
+
+    queryset = WikiMetric.objects.filter(date__gte=cutoff, product=product)
+    if locale:
+        queryset = queryset.filter(locale=locale)
+
+    # We keep one row per (code, locale, week). Weeks are returned
+    # ascending, so each series is already in ascending-date order.
+    rows = (
+        queryset.annotate(week=TruncWeek("date"))
+        .order_by("code", "locale", "week", "-date", "-id")
+        .distinct("code", "locale", "week")
+        .values("code", "locale", "date", "value", "week")
+    )
+
+    payload = {}
+    for row in rows:
+        by_locale = payload.setdefault(row["code"], {})
+        by_locale.setdefault(row["locale"] or "", []).append(
+            {"date": row["date"].isoformat(), "value": row["value"]}
+        )
+
+    return payload
+
+
+def _cache_key(product, locale):
+    product_key = product.slug if product else "all"
+    locale_key = locale or "all"
+    return f"wikimetrics:agg:{product_key}:{locale_key}"
+
+
+def get_wiki_metrics_data(product, locale=None):
+    """Return the cached metrics payload for the given product/locale.
+
+    `product` is a Product instance or None (the "All products" bucket).
+    `locale` limits the payload to a single locale (used by the per-locale
+    dashboard). The payload always covers a fixed one-year window.
+
+    Cached for WIKI_METRICS_CACHE_TIMEOUT. The data-refresh cron tasks call
+    warm_wiki_metrics_cache() to recompute and reset this entry, so in normal
+    operation the TTL is just a backstop for a missed cron run.
+    """
+    return cache.get_or_set(
+        _cache_key(product, locale),
+        lambda: _build_payload(product, locale),
+        WIKI_METRICS_CACHE_TIMEOUT,
+    )
+
+
+def warm_wiki_metrics_cache():
+    """Recompute and cache the aggregated (all-locales) payloads.
+
+    Called by the cron tasks after the WikiMetric rows are refreshed. Using
+    cache.set (not get_or_set) overwrites the existing entries and resets their
+    TTL, so the expensive all-locales query is never run on a user request and
+    there's no cold-cache stampede. The per-locale payloads are cheap and left
+    to populate on demand.
+    """
+    for product in [None, *Product.objects.filter(visible=True)]:
+        cache.set(
+            _cache_key(product, None),
+            _build_payload(product, None),
+            WIKI_METRICS_CACHE_TIMEOUT,
+        )

kitsune/dashboards/tasks.py

@@ -4,6 +4,7 @@
 from django.conf import settings
 from django.db import connection
 
+from kitsune.dashboards.metrics import warm_wiki_metrics_cache
 from kitsune.dashboards.models import (
     L10N_ACTIVE_CONTRIBUTORS_CODE,
     L10N_ALL_CODE,
@@ -104,6 +105,9 @@ def update_l10n_coverage_metrics() -> None:
                 code=L10N_ALL_CODE, locale=locale, product=product, date=today, value=percent
             )
 
+    # Warm the cached dashboard payloads with the freshly-computed data.
+    warm_wiki_metrics_cache()
+
 
 @shared_task
 @skip_if_read_only_mode
@@ -260,3 +264,6 @@ def update_l10n_contributor_metrics(day_isoformat: str | None = None) -> None:
                 date=previous_first_of_month,
                 value=num,
             )
+
+    # Warm the cached dashboard payloads with the freshly-computed data.
+    warm_wiki_metrics_cache()