diff --git a/.github/scripts/dependency_age.py b/.github/scripts/dependency_age.py index cfc8e707a63..ad0da39a049 100644 --- a/.github/scripts/dependency_age.py +++ b/.github/scripts/dependency_age.py @@ -5,6 +5,7 @@ import os import re import sys +import urllib.error import urllib.parse import urllib.request from dataclasses import dataclass @@ -18,7 +19,6 @@ DEFAULT_MIN_AGE_HOURS = 48 - @dataclass(frozen=True) class Candidate: version: str @@ -28,6 +28,7 @@ class Candidate: # Entry point for GitHub Actions workflows # select-gradle: get newest Gradle release that is at least MIN_DEPENDENCY_AGE_HOURS hours old # select-maven: get newest Maven artifact release that is at least MIN_DEPENDENCY_AGE_HOURS hours old +# validate-lockfiles: check that each new coordinate in the Gradle lockfiles is at least MIN_DEPENDENCY_AGE_HOURS hours old def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Dependency age helpers for GitHub workflows.") subparsers = parser.add_subparsers(dest="command", required=True) @@ -50,6 +51,15 @@ def parse_args() -> argparse.Namespace: help="Case-insensitive regex fragment used to exclude prerelease versions.", ) + validate = subparsers.add_parser("validate-lockfiles", help="Validate age of new coordinates in Gradle lockfiles.") + validate.add_argument("--baseline-dir", required=True) + validate.add_argument("--current-dir", default=".") + validate.add_argument("--metadata-file", help="JSON file mapping group:artifact:version to a timestamp override.") + validate.add_argument("--search-url", default=MAVEN_SEARCH_URL) + validate.add_argument("--min-age-hours", type=int, default=default_min_age_hours()) + validate.add_argument("--now") + validate.add_argument("--github-output", default=None) + return parser.parse_args() @@ -97,7 +107,7 @@ def parse_datetime(value: Any) -> datetime: except ValueError: pass - # ISO 8601: normalise Z and +HHMM → +HH:MM for fromisoformat + # ISO 8601: normalise Z and +HHMM -> +HH:MM for fromisoformat text = re.sub(r"([+-])(\d{2})(\d{2})$", r"\1\2:\3", text.replace("Z", "+00:00")) return datetime.fromisoformat(text).astimezone(timezone.utc) @@ -238,7 +248,7 @@ def load_maven_documents( return docs -# parse a version string into a tuple of ints for numeric comparison (e.g. "3.9.11" → (3, 9, 11)) +# parse a version string into a tuple of ints for numeric comparison (e.g. "3.9.11" -> (3, 9, 11)) def _version_sort_key(version: str) -> tuple: parts = [] for segment in re.split(r"([.\-])", version): @@ -293,12 +303,168 @@ def emit_selection_result( return 0 +# check that every new coordinate in the Gradle lockfiles is at least min_age_hours old +def validate_lockfiles(args: argparse.Namespace) -> int: + cutoff = now_utc(args.now) - timedelta(hours=args.min_age_hours) + baseline_dir = Path(args.baseline_dir) + current_dir = Path(args.current_dir) + metadata = load_metadata_overrides(args.metadata_file) + + changed = changed_lockfile_coordinates(baseline_dir=baseline_dir, current_dir=current_dir) + if not changed: + print("No dependency version changes detected across Gradle lockfiles.") + emit_outputs({"cutoff_at": format_datetime(cutoff), "reverted_files": 0}, args.github_output) + return 0 + + changed_by_file: dict[str, list[str]] = {} + for relative_path, gav in changed: + changed_by_file.setdefault(relative_path, []).append(gav) + + timestamp_cache: dict[str, tuple[datetime | None, str | None]] = {} + violations_by_file: dict[str, list[tuple[str, str]]] = {} + for relative_path, gavs in sorted(changed_by_file.items()): + for gav in gavs: + if gav not in timestamp_cache: + timestamp_cache[gav] = resolve_gav_timestamp(gav=gav, metadata=metadata, search_url=args.search_url) + published_at, reason = timestamp_cache[gav] + if published_at is None: + print(f"::warning file={relative_path}::{gav}: {reason} Skipping age check.") + continue + if published_at > cutoff: + violations_by_file.setdefault(relative_path, []).append( + (gav, f"Published at {format_datetime(published_at)}, cutoff {format_datetime(cutoff)}.") + ) + else: + print(f"Verified {gav} (published {format_datetime(published_at)}, cutoff {format_datetime(cutoff)})") + + if violations_by_file: + revert_lockfiles_to_baseline(violations_by_file=violations_by_file, baseline_dir=baseline_dir, current_dir=current_dir) + for relative_path, entries in sorted(violations_by_file.items()): + for gav, message in entries: + print(f"::warning file={relative_path}::{gav}: {message} Reverted lockfile to baseline.") + + reverted_files = len(violations_by_file) + emit_outputs({"cutoff_at": format_datetime(cutoff), "reverted_files": reverted_files}, args.github_output) + print(f"Validated {len(changed)} changed coordinate(s) across {len(changed_by_file)} lockfile(s). {reverted_files} lockfile(s) reverted.") + return 0 + + +# restore each violating lockfile to its baseline copy to keep the file consistent +def revert_lockfiles_to_baseline( + *, + violations_by_file: dict[str, list[tuple[str, str]]], + baseline_dir: Path, + current_dir: Path, +) -> None: + for relative_path in sorted(violations_by_file): + current_path = current_dir / relative_path + baseline_path = baseline_dir / relative_path + if baseline_path.exists(): + current_path.write_text(baseline_path.read_text(encoding="utf-8"), encoding="utf-8") + print(f"Reverted {relative_path} to baseline.") + else: + current_path.unlink(missing_ok=True) + print(f"Removed new lockfile {relative_path} (no baseline copy to restore).") + + +# look up the publish timestamp for a group:artifact:version coordinate in Maven Central +# returns (datetime, None) on success, (None, reason) when the timestamp cannot be determined +def resolve_gav_timestamp( + *, + gav: str, + metadata: dict[str, Any], + search_url: str, +) -> tuple[datetime | None, str | None]: + if gav in metadata: + return parse_metadata_override(gav, metadata[gav]) + + group_id, artifact_id, version = gav.split(":", 2) + query = urllib.parse.urlencode({ + "q": f'g:"{group_id}" AND a:"{artifact_id}" AND v:"{version}"', + "core": "gav", + "rows": 20, + "wt": "json", + }) + try: + payload = load_json(None, f"{search_url}?{query}") + docs = payload.get("response", {}).get("docs", []) + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError, ValueError): + return None, "Maven Central search was unreachable." + for doc in docs: + if doc.get("v") != version: + continue + timestamp = doc.get("timestamp") + if timestamp is None: + return None, "Maven Central search result did not include a publish timestamp." + return parse_datetime(timestamp), None + return None, f"No metadata found in Maven Central for {gav}." + + +# load optional metadata overrides from a JSON file (group:artifact:version -> timestamp or skip reason) +def load_metadata_overrides(path: str | None) -> dict[str, Any]: + if not path: + return {} + return load_json(path, None) + + +# parse a single metadata override value: a timestamp string/number, or a dict with "reason" to skip +def parse_metadata_override(gav: str, override: Any) -> tuple[datetime | None, str | None]: + if isinstance(override, dict): + if "reason" in override: + return None, str(override["reason"]) + for key in ("timestamp", "published_at", "timestamp_ms"): + if key in override: + return parse_datetime(override[key]), None + return None, f"Metadata override for {gav} is missing a timestamp." + if isinstance(override, (int, float, str)): + return parse_datetime(override), None + return None, f"Unsupported metadata override format for {gav}." + + +# diff baseline and current lockfile directories; return (relative_path, gav) for each new coordinate +def changed_lockfile_coordinates(*, baseline_dir: Path, current_dir: Path) -> list[tuple[str, str]]: + changed: list[tuple[str, str]] = [] + baseline_lockfiles = collect_lockfiles(baseline_dir) + current_lockfiles = collect_lockfiles(current_dir) + for relative_path in sorted(set(baseline_lockfiles) | set(current_lockfiles)): + before = baseline_lockfiles.get(relative_path, set()) + after = current_lockfiles.get(relative_path, set()) + for gav in sorted(after - before): + changed.append((relative_path, gav)) + return changed + + +# recursively find all gradle.lockfile paths under root and parse them into sets of coordinates +def collect_lockfiles(root: Path) -> dict[str, set[str]]: + if not root.exists(): + return {} + return { + str(path.relative_to(root)): parse_lockfile(path) + for path in root.rglob("gradle.lockfile") + } + + +# parse a lockfile into a set of group:artifact:version coordinates (skipping comments and empty lines) +def parse_lockfile(path: Path) -> set[str]: + coordinates: set[str] = set() + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + coordinate = line.split("=", 1)[0] + if coordinate.count(":") == 2: + coordinates.add(coordinate) + return coordinates + + def main() -> int: args = parse_args() if args.command == "select-gradle": return select_gradle_release(args) if args.command == "select-maven": return select_maven_release(args) + if args.command == "validate-lockfiles": + return validate_lockfiles(args) raise ValueError(f"Unsupported command: {args.command}") diff --git a/.github/scripts/tests/test_dependency_age.py b/.github/scripts/tests/test_dependency_age.py index 4ee3a8d0f50..93ab781b386 100644 --- a/.github/scripts/tests/test_dependency_age.py +++ b/.github/scripts/tests/test_dependency_age.py @@ -1,6 +1,9 @@ +import json import os import re +import shutil import subprocess +import tempfile import unittest from pathlib import Path @@ -10,7 +13,7 @@ FIXTURES = Path(__file__).resolve().parent / "fixtures" NOW = "2026-04-24T12:00:00Z" OUTPUT_PATTERN = re.compile( - r"^(cutoff_at|found|version|published_at|reason)=(.*)$" + r"^(cutoff_at|found|version|published_at|reason|reverted_files)=(.*)$" ) @@ -124,5 +127,134 @@ def test_exact_48_hour_boundary_is_accepted(self) -> None: self.assertEqual(outputs["published_at"], "2026-04-22T12:00:00Z") + def run_validate_lockfiles( + self, + *, + baseline: dict[str, str], + current: dict[str, str], + metadata: dict, + now: str = NOW, + ) -> tuple[subprocess.CompletedProcess[str], Path]: + """ + Run validate-lockfiles with in-memory lockfile content. + baseline/current map relative paths to file text. + All coordinates must be covered by metadata — any uncovered coordinate + hits the (unreachable) search URL and is warned+skipped. + """ + tmp = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, tmp, True) + baseline_dir = tmp / "before" + current_dir = tmp / "after" + metadata_file = tmp / "metadata.json" + + for rel_path, content in baseline.items(): + p = baseline_dir / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content, encoding="utf-8") + + for rel_path, content in current.items(): + p = current_dir / rel_path + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(content, encoding="utf-8") + + metadata_file.write_text(json.dumps(metadata), encoding="utf-8") + + result = self.run_script( + "validate-lockfiles", + "--baseline-dir", str(baseline_dir), + "--current-dir", str(current_dir), + "--metadata-file", str(metadata_file), + "--search-url", (tmp / "no-network").as_uri(), + "--now", now, + ) + return result, current_dir + + def test_validates_changed_lockfiles_when_all_updates_are_old_enough(self) -> None: + baseline_content = "# lockfile\ncom.example:lib-a:1.0.0=runtimeClasspath\ncom.example:lib-b:1.0.0=runtimeClasspath\n" + current_content = "# lockfile\ncom.example:lib-a:1.1.0=runtimeClasspath\ncom.example:lib-b:1.1.0=runtimeClasspath\n" + metadata = { + "com.example:lib-a:1.1.0": "2026-04-20T12:00:00Z", + "com.example:lib-b:1.1.0": "2026-04-20T11:00:00Z", + } + + result, current_dir = self.run_validate_lockfiles( + baseline={"module/gradle.lockfile": baseline_content}, + current={"module/gradle.lockfile": current_content}, + metadata=metadata, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + outputs = self.parse_outputs(result.stdout) + self.assertEqual(outputs["reverted_files"], "0") + self.assertEqual((current_dir / "module/gradle.lockfile").read_text(encoding="utf-8"), current_content) + + def test_reverts_lockfile_when_any_changed_dependency_is_too_new(self) -> None: + baseline_content = "# lockfile\ncom.example:lib-a:1.0.0=runtimeClasspath\ncom.example:lib-b:1.0.0=runtimeClasspath\n" + current_content = "# lockfile\ncom.example:lib-a:1.1.0=runtimeClasspath\ncom.example:lib-b:2.0.0=runtimeClasspath\n" + metadata = { + "com.example:lib-a:1.1.0": "2026-04-20T12:00:00Z", # old enough + "com.example:lib-b:2.0.0": "2026-04-24T11:00:00Z", # too new + } + + result, current_dir = self.run_validate_lockfiles( + baseline={"module/gradle.lockfile": baseline_content}, + current={"module/gradle.lockfile": current_content}, + metadata=metadata, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + outputs = self.parse_outputs(result.stdout) + self.assertEqual(outputs["reverted_files"], "1") + self.assertEqual((current_dir / "module/gradle.lockfile").read_text(encoding="utf-8"), baseline_content) + + def test_reverts_lockfile_when_one_of_multiple_coexisting_versions_is_too_new(self) -> None: + baseline_content = "# lockfile\ncom.typesafe:config:1.3.1=compileClasspath\ncom.typesafe:config:1.4.4=runtimeClasspath\n" + current_content = "# lockfile\ncom.typesafe:config:1.3.1=compileClasspath\ncom.typesafe:config:1.5.0=runtimeClasspath\n" + metadata = { + "com.typesafe:config:1.5.0": "2026-04-24T11:00:00Z", # too new + } + + result, current_dir = self.run_validate_lockfiles( + baseline={"module/gradle.lockfile": baseline_content}, + current={"module/gradle.lockfile": current_content}, + metadata=metadata, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((current_dir / "module/gradle.lockfile").read_text(encoding="utf-8"), baseline_content) + + def test_removes_brand_new_lockfile_with_too_new_dependency(self) -> None: + current_content = "# lockfile\ncom.example:brand-new:1.0.0=runtimeClasspath\n" + metadata = { + "com.example:brand-new:1.0.0": "2026-04-24T11:00:00Z", # too new + } + + result, current_dir = self.run_validate_lockfiles( + baseline={}, + current={"module/gradle.lockfile": current_content}, + metadata=metadata, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse((current_dir / "module/gradle.lockfile").exists()) + + def test_warns_and_skips_coordinate_when_metadata_lookup_fails(self) -> None: + # coordinate not in metadata -> hits unreachable search URL -> warns and skips (does not revert) + baseline_content = "# lockfile\ncom.example:lib:1.0.0=runtimeClasspath\n" + current_content = "# lockfile\ncom.example:lib:1.1.0=runtimeClasspath\n" + + result, current_dir = self.run_validate_lockfiles( + baseline={"module/gradle.lockfile": baseline_content}, + current={"module/gradle.lockfile": current_content}, + metadata={}, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + outputs = self.parse_outputs(result.stdout) + self.assertEqual(outputs["reverted_files"], "0") + self.assertIn("::warning", result.stdout) + self.assertEqual((current_dir / "module/gradle.lockfile").read_text(encoding="utf-8"), current_content) + + if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/update-gradle-dependencies.yaml b/.github/workflows/update-gradle-dependencies.yaml index 001c5b38d4e..a62af69cce1 100644 --- a/.github/workflows/update-gradle-dependencies.yaml +++ b/.github/workflows/update-gradle-dependencies.yaml @@ -8,6 +8,8 @@ jobs: update-gradle-dependencies: runs-on: ubuntu-latest name: Update Gradle dependencies + env: + MIN_DEPENDENCY_AGE_HOURS: 48 permissions: contents: read id-token: write # Required for OIDC token federation @@ -41,6 +43,11 @@ jobs: echo "core_branch=ci/update-gradle-dependencies-${DATE}" >> $GITHUB_OUTPUT echo "instrumentation_branch=ci/update-gradle-dependencies-instrumentation-${DATE}" >> $GITHUB_OUTPUT + - name: Snapshot current Gradle lock files + run: | + mkdir -p /tmp/gradle-lockfiles-before + find . -name 'gradle.lockfile' -exec cp --parents {} /tmp/gradle-lockfiles-before/ \; + - name: Update Gradle dependencies env: ORG_GRADLE_PROJECT_akkaRepositoryToken: ${{ secrets.AKKA_REPO_TOKEN }} @@ -49,6 +56,14 @@ jobs: GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx3G'" \ ./gradlew resolveAndLockAll --write-locks --parallel --stacktrace --no-daemon --max-workers=4 + - name: Validate changed lock files meet dependency age policy + run: | + python3 .github/scripts/dependency_age.py validate-lockfiles \ + --baseline-dir /tmp/gradle-lockfiles-before \ + --current-dir . \ + --min-age-hours "${MIN_DEPENDENCY_AGE_HOURS}" \ + --github-output "$GITHUB_OUTPUT" + - name: Save instrumentation lock files run: | mkdir -p /tmp/instrumentation-lockfiles