Skip to content

Commit 56503bd

Browse files
Ericson2314Mic92claude
committed
hydra-evaluator: Rewrite in Rust
Port hydra-evaluator from C++ to Rust, removing the last C++ executable from the project. The new implementation uses `sqlx` and `tokio`, matching the async patterns already established by hydra-queue-runner and hydra-builder. Build/packaging changes: - Remove all C++ dependencies from `subprojects/hydra/meson.build` - Move Rust packaging from `hydra-queue-runner/package.nix` to `subprojects/rust-package.nix` with named outputs (`queue_runner`, `builder`, `evaluator`) - Add `evaluatorExecutable` option to the NixOS web-app module, matching how the queue runner and builder are discovered via their own modules - Clean up stale references in `hydra-tests/meson.build` and `dev-shell.nix` `JobsetRow` uses `i32` for `lastCheckedTime` and `triggerTime` to match the `INT4` Postgres schema (`sqlx` refuses to decode `INT4` into `i64`). Post-eval DB updates are wrapped in a transaction, as in the C++ original. The DBI parser supports `sslmode`/`sslrootcert`/`sslcert`/ `sslkey` that `libpq` understood natively in the C++ version; unknown parameters are warned about rather than fatal. Satisfy the crate's own `#![deny(clippy::pedantic)]` gate: - use `fs_err` per workspace `disallowed_methods` policy - replace `as`-casts with `try_from` (schema is `INT4`, saturate at `i32::MAX`) - collapse nested `if let` chains Co-authored-by: Jörg Thalheim <[email protected]> Co-authored-by: Claude <[email protected]>
1 parent a11a46f commit 56503bd

19 files changed

Lines changed: 981 additions & 725 deletions

File tree

Cargo.lock

Lines changed: 18 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
[workspace]
22

3-
members = [ "subprojects/hydra-builder", "subprojects/hydra-queue-runner", "subprojects/crates/*" ]
3+
members = [
4+
"subprojects/hydra-builder",
5+
"subprojects/hydra-evaluator",
6+
"subprojects/hydra-queue-runner",
7+
"subprojects/crates/*",
8+
]
49
resolver = "2"
510

611
[workspace.package]

flake.nix

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,12 @@
6060
};
6161
hydra-linters = self'.callPackage ./subprojects/hydra-linters/package.nix {
6262
};
63-
hydra-queue-runner = self'.callPackage ./subprojects/hydra-queue-runner/package.nix {
63+
hydra-rust = self'.callPackage ./subprojects/rust-package.nix {
6464
inherit nixComponents;
6565
};
66+
hydra-queue-runner = self'.hydra-rust.queue_runner;
67+
hydra-builder = self'.hydra-rust.builder;
68+
hydra-evaluator = self'.hydra-rust.evaluator;
6669
});
6770

6871
treefmtConfig =

nixos-modules/default.nix

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ rec {
77
_file = ./default.nix;
88
imports = [ ./web-app.nix ];
99
services.hydra-dev.package = lib.mkDefault flakePackages.${pkgs.stdenv.hostPlatform.system}.hydra;
10+
services.hydra-dev.evaluatorExecutable = lib.mkDefault "${
11+
flakePackages.${pkgs.stdenv.hostPlatform.system}.hydra-evaluator
12+
}/bin/hydra-evaluator";
1013
};
1114

1215
postgresql = ./postgresql.nix;
@@ -28,7 +31,7 @@ rec {
2831
imports = [ ./linux-builder-module.nix ];
2932
services.hydra-queue-builder-dev.package =
3033
lib.mkDefault
31-
flakePackages.${pkgs.stdenv.hostPlatform.system}.hydra-queue-runner;
34+
flakePackages.${pkgs.stdenv.hostPlatform.system}.hydra-builder;
3235
};
3336

3437
darwin-builder =
@@ -38,7 +41,7 @@ rec {
3841
imports = [ ./darwin-builder-module.nix ];
3942
services.hydra-queue-builder-dev.package =
4043
lib.mkDefault
41-
flakePackages.${pkgs.stdenv.hostPlatform.system}.hydra-queue-runner;
44+
flakePackages.${pkgs.stdenv.hostPlatform.system}.hydra-builder;
4245
};
4346

4447
hydra =

nixos-modules/web-app.nix

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,11 @@ in
8282
description = "The Hydra package.";
8383
};
8484

85+
evaluatorExecutable = mkOption {
86+
type = types.path;
87+
description = "Path to the hydra-evaluator executable.";
88+
};
89+
8590
hydraURL = mkOption {
8691
type = types.str;
8792
description = ''
@@ -281,14 +286,17 @@ in
281286
];
282287
path = with pkgs; [
283288
hostname-debian
289+
# Because hydra-evaluator calls `hydra-eval-jobset`. If we
290+
# move that perl script into rust, then we can get rid of
291+
# this.
284292
cfg.package
285293
];
286294
environment = env // {
287295
HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator";
288296
};
289297
serviceConfig = {
290-
ExecStart = "@${cfg.package}/bin/hydra-evaluator hydra-evaluator";
291-
ExecStopPost = "${cfg.package}/bin/hydra-evaluator --unlock";
298+
ExecStart = "@${cfg.evaluatorExecutable} hydra-evaluator";
299+
ExecStopPost = "${cfg.evaluatorExecutable} --unlock";
292300
User = "hydra";
293301
Restart = "always";
294302
WorkingDirectory = baseDir;

packaging/dev-shell.nix

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ hydra.overrideAttrs (
6161
shellHook = ''
6262
pushd $(git rev-parse --show-toplevel) >/dev/null
6363
64-
PATH=$(pwd)/build/subprojects/hydra/hydra-evaluator:$(pwd)/subprojects/hydra/script:$PATH
64+
PATH=$(pwd)/subprojects/hydra/script:$PATH
6565
PERL5LIB=$(pwd)/subprojects/hydra/lib:$PERL5LIB
6666
export HYDRA_HOME="$(pwd)/subprojects/hydra/"
6767
mkdir -p .hydra-data

subprojects/crates/db/src/lib.rs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,23 @@ impl Database {
3636
})
3737
}
3838

39+
pub async fn new_with_options(
40+
options: sqlx::postgres::PgConnectOptions,
41+
max_connections: u32,
42+
) -> Result<Self, Error> {
43+
Ok(Self {
44+
pool: sqlx::postgres::PgPoolOptions::new()
45+
.max_connections(max_connections)
46+
.connect_with(options)
47+
.await?,
48+
})
49+
}
50+
51+
#[must_use]
52+
pub fn pool(&self) -> &sqlx::PgPool {
53+
&self.pool
54+
}
55+
3956
pub async fn get(&self) -> Result<Connection, Error> {
4057
let conn = self.pool.acquire().await?;
4158
Ok(Connection::new(conn))
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
[package]
2+
name = "hydra-evaluator"
3+
version.workspace = true
4+
edition = "2024"
5+
license = "GPL-3.0"
6+
rust-version.workspace = true
7+
8+
[dependencies]
9+
anyhow.workspace = true
10+
clap = { workspace = true, features = [ "derive" ] }
11+
fs-err.workspace = true
12+
futures.workspace = true
13+
sqlx = { workspace = true, features = [ "runtime-tokio", "tls-rustls-ring-webpki", "postgres" ] }
14+
tokio = { workspace = true, features = [ "full" ] }
15+
tokio-stream.workspace = true
16+
tracing.workspace = true
17+
18+
db = { path = "../crates/db" }
19+
hydra-tracing = { path = "../crates/tracing" }
Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
use std::collections::HashMap;
2+
3+
use anyhow::Context as _;
4+
use sqlx::postgres::{PgConnectOptions, PgSslMode};
5+
6+
#[derive(Debug)]
7+
pub(crate) struct HydraConfig {
8+
options: HashMap<String, String>,
9+
}
10+
11+
impl HydraConfig {
12+
pub(crate) fn load() -> Self {
13+
let mut options = HashMap::new();
14+
15+
let path = match std::env::var("HYDRA_CONFIG") {
16+
Ok(p) if !p.is_empty() => p,
17+
_ => return Self { options },
18+
};
19+
20+
let contents = match fs_err::read_to_string(&path) {
21+
Ok(c) => c,
22+
Err(e) => {
23+
tracing::warn!("could not read HYDRA_CONFIG at {path}: {e}");
24+
return Self { options };
25+
}
26+
};
27+
28+
for line in contents.lines() {
29+
// Strip comments
30+
let line = match line.find('#') {
31+
Some(pos) => &line[..pos],
32+
None => line,
33+
};
34+
let line = line.trim();
35+
36+
let Some(eq) = line.find('=') else {
37+
continue;
38+
};
39+
40+
let key = line[..eq].trim();
41+
let value = line[eq + 1..].trim();
42+
43+
if key.is_empty() {
44+
continue;
45+
}
46+
47+
options.insert(key.to_owned(), value.to_owned());
48+
}
49+
50+
Self { options }
51+
}
52+
53+
pub(crate) fn get_int(&self, key: &str, default: u64) -> u64 {
54+
self.options
55+
.get(key)
56+
.and_then(|v| v.parse().ok())
57+
.unwrap_or(default)
58+
}
59+
}
60+
61+
/// Parse a `HYDRA_DBI` environment variable into `PgConnectOptions`.
62+
///
63+
/// Accepts strings like `dbi:Pg:dbname=hydra;host=localhost;port=5432`.
64+
pub(crate) fn parse_hydra_dbi() -> anyhow::Result<PgConnectOptions> {
65+
let dbi = std::env::var("HYDRA_DBI").unwrap_or_else(|_| "dbi:Pg:dbname=hydra;".to_owned());
66+
parse_dbi(&dbi)
67+
}
68+
69+
fn parse_dbi(dbi: &str) -> anyhow::Result<PgConnectOptions> {
70+
let params = dbi
71+
.strip_prefix("dbi:Pg:")
72+
.or_else(|| dbi.strip_prefix("DBI:Pg:"))
73+
.context("$HYDRA_DBI does not denote a PostgreSQL database")?;
74+
75+
let mut opts = PgConnectOptions::new();
76+
77+
for pair in params.split(';').filter(|s| !s.is_empty()) {
78+
let (key, value) = pair
79+
.split_once('=')
80+
.with_context(|| format!("invalid DBI parameter: {pair}"))?;
81+
match key.trim() {
82+
"dbname" => opts = opts.database(value.trim()),
83+
"host" => opts = opts.host(value.trim()),
84+
"port" => {
85+
opts = opts.port(
86+
value
87+
.trim()
88+
.parse()
89+
.with_context(|| format!("invalid port: {value}"))?,
90+
);
91+
}
92+
"user" => opts = opts.username(value.trim()),
93+
"password" => opts = opts.password(value.trim()),
94+
"application_name" => opts = opts.application_name(value.trim()),
95+
// The C++ evaluator used libpq which understood these
96+
// natively; without explicit handling TLS-required
97+
// deployments would fail.
98+
"sslmode" => {
99+
let mode = match value.trim() {
100+
"disable" => PgSslMode::Disable,
101+
"allow" => PgSslMode::Allow,
102+
"prefer" => PgSslMode::Prefer,
103+
"require" => PgSslMode::Require,
104+
"verify-ca" => PgSslMode::VerifyCa,
105+
"verify-full" => PgSslMode::VerifyFull,
106+
v => anyhow::bail!("invalid sslmode: {v}"),
107+
};
108+
opts = opts.ssl_mode(mode);
109+
}
110+
"sslrootcert" => opts = opts.ssl_root_cert(value.trim()),
111+
"sslcert" => opts = opts.ssl_client_cert(value.trim()),
112+
"sslkey" => opts = opts.ssl_client_key(value.trim()),
113+
// Warn rather than bail on unknown parameters, to avoid
114+
// breaking on libpq keywords we haven't mapped yet.
115+
other => {
116+
tracing::warn!("ignoring unsupported DBI parameter: {other}");
117+
}
118+
}
119+
}
120+
121+
Ok(opts)
122+
}
123+
124+
#[cfg(test)]
125+
mod tests {
126+
use super::*;
127+
128+
#[test]
129+
fn parse_simple_dbi() {
130+
parse_dbi("dbi:Pg:dbname=hydra;host=localhost;port=5432").unwrap();
131+
}
132+
133+
#[test]
134+
fn parse_dbi_uppercase() {
135+
parse_dbi("DBI:Pg:dbname=testdb;").unwrap();
136+
}
137+
}

0 commit comments

Comments
 (0)