Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,6 @@ def _metric(self, item: dict, sample: TemplateSample) -> dict[str, float]:

fn_names_match = set(gt_fn_names) == set(pred_fn_name)
fn_name_accuracy_score = 1.0 if fn_names_match else 0.0
fn_name_and_args_accuracy_score = 0.0

try:
pred_fn_args_by_name: dict[str, dict] = {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,6 @@ def _fetch_all_pages_page_number(
This is used for endpoints with `default_pagination` in the stainless config.
"""
all_items: list[Any] = []
page_num = 1
total_pages_count = 0
total_results = 0
original_page_size = None
Expand Down
4 changes: 2 additions & 2 deletions packages/nmp_common/tests/secrets/test_secret_key.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,12 +124,12 @@ def test_secret_key_encryptor_config_validation():
# Invalid key (too short)
short_key = base64.b64encode(SHORT_TEST_KEY).decode()
with pytest.raises(ValueError):
config = SecretKeyEncryptorConfig(value=short_key)
SecretKeyEncryptorConfig(value=short_key)

# Invalid base64
invalid_base64_key = "not-a-valid-base64-string"
with pytest.raises(ValueError):
config = SecretKeyEncryptorConfig(value=invalid_base64_key)
SecretKeyEncryptorConfig(value=invalid_base64_key)


def test_secret_key_encryptor_config_from_env(monkeypatch):
Expand Down
35 changes: 16 additions & 19 deletions packages/nmp_testing/src/nmp/testing/e2e/evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import json
import os
from dataclasses import dataclass
from typing import cast

from nemo_platform import NeMoPlatform
from nemo_platform.types.evaluation import AggregatedMetricResult, MetricEvaluationJob, RowScore
Expand Down Expand Up @@ -100,23 +101,17 @@ def get_job_outputs(
job = sdk.evaluation.metric_jobs.retrieve(job_name, workspace=workspace)
job_status = sdk.evaluation.metric_jobs.get_status(job_name, workspace=workspace)

# Verify file download is functional
# Smoke-test the generic results.download endpoint (status + body readable).
# Content is validated below via the typed aggregate_scores/row_scores accessors.
results = sdk.evaluation.metric_jobs.results.list(job_name, workspace=workspace)
for result in results.data:
if result.name == AGGREGATE_SCORES_RESULT:
if result.name in (AGGREGATE_SCORES_RESULT, ROW_SCORES_RESULT):
Comment thread
marcusds marked this conversation as resolved.
response = sdk.evaluation.metric_jobs.results.download(
result.name,
job=job_name,
workspace=workspace,
)
aggregate_scores = json.loads(response.read())
elif result.name == ROW_SCORES_RESULT:
response = sdk.evaluation.metric_jobs.results.download(
result.name,
job=job_name,
workspace=workspace,
)
row_scores = [json.loads(line) for line in response.read().decode().strip().split("\n") if line.strip()]
response.read()

# Verify result entity is registered
job_result = sdk.evaluation.metric_job_results.retrieve(job_name, workspace=workspace)
Expand Down Expand Up @@ -168,7 +163,8 @@ def verify_job_completed_successfully(
f"Expect 100% progress completed for successful jobs: {outputs.job_status}"
)
if require_samples_processed:
assert outputs.job_status.status_details.get("samples_processed", 0) > 0, (
samples_processed = cast(int, outputs.job_status.status_details.get("samples_processed", 0))
assert samples_processed > 0, (
f"Expect samples_processed for custom jobs and industry metrics with limit_samples: {outputs.job_status}"
)

Expand All @@ -180,7 +176,8 @@ def verify_job_completed_successfully(
len_row_scores = len(outputs.row_scores)
assert len_row_scores > 0, "Row scores should not be empty"
if require_samples_processed:
assert outputs.job_status.status_details.get("samples_processed", 0) >= len_row_scores, (
samples_processed = cast(int, outputs.job_status.status_details.get("samples_processed", 0))
assert samples_processed >= len_row_scores, (
f"Expect samples_processed to match number of rows: {outputs.job_status}"
)

Expand Down Expand Up @@ -241,14 +238,14 @@ def create_dataset_fileset(
filename: str = "dataset.json",
) -> str:
"""Create a fileset with optional dataset schema metadata and upload rows."""
create_kwargs = {
"workspace": workspace,
"name": fileset_name,
}
if schema is not None:
create_kwargs["metadata"] = {"dataset": {"schema": schema}}

sdk.files.filesets.create(**create_kwargs)
sdk.files.filesets.create(
workspace=workspace,
name=fileset_name,
metadata={"dataset": {"schema": schema}},
)
else:
sdk.files.filesets.create(workspace=workspace, name=fileset_name)
return upload_dataset_to_fileset(sdk, workspace, fileset_name, rows, filename=filename)


Expand Down
35 changes: 35 additions & 0 deletions plugins/nemo-auditor/src/nemo_auditor/sdk_resources/_parent.py
Comment thread
marcusds marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Parent protocols for auditor SDK sub-resources.

Sub-resources (``configs``, ``targets``) only need their parent's HTTP client
and URL builder. Typing against these protocols instead of importing the
concrete ``AuditorPluginResource`` classes keeps the import graph acyclic.
"""

from __future__ import annotations

from typing import Protocol

import httpx


class AuditorResourceParent(Protocol):
"""Sync parent surface used by sub-resources."""

_http_client: httpx.Client

def _url(self, path: str) -> str:
"""Build the absolute request URL for ``path``."""
raise NotImplementedError


class AsyncAuditorResourceParent(Protocol):
"""Async parent surface used by sub-resources."""

_http_client: httpx.AsyncClient

def _url(self, path: str) -> str:
"""Build the absolute request URL for ``path``."""
raise NotImplementedError
10 changes: 3 additions & 7 deletions plugins/nemo-auditor/src/nemo_auditor/sdk_resources/configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@

from __future__ import annotations

from typing import TYPE_CHECKING

from nemo_auditor.api.v2.schemas import CreateAuditConfigRequest, UpdateAuditConfigRequest
from nemo_auditor.entities import (
AuditConfig,
Expand All @@ -20,9 +18,7 @@
AuditRunData,
AuditSystemData,
)

if TYPE_CHECKING:
from nemo_auditor.sdk import AsyncAuditorPluginResource, AuditorPluginResource
from nemo_auditor.sdk_resources._parent import AsyncAuditorResourceParent, AuditorResourceParent


def _build_create_body(
Expand Down Expand Up @@ -66,7 +62,7 @@ def _build_update_body(
class _ConfigResource:
"""Sync ``configs`` sub-resource — five CRUD verbs."""

def __init__(self, parent: AuditorPluginResource) -> None:
def __init__(self, parent: AuditorResourceParent) -> None:
self._parent = parent

def create(
Expand Down Expand Up @@ -152,7 +148,7 @@ def delete(self, *, workspace: str, name: str) -> None:
class _AsyncConfigResource:
"""Async ``configs`` sub-resource — mirrors :class:`_ConfigResource`."""

def __init__(self, parent: AsyncAuditorPluginResource) -> None:
def __init__(self, parent: AsyncAuditorResourceParent) -> None:
self._parent = parent

async def create(
Expand Down
10 changes: 4 additions & 6 deletions plugins/nemo-auditor/src/nemo_auditor/sdk_resources/targets.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,11 @@

from __future__ import annotations

from typing import TYPE_CHECKING, Any
from typing import Any

from nemo_auditor.api.v2.schemas import CreateAuditTargetRequest, UpdateAuditTargetRequest
from nemo_auditor.entities import AuditTarget

if TYPE_CHECKING:
from nemo_auditor.sdk import AsyncAuditorPluginResource, AuditorPluginResource
from nemo_auditor.sdk_resources._parent import AsyncAuditorResourceParent, AuditorResourceParent


def _build_create_body(
Expand Down Expand Up @@ -56,7 +54,7 @@ def _build_update_body(
class _TargetResource:
"""Sync ``targets`` sub-resource — five CRUD verbs."""

def __init__(self, parent: AuditorPluginResource) -> None:
def __init__(self, parent: AuditorResourceParent) -> None:
self._parent = parent

def create(
Expand Down Expand Up @@ -138,7 +136,7 @@ def delete(self, *, workspace: str, name: str) -> None:
class _AsyncTargetResource:
"""Async ``targets`` sub-resource — mirrors :class:`_TargetResource`."""

def __init__(self, parent: AsyncAuditorPluginResource) -> None:
def __init__(self, parent: AsyncAuditorResourceParent) -> None:
self._parent = parent

async def create(
Expand Down
3 changes: 1 addition & 2 deletions script/openapi_helper/openapi_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -887,8 +887,7 @@ def fix_schema(

except Exception as e:
print_verbose(f"Error: {str(e)}", style="bold red")
raise e
raise typer.Exit(1)
raise typer.Exit(1) from e


def rename_schema_references(spec: dict, old_name: str, new_name: str) -> None:
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 3 additions & 1 deletion services/core/jobs/tests/test_jobs_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ async def test_hello_world_jobs_list(test_client: AsyncClient):
assert response.status_code == 201, f"POST failed: {response.status_code} {response.text}"

response = await test_client.get("/apis/jobs/v2/workspaces/default/hello-world/jobs")
data = response.json()
assert response.status_code == 200, f"GET failed: {response.status_code} {response.text}"

response = await test_client.get("/apis/jobs/v2/workspaces/default/jobs")
data = response.json()
Expand Down Expand Up @@ -317,6 +317,7 @@ async def test_hello_world_jobs_list(test_client: AsyncClient):
"ownership": {"user": "fake-user", "service": "fake-ms"},
},
)
assert response.status_code == 201, f"POST failed: {response.status_code} {response.text}"
response = await test_client.get("/apis/jobs/v2/workspaces/default/jobs")
data = response.json()
assert len(data["data"]) == 2
Expand All @@ -341,6 +342,7 @@ async def test_hello_world_jobs_list(test_client: AsyncClient):
"ownership": {"user": "fake-user", "service": "fake-ms"},
},
)
assert response.status_code == 201, f"POST failed: {response.status_code} {response.text}"
response = await test_client.get("/apis/jobs/v2/workspaces/default/jobs")
data = response.json()
assert len(data["data"]) == 3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -473,10 +473,7 @@ def detect_mamba_config_from_cfg(
return detect_hybrid_mamba_via_introspection(cfg, n_layers, is_trusted)

# Priority 4: Pure Mamba model
if has_ssm_config:
return detect_pure_mamba_from_cfg(cfg, n_layers)

return None
return detect_pure_mamba_from_cfg(cfg, n_layers)


def _has_config_json(pretrained_or_path: str) -> bool:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
ModelSpec,
MoEConfig,
)
from pydantic import ValidationError

# =================================================================================================
# PYDANTIC MODEL TESTS
Expand Down Expand Up @@ -383,8 +384,8 @@ def test_precision_in_model_config_serialization():
def test_precision_required():
"""Test that precision field is required and must be provided."""
# Test that precision is required
try:
config = ModelSpec(
with pytest.raises(ValidationError):
ModelSpec(
checkpoint_model_name="test-model",
family="test",
num_layers=32,
Expand All @@ -398,10 +399,6 @@ def test_precision_required():
base_num_parameters=7_000_000_000,
# precision not provided - should fail
)
assert False, "Expected validation error for missing precision field"
except Exception:
# Expected - precision is required
pass

# Test that precision can be provided
config = ModelSpec(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,6 @@ def _run_head_with_driver(self, driver_script: str, driver_args: list[str]) -> i
Returns:
Exit code from driver execution
"""
exit_code = 1
try:
if not self._start_head_background():
raise RuntimeError("Failed to start Ray head node")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ async def to_dataset(dataset: str | URN | Dataset | None) -> Dataset:
return Dataset.model_validate(await response.json())
else:
raise ValueError(f"Unsupported dataset type: {type(dataset)}")
return dataset


def extract_path(dataset_files_url: str) -> str:
Expand Down
Loading