From d4784bbad1451840815f2282e42f771e96ff7cb4 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Mon, 7 Jul 2025 07:36:30 +0000 Subject: [PATCH 01/50] Add script to export config schema, add to pre commit hook --- .pre-commit-config.yaml | 7 ++++++- src/script/export_config_schema.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 src/script/export_config_schema.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e76ece1ac8..7857fc9e09 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,8 +24,13 @@ repos: types: [python] require_serial: true + - id: export-config-schema + name: export config schema + language: system + entry: python src/script/export_config_schema.py > config_schema.json + - repo: https://github.com/norwoodj/helm-docs - rev: "" + rev: "" hooks: - id: helm-docs-container args: diff --git a/src/script/export_config_schema.py b/src/script/export_config_schema.py new file mode 100644 index 0000000000..c002723839 --- /dev/null +++ b/src/script/export_config_schema.py @@ -0,0 +1,11 @@ +import json + +from blueapi.config import ApplicationConfig + + +def print_application_config_json_schema(): + print(json.dumps(ApplicationConfig.model_json_schema())) + + +if __name__ == "__main__": + print_application_config_json_schema() From 38e6599d02f964ff8f77867dca9a045380735260 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Mon, 7 Jul 2025 08:19:49 +0000 Subject: [PATCH 02/50] Change config_schema.json output location, add helm-schema to pre-commit --- .pre-commit-config.yaml | 22 +++++++++++++++++++++- src/script/export_config_schema.py | 4 +++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7857fc9e09..952ba8188c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,8 @@ repos: - id: export-config-schema name: export config schema language: system - entry: python src/script/export_config_schema.py > config_schema.json + entry: python + args: [src/script/export_config_schema.py] - repo: https://github.com/norwoodj/helm-docs rev: "" @@ -36,3 +37,22 @@ repos: args: # Make the tool search for charts only under the `helm` directory - --chart-search-root=helm + + - repo: https://github.com/dadav/helm-schema + rev: 0.18.1 + hooks: + - id: helm-schema + # for all available options: helm-schema -h + args: + # directory to search recursively within for charts + - --chart-search-root=./helm/blueapi + + # don't analyze dependencies + - --no-dependencies + + # add references to values file if not exist + - --add-schema-reference + + # list of fields to skip from being created by default + # e.g. generate a relatively permissive schema + # - "--skip-auto-generation=required,additionalProperties" diff --git a/src/script/export_config_schema.py b/src/script/export_config_schema.py index c002723839..8b76118f7b 100644 --- a/src/script/export_config_schema.py +++ b/src/script/export_config_schema.py @@ -4,7 +4,9 @@ def print_application_config_json_schema(): - print(json.dumps(ApplicationConfig.model_json_schema())) + # print(json.dumps(ApplicationConfig.model_json_schema())) + with open("helm/blueapi/config_schema.json", "w") as file: + file.write(json.dumps(ApplicationConfig.model_json_schema())) if __name__ == "__main__": From 8b6b32561bf393af117898fc71c4c29390d3b26b Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Mon, 7 Jul 2025 08:20:37 +0000 Subject: [PATCH 03/50] Add jsonschema --- helm/blueapi/jsonschema | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 helm/blueapi/jsonschema diff --git a/helm/blueapi/jsonschema b/helm/blueapi/jsonschema new file mode 100644 index 0000000000..d8f1065068 --- /dev/null +++ b/helm/blueapi/jsonschema @@ -0,0 +1,3 @@ +# @schema +# $ref: ./config_schema.json +# @schema From bc35cae8719a0227ca6d5583e42868b0adb1252c Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Mon, 7 Jul 2025 14:25:14 +0000 Subject: [PATCH 04/50] Switch helm schema generator to helm-values-schema-generator --- .pre-commit-config.yaml | 19 +++---------------- helm/blueapi/.schema.yaml | 23 +++++++++++++++++++++++ helm/blueapi/jsonschema | 3 --- 3 files changed, 26 insertions(+), 19 deletions(-) create mode 100644 helm/blueapi/.schema.yaml delete mode 100644 helm/blueapi/jsonschema diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 952ba8188c..945d6358d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,21 +38,8 @@ repos: # Make the tool search for charts only under the `helm` directory - --chart-search-root=helm - - repo: https://github.com/dadav/helm-schema - rev: 0.18.1 + - repo: https://github.com/losisin/helm-values-schema-json + rev: v1.7.2 hooks: - id: helm-schema - # for all available options: helm-schema -h - args: - # directory to search recursively within for charts - - --chart-search-root=./helm/blueapi - - # don't analyze dependencies - - --no-dependencies - - # add references to values file if not exist - - --add-schema-reference - - # list of fields to skip from being created by default - # e.g. generate a relatively permissive schema - # - "--skip-auto-generation=required,additionalProperties" + args: ["--values", "values.yaml"] diff --git a/helm/blueapi/.schema.yaml b/helm/blueapi/.schema.yaml new file mode 100644 index 0000000000..ee759d877b --- /dev/null +++ b/helm/blueapi/.schema.yaml @@ -0,0 +1,23 @@ +# .schema.yaml + +values: + - values.yaml + +draft: 2020 +indent: 4 +output: values.schema.json + +bundle: true +bundleRoot: . +bundleWithoutID: false + +useHelmDocs: false + +noAdditionalProperties: false + +schemaRoot: + id: https://example.com/schema + title: Helm Values Schema + description: Schema for Helm values + additionalProperties: true + ref: config_schema.json diff --git a/helm/blueapi/jsonschema b/helm/blueapi/jsonschema deleted file mode 100644 index d8f1065068..0000000000 --- a/helm/blueapi/jsonschema +++ /dev/null @@ -1,3 +0,0 @@ -# @schema -# $ref: ./config_schema.json -# @schema From fda27f6c2205612acd4a1cf46ce6a2401d0f2a47 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 08:03:28 +0000 Subject: [PATCH 05/50] Annotate worker with schema ref, remove additionalProperties --- helm/blueapi/.schema.yaml | 3 +-- helm/blueapi/values.yaml | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm/blueapi/.schema.yaml b/helm/blueapi/.schema.yaml index ee759d877b..5f452a4333 100644 --- a/helm/blueapi/.schema.yaml +++ b/helm/blueapi/.schema.yaml @@ -19,5 +19,4 @@ schemaRoot: id: https://example.com/schema title: Helm Values Schema description: Schema for Helm values - additionalProperties: true - ref: config_schema.json + additionalProperties: false diff --git a/helm/blueapi/values.yaml b/helm/blueapi/values.yaml index 242a0dcdc6..d136edc460 100644 --- a/helm/blueapi/values.yaml +++ b/helm/blueapi/values.yaml @@ -170,12 +170,13 @@ tracing: host: http://opentelemetry-collector.tracing port: 4318 +# @schema $ref: config_schema.json # -- Config for the worker goes here, will be mounted into a config file worker: api: # -- 0.0.0.0 required to allow non-loopback traffic # If using hostNetwork, the port must be free on the host - url: http://0.0.0.0:8000/ + url: http://0.0.0.0:8000/ env: # -- modules (must be installed in the venv) to fetch devices/plans from sources: From 23e1d44f5d8b8d7b394fbddaddc0aa0504dd6db0 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 08:59:46 +0000 Subject: [PATCH 06/50] Add install helma-values-schema-json plugin as step in Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 2abb29bc31..370215baef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/s chmod 700 get_helm.sh; \ ./get_helm.sh; \ rm get_helm.sh +RUN helm plugin install https://github.com/losisin/helm-values-schema-json.git # Set up a virtual environment and put it in PATH RUN python -m venv /venv From 4d1c83deae373f25d530599f65622ca83443d8d9 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 09:15:31 +0000 Subject: [PATCH 07/50] Add accidentally removed whitespace --- helm/blueapi/config_schema.json | 1 + helm/blueapi/values.schema.json | 820 ++++++++++++++++++++++++++++++++ helm/blueapi/values.yaml | 2 +- 3 files changed, 822 insertions(+), 1 deletion(-) create mode 100644 helm/blueapi/config_schema.json create mode 100644 helm/blueapi/values.schema.json diff --git a/helm/blueapi/config_schema.json b/helm/blueapi/config_schema.json new file mode 100644 index 0000000000..4fea1eb91e --- /dev/null +++ b/helm/blueapi/config_schema.json @@ -0,0 +1 @@ +{"$defs": {"BasicAuthentication": {"additionalProperties": false, "description": "User credentials for basic authentication", "properties": {"username": {"description": "Unique identifier for user", "title": "Username", "type": "string"}, "password": {"description": "Password to verify user's identity", "title": "Password", "type": "string"}}, "required": ["username", "password"], "title": "BasicAuthentication", "type": "object"}, "CORSConfig": {"additionalProperties": false, "properties": {"origins": {"items": {"type": "string"}, "title": "Origins", "type": "array"}, "allow_credentials": {"default": false, "title": "Allow Credentials", "type": "boolean"}, "allow_methods": {"default": ["*"], "items": {"type": "string"}, "title": "Allow Methods", "type": "array"}, "allow_headers": {"default": ["*"], "items": {"type": "string"}, "title": "Allow Headers", "type": "array"}}, "required": ["origins"], "title": "CORSConfig", "type": "object"}, "EnvironmentConfig": {"additionalProperties": false, "description": "Config for the RunEngine environment", "properties": {"sources": {"default": [{"kind": "planFunctions", "module": "dodal.plans"}, {"kind": "planFunctions", "module": "dodal.plan_stubs.wrapped"}], "items": {"$ref": "#/$defs/Source"}, "title": "Sources", "type": "array"}, "events": {"$ref": "#/$defs/WorkerEventConfig"}, "metadata": {"anyOf": [{"$ref": "#/$defs/MetadataConfig"}, {"type": "null"}], "default": null}}, "title": "EnvironmentConfig", "type": "object"}, "GraylogConfig": {"additionalProperties": false, "properties": {"enabled": {"default": false, "title": "Enabled", "type": "boolean"}, "url": {"default": "tcp://localhost:5555", "format": "uri", "minLength": 1, "title": "Url", "type": "string"}}, "title": "GraylogConfig", "type": "object"}, "LoggingConfig": {"additionalProperties": false, "properties": {"level": {"default": "INFO", "enum": ["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], "title": "Level", "type": "string"}, "graylog": {"$ref": "#/$defs/GraylogConfig", "default": {"enabled": false, "url": "tcp://localhost:5555"}}}, "title": "LoggingConfig", "type": "object"}, "MetadataConfig": {"additionalProperties": false, "properties": {"instrument_session": {"title": "Instrument Session", "type": "string"}, "instrument": {"title": "Instrument", "type": "string"}}, "required": ["instrument_session", "instrument"], "title": "MetadataConfig", "type": "object"}, "NumtrackerConfig": {"additionalProperties": false, "properties": {"url": {"default": "http://localhost:8002/graphql", "format": "uri", "maxLength": 2083, "minLength": 1, "title": "Url", "type": "string"}}, "title": "NumtrackerConfig", "type": "object"}, "OIDCConfig": {"additionalProperties": false, "properties": {"well_known_url": {"description": "URL to fetch OIDC config from the provider", "title": "Well Known Url", "type": "string"}, "client_id": {"description": "Client ID", "title": "Client Id", "type": "string"}, "client_audience": {"default": "blueapi", "description": "Client Audience(s)", "title": "Client Audience", "type": "string"}}, "required": ["well_known_url", "client_id"], "title": "OIDCConfig", "type": "object"}, "RestConfig": {"additionalProperties": false, "properties": {"url": {"default": "http://localhost:8000/", "format": "uri", "maxLength": 2083, "minLength": 1, "title": "Url", "type": "string"}, "cors": {"anyOf": [{"$ref": "#/$defs/CORSConfig"}, {"type": "null"}], "default": null}}, "title": "RestConfig", "type": "object"}, "ScratchConfig": {"additionalProperties": false, "properties": {"root": {"default": "/tmp/scratch/blueapi", "description": "The root directory of the scratch area, all repositories will be cloned under this directory.", "format": "path", "title": "Root", "type": "string"}, "required_gid": {"anyOf": [{"type": "integer"}, {"type": "null"}], "default": null, "description": "\nRequired owner GID for the scratch directory. If supplied, the setup-scratch\ncommand will check the scratch area ownership and raise an error if it is\nnot owned by , or if it does not have SGID permission bit set.\n", "title": "Required Gid"}, "repositories": {"description": "Details of repositories to be cloned and imported into blueapi", "items": {"$ref": "#/$defs/ScratchRepository"}, "title": "Repositories", "type": "array"}}, "title": "ScratchConfig", "type": "object"}, "ScratchRepository": {"additionalProperties": false, "properties": {"name": {"default": "example", "description": "Unique name for this repository in the scratch directory", "title": "Name", "type": "string"}, "remote_url": {"default": "https://github.com/example/example.git", "description": "URL to clone from", "title": "Remote Url", "type": "string"}}, "title": "ScratchRepository", "type": "object"}, "Source": {"additionalProperties": false, "properties": {"kind": {"$ref": "#/$defs/SourceKind"}, "module": {"anyOf": [{"format": "path", "type": "string"}, {"type": "string"}], "title": "Module"}}, "required": ["kind", "module"], "title": "Source", "type": "object"}, "SourceKind": {"enum": ["planFunctions", "deviceFunctions", "dodal"], "title": "SourceKind", "type": "string"}, "StompConfig": {"additionalProperties": false, "description": "Config for connecting to stomp broker", "properties": {"enabled": {"default": false, "description": "True if blueapi should connect to stomp for asynchronous event publishing", "title": "Enabled", "type": "boolean"}, "url": {"default": "tcp://localhost:61613", "format": "uri", "minLength": 1, "title": "Url", "type": "string"}, "auth": {"anyOf": [{"$ref": "#/$defs/BasicAuthentication"}, {"type": "null"}], "default": null, "description": "Auth information for communicating with STOMP broker, if required"}}, "title": "StompConfig", "type": "object"}, "WorkerEventConfig": {"additionalProperties": false, "description": "Config for event broadcasting via the message bus", "properties": {"broadcast_status_events": {"default": true, "title": "Broadcast Status Events", "type": "boolean"}}, "title": "WorkerEventConfig", "type": "object"}}, "additionalProperties": false, "description": "Config for the worker application as a whole. Root of\nconfig tree.", "properties": {"stomp": {"$ref": "#/$defs/StompConfig"}, "env": {"$ref": "#/$defs/EnvironmentConfig"}, "logging": {"$ref": "#/$defs/LoggingConfig"}, "api": {"$ref": "#/$defs/RestConfig"}, "scratch": {"anyOf": [{"$ref": "#/$defs/ScratchConfig"}, {"type": "null"}], "default": null}, "oidc": {"anyOf": [{"$ref": "#/$defs/OIDCConfig"}, {"type": "null"}], "default": null}, "auth_token_path": {"anyOf": [{"format": "path", "type": "string"}, {"type": "null"}], "default": null, "title": "Auth Token Path"}, "numtracker": {"anyOf": [{"$ref": "#/$defs/NumtrackerConfig"}, {"type": "null"}], "default": null}}, "title": "ApplicationConfig", "type": "object"} \ No newline at end of file diff --git a/helm/blueapi/values.schema.json b/helm/blueapi/values.schema.json new file mode 100644 index 0000000000..1f5658faac --- /dev/null +++ b/helm/blueapi/values.schema.json @@ -0,0 +1,820 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/schema", + "title": "Helm Values Schema", + "description": "Schema for Helm values", + "type": "object", + "properties": { + "affinity": { + "type": "object" + }, + "debug": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "extraEnvVars": { + "type": "array" + }, + "fullnameOverride": { + "type": "string" + }, + "hostNetwork": { + "type": "boolean" + }, + "image": { + "type": "object", + "properties": { + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "imagePullSecrets": { + "type": "array" + }, + "ingress": { + "type": "object", + "properties": { + "annotations": { + "type": "object" + }, + "className": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "hosts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "host": { + "type": "string" + }, + "paths": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "pathType": { + "type": "string" + } + } + } + } + } + } + }, + "tls": { + "type": "array" + } + } + }, + "initContainer": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "persistentVolume": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "existingClaimName": { + "type": "string" + } + } + } + } + }, + "livenessProbe": { + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "port": { + "type": "string" + } + } + }, + "periodSeconds": { + "type": "integer" + } + } + }, + "nameOverride": { + "type": "string" + }, + "nodeSelector": { + "type": "object" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "podSecurityContext": { + "type": "object" + }, + "readinessProbe": { + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "port": { + "type": "string" + } + } + }, + "periodSeconds": { + "type": "integer" + } + } + }, + "resources": { + "type": "object", + "properties": { + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + } + }, + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + } + } + } + }, + "restartOnConfigChange": { + "type": "boolean" + }, + "securityContext": { + "type": "object", + "properties": { + "runAsNonRoot": { + "type": "boolean" + }, + "runAsUser": { + "type": "integer" + } + } + }, + "service": { + "type": "object", + "properties": { + "port": { + "type": "integer" + }, + "type": { + "type": "string" + } + } + }, + "serviceAccount": { + "type": "object", + "properties": { + "annotations": { + "type": "object" + }, + "automount": { + "type": "boolean" + }, + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + } + }, + "startupProbe": { + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "port": { + "type": "string" + } + } + }, + "periodSeconds": { + "type": "integer" + } + } + }, + "tolerations": { + "type": "array" + }, + "tracing": { + "type": "object", + "properties": { + "otlp": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "protocol": { + "type": "string" + }, + "server": { + "type": "object", + "properties": { + "host": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + } + } + } + } + }, + "volumeMounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "mountPath": { + "type": "string" + }, + "name": { + "type": "string" + }, + "readOnly": { + "type": "boolean" + } + } + } + }, + "volumes": { + "type": "array" + }, + "worker": { + "$ref": "config_schema.json", + "type": "object", + "properties": { + "api": { + "type": "object", + "properties": { + "url": { + "type": "string" + } + } + }, + "env": { + "type": "object", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "module": { + "type": "string" + } + } + } + } + } + }, + "logging": { + "type": "object", + "properties": { + "graylog": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "url": { + "type": "string" + } + } + }, + "level": { + "type": "string" + } + } + }, + "scratch": { + "type": "object", + "properties": { + "repositories": { + "type": "array" + }, + "root": { + "type": "string" + } + } + }, + "stomp": { + "type": "object", + "properties": { + "auth": { + "type": "object", + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, + "enabled": { + "type": "boolean" + }, + "url": { + "type": "string" + } + } + } + } + } + }, + "additionalProperties": false, + "$defs": { + "config_schema.json": { + "$id": "config_schema.json", + "title": "ApplicationConfig", + "description": "Config for the worker application as a whole. Root of\nconfig tree.", + "type": "object", + "properties": { + "api": { + "$ref": "#/$defs/RestConfig" + }, + "auth_token_path": { + "title": "Auth Token Path", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "env": { + "$ref": "#/$defs/EnvironmentConfig" + }, + "logging": { + "$ref": "#/$defs/LoggingConfig" + }, + "numtracker": { + "anyOf": [ + { + "$ref": "#/$defs/NumtrackerConfig" + }, + { + "type": "null" + } + ] + }, + "oidc": { + "anyOf": [ + { + "$ref": "#/$defs/OIDCConfig" + }, + { + "type": "null" + } + ] + }, + "scratch": { + "anyOf": [ + { + "$ref": "#/$defs/ScratchConfig" + }, + { + "type": "null" + } + ] + }, + "stomp": { + "$ref": "#/$defs/StompConfig" + } + }, + "additionalProperties": false, + "$defs": { + "BasicAuthentication": { + "title": "BasicAuthentication", + "description": "User credentials for basic authentication", + "type": "object", + "required": [ + "username", + "password" + ], + "properties": { + "password": { + "title": "Password", + "description": "Password to verify user's identity", + "type": "string" + }, + "username": { + "title": "Username", + "description": "Unique identifier for user", + "type": "string" + } + }, + "additionalProperties": false + }, + "CORSConfig": { + "title": "CORSConfig", + "type": "object", + "required": [ + "origins" + ], + "properties": { + "allow_credentials": { + "title": "Allow Credentials", + "default": false, + "type": "boolean" + }, + "allow_headers": { + "title": "Allow Headers", + "default": [ + "*" + ], + "type": "array", + "items": { + "type": "string" + } + }, + "allow_methods": { + "title": "Allow Methods", + "default": [ + "*" + ], + "type": "array", + "items": { + "type": "string" + } + }, + "origins": { + "title": "Origins", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "EnvironmentConfig": { + "title": "EnvironmentConfig", + "description": "Config for the RunEngine environment", + "type": "object", + "properties": { + "events": { + "$ref": "#/$defs/WorkerEventConfig" + }, + "metadata": { + "anyOf": [ + { + "$ref": "#/$defs/MetadataConfig" + }, + { + "type": "null" + } + ] + }, + "sources": { + "title": "Sources", + "default": [ + { + "kind": "planFunctions", + "module": "dodal.plans" + }, + { + "kind": "planFunctions", + "module": "dodal.plan_stubs.wrapped" + } + ], + "type": "array", + "items": { + "$ref": "#/$defs/Source" + } + } + }, + "additionalProperties": false + }, + "GraylogConfig": { + "title": "GraylogConfig", + "type": "object", + "properties": { + "enabled": { + "title": "Enabled", + "default": false, + "type": "boolean" + }, + "url": { + "title": "Url", + "default": "tcp://localhost:5555", + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "LoggingConfig": { + "title": "LoggingConfig", + "type": "object", + "properties": { + "graylog": { + "default": { + "enabled": false, + "url": "tcp://localhost:5555" + }, + "$ref": "#/$defs/GraylogConfig" + }, + "level": { + "title": "Level", + "default": "INFO", + "type": "string", + "enum": [ + "NOTSET", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ] + } + }, + "additionalProperties": false + }, + "MetadataConfig": { + "title": "MetadataConfig", + "type": "object", + "required": [ + "instrument_session", + "instrument" + ], + "properties": { + "instrument": { + "title": "Instrument", + "type": "string" + }, + "instrument_session": { + "title": "Instrument Session", + "type": "string" + } + }, + "additionalProperties": false + }, + "NumtrackerConfig": { + "title": "NumtrackerConfig", + "type": "object", + "properties": { + "url": { + "title": "Url", + "default": "http://localhost:8002/graphql", + "type": "string", + "maxLength": 2083, + "minLength": 1 + } + }, + "additionalProperties": false + }, + "OIDCConfig": { + "title": "OIDCConfig", + "type": "object", + "required": [ + "well_known_url", + "client_id" + ], + "properties": { + "client_audience": { + "title": "Client Audience", + "description": "Client Audience(s)", + "default": "blueapi", + "type": "string" + }, + "client_id": { + "title": "Client Id", + "description": "Client ID", + "type": "string" + }, + "well_known_url": { + "title": "Well Known Url", + "description": "URL to fetch OIDC config from the provider", + "type": "string" + } + }, + "additionalProperties": false + }, + "RestConfig": { + "title": "RestConfig", + "type": "object", + "properties": { + "cors": { + "anyOf": [ + { + "$ref": "#/$defs/CORSConfig" + }, + { + "type": "null" + } + ] + }, + "url": { + "title": "Url", + "default": "http://localhost:8000/", + "type": "string", + "maxLength": 2083, + "minLength": 1 + } + }, + "additionalProperties": false + }, + "ScratchConfig": { + "title": "ScratchConfig", + "type": "object", + "properties": { + "repositories": { + "title": "Repositories", + "description": "Details of repositories to be cloned and imported into blueapi", + "type": "array", + "items": { + "$ref": "#/$defs/ScratchRepository" + } + }, + "required_gid": { + "title": "Required Gid", + "description": "\nRequired owner GID for the scratch directory. If supplied, the setup-scratch\ncommand will check the scratch area ownership and raise an error if it is\nnot owned by \u003cGID\u003e, or if it does not have SGID permission bit set.\n", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "root": { + "title": "Root", + "description": "The root directory of the scratch area, all repositories will be cloned under this directory.", + "default": "/tmp/scratch/blueapi", + "type": "string" + } + }, + "additionalProperties": false + }, + "ScratchRepository": { + "title": "ScratchRepository", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "Unique name for this repository in the scratch directory", + "default": "example", + "type": "string" + }, + "remote_url": { + "title": "Remote Url", + "description": "URL to clone from", + "default": "https://github.com/example/example.git", + "type": "string" + } + }, + "additionalProperties": false + }, + "Source": { + "title": "Source", + "type": "object", + "required": [ + "kind", + "module" + ], + "properties": { + "kind": { + "$ref": "#/$defs/SourceKind" + }, + "module": { + "title": "Module", + "anyOf": [ + { + "type": "string" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + "SourceKind": { + "title": "SourceKind", + "type": "string", + "enum": [ + "planFunctions", + "deviceFunctions", + "dodal" + ] + }, + "StompConfig": { + "title": "StompConfig", + "description": "Config for connecting to stomp broker", + "type": "object", + "properties": { + "auth": { + "description": "Auth information for communicating with STOMP broker, if required", + "anyOf": [ + { + "$ref": "#/$defs/BasicAuthentication" + }, + { + "type": "null" + } + ] + }, + "enabled": { + "title": "Enabled", + "description": "True if blueapi should connect to stomp for asynchronous event publishing", + "default": false, + "type": "boolean" + }, + "url": { + "title": "Url", + "default": "tcp://localhost:61613", + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "WorkerEventConfig": { + "title": "WorkerEventConfig", + "description": "Config for event broadcasting via the message bus", + "type": "object", + "properties": { + "broadcast_status_events": { + "title": "Broadcast Status Events", + "default": true, + "type": "boolean" + } + }, + "additionalProperties": false + } + } + } + } +} diff --git a/helm/blueapi/values.yaml b/helm/blueapi/values.yaml index d136edc460..47b8296c3d 100644 --- a/helm/blueapi/values.yaml +++ b/helm/blueapi/values.yaml @@ -176,7 +176,7 @@ worker: api: # -- 0.0.0.0 required to allow non-loopback traffic # If using hostNetwork, the port must be free on the host - url: http://0.0.0.0:8000/ + url: http://0.0.0.0:8000/ env: # -- modules (must be installed in the venv) to fetch devices/plans from sources: From 627b307fad3a2ed1b1c516d43458a7e2cf58b717 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 09:50:34 +0000 Subject: [PATCH 08/50] Add config-schema option to cli --- src/blueapi/cli/cli.py | 24 ++++++++++++++++++++++++ src/blueapi/config.py | 4 ++++ 2 files changed, 28 insertions(+) diff --git a/src/blueapi/cli/cli.py b/src/blueapi/cli/cli.py index afc94c01ef..50d2c769b4 100644 --- a/src/blueapi/cli/cli.py +++ b/src/blueapi/cli/cli.py @@ -106,6 +106,30 @@ def schema(output: Path | None = None, update: bool = False) -> None: print_schema_as_yaml(schema) +@click.option( + "-o", "--output", type=Path, help="Path to file to save the config schema" +) +@click.option( + "-u", + "--update", + type=bool, + is_flag=True, + help="[Development only] update the config schema in the documentation", +) +@main.command(name="config-schema") +def config_schema(output: Path | None = None, update: bool = False) -> None: + """Generates a json schema of the ApplicationConfig pydantic basemodel""" + schema = ApplicationConfig.model_json_schema() + + if update: + output = config.CONFIG_SCHEMA_LOCATION + if output is not None: + with open(output, "w") as file: + json.dump(schema, file) + else: + print(json.dumps(schema)) + + @main.command(name="serve") @click.pass_obj def start_application(obj: dict): diff --git a/src/blueapi/config.py b/src/blueapi/config.py index 4ac57347b6..f95894e313 100644 --- a/src/blueapi/config.py +++ b/src/blueapi/config.py @@ -28,6 +28,10 @@ FORBIDDEN_OWN_REMOTE_URL = "https://github.com/DiamondLightSource/blueapi.git" +CONFIG_SCHEMA_LOCATION = ( + Path(__file__).parents[2] / "helm" / "blueapi" / "config_schema.json" +) + def _expand_env(loader: yaml.Loader, node: yaml.ScalarNode) -> str: value = loader.construct_scalar(node) From aa3e82a6b585ec345affc9419ccc29ca9baa5e46 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 09:51:41 +0000 Subject: [PATCH 09/50] Change pre commit to use blueapi config-schema rather than custom script --- .pre-commit-config.yaml | 4 ++-- src/script/export_config_schema.py | 13 ------------- 2 files changed, 2 insertions(+), 15 deletions(-) delete mode 100644 src/script/export_config_schema.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 945d6358d0..12fdfd761a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,8 +27,8 @@ repos: - id: export-config-schema name: export config schema language: system - entry: python - args: [src/script/export_config_schema.py] + entry: blueapi + args: [config-schema, -u] - repo: https://github.com/norwoodj/helm-docs rev: "" diff --git a/src/script/export_config_schema.py b/src/script/export_config_schema.py deleted file mode 100644 index 8b76118f7b..0000000000 --- a/src/script/export_config_schema.py +++ /dev/null @@ -1,13 +0,0 @@ -import json - -from blueapi.config import ApplicationConfig - - -def print_application_config_json_schema(): - # print(json.dumps(ApplicationConfig.model_json_schema())) - with open("helm/blueapi/config_schema.json", "w") as file: - file.write(json.dumps(ApplicationConfig.model_json_schema())) - - -if __name__ == "__main__": - print_application_config_json_schema() From 817fce303f6c1c86ecf56bfbfb1f63592be48c83 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 10:11:27 +0000 Subject: [PATCH 10/50] Fix broken export-config-schema hook --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 12fdfd761a..4dc291e420 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,10 +25,10 @@ repos: require_serial: true - id: export-config-schema + language: python name: export config schema - language: system - entry: blueapi - args: [config-schema, -u] + entry: blueapi config-schema -u + pass_filenames: false - repo: https://github.com/norwoodj/helm-docs rev: "" From ec4948cc5772712687581fbf613ffe654004658e Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 10:15:49 +0000 Subject: [PATCH 11/50] Add file dependencies to export-config-schema --- .pre-commit-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4dc291e420..bf8501115e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,9 +25,10 @@ repos: require_serial: true - id: export-config-schema - language: python name: export config schema entry: blueapi config-schema -u + language: python + files: "src/blueapi/config.py|src/blueapi/utils/*" pass_filenames: false - repo: https://github.com/norwoodj/helm-docs From 5cdd60fbf365bad4e42c010ebe93dabd2005f134 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 12:40:16 +0000 Subject: [PATCH 12/50] Add github action to install relevant helm plugin, invoke in _test.yml and _tox.yml --- .github/actions/install_helm_plugins/action.yml | 9 +++++++++ .github/workflows/_test.yml | 3 +++ .github/workflows/_tox.yml | 5 ++++- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 .github/actions/install_helm_plugins/action.yml diff --git a/.github/actions/install_helm_plugins/action.yml b/.github/actions/install_helm_plugins/action.yml new file mode 100644 index 0000000000..033d2f0994 --- /dev/null +++ b/.github/actions/install_helm_plugins/action.yml @@ -0,0 +1,9 @@ +name: Install helm plugins +description: Installs helm-values-schema-json plugin + +runs: + using: composite + steps: + - name: Install helm-values-schema-json + run: helm plugin install https://github.com/losisin/helm-values-schema-json.git + shell: bash diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index ff5c2aea30..f580d8f691 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -53,6 +53,9 @@ jobs: with: python-version: ${{ inputs.python-version }} pip-install: ".[dev]" + + - name: Install helm plugins + uses: ./.github/actions/install_helm_plugins - name: Run tests run: tox -e tests diff --git a/.github/workflows/_tox.yml b/.github/workflows/_tox.yml index ad74c9acb7..a95ce8cdd7 100644 --- a/.github/workflows/_tox.yml +++ b/.github/workflows/_tox.yml @@ -17,6 +17,9 @@ jobs: - name: Install python packages uses: ./.github/actions/install_requirements - + + - name: Install helm plugins + uses: ./.github/actions/install_helm_plugins + - name: Run tox run: tox -e ${{ inputs.tox }} From ee81b5939f95fb39bdd47a92cca92a6eae307d1d Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 12:52:25 +0000 Subject: [PATCH 13/50] Make config-schema pretty print to file --- helm/blueapi/config_schema.json | 440 +++++++++++++++++++++++++++++++- src/blueapi/cli/cli.py | 3 +- 2 files changed, 441 insertions(+), 2 deletions(-) diff --git a/helm/blueapi/config_schema.json b/helm/blueapi/config_schema.json index 4fea1eb91e..3ea380b111 100644 --- a/helm/blueapi/config_schema.json +++ b/helm/blueapi/config_schema.json @@ -1 +1,439 @@ -{"$defs": {"BasicAuthentication": {"additionalProperties": false, "description": "User credentials for basic authentication", "properties": {"username": {"description": "Unique identifier for user", "title": "Username", "type": "string"}, "password": {"description": "Password to verify user's identity", "title": "Password", "type": "string"}}, "required": ["username", "password"], "title": "BasicAuthentication", "type": "object"}, "CORSConfig": {"additionalProperties": false, "properties": {"origins": {"items": {"type": "string"}, "title": "Origins", "type": "array"}, "allow_credentials": {"default": false, "title": "Allow Credentials", "type": "boolean"}, "allow_methods": {"default": ["*"], "items": {"type": "string"}, "title": "Allow Methods", "type": "array"}, "allow_headers": {"default": ["*"], "items": {"type": "string"}, "title": "Allow Headers", "type": "array"}}, "required": ["origins"], "title": "CORSConfig", "type": "object"}, "EnvironmentConfig": {"additionalProperties": false, "description": "Config for the RunEngine environment", "properties": {"sources": {"default": [{"kind": "planFunctions", "module": "dodal.plans"}, {"kind": "planFunctions", "module": "dodal.plan_stubs.wrapped"}], "items": {"$ref": "#/$defs/Source"}, "title": "Sources", "type": "array"}, "events": {"$ref": "#/$defs/WorkerEventConfig"}, "metadata": {"anyOf": [{"$ref": "#/$defs/MetadataConfig"}, {"type": "null"}], "default": null}}, "title": "EnvironmentConfig", "type": "object"}, "GraylogConfig": {"additionalProperties": false, "properties": {"enabled": {"default": false, "title": "Enabled", "type": "boolean"}, "url": {"default": "tcp://localhost:5555", "format": "uri", "minLength": 1, "title": "Url", "type": "string"}}, "title": "GraylogConfig", "type": "object"}, "LoggingConfig": {"additionalProperties": false, "properties": {"level": {"default": "INFO", "enum": ["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], "title": "Level", "type": "string"}, "graylog": {"$ref": "#/$defs/GraylogConfig", "default": {"enabled": false, "url": "tcp://localhost:5555"}}}, "title": "LoggingConfig", "type": "object"}, "MetadataConfig": {"additionalProperties": false, "properties": {"instrument_session": {"title": "Instrument Session", "type": "string"}, "instrument": {"title": "Instrument", "type": "string"}}, "required": ["instrument_session", "instrument"], "title": "MetadataConfig", "type": "object"}, "NumtrackerConfig": {"additionalProperties": false, "properties": {"url": {"default": "http://localhost:8002/graphql", "format": "uri", "maxLength": 2083, "minLength": 1, "title": "Url", "type": "string"}}, "title": "NumtrackerConfig", "type": "object"}, "OIDCConfig": {"additionalProperties": false, "properties": {"well_known_url": {"description": "URL to fetch OIDC config from the provider", "title": "Well Known Url", "type": "string"}, "client_id": {"description": "Client ID", "title": "Client Id", "type": "string"}, "client_audience": {"default": "blueapi", "description": "Client Audience(s)", "title": "Client Audience", "type": "string"}}, "required": ["well_known_url", "client_id"], "title": "OIDCConfig", "type": "object"}, "RestConfig": {"additionalProperties": false, "properties": {"url": {"default": "http://localhost:8000/", "format": "uri", "maxLength": 2083, "minLength": 1, "title": "Url", "type": "string"}, "cors": {"anyOf": [{"$ref": "#/$defs/CORSConfig"}, {"type": "null"}], "default": null}}, "title": "RestConfig", "type": "object"}, "ScratchConfig": {"additionalProperties": false, "properties": {"root": {"default": "/tmp/scratch/blueapi", "description": "The root directory of the scratch area, all repositories will be cloned under this directory.", "format": "path", "title": "Root", "type": "string"}, "required_gid": {"anyOf": [{"type": "integer"}, {"type": "null"}], "default": null, "description": "\nRequired owner GID for the scratch directory. If supplied, the setup-scratch\ncommand will check the scratch area ownership and raise an error if it is\nnot owned by , or if it does not have SGID permission bit set.\n", "title": "Required Gid"}, "repositories": {"description": "Details of repositories to be cloned and imported into blueapi", "items": {"$ref": "#/$defs/ScratchRepository"}, "title": "Repositories", "type": "array"}}, "title": "ScratchConfig", "type": "object"}, "ScratchRepository": {"additionalProperties": false, "properties": {"name": {"default": "example", "description": "Unique name for this repository in the scratch directory", "title": "Name", "type": "string"}, "remote_url": {"default": "https://github.com/example/example.git", "description": "URL to clone from", "title": "Remote Url", "type": "string"}}, "title": "ScratchRepository", "type": "object"}, "Source": {"additionalProperties": false, "properties": {"kind": {"$ref": "#/$defs/SourceKind"}, "module": {"anyOf": [{"format": "path", "type": "string"}, {"type": "string"}], "title": "Module"}}, "required": ["kind", "module"], "title": "Source", "type": "object"}, "SourceKind": {"enum": ["planFunctions", "deviceFunctions", "dodal"], "title": "SourceKind", "type": "string"}, "StompConfig": {"additionalProperties": false, "description": "Config for connecting to stomp broker", "properties": {"enabled": {"default": false, "description": "True if blueapi should connect to stomp for asynchronous event publishing", "title": "Enabled", "type": "boolean"}, "url": {"default": "tcp://localhost:61613", "format": "uri", "minLength": 1, "title": "Url", "type": "string"}, "auth": {"anyOf": [{"$ref": "#/$defs/BasicAuthentication"}, {"type": "null"}], "default": null, "description": "Auth information for communicating with STOMP broker, if required"}}, "title": "StompConfig", "type": "object"}, "WorkerEventConfig": {"additionalProperties": false, "description": "Config for event broadcasting via the message bus", "properties": {"broadcast_status_events": {"default": true, "title": "Broadcast Status Events", "type": "boolean"}}, "title": "WorkerEventConfig", "type": "object"}}, "additionalProperties": false, "description": "Config for the worker application as a whole. Root of\nconfig tree.", "properties": {"stomp": {"$ref": "#/$defs/StompConfig"}, "env": {"$ref": "#/$defs/EnvironmentConfig"}, "logging": {"$ref": "#/$defs/LoggingConfig"}, "api": {"$ref": "#/$defs/RestConfig"}, "scratch": {"anyOf": [{"$ref": "#/$defs/ScratchConfig"}, {"type": "null"}], "default": null}, "oidc": {"anyOf": [{"$ref": "#/$defs/OIDCConfig"}, {"type": "null"}], "default": null}, "auth_token_path": {"anyOf": [{"format": "path", "type": "string"}, {"type": "null"}], "default": null, "title": "Auth Token Path"}, "numtracker": {"anyOf": [{"$ref": "#/$defs/NumtrackerConfig"}, {"type": "null"}], "default": null}}, "title": "ApplicationConfig", "type": "object"} \ No newline at end of file +{ + "$defs": { + "BasicAuthentication": { + "additionalProperties": false, + "description": "User credentials for basic authentication", + "properties": { + "username": { + "description": "Unique identifier for user", + "title": "Username", + "type": "string" + }, + "password": { + "description": "Password to verify user's identity", + "title": "Password", + "type": "string" + } + }, + "required": [ + "username", + "password" + ], + "title": "BasicAuthentication", + "type": "object" + }, + "CORSConfig": { + "additionalProperties": false, + "properties": { + "origins": { + "items": { + "type": "string" + }, + "title": "Origins", + "type": "array" + }, + "allow_credentials": { + "default": false, + "title": "Allow Credentials", + "type": "boolean" + }, + "allow_methods": { + "default": [ + "*" + ], + "items": { + "type": "string" + }, + "title": "Allow Methods", + "type": "array" + }, + "allow_headers": { + "default": [ + "*" + ], + "items": { + "type": "string" + }, + "title": "Allow Headers", + "type": "array" + } + }, + "required": [ + "origins" + ], + "title": "CORSConfig", + "type": "object" + }, + "EnvironmentConfig": { + "additionalProperties": false, + "description": "Config for the RunEngine environment", + "properties": { + "sources": { + "default": [ + { + "kind": "planFunctions", + "module": "dodal.plans" + }, + { + "kind": "planFunctions", + "module": "dodal.plan_stubs.wrapped" + } + ], + "items": { + "$ref": "#/$defs/Source" + }, + "title": "Sources", + "type": "array" + }, + "events": { + "$ref": "#/$defs/WorkerEventConfig" + }, + "metadata": { + "anyOf": [ + { + "$ref": "#/$defs/MetadataConfig" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "EnvironmentConfig", + "type": "object" + }, + "GraylogConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + }, + "url": { + "default": "tcp://localhost:5555", + "format": "uri", + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "title": "GraylogConfig", + "type": "object" + }, + "LoggingConfig": { + "additionalProperties": false, + "properties": { + "level": { + "default": "INFO", + "enum": [ + "NOTSET", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "title": "Level", + "type": "string" + }, + "graylog": { + "$ref": "#/$defs/GraylogConfig", + "default": { + "enabled": false, + "url": "tcp://localhost:5555" + } + } + }, + "title": "LoggingConfig", + "type": "object" + }, + "MetadataConfig": { + "additionalProperties": false, + "properties": { + "instrument_session": { + "title": "Instrument Session", + "type": "string" + }, + "instrument": { + "title": "Instrument", + "type": "string" + } + }, + "required": [ + "instrument_session", + "instrument" + ], + "title": "MetadataConfig", + "type": "object" + }, + "NumtrackerConfig": { + "additionalProperties": false, + "properties": { + "url": { + "default": "http://localhost:8002/graphql", + "format": "uri", + "maxLength": 2083, + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "title": "NumtrackerConfig", + "type": "object" + }, + "OIDCConfig": { + "additionalProperties": false, + "properties": { + "well_known_url": { + "description": "URL to fetch OIDC config from the provider", + "title": "Well Known Url", + "type": "string" + }, + "client_id": { + "description": "Client ID", + "title": "Client Id", + "type": "string" + }, + "client_audience": { + "default": "blueapi", + "description": "Client Audience(s)", + "title": "Client Audience", + "type": "string" + } + }, + "required": [ + "well_known_url", + "client_id" + ], + "title": "OIDCConfig", + "type": "object" + }, + "RestConfig": { + "additionalProperties": false, + "properties": { + "url": { + "default": "http://localhost:8000/", + "format": "uri", + "maxLength": 2083, + "minLength": 1, + "title": "Url", + "type": "string" + }, + "cors": { + "anyOf": [ + { + "$ref": "#/$defs/CORSConfig" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "RestConfig", + "type": "object" + }, + "ScratchConfig": { + "additionalProperties": false, + "properties": { + "root": { + "default": "/tmp/scratch/blueapi", + "description": "The root directory of the scratch area, all repositories will be cloned under this directory.", + "format": "path", + "title": "Root", + "type": "string" + }, + "required_gid": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "\nRequired owner GID for the scratch directory. If supplied, the setup-scratch\ncommand will check the scratch area ownership and raise an error if it is\nnot owned by , or if it does not have SGID permission bit set.\n", + "title": "Required Gid" + }, + "repositories": { + "description": "Details of repositories to be cloned and imported into blueapi", + "items": { + "$ref": "#/$defs/ScratchRepository" + }, + "title": "Repositories", + "type": "array" + } + }, + "title": "ScratchConfig", + "type": "object" + }, + "ScratchRepository": { + "additionalProperties": false, + "properties": { + "name": { + "default": "example", + "description": "Unique name for this repository in the scratch directory", + "title": "Name", + "type": "string" + }, + "remote_url": { + "default": "https://github.com/example/example.git", + "description": "URL to clone from", + "title": "Remote Url", + "type": "string" + } + }, + "title": "ScratchRepository", + "type": "object" + }, + "Source": { + "additionalProperties": false, + "properties": { + "kind": { + "$ref": "#/$defs/SourceKind" + }, + "module": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "string" + } + ], + "title": "Module" + } + }, + "required": [ + "kind", + "module" + ], + "title": "Source", + "type": "object" + }, + "SourceKind": { + "enum": [ + "planFunctions", + "deviceFunctions", + "dodal" + ], + "title": "SourceKind", + "type": "string" + }, + "StompConfig": { + "additionalProperties": false, + "description": "Config for connecting to stomp broker", + "properties": { + "enabled": { + "default": false, + "description": "True if blueapi should connect to stomp for asynchronous event publishing", + "title": "Enabled", + "type": "boolean" + }, + "url": { + "default": "tcp://localhost:61613", + "format": "uri", + "minLength": 1, + "title": "Url", + "type": "string" + }, + "auth": { + "anyOf": [ + { + "$ref": "#/$defs/BasicAuthentication" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Auth information for communicating with STOMP broker, if required" + } + }, + "title": "StompConfig", + "type": "object" + }, + "WorkerEventConfig": { + "additionalProperties": false, + "description": "Config for event broadcasting via the message bus", + "properties": { + "broadcast_status_events": { + "default": true, + "title": "Broadcast Status Events", + "type": "boolean" + } + }, + "title": "WorkerEventConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Config for the worker application as a whole. Root of\nconfig tree.", + "properties": { + "stomp": { + "$ref": "#/$defs/StompConfig" + }, + "env": { + "$ref": "#/$defs/EnvironmentConfig" + }, + "logging": { + "$ref": "#/$defs/LoggingConfig" + }, + "api": { + "$ref": "#/$defs/RestConfig" + }, + "scratch": { + "anyOf": [ + { + "$ref": "#/$defs/ScratchConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "oidc": { + "anyOf": [ + { + "$ref": "#/$defs/OIDCConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "auth_token_path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auth Token Path" + }, + "numtracker": { + "anyOf": [ + { + "$ref": "#/$defs/NumtrackerConfig" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "ApplicationConfig", + "type": "object" +} diff --git a/src/blueapi/cli/cli.py b/src/blueapi/cli/cli.py index 50d2c769b4..41fe225fdb 100644 --- a/src/blueapi/cli/cli.py +++ b/src/blueapi/cli/cli.py @@ -125,7 +125,8 @@ def config_schema(output: Path | None = None, update: bool = False) -> None: output = config.CONFIG_SCHEMA_LOCATION if output is not None: with open(output, "w") as file: - json.dump(schema, file) + json.dump(schema, file, indent=4) + file.write("\n") else: print(json.dumps(schema)) From 04d76a7a79d7ffb3c1a6deb81db73aa8e2f68782 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 13:14:10 +0000 Subject: [PATCH 14/50] Add empty initResources object to inform schema --- helm/blueapi/values.schema.json | 3 +++ helm/blueapi/values.yaml | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/helm/blueapi/values.schema.json b/helm/blueapi/values.schema.json index 1f5658faac..e7bd6f2e5a 100644 --- a/helm/blueapi/values.schema.json +++ b/helm/blueapi/values.schema.json @@ -103,6 +103,9 @@ } } }, + "initResources": { + "type": "object" + }, "livenessProbe": { "type": "object", "properties": { diff --git a/helm/blueapi/values.yaml b/helm/blueapi/values.yaml index 47b8296c3d..a747d7b950 100644 --- a/helm/blueapi/values.yaml +++ b/helm/blueapi/values.yaml @@ -94,6 +94,10 @@ resources: requests: cpu: 200m memory: 400Mi + + +# -- Override resources for init container. By default copies resources of main container. +initResources: {} # This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ # -- Liveness probe, if configured kubernetes will kill the pod and start a new one if failed consecutively. From 42eeea8e7b11589cacb41f401765bee7e077b317 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 13:16:00 +0000 Subject: [PATCH 15/50] Update docstring --- src/blueapi/cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blueapi/cli/cli.py b/src/blueapi/cli/cli.py index 41fe225fdb..661272178c 100644 --- a/src/blueapi/cli/cli.py +++ b/src/blueapi/cli/cli.py @@ -118,7 +118,7 @@ def schema(output: Path | None = None, update: bool = False) -> None: ) @main.command(name="config-schema") def config_schema(output: Path | None = None, update: bool = False) -> None: - """Generates a json schema of the ApplicationConfig pydantic basemodel""" + """Generates a json schema from the ApplicationConfig pydantic basemodel""" schema = ApplicationConfig.model_json_schema() if update: From df3b8a199a3965b7d92e13113f39399818d8e4ef Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 13:28:05 +0000 Subject: [PATCH 16/50] Fix export-config-schema regex (hopefully..) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bf8501115e..4366e57507 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: name: export config schema entry: blueapi config-schema -u language: python - files: "src/blueapi/config.py|src/blueapi/utils/*" + files: "src\/blueapi\/config.py|src\/blueapi\/utils\/" pass_filenames: false - repo: https://github.com/norwoodj/helm-docs From d12d18afc47dba298389eee9634de4d1d4ecb843 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 14:01:59 +0000 Subject: [PATCH 17/50] Update helm README.md --- helm/blueapi/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/helm/blueapi/README.md b/helm/blueapi/README.md index 8a77201e94..3de42135f5 100644 --- a/helm/blueapi/README.md +++ b/helm/blueapi/README.md @@ -22,6 +22,7 @@ A Helm chart deploying a worker pod that runs Bluesky plans | initContainer | object | `{"enabled":false,"persistentVolume":{"enabled":false,"existingClaimName":""}}` | Configure the initContainer that checks out the scratch configuration repositories | | initContainer.persistentVolume.enabled | bool | `false` | Whether to use a persistent volume in the cluster or check out onto the mounted host filesystem If persistentVolume.enabled: False, mounts scratch.root as scratch.root in the container | | initContainer.persistentVolume.existingClaimName | string | `""` | May be set to an existing persistent volume claim to re-use the volume, else a new one is created for each blueapi release | +| initResources | object | `{}` | Override resources for init container. By default copies resources of main container. | | livenessProbe | object | `{"failureThreshold":3,"httpGet":{"path":"/healthz","port":"http"},"periodSeconds":10}` | Liveness probe, if configured kubernetes will kill the pod and start a new one if failed consecutively. This is automatically disabled when in debug mode. | | nameOverride | string | `""` | | | nodeSelector | object | `{}` | May be required to run on specific nodes (e.g. the control machine) | @@ -51,5 +52,3 @@ A Helm chart deploying a worker pod that runs Bluesky plans | worker.scratch | object | `{"repositories":[],"root":"/blueapi-plugins/scratch"}` | If initContainer is enabled the default branch of python projects in this section are installed into the venv *without their dependencies* | | worker.stomp | object | `{"auth":{"password":"guest","username":"guest"},"enabled":false,"url":"tcp://rabbitmq:61613/"}` | Message bus configuration for returning status to GDA/forwarding documents downstream Password may be in the form ${ENV_VAR} to be fetched from an environment variable e.g. mounted from a SealedSecret | ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) From e5a4181c9c00e022fb1eef7e9d51236f54ee05fb Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 14:03:38 +0000 Subject: [PATCH 18/50] Change helm-docs-container to helm-docs-built --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e76ece1ac8..5ea7ba363d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: - repo: https://github.com/norwoodj/helm-docs rev: "" hooks: - - id: helm-docs-container + - id: helm-docs-built args: # Make the tool search for charts only under the `helm` directory - --chart-search-root=helm From 085954c1c0018bc6b4fa944cc90b10dcf0947fc0 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 14:38:15 +0000 Subject: [PATCH 19/50] Remove extra line at end of README.md --- helm/blueapi/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/helm/blueapi/README.md b/helm/blueapi/README.md index 3de42135f5..2c841a9556 100644 --- a/helm/blueapi/README.md +++ b/helm/blueapi/README.md @@ -51,4 +51,3 @@ A Helm chart deploying a worker pod that runs Bluesky plans | worker.logging | object | `{"graylog":{"enabled":false,"url":"tcp://graylog-log-target.diamond.ac.uk:12232/"},"level":"INFO"}` | Configures logging. Port 12231 is the `dodal` input on graylog which will be renamed `blueapi` | | worker.scratch | object | `{"repositories":[],"root":"/blueapi-plugins/scratch"}` | If initContainer is enabled the default branch of python projects in this section are installed into the venv *without their dependencies* | | worker.stomp | object | `{"auth":{"password":"guest","username":"guest"},"enabled":false,"url":"tcp://rabbitmq:61613/"}` | Message bus configuration for returning status to GDA/forwarding documents downstream Password may be in the form ${ENV_VAR} to be fetched from an environment variable e.g. mounted from a SealedSecret | - From 1bb6b87006996f49603e2617ea82daf3ccc8a941 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 14:39:55 +0000 Subject: [PATCH 20/50] Remove extra line at end of README.md --- helm/blueapi/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/helm/blueapi/README.md b/helm/blueapi/README.md index 8a77201e94..4c3129efdd 100644 --- a/helm/blueapi/README.md +++ b/helm/blueapi/README.md @@ -50,6 +50,3 @@ A Helm chart deploying a worker pod that runs Bluesky plans | worker.logging | object | `{"graylog":{"enabled":false,"url":"tcp://graylog-log-target.diamond.ac.uk:12232/"},"level":"INFO"}` | Configures logging. Port 12231 is the `dodal` input on graylog which will be renamed `blueapi` | | worker.scratch | object | `{"repositories":[],"root":"/blueapi-plugins/scratch"}` | If initContainer is enabled the default branch of python projects in this section are installed into the venv *without their dependencies* | | worker.stomp | object | `{"auth":{"password":"guest","username":"guest"},"enabled":false,"url":"tcp://rabbitmq:61613/"}` | Message bus configuration for returning status to GDA/forwarding documents downstream Password may be in the form ${ENV_VAR} to be fetched from an environment variable e.g. mounted from a SealedSecret | - ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.14.2](https://github.com/norwoodj/helm-docs/releases/v1.14.2) From c53086ae10de629a8ab2ce94d3299d2b324e0854 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 15:02:55 +0000 Subject: [PATCH 21/50] Exclude helm README.md from end-of-file-fixer --- .pre-commit-config.yaml | 1 + helm/blueapi/README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5ea7ba363d..4d316e88a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,7 @@ repos: exclude: ^helm\/.*\/templates\/.*|catalog-info.yaml - id: check-merge-conflict - id: end-of-file-fixer + exclude: "helm/blueapi/README.md" - repo: local hooks: diff --git a/helm/blueapi/README.md b/helm/blueapi/README.md index 4c3129efdd..ac28edc023 100644 --- a/helm/blueapi/README.md +++ b/helm/blueapi/README.md @@ -50,3 +50,4 @@ A Helm chart deploying a worker pod that runs Bluesky plans | worker.logging | object | `{"graylog":{"enabled":false,"url":"tcp://graylog-log-target.diamond.ac.uk:12232/"},"level":"INFO"}` | Configures logging. Port 12231 is the `dodal` input on graylog which will be renamed `blueapi` | | worker.scratch | object | `{"repositories":[],"root":"/blueapi-plugins/scratch"}` | If initContainer is enabled the default branch of python projects in this section are installed into the venv *without their dependencies* | | worker.stomp | object | `{"auth":{"password":"guest","username":"guest"},"enabled":false,"url":"tcp://rabbitmq:61613/"}` | Message bus configuration for returning status to GDA/forwarding documents downstream Password may be in the form ${ENV_VAR} to be fetched from an environment variable e.g. mounted from a SealedSecret | + From 8642a459fd4930bc08d6c9ea46dc8a3f1b89fc9d Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Tue, 8 Jul 2025 15:07:59 +0000 Subject: [PATCH 22/50] Fix regex --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d316e88a0..57781e5016 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: exclude: ^helm\/.*\/templates\/.*|catalog-info.yaml - id: check-merge-conflict - id: end-of-file-fixer - exclude: "helm/blueapi/README.md" + exclude: "helm\/blueapi\/README.md" - repo: local hooks: From 67c02d006f51cdc57a23b5a169dcac719ce1540e Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Wed, 9 Jul 2025 07:28:31 +0000 Subject: [PATCH 23/50] Regenerate README.md --- helm/blueapi/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/blueapi/README.md b/helm/blueapi/README.md index 2c841a9556..3de42135f5 100644 --- a/helm/blueapi/README.md +++ b/helm/blueapi/README.md @@ -51,3 +51,4 @@ A Helm chart deploying a worker pod that runs Bluesky plans | worker.logging | object | `{"graylog":{"enabled":false,"url":"tcp://graylog-log-target.diamond.ac.uk:12232/"},"level":"INFO"}` | Configures logging. Port 12231 is the `dodal` input on graylog which will be renamed `blueapi` | | worker.scratch | object | `{"repositories":[],"root":"/blueapi-plugins/scratch"}` | If initContainer is enabled the default branch of python projects in this section are installed into the venv *without their dependencies* | | worker.stomp | object | `{"auth":{"password":"guest","username":"guest"},"enabled":false,"url":"tcp://rabbitmq:61613/"}` | Message bus configuration for returning status to GDA/forwarding documents downstream Password may be in the form ${ENV_VAR} to be fetched from an environment variable e.g. mounted from a SealedSecret | + From 3fed2cf19c1058e6da8072d59833318ecc908ba8 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Wed, 9 Jul 2025 07:31:27 +0000 Subject: [PATCH 24/50] De-abstract installing helm plugins --- .github/actions/install_helm_plugins/action.yml | 9 --------- .github/workflows/_test.yml | 2 +- .github/workflows/_tox.yml | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) delete mode 100644 .github/actions/install_helm_plugins/action.yml diff --git a/.github/actions/install_helm_plugins/action.yml b/.github/actions/install_helm_plugins/action.yml deleted file mode 100644 index 033d2f0994..0000000000 --- a/.github/actions/install_helm_plugins/action.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: Install helm plugins -description: Installs helm-values-schema-json plugin - -runs: - using: composite - steps: - - name: Install helm-values-schema-json - run: helm plugin install https://github.com/losisin/helm-values-schema-json.git - shell: bash diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index f580d8f691..56fffde97f 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -55,7 +55,7 @@ jobs: pip-install: ".[dev]" - name: Install helm plugins - uses: ./.github/actions/install_helm_plugins + run: helm plugin install https://github.com/losisin/helm-values-schema-json.git - name: Run tests run: tox -e tests diff --git a/.github/workflows/_tox.yml b/.github/workflows/_tox.yml index a95ce8cdd7..94fe0e57f2 100644 --- a/.github/workflows/_tox.yml +++ b/.github/workflows/_tox.yml @@ -19,7 +19,7 @@ jobs: uses: ./.github/actions/install_requirements - name: Install helm plugins - uses: ./.github/actions/install_helm_plugins + run: helm plugin install https://github.com/losisin/helm-values-schema-json.git - name: Run tox run: tox -e ${{ inputs.tox }} From ced668ee966c475c2846614755e922e956c2c0fd Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Wed, 9 Jul 2025 07:58:06 +0000 Subject: [PATCH 25/50] Add test_config_schema_updated --- tests/unit_tests/test_config.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py index 1367e85a0a..885c8493ca 100644 --- a/tests/unit_tests/test_config.py +++ b/tests/unit_tests/test_config.py @@ -11,9 +11,16 @@ import responses import yaml from bluesky_stomp.models import BasicAuthentication +from deepdiff import DeepDiff +from prompt_toolkit import Application from pydantic import BaseModel, Field -from blueapi.config import ApplicationConfig, ConfigLoader, OIDCConfig +from blueapi.config import ( + CONFIG_SCHEMA_LOCATION, + ApplicationConfig, + ConfigLoader, + OIDCConfig, +) from blueapi.utils import InvalidConfigError @@ -500,3 +507,14 @@ def validate_field_annotations(model_class: Any, model_field: str) -> None: check_no_extra_fields(annotation) else: check_no_extra_fields(extracted_annotations) + + +@pytest.mark.skipif( + not CONFIG_SCHEMA_LOCATION.exists(), + reason="If the schema file does not exist, the test is being run" + " with a non-editable install", +) +def test_config_schema_updated() -> None: + with CONFIG_SCHEMA_LOCATION.open("r") as stream: + config_schema = json.load(stream) + assert DeepDiff(config_schema, ApplicationConfig.model_json_schema()) == {} From 89dbd270058ca655aa5298dc8f3f7a20696d602f Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Wed, 9 Jul 2025 07:58:50 +0000 Subject: [PATCH 26/50] Remove export-config-schema pre-commit hook --- .pre-commit-config.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e09882252e..fb12df76cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,13 +25,6 @@ repos: types: [python] require_serial: true - - id: export-config-schema - name: export config schema - entry: blueapi config-schema -u - language: python - files: "src\/blueapi\/config.py|src\/blueapi\/utils\/" - pass_filenames: false - - repo: https://github.com/norwoodj/helm-docs rev: "" hooks: From 33fc7a3eef2c981abaecf0ff3f15f6f39f9919f3 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Wed, 9 Jul 2025 07:59:38 +0000 Subject: [PATCH 27/50] Remove unused import --- tests/unit_tests/test_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py index 885c8493ca..7e7e942427 100644 --- a/tests/unit_tests/test_config.py +++ b/tests/unit_tests/test_config.py @@ -12,7 +12,6 @@ import yaml from bluesky_stomp.models import BasicAuthentication from deepdiff import DeepDiff -from prompt_toolkit import Application from pydantic import BaseModel, Field from blueapi.config import ( From dabb2f676dcaf03fd5a1160eb6b675de12c44b15 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Wed, 9 Jul 2025 08:09:13 +0000 Subject: [PATCH 28/50] Add exact path to values.yaml --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb12df76cf..3565c15ebb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,4 +37,4 @@ repos: rev: v1.7.2 hooks: - id: helm-schema - args: ["--values", "values.yaml"] + args: ["--values", "helm/blueapi/values.yaml"] From 425ddae800c6e0e209b09f74765ad1509ebcc98d Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Wed, 9 Jul 2025 08:14:28 +0000 Subject: [PATCH 29/50] Add fail message to test_config_schema_updated --- tests/unit_tests/test_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py index 7e7e942427..ad97d987c3 100644 --- a/tests/unit_tests/test_config.py +++ b/tests/unit_tests/test_config.py @@ -516,4 +516,7 @@ def validate_field_annotations(model_class: Any, model_field: str) -> None: def test_config_schema_updated() -> None: with CONFIG_SCHEMA_LOCATION.open("r") as stream: config_schema = json.load(stream) - assert DeepDiff(config_schema, ApplicationConfig.model_json_schema()) == {} + assert DeepDiff(config_schema, ApplicationConfig.model_json_schema()) == {}, ( + f"Config schema is out of date with schema at {CONFIG_SCHEMA_LOCATION}. \ + You may need to run `blueapi config-schema -u`" + ) From e135689aea7e3a82eac0d028927192a89199f9ba Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Wed, 9 Jul 2025 11:43:39 +0000 Subject: [PATCH 30/50] Refactor config_schema --- src/blueapi/cli/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/blueapi/cli/cli.py b/src/blueapi/cli/cli.py index 9bc23e8ea4..6468e2cca0 100644 --- a/src/blueapi/cli/cli.py +++ b/src/blueapi/cli/cli.py @@ -126,9 +126,9 @@ def config_schema(output: Path | None = None, update: bool = False) -> None: if update: output = config.CONFIG_SCHEMA_LOCATION if output is not None: - with open(output, "w") as file: - json.dump(schema, file, indent=4) - file.write("\n") + with output.open("w") as stream: + json.dump(schema, stream, indent=4) + stream.write("\n") else: print(json.dumps(schema)) From f62dc6a66ca02b9cd6009c96e91c6eed0437e493 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Thu, 10 Jul 2025 09:40:54 +0000 Subject: [PATCH 31/50] Reword error message --- tests/unit_tests/test_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py index ad97d987c3..a8eaaf1fad 100644 --- a/tests/unit_tests/test_config.py +++ b/tests/unit_tests/test_config.py @@ -517,6 +517,6 @@ def test_config_schema_updated() -> None: with CONFIG_SCHEMA_LOCATION.open("r") as stream: config_schema = json.load(stream) assert DeepDiff(config_schema, ApplicationConfig.model_json_schema()) == {}, ( - f"Config schema is out of date with schema at {CONFIG_SCHEMA_LOCATION}. \ - You may need to run `blueapi config-schema -u`" + f"ApplicationConfig model is out of date with schema at \ + {CONFIG_SCHEMA_LOCATION}. You may need to run `blueapi config-schema -u`" ) From 255154865ef5cc38b294a0012dbd1a47fe1cff0d Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Thu, 10 Jul 2025 09:41:18 +0000 Subject: [PATCH 32/50] Add test_config_schema --- tests/unit_tests/test_cli.py | 47 +++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index a403fb829b..c4000e5cac 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -17,6 +17,7 @@ from bluesky_stomp.messaging import StompClient from bluesky_stomp.models import MessageTopic from click.testing import CliRunner +from deepdiff import DeepDiff from opentelemetry import trace from ophyd_async.core import AsyncStatus from pydantic import BaseModel @@ -25,7 +26,7 @@ from stomp.connect import StompConnection11 as Connection from blueapi import __version__ -from blueapi.cli.cli import main +from blueapi.cli.cli import config_schema, main from blueapi.cli.format import OutputFormat, fmt_dict from blueapi.client.event_bus import BlueskyStreamingError from blueapi.client.rest import ( @@ -36,6 +37,7 @@ UnknownPlan, ) from blueapi.config import ( + CONFIG_SCHEMA_LOCATION, ApplicationConfig, ScratchConfig, ScratchRepository, @@ -1281,3 +1283,46 @@ def test_python_env_output_formatting(): """) _assert_matching_formatting(OutputFormat.FULL, empty_python_env, full) + + +@pytest.mark.parametrize("output_flag", [True, False]) +@pytest.mark.parametrize("update", [True, False]) +@patch("blueapi.cli.cli.config.CONFIG_SCHEMA_LOCATION") +def test_config_schema( + config_schema_location_mock: Mock, + runner: CliRunner, + output_flag: bool, + update: bool, + tmp_path: Path, +): + args = ["config-schema"] + + tmp_path = tmp_path / "foo.json" + + if output_flag: + # args.append(f"-o {output}") + args.append("-o") + args.append(f"{tmp_path}") + + if update: + args.append("-u") + + result = runner.invoke( + main, + args, + ) + + if output_flag and (not update): + with tmp_path.open("r") as stream: + assert ( + DeepDiff(json.load(stream), ApplicationConfig.model_json_schema()) == {} + ) + + elif update: + config_schema_location_mock.open.assert_called() + with config_schema_location_mock.open() as stream: + stream.write.assert_called() + + else: + with CONFIG_SCHEMA_LOCATION.open("r") as stream: + assert DeepDiff(json.loads(result.output), json.load(stream)) == {} From f5da1bd04a69c0247123cba452724d79d3899c61 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Thu, 10 Jul 2025 09:44:08 +0000 Subject: [PATCH 33/50] Fix merge issue --- helm/blueapi/README.md | 4 ---- tests/unit_tests/test_cli.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/helm/blueapi/README.md b/helm/blueapi/README.md index 560a717ce4..2c841a9556 100644 --- a/helm/blueapi/README.md +++ b/helm/blueapi/README.md @@ -51,7 +51,3 @@ A Helm chart deploying a worker pod that runs Bluesky plans | worker.logging | object | `{"graylog":{"enabled":false,"url":"tcp://graylog-log-target.diamond.ac.uk:12232/"},"level":"INFO"}` | Configures logging. Port 12231 is the `dodal` input on graylog which will be renamed `blueapi` | | worker.scratch | object | `{"repositories":[],"root":"/blueapi-plugins/scratch"}` | If initContainer is enabled the default branch of python projects in this section are installed into the venv *without their dependencies* | | worker.stomp | object | `{"auth":{"password":"guest","username":"guest"},"enabled":false,"url":"tcp://rabbitmq:61613/"}` | Message bus configuration for returning status to GDA/forwarding documents downstream Password may be in the form ${ENV_VAR} to be fetched from an environment variable e.g. mounted from a SealedSecret | -<<<<<<< HEAD - -======= ->>>>>>> main diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index c4000e5cac..8b0ed5a154 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -26,7 +26,7 @@ from stomp.connect import StompConnection11 as Connection from blueapi import __version__ -from blueapi.cli.cli import config_schema, main +from blueapi.cli.cli import main from blueapi.cli.format import OutputFormat, fmt_dict from blueapi.client.event_bus import BlueskyStreamingError from blueapi.client.rest import ( From c9b505485fba0932672583104ecc183f28ff32e4 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Thu, 10 Jul 2025 10:02:26 +0000 Subject: [PATCH 34/50] Remove test_config_schema reliance on config_schema.json --- tests/unit_tests/test_cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index 8b0ed5a154..b12b5e7946 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -1324,5 +1324,7 @@ def test_config_schema( stream.write.assert_called() else: - with CONFIG_SCHEMA_LOCATION.open("r") as stream: - assert DeepDiff(json.loads(result.output), json.load(stream)) == {} + assert ( + DeepDiff(json.loads(result.output), ApplicationConfig.model_json_schema()) + == {} + ) From bbf0dcb9dbbc3fe87fa40aded69d07a97c73d6e5 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Thu, 10 Jul 2025 10:04:14 +0000 Subject: [PATCH 35/50] Lint --- tests/unit_tests/test_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index b12b5e7946..237f8f9f0b 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -37,7 +37,6 @@ UnknownPlan, ) from blueapi.config import ( - CONFIG_SCHEMA_LOCATION, ApplicationConfig, ScratchConfig, ScratchRepository, From 92aa5092b5444978da1c4ab99809ab4fee657449 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes <65790536+dan-fernandes@users.noreply.github.com> Date: Wed, 23 Jul 2025 14:17:16 +0100 Subject: [PATCH 36/50] Update .pre-commit-config.yaml Co-authored-by: Joseph Ware <53935796+DiamondJoseph@users.noreply.github.com> --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3565c15ebb..6755bdc022 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,6 @@ repos: exclude: ^helm\/.*\/templates\/.*|catalog-info.yaml - id: check-merge-conflict - id: end-of-file-fixer - exclude: "helm\/blueapi\/README.md" - repo: local hooks: From a422ed39e6a6b6b5823336f8503980214e24f420 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes <65790536+dan-fernandes@users.noreply.github.com> Date: Wed, 23 Jul 2025 14:19:07 +0100 Subject: [PATCH 37/50] Remove comment Co-authored-by: Joseph Ware <53935796+DiamondJoseph@users.noreply.github.com> --- tests/unit_tests/test_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index 237f8f9f0b..08e62c62ff 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -1299,7 +1299,6 @@ def test_config_schema( tmp_path = tmp_path / "foo.json" if output_flag: - # args.append(f"-o {output}") args.append("-o") args.append(f"{tmp_path}") From ca373085af01d43f4a4827929109d6d40c0b5e22 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes <65790536+dan-fernandes@users.noreply.github.com> Date: Wed, 23 Jul 2025 14:21:37 +0100 Subject: [PATCH 38/50] Remove unecessary DeepDiff use Co-authored-by: Joseph Ware <53935796+DiamondJoseph@users.noreply.github.com> --- tests/unit_tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py index a8eaaf1fad..ddb4db0d88 100644 --- a/tests/unit_tests/test_config.py +++ b/tests/unit_tests/test_config.py @@ -516,7 +516,7 @@ def validate_field_annotations(model_class: Any, model_field: str) -> None: def test_config_schema_updated() -> None: with CONFIG_SCHEMA_LOCATION.open("r") as stream: config_schema = json.load(stream) - assert DeepDiff(config_schema, ApplicationConfig.model_json_schema()) == {}, ( + assert config_schema == ApplicationConfig.model_json_schema(), ( f"ApplicationConfig model is out of date with schema at \ {CONFIG_SCHEMA_LOCATION}. You may need to run `blueapi config-schema -u`" ) From f0971617036e0fd99dde080c6bb73c4752f83f82 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Thu, 24 Jul 2025 08:04:58 +0000 Subject: [PATCH 39/50] Refactor test_config_schema to remove dependency on deepdiff --- tests/unit_tests/test_cli.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index 08e62c62ff..93d87dd332 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -17,7 +17,6 @@ from bluesky_stomp.messaging import StompClient from bluesky_stomp.models import MessageTopic from click.testing import CliRunner -from deepdiff import DeepDiff from opentelemetry import trace from ophyd_async.core import AsyncStatus from pydantic import BaseModel @@ -1310,19 +1309,13 @@ def test_config_schema( args, ) + expected = ApplicationConfig.model_json_schema() if output_flag and (not update): with tmp_path.open("r") as stream: - assert ( - DeepDiff(json.load(stream), ApplicationConfig.model_json_schema()) == {} - ) - + assert json.load(stream) == expected elif update: config_schema_location_mock.open.assert_called() with config_schema_location_mock.open() as stream: stream.write.assert_called() - else: - assert ( - DeepDiff(json.loads(result.output), ApplicationConfig.model_json_schema()) - == {} - ) + assert json.loads(result.output) == expected From f12d4331ff6e57c175c6214164d1aecda79acfa8 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Thu, 24 Jul 2025 08:07:57 +0000 Subject: [PATCH 40/50] Remove unused improt --- tests/unit_tests/test_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py index ddb4db0d88..bb35d60680 100644 --- a/tests/unit_tests/test_config.py +++ b/tests/unit_tests/test_config.py @@ -11,7 +11,6 @@ import responses import yaml from bluesky_stomp.models import BasicAuthentication -from deepdiff import DeepDiff from pydantic import BaseModel, Field from blueapi.config import ( From e8d9bc9ed5d2256e983fe342cb53bfb6e57d5301 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Thu, 24 Jul 2025 08:14:02 +0000 Subject: [PATCH 41/50] Remove default fields --- helm/blueapi/.schema.yaml | 4 +- helm/blueapi/values.schema.json | 121 +++++++++++++++++++++----------- 2 files changed, 81 insertions(+), 44 deletions(-) diff --git a/helm/blueapi/.schema.yaml b/helm/blueapi/.schema.yaml index 5f452a4333..0196d005f2 100644 --- a/helm/blueapi/.schema.yaml +++ b/helm/blueapi/.schema.yaml @@ -13,10 +13,8 @@ bundleWithoutID: false useHelmDocs: false -noAdditionalProperties: false +noAdditionalProperties: true schemaRoot: - id: https://example.com/schema title: Helm Values Schema description: Schema for Helm values - additionalProperties: false diff --git a/helm/blueapi/values.schema.json b/helm/blueapi/values.schema.json index e7bd6f2e5a..d996cc689e 100644 --- a/helm/blueapi/values.schema.json +++ b/helm/blueapi/values.schema.json @@ -1,12 +1,12 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://example.com/schema", "title": "Helm Values Schema", "description": "Schema for Helm values", "type": "object", "properties": { "affinity": { - "type": "object" + "type": "object", + "additionalProperties": false }, "debug": { "type": "object", @@ -14,7 +14,8 @@ "enabled": { "type": "boolean" } - } + }, + "additionalProperties": false }, "extraEnvVars": { "type": "array" @@ -37,7 +38,8 @@ "tag": { "type": "string" } - } + }, + "additionalProperties": false }, "imagePullSecrets": { "type": "array" @@ -46,7 +48,8 @@ "type": "object", "properties": { "annotations": { - "type": "object" + "type": "object", + "additionalProperties": false }, "className": { "type": "string" @@ -73,16 +76,19 @@ "pathType": { "type": "string" } - } + }, + "additionalProperties": false } } - } + }, + "additionalProperties": false } }, "tls": { "type": "array" } - } + }, + "additionalProperties": false }, "initContainer": { "type": "object", @@ -99,12 +105,15 @@ "existingClaimName": { "type": "string" } - } + }, + "additionalProperties": false } - } + }, + "additionalProperties": false }, "initResources": { - "type": "object" + "type": "object", + "additionalProperties": false }, "livenessProbe": { "type": "object", @@ -121,27 +130,33 @@ "port": { "type": "string" } - } + }, + "additionalProperties": false }, "periodSeconds": { "type": "integer" } - } + }, + "additionalProperties": false }, "nameOverride": { "type": "string" }, "nodeSelector": { - "type": "object" + "type": "object", + "additionalProperties": false }, "podAnnotations": { - "type": "object" + "type": "object", + "additionalProperties": false }, "podLabels": { - "type": "object" + "type": "object", + "additionalProperties": false }, "podSecurityContext": { - "type": "object" + "type": "object", + "additionalProperties": false }, "readinessProbe": { "type": "object", @@ -158,12 +173,14 @@ "port": { "type": "string" } - } + }, + "additionalProperties": false }, "periodSeconds": { "type": "integer" } - } + }, + "additionalProperties": false }, "resources": { "type": "object", @@ -177,7 +194,8 @@ "memory": { "type": "string" } - } + }, + "additionalProperties": false }, "requests": { "type": "object", @@ -188,9 +206,11 @@ "memory": { "type": "string" } - } + }, + "additionalProperties": false } - } + }, + "additionalProperties": false }, "restartOnConfigChange": { "type": "boolean" @@ -204,7 +224,8 @@ "runAsUser": { "type": "integer" } - } + }, + "additionalProperties": false }, "service": { "type": "object", @@ -215,13 +236,15 @@ "type": { "type": "string" } - } + }, + "additionalProperties": false }, "serviceAccount": { "type": "object", "properties": { "annotations": { - "type": "object" + "type": "object", + "additionalProperties": false }, "automount": { "type": "boolean" @@ -232,7 +255,8 @@ "name": { "type": "string" } - } + }, + "additionalProperties": false }, "startupProbe": { "type": "object", @@ -249,12 +273,14 @@ "port": { "type": "string" } - } + }, + "additionalProperties": false }, "periodSeconds": { "type": "integer" } - } + }, + "additionalProperties": false }, "tolerations": { "type": "array" @@ -280,11 +306,14 @@ "port": { "type": "integer" } - } + }, + "additionalProperties": false } - } + }, + "additionalProperties": false } - } + }, + "additionalProperties": false }, "volumeMounts": { "type": "array", @@ -300,7 +329,8 @@ "readOnly": { "type": "boolean" } - } + }, + "additionalProperties": false } }, "volumes": { @@ -316,7 +346,8 @@ "url": { "type": "string" } - } + }, + "additionalProperties": false }, "env": { "type": "object", @@ -332,10 +363,12 @@ "module": { "type": "string" } - } + }, + "additionalProperties": false } } - } + }, + "additionalProperties": false }, "logging": { "type": "object", @@ -349,12 +382,14 @@ "url": { "type": "string" } - } + }, + "additionalProperties": false }, "level": { "type": "string" } - } + }, + "additionalProperties": false }, "scratch": { "type": "object", @@ -365,7 +400,8 @@ "root": { "type": "string" } - } + }, + "additionalProperties": false }, "stomp": { "type": "object", @@ -379,7 +415,8 @@ "username": { "type": "string" } - } + }, + "additionalProperties": false }, "enabled": { "type": "boolean" @@ -387,9 +424,11 @@ "url": { "type": "string" } - } + }, + "additionalProperties": false } - } + }, + "additionalProperties": false } }, "additionalProperties": false, From ab5e7cbb85693559d2b504e47944d61eaa7fff42 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Thu, 24 Jul 2025 08:37:13 +0000 Subject: [PATCH 42/50] Update additional properties --- helm/blueapi/.schema.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/helm/blueapi/.schema.yaml b/helm/blueapi/.schema.yaml index 0196d005f2..af02eff067 100644 --- a/helm/blueapi/.schema.yaml +++ b/helm/blueapi/.schema.yaml @@ -13,8 +13,11 @@ bundleWithoutID: false useHelmDocs: false -noAdditionalProperties: true +# Additional properties must be allowed for eg. initResources: +noAdditionalProperties: false schemaRoot: title: Helm Values Schema description: Schema for Helm values + # No additional properties in schema root for tighter protection: + additionalProperties: false From f194bfe76e88b1198f61f7bc49faf76e985b200d Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Thu, 24 Jul 2025 08:38:12 +0000 Subject: [PATCH 43/50] Update values.schema.json --- helm/blueapi/values.schema.json | 120 +++++++++++--------------------- 1 file changed, 40 insertions(+), 80 deletions(-) diff --git a/helm/blueapi/values.schema.json b/helm/blueapi/values.schema.json index d996cc689e..8ed16d2552 100644 --- a/helm/blueapi/values.schema.json +++ b/helm/blueapi/values.schema.json @@ -5,8 +5,7 @@ "type": "object", "properties": { "affinity": { - "type": "object", - "additionalProperties": false + "type": "object" }, "debug": { "type": "object", @@ -14,8 +13,7 @@ "enabled": { "type": "boolean" } - }, - "additionalProperties": false + } }, "extraEnvVars": { "type": "array" @@ -38,8 +36,7 @@ "tag": { "type": "string" } - }, - "additionalProperties": false + } }, "imagePullSecrets": { "type": "array" @@ -48,8 +45,7 @@ "type": "object", "properties": { "annotations": { - "type": "object", - "additionalProperties": false + "type": "object" }, "className": { "type": "string" @@ -76,19 +72,16 @@ "pathType": { "type": "string" } - }, - "additionalProperties": false + } } } - }, - "additionalProperties": false + } } }, "tls": { "type": "array" } - }, - "additionalProperties": false + } }, "initContainer": { "type": "object", @@ -105,15 +98,12 @@ "existingClaimName": { "type": "string" } - }, - "additionalProperties": false + } } - }, - "additionalProperties": false + } }, "initResources": { - "type": "object", - "additionalProperties": false + "type": "object" }, "livenessProbe": { "type": "object", @@ -130,33 +120,27 @@ "port": { "type": "string" } - }, - "additionalProperties": false + } }, "periodSeconds": { "type": "integer" } - }, - "additionalProperties": false + } }, "nameOverride": { "type": "string" }, "nodeSelector": { - "type": "object", - "additionalProperties": false + "type": "object" }, "podAnnotations": { - "type": "object", - "additionalProperties": false + "type": "object" }, "podLabels": { - "type": "object", - "additionalProperties": false + "type": "object" }, "podSecurityContext": { - "type": "object", - "additionalProperties": false + "type": "object" }, "readinessProbe": { "type": "object", @@ -173,14 +157,12 @@ "port": { "type": "string" } - }, - "additionalProperties": false + } }, "periodSeconds": { "type": "integer" } - }, - "additionalProperties": false + } }, "resources": { "type": "object", @@ -194,8 +176,7 @@ "memory": { "type": "string" } - }, - "additionalProperties": false + } }, "requests": { "type": "object", @@ -206,11 +187,9 @@ "memory": { "type": "string" } - }, - "additionalProperties": false + } } - }, - "additionalProperties": false + } }, "restartOnConfigChange": { "type": "boolean" @@ -224,8 +203,7 @@ "runAsUser": { "type": "integer" } - }, - "additionalProperties": false + } }, "service": { "type": "object", @@ -236,15 +214,13 @@ "type": { "type": "string" } - }, - "additionalProperties": false + } }, "serviceAccount": { "type": "object", "properties": { "annotations": { - "type": "object", - "additionalProperties": false + "type": "object" }, "automount": { "type": "boolean" @@ -255,8 +231,7 @@ "name": { "type": "string" } - }, - "additionalProperties": false + } }, "startupProbe": { "type": "object", @@ -273,14 +248,12 @@ "port": { "type": "string" } - }, - "additionalProperties": false + } }, "periodSeconds": { "type": "integer" } - }, - "additionalProperties": false + } }, "tolerations": { "type": "array" @@ -306,14 +279,11 @@ "port": { "type": "integer" } - }, - "additionalProperties": false + } } - }, - "additionalProperties": false + } } - }, - "additionalProperties": false + } }, "volumeMounts": { "type": "array", @@ -329,8 +299,7 @@ "readOnly": { "type": "boolean" } - }, - "additionalProperties": false + } } }, "volumes": { @@ -346,8 +315,7 @@ "url": { "type": "string" } - }, - "additionalProperties": false + } }, "env": { "type": "object", @@ -363,12 +331,10 @@ "module": { "type": "string" } - }, - "additionalProperties": false + } } } - }, - "additionalProperties": false + } }, "logging": { "type": "object", @@ -382,14 +348,12 @@ "url": { "type": "string" } - }, - "additionalProperties": false + } }, "level": { "type": "string" } - }, - "additionalProperties": false + } }, "scratch": { "type": "object", @@ -400,8 +364,7 @@ "root": { "type": "string" } - }, - "additionalProperties": false + } }, "stomp": { "type": "object", @@ -415,8 +378,7 @@ "username": { "type": "string" } - }, - "additionalProperties": false + } }, "enabled": { "type": "boolean" @@ -424,11 +386,9 @@ "url": { "type": "string" } - }, - "additionalProperties": false + } } - }, - "additionalProperties": false + } } }, "additionalProperties": false, From 1e47fcc0aa9d67fd3d4cfc617843bf9545e66132 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Mon, 28 Jul 2025 13:30:06 +0000 Subject: [PATCH 44/50] Update dev-requirements.txt --- dev-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index cec6c48cd9..d41e602ad3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,6 +2,7 @@ accessible-pygments==0.0.5 aioca==1.8.1 aiofiles==24.1.0 aiohappyeyeballs==2.6.1 +aiohttp==3.12.13 aiosignal==1.3.2 alabaster==1.0.0 annotated-types==0.7.0 @@ -11,6 +12,7 @@ attrs==25.3.0 babel==2.17.0 beautifulsoup4==4.13.4 bidict==0.23.1 +-e git+ssh://git@github.com/DiamondLightSource/blueapi.git@f194bfe76e88b1198f61f7bc49faf76e985b200d#egg=blueapi bluesky==1.13.1 bluesky-stomp==0.1.6 certifi==2025.1.31 From 593305f336985acdd9cc008f054a1061637270ed Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Mon, 28 Jul 2025 13:32:23 +0000 Subject: [PATCH 45/50] Remove editable packages from dev-requirements --- dev-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index d41e602ad3..4c4deb08f6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -12,7 +12,6 @@ attrs==25.3.0 babel==2.17.0 beautifulsoup4==4.13.4 bidict==0.23.1 --e git+ssh://git@github.com/DiamondLightSource/blueapi.git@f194bfe76e88b1198f61f7bc49faf76e985b200d#egg=blueapi bluesky==1.13.1 bluesky-stomp==0.1.6 certifi==2025.1.31 From 24d1dead08e8c972aa8b255069ba3c6c58563c46 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Mon, 28 Jul 2025 13:53:44 +0000 Subject: [PATCH 46/50] Remove duplicate requirements --- dev-requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 4f77f7cc4d..58c41d348c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,8 +2,6 @@ accessible-pygments==0.0.5 aioca==1.8.1 aiofiles==24.1.0 aiohappyeyeballs==2.6.1 -aiohttp==3.12.13 -aiosignal==1.3.2 aiohttp==3.12.14 aiosignal==1.4.0 alabaster==1.0.0 From 1dbab6c382c7f7bf909825449008cfbc73d26a79 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Mon, 28 Jul 2025 14:23:15 +0000 Subject: [PATCH 47/50] Update config_schema.json --- helm/blueapi/config_schema.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/helm/blueapi/config_schema.json b/helm/blueapi/config_schema.json index 3ea380b111..e8529d3d73 100644 --- a/helm/blueapi/config_schema.json +++ b/helm/blueapi/config_schema.json @@ -152,17 +152,12 @@ "MetadataConfig": { "additionalProperties": false, "properties": { - "instrument_session": { - "title": "Instrument Session", - "type": "string" - }, "instrument": { "title": "Instrument", "type": "string" } }, "required": [ - "instrument_session", "instrument" ], "title": "MetadataConfig", From 1ddd98e148293aadccb4c809e9bb92969a348fc0 Mon Sep 17 00:00:00 2001 From: Daniel Fernandes Date: Mon, 28 Jul 2025 14:29:55 +0000 Subject: [PATCH 48/50] Remove helm plugin install from _test.yml --- .github/workflows/_test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 56fffde97f..5aa8ad6d24 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -54,9 +54,6 @@ jobs: python-version: ${{ inputs.python-version }} pip-install: ".[dev]" - - name: Install helm plugins - run: helm plugin install https://github.com/losisin/helm-values-schema-json.git - - name: Run tests run: tox -e tests From 20a4ded63a83c84d1cb725aa92ea4ff139a0d334 Mon Sep 17 00:00:00 2001 From: Joseph Ware <53935796+DiamondJoseph@users.noreply.github.com> Date: Mon, 11 Aug 2025 11:38:06 +0100 Subject: [PATCH 49/50] Pass required config through .schema.yaml --- .pre-commit-config.yaml | 4 +++- helm/blueapi/.schema.yaml | 21 ++++++++++--------- helm/blueapi/values.schema.json | 36 ++++++++++++++++++++++++++------- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d48d731404..1edb6f7a79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,4 +41,6 @@ repos: rev: v2.2.1 hooks: - id: helm-schema - args: ["--values", "helm/blueapi/values.yaml"] + args: + - "--config" + - "helm/blueapi/.schema.yaml" diff --git a/helm/blueapi/.schema.yaml b/helm/blueapi/.schema.yaml index af02eff067..dec9c80b44 100644 --- a/helm/blueapi/.schema.yaml +++ b/helm/blueapi/.schema.yaml @@ -1,23 +1,24 @@ # .schema.yaml +# Define input, output and source for $refs relative to repository root for pre-commit values: - - values.yaml + - helm/blueapi/values.yaml -draft: 2020 -indent: 4 -output: values.schema.json +output: helm/blueapi/values.schema.json +bundleRoot: helm/blueapi/ + +# Include $refs bundle: true -bundleRoot: . -bundleWithoutID: false -useHelmDocs: false +# Include comments for the helm-docs plugin into the schema, to allow e.g. documentation in VSCode +useHelmDocs: true -# Additional properties must be allowed for eg. initResources: +# Allow additional properties for eg. initResources, different types of volumes/volumeMounts noAdditionalProperties: false schemaRoot: - title: Helm Values Schema - description: Schema for Helm values + title: Blueapi Helm chart scheam + description: Schema to allow validation of values passed to Blueapi Helm chart # No additional properties in schema root for tighter protection: additionalProperties: false diff --git a/helm/blueapi/values.schema.json b/helm/blueapi/values.schema.json index 8ed16d2552..da94b9dada 100644 --- a/helm/blueapi/values.schema.json +++ b/helm/blueapi/values.schema.json @@ -1,27 +1,31 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Helm Values Schema", - "description": "Schema for Helm values", + "title": "Blueapi Helm chart scheam", + "description": "Schema to allow validation of values passed to Blueapi Helm chart", "type": "object", "properties": { "affinity": { + "description": "May be required to run on specific nodes (e.g. the control machine)", "type": "object" }, "debug": { "type": "object", "properties": { "enabled": { + "description": "If enabled, disables liveness and readiness probes, and does not start the service on startup This allows connecting to the pod and starting the service manually to allow debugging on the cluster", "type": "boolean" } } }, "extraEnvVars": { + "description": "Additional envVars to mount to the pod", "type": "array" }, "fullnameOverride": { "type": "string" }, "hostNetwork": { + "description": "May be needed for EPICS depending on gateway configuration", "type": "boolean" }, "image": { @@ -31,6 +35,7 @@ "type": "string" }, "repository": { + "description": "To use a container image that extends the blueapi one, set it here", "type": "string" }, "tag": { @@ -42,6 +47,7 @@ "type": "array" }, "ingress": { + "description": "Configuring and enabling an ingress allows blueapi to be served at a nicer address, e.g. ixx-blueapi.diamond.ac.uk", "type": "object", "properties": { "annotations": { @@ -84,6 +90,7 @@ } }, "initContainer": { + "description": "Configure the initContainer that checks out the scratch configuration repositories", "type": "object", "properties": { "enabled": { @@ -93,9 +100,11 @@ "type": "object", "properties": { "enabled": { + "description": "Whether to use a persistent volume in the cluster or check out onto the mounted host filesystem If persistentVolume.enabled: False, mounts scratch.root as scratch.root in the container", "type": "boolean" }, "existingClaimName": { + "description": "May be set to an existing persistent volume claim to re-use the volume, else a new one is created for each blueapi release", "type": "string" } } @@ -103,9 +112,11 @@ } }, "initResources": { + "description": "Override resources for init container. By default copies resources of main container.", "type": "object" }, "livenessProbe": { + "description": "Liveness probe, if configured kubernetes will kill the pod and start a new one if failed consecutively. This is automatically disabled when in debug mode.", "type": "object", "properties": { "failureThreshold": { @@ -131,6 +142,7 @@ "type": "string" }, "nodeSelector": { + "description": "May be required to run on specific nodes (e.g. the control machine)", "type": "object" }, "podAnnotations": { @@ -143,6 +155,7 @@ "type": "object" }, "readinessProbe": { + "description": "Readiness probe, if configured kubernetes will not route traffic to this pod if failed consecutively. This could allow the service time to recover if it is being overwhelmed by traffic, but without the to ability to load balance or scale up/outwards, upstream services will need to know to back off. This is automatically disabled when in debug mode.", "type": "object", "properties": { "failureThreshold": { @@ -165,6 +178,7 @@ } }, "resources": { + "description": "Sets the compute resources available to the pod. These defaults are appropriate when using debug mode or an internal PVC and therefore running VS Code server in the pod. In the Diamond cluster, requests must be \u003e= 0.1*limits When not using either of the above, the limits may be lowered. When idle but connected, blueapi consumes ~400MB of memory and 1% cpu and may struggle when allocated less.", "type": "object", "properties": { "limits": { @@ -192,6 +206,7 @@ } }, "restartOnConfigChange": { + "description": "If enabled the blueapi pod will restart on changes to `worker`", "type": "boolean" }, "securityContext": { @@ -212,6 +227,7 @@ "type": "integer" }, "type": { + "description": "To make blueapi available on an IP outside of the cluster prior to an Ingress being created, change this to LoadBalancer", "type": "string" } } @@ -234,6 +250,7 @@ } }, "startupProbe": { + "description": "A more lenient livenessProbe to allow the service to start fully. This is automatically disabled when in debug mode.", "type": "object", "properties": { "failureThreshold": { @@ -256,9 +273,11 @@ } }, "tolerations": { + "description": "May be required to run on specific nodes (e.g. the control machine)", "type": "array" }, "tracing": { + "description": "Configure tracing: opentelemetry-collector.tracing should be available in all Diamond clusters", "type": "object", "properties": { "otlp": { @@ -286,6 +305,7 @@ } }, "volumeMounts": { + "description": "Additional volumeMounts on the output StatefulSet definition. Define how volumes are mounted to the container referenced by using the same name.", "type": "array", "items": { "type": "object", @@ -303,9 +323,11 @@ } }, "volumes": { + "description": "Additional volumes on the output StatefulSet definition. Define volumes from e.g. Secrets, ConfigMaps or the Filesystem", "type": "array" }, "worker": { + "description": "Config for the worker goes here, will be mounted into a config file", "$ref": "config_schema.json", "type": "object", "properties": { @@ -313,6 +335,7 @@ "type": "object", "properties": { "url": { + "description": "0.0.0.0 required to allow non-loopback traffic If using hostNetwork, the port must be free on the host", "type": "string" } } @@ -321,6 +344,7 @@ "type": "object", "properties": { "sources": { + "description": "modules (must be installed in the venv) to fetch devices/plans from", "type": "array", "items": { "type": "object", @@ -337,6 +361,7 @@ } }, "logging": { + "description": "Configures logging. Port 12231 is the `dodal` input on graylog which will be renamed `blueapi`", "type": "object", "properties": { "graylog": { @@ -356,6 +381,7 @@ } }, "scratch": { + "description": "If initContainer is enabled the default branch of python projects in this section are installed into the venv *without their dependencies*", "type": "object", "properties": { "repositories": { @@ -367,6 +393,7 @@ } }, "stomp": { + "description": "Message bus configuration for returning status to GDA/forwarding documents downstream Password may be in the form ${ENV_VAR} to be fetched from an environment variable e.g. mounted from a SealedSecret", "type": "object", "properties": { "auth": { @@ -606,17 +633,12 @@ "title": "MetadataConfig", "type": "object", "required": [ - "instrument_session", "instrument" ], "properties": { "instrument": { "title": "Instrument", "type": "string" - }, - "instrument_session": { - "title": "Instrument Session", - "type": "string" } }, "additionalProperties": false From 3fdb00285d388f20fa63cd5085e305694224caa1 Mon Sep 17 00:00:00 2001 From: Joseph Ware <53935796+DiamondJoseph@users.noreply.github.com> Date: Mon, 11 Aug 2025 16:23:48 +0100 Subject: [PATCH 50/50] Apply suggestions from code review --- .github/workflows/_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 5aa8ad6d24..2db9ff0ce0 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -53,7 +53,6 @@ jobs: with: python-version: ${{ inputs.python-version }} pip-install: ".[dev]" - - name: Run tests run: tox -e tests