diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index ff5c2aea30..2db9ff0ce0 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -53,7 +53,6 @@ jobs: with: python-version: ${{ inputs.python-version }} pip-install: ".[dev]" - - name: Run tests run: tox -e tests diff --git a/.github/workflows/_tox.yml b/.github/workflows/_tox.yml index ad74c9acb7..94fe0e57f2 100644 --- a/.github/workflows/_tox.yml +++ b/.github/workflows/_tox.yml @@ -17,6 +17,9 @@ jobs: - name: Install python packages uses: ./.github/actions/install_requirements - + + - name: Install helm plugins + run: helm plugin install https://github.com/losisin/helm-values-schema-json.git + - name: Run tox run: tox -e ${{ inputs.tox }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4488ffa823..446f5ca8c9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: require_serial: true - repo: https://github.com/norwoodj/helm-docs - rev: "" + rev: "" hooks: - id: helm-docs-built args: @@ -37,3 +37,11 @@ repos: rev: v8.28.0 hooks: - id: gitleaks + + - repo: https://github.com/losisin/helm-values-schema-json + rev: v2.2.1 + hooks: + - id: helm-schema + args: + - "--config" + - "helm/blueapi/.schema.yaml" diff --git a/Dockerfile b/Dockerfile index eaaba6a5d8..d93e83103f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/s chmod 700 get_helm.sh; \ ./get_helm.sh; \ rm get_helm.sh +RUN helm plugin install https://github.com/losisin/helm-values-schema-json.git # Set up a virtual environment and put it in PATH RUN python -m venv /venv diff --git a/helm/blueapi/.schema.yaml b/helm/blueapi/.schema.yaml new file mode 100644 index 0000000000..dec9c80b44 --- /dev/null +++ b/helm/blueapi/.schema.yaml @@ -0,0 +1,24 @@ +# .schema.yaml + +# Define input, output and source for $refs relative to repository root for pre-commit +values: + - helm/blueapi/values.yaml + +output: helm/blueapi/values.schema.json + +bundleRoot: helm/blueapi/ + +# Include $refs +bundle: true + +# Include comments for the helm-docs plugin into the schema, to allow e.g. documentation in VSCode +useHelmDocs: true + +# Allow additional properties for eg. initResources, different types of volumes/volumeMounts +noAdditionalProperties: false + +schemaRoot: + title: Blueapi Helm chart scheam + description: Schema to allow validation of values passed to Blueapi Helm chart + # No additional properties in schema root for tighter protection: + additionalProperties: false diff --git a/helm/blueapi/README.md b/helm/blueapi/README.md index 455028375c..effe4a7e94 100644 --- a/helm/blueapi/README.md +++ b/helm/blueapi/README.md @@ -22,6 +22,7 @@ A Helm chart deploying a worker pod that runs Bluesky plans | initContainer | object | `{"enabled":false,"persistentVolume":{"enabled":false,"existingClaimName":""}}` | Configure the initContainer that checks out the scratch configuration repositories | | initContainer.persistentVolume.enabled | bool | `false` | Whether to use a persistent volume in the cluster or check out onto the mounted host filesystem If persistentVolume.enabled: False, mounts scratch.root as scratch.root in the container | | initContainer.persistentVolume.existingClaimName | string | `""` | May be set to an existing persistent volume claim to re-use the volume, else a new one is created for each blueapi release | +| initResources | object | `{}` | Override resources for init container. By default copies resources of main container. | | livenessProbe | object | `{"failureThreshold":3,"httpGet":{"path":"/healthz","port":"http"},"periodSeconds":10}` | Liveness probe, if configured kubernetes will kill the pod and start a new one if failed consecutively. This is automatically disabled when in debug mode. | | nameOverride | string | `""` | | | nodeSelector | object | `{}` | May be required to run on specific nodes (e.g. the control machine) | diff --git a/helm/blueapi/config_schema.json b/helm/blueapi/config_schema.json new file mode 100644 index 0000000000..e8529d3d73 --- /dev/null +++ b/helm/blueapi/config_schema.json @@ -0,0 +1,434 @@ +{ + "$defs": { + "BasicAuthentication": { + "additionalProperties": false, + "description": "User credentials for basic authentication", + "properties": { + "username": { + "description": "Unique identifier for user", + "title": "Username", + "type": "string" + }, + "password": { + "description": "Password to verify user's identity", + "title": "Password", + "type": "string" + } + }, + "required": [ + "username", + "password" + ], + "title": "BasicAuthentication", + "type": "object" + }, + "CORSConfig": { + "additionalProperties": false, + "properties": { + "origins": { + "items": { + "type": "string" + }, + "title": "Origins", + "type": "array" + }, + "allow_credentials": { + "default": false, + "title": "Allow Credentials", + "type": "boolean" + }, + "allow_methods": { + "default": [ + "*" + ], + "items": { + "type": "string" + }, + "title": "Allow Methods", + "type": "array" + }, + "allow_headers": { + "default": [ + "*" + ], + "items": { + "type": "string" + }, + "title": "Allow Headers", + "type": "array" + } + }, + "required": [ + "origins" + ], + "title": "CORSConfig", + "type": "object" + }, + "EnvironmentConfig": { + "additionalProperties": false, + "description": "Config for the RunEngine environment", + "properties": { + "sources": { + "default": [ + { + "kind": "planFunctions", + "module": "dodal.plans" + }, + { + "kind": "planFunctions", + "module": "dodal.plan_stubs.wrapped" + } + ], + "items": { + "$ref": "#/$defs/Source" + }, + "title": "Sources", + "type": "array" + }, + "events": { + "$ref": "#/$defs/WorkerEventConfig" + }, + "metadata": { + "anyOf": [ + { + "$ref": "#/$defs/MetadataConfig" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "EnvironmentConfig", + "type": "object" + }, + "GraylogConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "title": "Enabled", + "type": "boolean" + }, + "url": { + "default": "tcp://localhost:5555", + "format": "uri", + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "title": "GraylogConfig", + "type": "object" + }, + "LoggingConfig": { + "additionalProperties": false, + "properties": { + "level": { + "default": "INFO", + "enum": [ + "NOTSET", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "title": "Level", + "type": "string" + }, + "graylog": { + "$ref": "#/$defs/GraylogConfig", + "default": { + "enabled": false, + "url": "tcp://localhost:5555" + } + } + }, + "title": "LoggingConfig", + "type": "object" + }, + "MetadataConfig": { + "additionalProperties": false, + "properties": { + "instrument": { + "title": "Instrument", + "type": "string" + } + }, + "required": [ + "instrument" + ], + "title": "MetadataConfig", + "type": "object" + }, + "NumtrackerConfig": { + "additionalProperties": false, + "properties": { + "url": { + "default": "http://localhost:8002/graphql", + "format": "uri", + "maxLength": 2083, + "minLength": 1, + "title": "Url", + "type": "string" + } + }, + "title": "NumtrackerConfig", + "type": "object" + }, + "OIDCConfig": { + "additionalProperties": false, + "properties": { + "well_known_url": { + "description": "URL to fetch OIDC config from the provider", + "title": "Well Known Url", + "type": "string" + }, + "client_id": { + "description": "Client ID", + "title": "Client Id", + "type": "string" + }, + "client_audience": { + "default": "blueapi", + "description": "Client Audience(s)", + "title": "Client Audience", + "type": "string" + } + }, + "required": [ + "well_known_url", + "client_id" + ], + "title": "OIDCConfig", + "type": "object" + }, + "RestConfig": { + "additionalProperties": false, + "properties": { + "url": { + "default": "http://localhost:8000/", + "format": "uri", + "maxLength": 2083, + "minLength": 1, + "title": "Url", + "type": "string" + }, + "cors": { + "anyOf": [ + { + "$ref": "#/$defs/CORSConfig" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "RestConfig", + "type": "object" + }, + "ScratchConfig": { + "additionalProperties": false, + "properties": { + "root": { + "default": "/tmp/scratch/blueapi", + "description": "The root directory of the scratch area, all repositories will be cloned under this directory.", + "format": "path", + "title": "Root", + "type": "string" + }, + "required_gid": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "\nRequired owner GID for the scratch directory. If supplied, the setup-scratch\ncommand will check the scratch area ownership and raise an error if it is\nnot owned by , or if it does not have SGID permission bit set.\n", + "title": "Required Gid" + }, + "repositories": { + "description": "Details of repositories to be cloned and imported into blueapi", + "items": { + "$ref": "#/$defs/ScratchRepository" + }, + "title": "Repositories", + "type": "array" + } + }, + "title": "ScratchConfig", + "type": "object" + }, + "ScratchRepository": { + "additionalProperties": false, + "properties": { + "name": { + "default": "example", + "description": "Unique name for this repository in the scratch directory", + "title": "Name", + "type": "string" + }, + "remote_url": { + "default": "https://github.com/example/example.git", + "description": "URL to clone from", + "title": "Remote Url", + "type": "string" + } + }, + "title": "ScratchRepository", + "type": "object" + }, + "Source": { + "additionalProperties": false, + "properties": { + "kind": { + "$ref": "#/$defs/SourceKind" + }, + "module": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "string" + } + ], + "title": "Module" + } + }, + "required": [ + "kind", + "module" + ], + "title": "Source", + "type": "object" + }, + "SourceKind": { + "enum": [ + "planFunctions", + "deviceFunctions", + "dodal" + ], + "title": "SourceKind", + "type": "string" + }, + "StompConfig": { + "additionalProperties": false, + "description": "Config for connecting to stomp broker", + "properties": { + "enabled": { + "default": false, + "description": "True if blueapi should connect to stomp for asynchronous event publishing", + "title": "Enabled", + "type": "boolean" + }, + "url": { + "default": "tcp://localhost:61613", + "format": "uri", + "minLength": 1, + "title": "Url", + "type": "string" + }, + "auth": { + "anyOf": [ + { + "$ref": "#/$defs/BasicAuthentication" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Auth information for communicating with STOMP broker, if required" + } + }, + "title": "StompConfig", + "type": "object" + }, + "WorkerEventConfig": { + "additionalProperties": false, + "description": "Config for event broadcasting via the message bus", + "properties": { + "broadcast_status_events": { + "default": true, + "title": "Broadcast Status Events", + "type": "boolean" + } + }, + "title": "WorkerEventConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Config for the worker application as a whole. Root of\nconfig tree.", + "properties": { + "stomp": { + "$ref": "#/$defs/StompConfig" + }, + "env": { + "$ref": "#/$defs/EnvironmentConfig" + }, + "logging": { + "$ref": "#/$defs/LoggingConfig" + }, + "api": { + "$ref": "#/$defs/RestConfig" + }, + "scratch": { + "anyOf": [ + { + "$ref": "#/$defs/ScratchConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "oidc": { + "anyOf": [ + { + "$ref": "#/$defs/OIDCConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "auth_token_path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Auth Token Path" + }, + "numtracker": { + "anyOf": [ + { + "$ref": "#/$defs/NumtrackerConfig" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "ApplicationConfig", + "type": "object" +} diff --git a/helm/blueapi/values.schema.json b/helm/blueapi/values.schema.json new file mode 100644 index 0000000000..da94b9dada --- /dev/null +++ b/helm/blueapi/values.schema.json @@ -0,0 +1,844 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Blueapi Helm chart scheam", + "description": "Schema to allow validation of values passed to Blueapi Helm chart", + "type": "object", + "properties": { + "affinity": { + "description": "May be required to run on specific nodes (e.g. the control machine)", + "type": "object" + }, + "debug": { + "type": "object", + "properties": { + "enabled": { + "description": "If enabled, disables liveness and readiness probes, and does not start the service on startup This allows connecting to the pod and starting the service manually to allow debugging on the cluster", + "type": "boolean" + } + } + }, + "extraEnvVars": { + "description": "Additional envVars to mount to the pod", + "type": "array" + }, + "fullnameOverride": { + "type": "string" + }, + "hostNetwork": { + "description": "May be needed for EPICS depending on gateway configuration", + "type": "boolean" + }, + "image": { + "type": "object", + "properties": { + "pullPolicy": { + "type": "string" + }, + "repository": { + "description": "To use a container image that extends the blueapi one, set it here", + "type": "string" + }, + "tag": { + "type": "string" + } + } + }, + "imagePullSecrets": { + "type": "array" + }, + "ingress": { + "description": "Configuring and enabling an ingress allows blueapi to be served at a nicer address, e.g. ixx-blueapi.diamond.ac.uk", + "type": "object", + "properties": { + "annotations": { + "type": "object" + }, + "className": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "hosts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "host": { + "type": "string" + }, + "paths": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "pathType": { + "type": "string" + } + } + } + } + } + } + }, + "tls": { + "type": "array" + } + } + }, + "initContainer": { + "description": "Configure the initContainer that checks out the scratch configuration repositories", + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "persistentVolume": { + "type": "object", + "properties": { + "enabled": { + "description": "Whether to use a persistent volume in the cluster or check out onto the mounted host filesystem If persistentVolume.enabled: False, mounts scratch.root as scratch.root in the container", + "type": "boolean" + }, + "existingClaimName": { + "description": "May be set to an existing persistent volume claim to re-use the volume, else a new one is created for each blueapi release", + "type": "string" + } + } + } + } + }, + "initResources": { + "description": "Override resources for init container. By default copies resources of main container.", + "type": "object" + }, + "livenessProbe": { + "description": "Liveness probe, if configured kubernetes will kill the pod and start a new one if failed consecutively. This is automatically disabled when in debug mode.", + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "port": { + "type": "string" + } + } + }, + "periodSeconds": { + "type": "integer" + } + } + }, + "nameOverride": { + "type": "string" + }, + "nodeSelector": { + "description": "May be required to run on specific nodes (e.g. the control machine)", + "type": "object" + }, + "podAnnotations": { + "type": "object" + }, + "podLabels": { + "type": "object" + }, + "podSecurityContext": { + "type": "object" + }, + "readinessProbe": { + "description": "Readiness probe, if configured kubernetes will not route traffic to this pod if failed consecutively. This could allow the service time to recover if it is being overwhelmed by traffic, but without the to ability to load balance or scale up/outwards, upstream services will need to know to back off. This is automatically disabled when in debug mode.", + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "port": { + "type": "string" + } + } + }, + "periodSeconds": { + "type": "integer" + } + } + }, + "resources": { + "description": "Sets the compute resources available to the pod. These defaults are appropriate when using debug mode or an internal PVC and therefore running VS Code server in the pod. In the Diamond cluster, requests must be \u003e= 0.1*limits When not using either of the above, the limits may be lowered. When idle but connected, blueapi consumes ~400MB of memory and 1% cpu and may struggle when allocated less.", + "type": "object", + "properties": { + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + } + }, + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + } + } + } + }, + "restartOnConfigChange": { + "description": "If enabled the blueapi pod will restart on changes to `worker`", + "type": "boolean" + }, + "securityContext": { + "type": "object", + "properties": { + "runAsNonRoot": { + "type": "boolean" + }, + "runAsUser": { + "type": "integer" + } + } + }, + "service": { + "type": "object", + "properties": { + "port": { + "type": "integer" + }, + "type": { + "description": "To make blueapi available on an IP outside of the cluster prior to an Ingress being created, change this to LoadBalancer", + "type": "string" + } + } + }, + "serviceAccount": { + "type": "object", + "properties": { + "annotations": { + "type": "object" + }, + "automount": { + "type": "boolean" + }, + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + } + }, + "startupProbe": { + "description": "A more lenient livenessProbe to allow the service to start fully. This is automatically disabled when in debug mode.", + "type": "object", + "properties": { + "failureThreshold": { + "type": "integer" + }, + "httpGet": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "port": { + "type": "string" + } + } + }, + "periodSeconds": { + "type": "integer" + } + } + }, + "tolerations": { + "description": "May be required to run on specific nodes (e.g. the control machine)", + "type": "array" + }, + "tracing": { + "description": "Configure tracing: opentelemetry-collector.tracing should be available in all Diamond clusters", + "type": "object", + "properties": { + "otlp": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "protocol": { + "type": "string" + }, + "server": { + "type": "object", + "properties": { + "host": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + } + } + } + } + }, + "volumeMounts": { + "description": "Additional volumeMounts on the output StatefulSet definition. Define how volumes are mounted to the container referenced by using the same name.", + "type": "array", + "items": { + "type": "object", + "properties": { + "mountPath": { + "type": "string" + }, + "name": { + "type": "string" + }, + "readOnly": { + "type": "boolean" + } + } + } + }, + "volumes": { + "description": "Additional volumes on the output StatefulSet definition. Define volumes from e.g. Secrets, ConfigMaps or the Filesystem", + "type": "array" + }, + "worker": { + "description": "Config for the worker goes here, will be mounted into a config file", + "$ref": "config_schema.json", + "type": "object", + "properties": { + "api": { + "type": "object", + "properties": { + "url": { + "description": "0.0.0.0 required to allow non-loopback traffic If using hostNetwork, the port must be free on the host", + "type": "string" + } + } + }, + "env": { + "type": "object", + "properties": { + "sources": { + "description": "modules (must be installed in the venv) to fetch devices/plans from", + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "module": { + "type": "string" + } + } + } + } + } + }, + "logging": { + "description": "Configures logging. Port 12231 is the `dodal` input on graylog which will be renamed `blueapi`", + "type": "object", + "properties": { + "graylog": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "url": { + "type": "string" + } + } + }, + "level": { + "type": "string" + } + } + }, + "scratch": { + "description": "If initContainer is enabled the default branch of python projects in this section are installed into the venv *without their dependencies*", + "type": "object", + "properties": { + "repositories": { + "type": "array" + }, + "root": { + "type": "string" + } + } + }, + "stomp": { + "description": "Message bus configuration for returning status to GDA/forwarding documents downstream Password may be in the form ${ENV_VAR} to be fetched from an environment variable e.g. mounted from a SealedSecret", + "type": "object", + "properties": { + "auth": { + "type": "object", + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, + "enabled": { + "type": "boolean" + }, + "url": { + "type": "string" + } + } + } + } + } + }, + "additionalProperties": false, + "$defs": { + "config_schema.json": { + "$id": "config_schema.json", + "title": "ApplicationConfig", + "description": "Config for the worker application as a whole. Root of\nconfig tree.", + "type": "object", + "properties": { + "api": { + "$ref": "#/$defs/RestConfig" + }, + "auth_token_path": { + "title": "Auth Token Path", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "env": { + "$ref": "#/$defs/EnvironmentConfig" + }, + "logging": { + "$ref": "#/$defs/LoggingConfig" + }, + "numtracker": { + "anyOf": [ + { + "$ref": "#/$defs/NumtrackerConfig" + }, + { + "type": "null" + } + ] + }, + "oidc": { + "anyOf": [ + { + "$ref": "#/$defs/OIDCConfig" + }, + { + "type": "null" + } + ] + }, + "scratch": { + "anyOf": [ + { + "$ref": "#/$defs/ScratchConfig" + }, + { + "type": "null" + } + ] + }, + "stomp": { + "$ref": "#/$defs/StompConfig" + } + }, + "additionalProperties": false, + "$defs": { + "BasicAuthentication": { + "title": "BasicAuthentication", + "description": "User credentials for basic authentication", + "type": "object", + "required": [ + "username", + "password" + ], + "properties": { + "password": { + "title": "Password", + "description": "Password to verify user's identity", + "type": "string" + }, + "username": { + "title": "Username", + "description": "Unique identifier for user", + "type": "string" + } + }, + "additionalProperties": false + }, + "CORSConfig": { + "title": "CORSConfig", + "type": "object", + "required": [ + "origins" + ], + "properties": { + "allow_credentials": { + "title": "Allow Credentials", + "default": false, + "type": "boolean" + }, + "allow_headers": { + "title": "Allow Headers", + "default": [ + "*" + ], + "type": "array", + "items": { + "type": "string" + } + }, + "allow_methods": { + "title": "Allow Methods", + "default": [ + "*" + ], + "type": "array", + "items": { + "type": "string" + } + }, + "origins": { + "title": "Origins", + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "EnvironmentConfig": { + "title": "EnvironmentConfig", + "description": "Config for the RunEngine environment", + "type": "object", + "properties": { + "events": { + "$ref": "#/$defs/WorkerEventConfig" + }, + "metadata": { + "anyOf": [ + { + "$ref": "#/$defs/MetadataConfig" + }, + { + "type": "null" + } + ] + }, + "sources": { + "title": "Sources", + "default": [ + { + "kind": "planFunctions", + "module": "dodal.plans" + }, + { + "kind": "planFunctions", + "module": "dodal.plan_stubs.wrapped" + } + ], + "type": "array", + "items": { + "$ref": "#/$defs/Source" + } + } + }, + "additionalProperties": false + }, + "GraylogConfig": { + "title": "GraylogConfig", + "type": "object", + "properties": { + "enabled": { + "title": "Enabled", + "default": false, + "type": "boolean" + }, + "url": { + "title": "Url", + "default": "tcp://localhost:5555", + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "LoggingConfig": { + "title": "LoggingConfig", + "type": "object", + "properties": { + "graylog": { + "default": { + "enabled": false, + "url": "tcp://localhost:5555" + }, + "$ref": "#/$defs/GraylogConfig" + }, + "level": { + "title": "Level", + "default": "INFO", + "type": "string", + "enum": [ + "NOTSET", + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ] + } + }, + "additionalProperties": false + }, + "MetadataConfig": { + "title": "MetadataConfig", + "type": "object", + "required": [ + "instrument" + ], + "properties": { + "instrument": { + "title": "Instrument", + "type": "string" + } + }, + "additionalProperties": false + }, + "NumtrackerConfig": { + "title": "NumtrackerConfig", + "type": "object", + "properties": { + "url": { + "title": "Url", + "default": "http://localhost:8002/graphql", + "type": "string", + "maxLength": 2083, + "minLength": 1 + } + }, + "additionalProperties": false + }, + "OIDCConfig": { + "title": "OIDCConfig", + "type": "object", + "required": [ + "well_known_url", + "client_id" + ], + "properties": { + "client_audience": { + "title": "Client Audience", + "description": "Client Audience(s)", + "default": "blueapi", + "type": "string" + }, + "client_id": { + "title": "Client Id", + "description": "Client ID", + "type": "string" + }, + "well_known_url": { + "title": "Well Known Url", + "description": "URL to fetch OIDC config from the provider", + "type": "string" + } + }, + "additionalProperties": false + }, + "RestConfig": { + "title": "RestConfig", + "type": "object", + "properties": { + "cors": { + "anyOf": [ + { + "$ref": "#/$defs/CORSConfig" + }, + { + "type": "null" + } + ] + }, + "url": { + "title": "Url", + "default": "http://localhost:8000/", + "type": "string", + "maxLength": 2083, + "minLength": 1 + } + }, + "additionalProperties": false + }, + "ScratchConfig": { + "title": "ScratchConfig", + "type": "object", + "properties": { + "repositories": { + "title": "Repositories", + "description": "Details of repositories to be cloned and imported into blueapi", + "type": "array", + "items": { + "$ref": "#/$defs/ScratchRepository" + } + }, + "required_gid": { + "title": "Required Gid", + "description": "\nRequired owner GID for the scratch directory. If supplied, the setup-scratch\ncommand will check the scratch area ownership and raise an error if it is\nnot owned by \u003cGID\u003e, or if it does not have SGID permission bit set.\n", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "root": { + "title": "Root", + "description": "The root directory of the scratch area, all repositories will be cloned under this directory.", + "default": "/tmp/scratch/blueapi", + "type": "string" + } + }, + "additionalProperties": false + }, + "ScratchRepository": { + "title": "ScratchRepository", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "Unique name for this repository in the scratch directory", + "default": "example", + "type": "string" + }, + "remote_url": { + "title": "Remote Url", + "description": "URL to clone from", + "default": "https://github.com/example/example.git", + "type": "string" + } + }, + "additionalProperties": false + }, + "Source": { + "title": "Source", + "type": "object", + "required": [ + "kind", + "module" + ], + "properties": { + "kind": { + "$ref": "#/$defs/SourceKind" + }, + "module": { + "title": "Module", + "anyOf": [ + { + "type": "string" + }, + { + "type": "string" + } + ] + } + }, + "additionalProperties": false + }, + "SourceKind": { + "title": "SourceKind", + "type": "string", + "enum": [ + "planFunctions", + "deviceFunctions", + "dodal" + ] + }, + "StompConfig": { + "title": "StompConfig", + "description": "Config for connecting to stomp broker", + "type": "object", + "properties": { + "auth": { + "description": "Auth information for communicating with STOMP broker, if required", + "anyOf": [ + { + "$ref": "#/$defs/BasicAuthentication" + }, + { + "type": "null" + } + ] + }, + "enabled": { + "title": "Enabled", + "description": "True if blueapi should connect to stomp for asynchronous event publishing", + "default": false, + "type": "boolean" + }, + "url": { + "title": "Url", + "default": "tcp://localhost:61613", + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "WorkerEventConfig": { + "title": "WorkerEventConfig", + "description": "Config for event broadcasting via the message bus", + "type": "object", + "properties": { + "broadcast_status_events": { + "title": "Broadcast Status Events", + "default": true, + "type": "boolean" + } + }, + "additionalProperties": false + } + } + } + } +} diff --git a/helm/blueapi/values.yaml b/helm/blueapi/values.yaml index 670cae8f5e..38328f8fa1 100644 --- a/helm/blueapi/values.yaml +++ b/helm/blueapi/values.yaml @@ -94,6 +94,10 @@ resources: requests: cpu: 200m memory: 400Mi + + +# -- Override resources for init container. By default copies resources of main container. +initResources: {} # This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ # -- Liveness probe, if configured kubernetes will kill the pod and start a new one if failed consecutively. @@ -170,6 +174,7 @@ tracing: host: http://opentelemetry-collector.tracing port: 4318 +# @schema $ref: config_schema.json # -- Config for the worker goes here, will be mounted into a config file worker: api: diff --git a/src/blueapi/cli/cli.py b/src/blueapi/cli/cli.py index 47914ba480..6468e2cca0 100644 --- a/src/blueapi/cli/cli.py +++ b/src/blueapi/cli/cli.py @@ -108,6 +108,31 @@ def schema(output: Path | None = None, update: bool = False) -> None: print_schema_as_yaml(schema) +@click.option( + "-o", "--output", type=Path, help="Path to file to save the config schema" +) +@click.option( + "-u", + "--update", + type=bool, + is_flag=True, + help="[Development only] update the config schema in the documentation", +) +@main.command(name="config-schema") +def config_schema(output: Path | None = None, update: bool = False) -> None: + """Generates a json schema from the ApplicationConfig pydantic basemodel""" + schema = ApplicationConfig.model_json_schema() + + if update: + output = config.CONFIG_SCHEMA_LOCATION + if output is not None: + with output.open("w") as stream: + json.dump(schema, stream, indent=4) + stream.write("\n") + else: + print(json.dumps(schema)) + + @main.command(name="serve") @click.pass_obj def start_application(obj: dict): diff --git a/src/blueapi/config.py b/src/blueapi/config.py index 977722cff7..1976f0f681 100644 --- a/src/blueapi/config.py +++ b/src/blueapi/config.py @@ -28,6 +28,10 @@ FORBIDDEN_OWN_REMOTE_URL = "https://github.com/DiamondLightSource/blueapi.git" +CONFIG_SCHEMA_LOCATION = ( + Path(__file__).parents[2] / "helm" / "blueapi" / "config_schema.json" +) + def _expand_env(loader: yaml.Loader, node: yaml.ScalarNode) -> str: value = loader.construct_scalar(node) diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index a403fb829b..93d87dd332 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -1281,3 +1281,41 @@ def test_python_env_output_formatting(): """) _assert_matching_formatting(OutputFormat.FULL, empty_python_env, full) + + +@pytest.mark.parametrize("output_flag", [True, False]) +@pytest.mark.parametrize("update", [True, False]) +@patch("blueapi.cli.cli.config.CONFIG_SCHEMA_LOCATION") +def test_config_schema( + config_schema_location_mock: Mock, + runner: CliRunner, + output_flag: bool, + update: bool, + tmp_path: Path, +): + args = ["config-schema"] + + tmp_path = tmp_path / "foo.json" + + if output_flag: + args.append("-o") + args.append(f"{tmp_path}") + + if update: + args.append("-u") + + result = runner.invoke( + main, + args, + ) + + expected = ApplicationConfig.model_json_schema() + if output_flag and (not update): + with tmp_path.open("r") as stream: + assert json.load(stream) == expected + elif update: + config_schema_location_mock.open.assert_called() + with config_schema_location_mock.open() as stream: + stream.write.assert_called() + else: + assert json.loads(result.output) == expected diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py index 339644e958..bd841743b2 100644 --- a/tests/unit_tests/test_config.py +++ b/tests/unit_tests/test_config.py @@ -13,7 +13,12 @@ from bluesky_stomp.models import BasicAuthentication from pydantic import BaseModel, Field -from blueapi.config import ApplicationConfig, ConfigLoader, OIDCConfig +from blueapi.config import ( + CONFIG_SCHEMA_LOCATION, + ApplicationConfig, + ConfigLoader, + OIDCConfig, +) from blueapi.utils import InvalidConfigError @@ -497,3 +502,17 @@ def validate_field_annotations(model_class: Any, model_field: str) -> None: check_no_extra_fields(annotation) else: check_no_extra_fields(extracted_annotations) + + +@pytest.mark.skipif( + not CONFIG_SCHEMA_LOCATION.exists(), + reason="If the schema file does not exist, the test is being run" + " with a non-editable install", +) +def test_config_schema_updated() -> None: + with CONFIG_SCHEMA_LOCATION.open("r") as stream: + config_schema = json.load(stream) + assert config_schema == ApplicationConfig.model_json_schema(), ( + f"ApplicationConfig model is out of date with schema at \ + {CONFIG_SCHEMA_LOCATION}. You may need to run `blueapi config-schema -u`" + )