diff --git a/CHANGELOG.md b/CHANGELOG.md index 0900214..5bc0e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `GraphSerializer.json`: explicit serialization like python's `json` module does - `GraphSerializer.json_pickle` (default for `JsonProxy`): preserve JSON type, pickle others types - `GraphSerializer.hdf5_pickle` (default for `NexusProxy`): preserve JSON+numpy type, pickle others types +- Add new field `inputs` to the spec + ### Fixed diff --git a/doc/reference/specs.rst b/doc/reference/specs.rst index 1b56b75..d937d64 100644 --- a/doc/reference/specs.rst +++ b/doc/reference/specs.rst @@ -49,7 +49,8 @@ Graph attributes ^^^^^^^^^^^^^^^^ * *id* (optional): graph identifier unique to a database of graphs (Default: "notspecified") * *label* (optional): non-unique label to be used when identifying a graph for human consumption -* *schema_version* (optional): the schema version of this graph representation (Default: "1.0") +* *schema_version* (optional): the schema version of this graph representation (Default: "1.2") +* *inputs* (optional): inputs of the workflow itself. Must be a JSON schema representing a subset of the node inputs. * *requirements* (optional): a list of projects that should be present in the Python environment for the graph to be executed. * *input_nodes* (optional): nodes that are expected to be used as link targets when the graph diff --git a/pyproject.toml b/pyproject.toml index e424a81..0c11a50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "pydantic >= 2", "typing_extensions;python_version < '3.11'", "backports.strenum;python_version < '3.11'", + "jsonschema", ] [project.urls] diff --git a/src/ewokscore/_serialization/common/utils/constants.py b/src/ewokscore/_serialization/common/utils/constants.py index 43ad693..7779227 100644 --- a/src/ewokscore/_serialization/common/utils/constants.py +++ b/src/ewokscore/_serialization/common/utils/constants.py @@ -1,2 +1,2 @@ -EWOKS_KEY = "__ewoks__" +EWOKS_KEY = "__ewoks_type__" EWOKS_FORMAT_KEY = "__ewoks_serialize__" diff --git a/src/ewokscore/graph/schema/__init__.py b/src/ewokscore/graph/schema/__init__.py index 0703744..540e6cc 100644 --- a/src/ewokscore/graph/schema/__init__.py +++ b/src/ewokscore/graph/schema/__init__.py @@ -8,6 +8,7 @@ from .metadata import SchemaMetadata from .update import from_v1_0_to_v1_1 +from .update import from_v1_1_to_v1_2 from .update import v0_update _VERSIONS = None @@ -21,7 +22,8 @@ def get_versions() -> Dict[Version, SchemaMetadata]: _VERSIONS = { parse_version("0.0"): SchemaMetadata(("0.0", "0.0.1"), v0_update), parse_version("1.0"): SchemaMetadata(("0.1.0-rc", None), from_v1_0_to_v1_1), - parse_version("1.1"): SchemaMetadata(("0.1.0-rc", None), None), + parse_version("1.1"): SchemaMetadata(("0.1.0-rc", None), from_v1_1_to_v1_2), + parse_version("1.2"): SchemaMetadata(("0.1.0-rc1", None), None), } return _VERSIONS diff --git a/src/ewokscore/graph/schema/model.py b/src/ewokscore/graph/schema/model.py index c6afcd0..08fa7af 100644 --- a/src/ewokscore/graph/schema/model.py +++ b/src/ewokscore/graph/schema/model.py @@ -1,5 +1,6 @@ import sys from typing import Any +from typing import Dict from typing import Hashable from typing import Literal from typing import Optional @@ -17,8 +18,12 @@ from typing import Self +from jsonschema import Draft7Validator +from jsonschema.exceptions import SchemaError from pydantic import BaseModel from pydantic import Field +from pydantic import ValidationError +from pydantic import field_validator from pydantic import model_validator from . import LATEST_VERSION @@ -109,6 +114,14 @@ class PpfPortNode(_EwoksBaseNode): ] +class EwoksWorkflowInput(BaseModel): + name: str + task_identifier: Optional[str] = None + id: Optional[NodeId] = None + label: Optional[str] = None + all_nodes: bool = False + + class EwoksNodeAlias(EwoksNodeAttributes): id: NodeId node: NodeId @@ -123,6 +136,23 @@ class EwoksGraphAttributes(BaseModel): requirements: Sequence[str] = [] input_nodes: Sequence[EwoksNodeAlias] = [] output_nodes: Sequence[EwoksNodeAlias] = [] + inputs: Dict[str, Any] = {} + + @field_validator("inputs") + @classmethod + def validate_schema(cls, inputs): + try: + Draft7Validator.check_schema(inputs) + except SchemaError: + raise + + properties = inputs.get("properties", {}) + for input in properties.values(): + try: + EwoksWorkflowInput(**input.get("__ewoks__")) + except ValidationError: + raise + return inputs class EwoksGraph(BaseModel): diff --git a/src/ewokscore/graph/schema/update.py b/src/ewokscore/graph/schema/update.py index b89f601..7d17c5f 100644 --- a/src/ewokscore/graph/schema/update.py +++ b/src/ewokscore/graph/schema/update.py @@ -9,3 +9,8 @@ def v0_update(graph: networkx.DiGraph) -> None: def from_v1_0_to_v1_1(graph: networkx.DiGraph) -> None: """This version does not have the requirements field.""" graph.graph["schema_version"] = "1.1" + + +def from_v1_1_to_v1_2(graph: networkx.DiGraph) -> None: + """This version does not have the inputs field.""" + graph.graph["schema_version"] = "1.2" diff --git a/src/ewokscore/tests/examples/graphs/__init__.py b/src/ewokscore/tests/examples/graphs/__init__.py index d1ee2e6..ae16eed 100644 --- a/src/ewokscore/tests/examples/graphs/__init__.py +++ b/src/ewokscore/tests/examples/graphs/__init__.py @@ -34,7 +34,7 @@ def wrapper(): attrs = g.setdefault("graph", dict()) assert attrs.get("id") == name assert attrs.get("label") == name - assert attrs.get("schema_version") == "1.1" + assert attrs.get("schema_version") == "1.2" return g, result if _ALL_GRAPHS is None: diff --git a/src/ewokscore/tests/examples/graphs/acyclic1.py b/src/ewokscore/tests/examples/graphs/acyclic1.py index f3d2f8e..2de971d 100644 --- a/src/ewokscore/tests/examples/graphs/acyclic1.py +++ b/src/ewokscore/tests/examples/graphs/acyclic1.py @@ -133,7 +133,7 @@ def acyclic1(): }, ] } - graph = {"id": "acyclic1", "label": "acyclic1", "schema_version": "1.1", "ows": ows} + graph = {"id": "acyclic1", "label": "acyclic1", "schema_version": "1.2", "ows": ows} taskgraph = {"graph": graph, "links": links, "nodes": nodes} diff --git a/src/ewokscore/tests/examples/graphs/acyclic2.py b/src/ewokscore/tests/examples/graphs/acyclic2.py index 7462c34..f36cd2a 100644 --- a/src/ewokscore/tests/examples/graphs/acyclic2.py +++ b/src/ewokscore/tests/examples/graphs/acyclic2.py @@ -3,7 +3,7 @@ @graph def acyclic2(): - graph = {"id": "acyclic2", "label": "acyclic2", "schema_version": "1.1"} + graph = {"id": "acyclic2", "label": "acyclic2", "schema_version": "1.2"} task = "ewokscore.tests.examples.tasks.errorsumtask.ErrorSumTask" nodes = [ diff --git a/src/ewokscore/tests/examples/graphs/acyclic3.py b/src/ewokscore/tests/examples/graphs/acyclic3.py index a853ebd..2f4f629 100644 --- a/src/ewokscore/tests/examples/graphs/acyclic3.py +++ b/src/ewokscore/tests/examples/graphs/acyclic3.py @@ -4,7 +4,7 @@ @graph def acyclic3(): - graph = {"id": "acyclic3", "label": "acyclic3", "schema_version": "1.1"} + graph = {"id": "acyclic3", "label": "acyclic3", "schema_version": "1.2"} task = "ewokscore.tests.examples.tasks.sumtask.SumTask" nodes = [ diff --git a/src/ewokscore/tests/examples/graphs/cyclic1.py b/src/ewokscore/tests/examples/graphs/cyclic1.py index a770675..ab30a30 100644 --- a/src/ewokscore/tests/examples/graphs/cyclic1.py +++ b/src/ewokscore/tests/examples/graphs/cyclic1.py @@ -3,7 +3,7 @@ @graph def cyclic1(): - graph = {"id": "cyclic1", "label": "cyclic1", "schema_version": "1.1"} + graph = {"id": "cyclic1", "label": "cyclic1", "schema_version": "1.2"} task = "ewokscore.tests.examples.tasks.condsumtask.CondSumTask" nodes = [ diff --git a/src/ewokscore/tests/examples/graphs/demo.py b/src/ewokscore/tests/examples/graphs/demo.py index 8e11092..705eeae 100644 --- a/src/ewokscore/tests/examples/graphs/demo.py +++ b/src/ewokscore/tests/examples/graphs/demo.py @@ -3,7 +3,25 @@ @graph def demo(): - graph = {"id": "demo", "label": "demo", "schema_version": "1.1"} + graph = { + "id": "demo", + "label": "demo", + "schema_version": "1.2", + "inputs": { + "properties": { + "task0a": { + "type": "number", + "__ewoks__": {"name": "a", "id": "task0"}, + }, + "task0b": { + "type": "number", + "default": 0, + "__ewoks__": {"name": "b", "id": 2}, + }, + }, + "required": ["task0a"], + }, + } sumtask = "ewokscore.tests.examples.tasks.sumtask.SumTask" sumlist = "ewokscore.tests.examples.tasks.sumlist.SumList" @@ -120,7 +138,11 @@ def demo(): }, ] - taskgraph = {"graph": graph, "links": links, "nodes": nodes} + taskgraph = { + "graph": graph, + "links": links, + "nodes": nodes, + } expected_results = { "task0": {"sum": 3}, diff --git a/src/ewokscore/tests/examples/graphs/empty.py b/src/ewokscore/tests/examples/graphs/empty.py index 850d2d4..ec9d7e8 100644 --- a/src/ewokscore/tests/examples/graphs/empty.py +++ b/src/ewokscore/tests/examples/graphs/empty.py @@ -3,5 +3,5 @@ @graph def empty(): - graph = {"id": "empty", "label": "empty", "schema_version": "1.1"} + graph = {"id": "empty", "label": "empty", "schema_version": "1.2"} return {"graph": graph}, dict() diff --git a/src/ewokscore/tests/examples/graphs/self_trigger.py b/src/ewokscore/tests/examples/graphs/self_trigger.py index b5f025f..35dd878 100644 --- a/src/ewokscore/tests/examples/graphs/self_trigger.py +++ b/src/ewokscore/tests/examples/graphs/self_trigger.py @@ -3,7 +3,7 @@ @graph def self_trigger(): - graph = {"id": "self_trigger", "label": "self_trigger", "schema_version": "1.1"} + graph = {"id": "self_trigger", "label": "self_trigger", "schema_version": "1.2"} task = "ewokscore.tests.examples.tasks.condsumtask.CondSumTask" nodes = [ diff --git a/src/ewokscore/tests/examples/graphs/triangle1.py b/src/ewokscore/tests/examples/graphs/triangle1.py index 91234d9..137793f 100644 --- a/src/ewokscore/tests/examples/graphs/triangle1.py +++ b/src/ewokscore/tests/examples/graphs/triangle1.py @@ -3,7 +3,7 @@ @graph def triangle1(): - graph = {"id": "triangle1", "label": "triangle1", "schema_version": "1.1"} + graph = {"id": "triangle1", "label": "triangle1", "schema_version": "1.2"} task = "ewokscore.tests.examples.tasks.condsumtask.CondSumTask" nodes = [ diff --git a/src/ewokscore/tests/serialization/test_graph.py b/src/ewokscore/tests/serialization/test_graph.py index d226f9c..23eda74 100644 --- a/src/ewokscore/tests/serialization/test_graph.py +++ b/src/ewokscore/tests/serialization/test_graph.py @@ -4,7 +4,11 @@ import pytest import yaml +from ..._serialization.common.utils.types import GraphSerializer +from ...bindings import save_graph from ...graph import load_graph +from ..test_examples import get_graph +from ..test_examples import graph_names @pytest.mark.parametrize("with_ext", [True, False]) @@ -56,6 +60,17 @@ def test_graph_discovery_json_module(with_representation): assert ewoksgraph.graph.graph["id"] == "ewokscore.tests.examples.loadtest.graph" +@pytest.mark.parametrize("graph_name", graph_names()) +@pytest.mark.parametrize("serializer", [None, *GraphSerializer]) +def test_graph_save(tmp_path, graph_name, serializer): + graph_dict, _ = get_graph("demo") + graph = load_graph(graph_dict, representation="json_dict") + save_graph( + graph, destination=tmp_path / f"{graph_name}.json", serializer=serializer + ) + assert (tmp_path / f"{graph_name}.json").exists() + + def _dump_graph_and_subgraph(tmp_path, format, with_ext): if format == "yaml": dump = yaml.dump diff --git a/src/ewokscore/tests/test_model.py b/src/ewokscore/tests/test_model.py index 0ed7518..0f0ef71 100644 --- a/src/ewokscore/tests/test_model.py +++ b/src/ewokscore/tests/test_model.py @@ -1,5 +1,6 @@ import pydantic import pytest +from jsonschema.exceptions import SchemaError from ewokscore import load_graph from ewokscore.graph.schema.model import EwoksGraph @@ -127,3 +128,45 @@ def test_link_with_datamapping_and_map_all_data(): match="1 validation error for EwoksGraph\nlinks.0\n", ): EwoksGraph(**graph_dict) + + +def test_input_schema(): + # Wrong type in schema + graph_dict = { + "graph": { + "id": "required", + "inputs": {"properties": {"a": {"type": "oo", "__ewoks__": {"name": "a"}}}}, + }, + "nodes": [ + { + "id": "node1", + "task_type": "class", + "task_identifier": "task1", + }, + ], + "links": [], + } + with pytest.raises(SchemaError): + EwoksGraph(**graph_dict) + + # Missing name in __ewoks__ + graph_dict = { + "graph": { + "id": "required", + "inputs": {"properties": {"a": {"type": "number", "__ewoks__": {}}}}, + }, + "nodes": [ + { + "id": "node1", + "task_type": "class", + "task_identifier": "task1", + }, + ], + "links": [], + } + + with pytest.raises( + pydantic.ValidationError, + match="1 validation error for EwoksGraph\ngraph.inputs.name\n", + ): + EwoksGraph(**graph_dict)