Skip to content

perf(profiles): ⚡️ speed up SPARQL target queries on medium/large crates #191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kikkomep wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

perf(profiles): ⚡️ speed up SPARQL target queries on medium/large crates #191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
be6aa6f
perf(profiles): ⚡️ eliminate cross products in SPARQL target queries
kikkomep Aug 6, 2026
88ded67
perf(profiles): ⚡️ deduplicate contextual entity target before filtering
kikkomep Aug 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions rocrate_validator/profiles/ro-crate/1.1/must/4_data_entity_metadata.ttl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,11 @@ ro-crate:FileDataEntity a sh:NodeShape ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
Comment on lines +50 to +53

@elichad elichad Aug 7, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be really useful to document this as a recommended pattern for profile developers when they need to fetch the root in a query

@elichad elichad Aug 7, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also: the wording of this PR is very jargon-heavy, and I found it difficult to parse until I read the code myself ("Cartesian products in target queries" in particular meant nothing to me). It would help in future if you could describe the solution in simpler language, to make it easier for others to review the code, understand why it works, and (in my case) understand what further changes might be needed downstream.

?this a schema:MediaObject .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
FILTER(!STRSTARTS(STR(?this), CONCAT(STR(?root), "#")))
}
"""
Expand Down Expand Up @@ -86,10 +88,12 @@ ro-crate:DirectoryDataEntity a sh:NodeShape ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
?this a schema:Dataset .
?metadatafile schema:about ?root .
# Exclude all dataset entities that ends with `./#<something>`
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
FILTER(?this != ?root)
FILTER(!STRSTARTS(STR(?this), CONCAT(STR(?root), "#")))
}
Expand Down Expand Up @@ -157,9 +161,11 @@ ro-crate:GenericDataEntityRequiredProperties a sh:NodeShape ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root ?metadatafile WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
?root schema:hasPart ?this .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
FILTER(?this != ?root)
FILTER(?this != ?metadatafile)
FILTER NOT EXISTS {
Expand Down
18 changes: 12 additions & 6 deletions rocrate_validator/profiles/ro-crate/1.2/must/4_data_entity_metadata.ttl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,11 @@ ro-crate:FileDataEntity a sh:NodeShape ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
?this a schema:MediaObject .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
FILTER(!STRSTARTS(STR(?this), CONCAT(STR(?root), "#")))
}
"""
Expand Down Expand Up @@ -93,10 +95,12 @@ ro-crate:DirectoryDataEntity a sh:NodeShape ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
?this a schema:Dataset .
?metadatafile schema:about ?root .
# Exclude all dataset entities that ends with `./#<something>`
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
FILTER(?this != ?root)
FILTER(!STRSTARTS(STR(?this), CONCAT(STR(?root), "#")))
}
Expand Down Expand Up @@ -161,9 +165,11 @@ ro-crate:GenericDataEntityRequiredProperties a sh:NodeShape ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root ?metadatafile WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
?root schema:hasPart ?this .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
FILTER(?this != ?root)
FILTER(?this != ?metadatafile)
FILTER NOT EXISTS {
Expand Down
12 changes: 9 additions & 3 deletions rocrate_validator/profiles/ro-crate/1.2/must/6_contextual_entity_metadata.ttl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,20 @@ ro-crate:ContextualEntityDefinition a sh:NodeShape, validator:HiddenShape ;
a sh:SPARQLTarget ;
sh:prefixes ro-crate:sparqlPrefixes ;
sh:select """
SELECT DISTINCT ?this
SELECT ?this
WHERE {
# Structural criteria: IRI or blank node, subject of some triple.
# SHACL rules from earlier shapes (e.g. CreativeWorkAuthorDefinition)
# tag even referenced-but-undescribed entities so they become
# subjects and are correctly classified as Contextual Entities.
?this ?p ?o .
FILTER(isIRI(?this) || isBlank(?this))
#
# The distinct subjects are materialised by a subquery FIRST so the
# exclusion FILTERs below run once per entity instead of once per
# triple (~2.5k subjects vs ~40k triples on a large crate).
{ SELECT DISTINCT ?this WHERE {
?this ?p ?o .
FILTER(isIRI(?this) || isBlank(?this))
} }

# Structural exclusions — Metadata Descriptor, Root Data Entity,
# Data Entities (File/Dataset or ro-crate:DataEntity), Ontology.
Expand Down
18 changes: 12 additions & 6 deletions rocrate_validator/profiles/ro-crate/1.2/must/8_workflow_metadata.ttl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ ro-crate:SoftwareSourceCodeTarget a sh:SPARQLTarget ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
Comment on lines +29 to +32

@elichad elichad Aug 7, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see ?metadatafile or ?root referenced elsewhere in this target - is this part necessary?

?this a schema:SoftwareSourceCode .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
}
""" .

Expand All @@ -48,9 +50,11 @@ ro-crate:ScriptDefinition a sh:NodeShape, validator:HiddenShape ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
Comment on lines +53 to +56

@elichad elichad Aug 7, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same as above - I don't see ?metadatafile or ?root referenced elsewhere in this target - is this part necessary?

?this a schema:SoftwareSourceCode .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
FILTER NOT EXISTS { ?this a bioschemas:ComputationalWorkflow }
}
"""
Expand Down Expand Up @@ -78,9 +82,11 @@ ro-crate:WorkflowDefinition a sh:NodeShape, validator:HiddenShape ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
Comment on lines +85 to +88

@elichad elichad Aug 7, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same as above - I don't see ?metadatafile or ?root referenced elsewhere in this target - is this part necessary?

?this a bioschemas:ComputationalWorkflow .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
}
"""
] ;
Expand Down
9 changes: 5 additions & 4 deletions rocrate_validator/profiles/ro-crate/1.2/should/0_entity_metadata.ttl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ ro-crate:ROCrateMetadataEntityRecommendedType a sh:NodeShape ;
WHERE {
?this a ?type .

# Exclude Root and RO-Crate Metadata File entities
?root a schema:Dataset .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
{ SELECT DISTINCT ?root ?metadatafile WHERE {
?root a schema:Dataset .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
FILTER(?this != ?metadatafile)
FILTER (?this != ?root)
# Exclude entities with non-IRI identifiers or those from specific namespaces
Expand Down
6 changes: 4 additions & 2 deletions rocrate_validator/profiles/ro-crate/1.2/should/4_data_entity_metadata.ttl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,9 +164,11 @@ ro-crate:WebDirectoryDistributionRecommended a sh:NodeShape ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
?this a schema:Dataset .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
FILTER(?this != ?root)
FILTER(!STRSTARTS(STR(?this), CONCAT(STR(?root), "#")))
FILTER regex(str(?this), "^https?://", "i")
Expand Down
6 changes: 4 additions & 2 deletions rocrate_validator/profiles/ro-crate/1.2/should/4_dataset_data_entity.ttl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ ro-crate:LocalDatasetTarget a sh:SPARQLTarget ;
sh:select """
SELECT ?this
WHERE {
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
?this a schema:Dataset .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
FILTER(?this != ?root)
FILTER(!STRSTARTS(STR(?this), CONCAT(STR(?root), "#")))
FILTER(!regex(str(?this), "^https?://", "i"))
Expand Down
12 changes: 8 additions & 4 deletions rocrate_validator/profiles/ro-crate/1.2/should/4_referenced_rocrate.ttl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,10 @@ ro-crate:ReferencedROCrateDataEntityTarget a sh:SPARQLTarget ;
?this a schema:Dataset ;
dct:conformsTo ?profile .
FILTER(STRSTARTS(STR(?profile), "https://w3id.org/ro/crate"))
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
FILTER(?this != ?root)
}
""" .
Expand All @@ -56,8 +58,10 @@ ro-crate:ReferencedROCrateMetadataDescriptorTarget a sh:SPARQLTarget ;
dct:conformsTo ?profile ;
schema:subjectOf ?this .
FILTER(STRSTARTS(STR(?profile), "https://w3id.org/ro/crate"))
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
FILTER(?crate != ?root)
}
""" .
Expand Down
6 changes: 4 additions & 2 deletions rocrate_validator/profiles/ro-crate/1.2/should/8_workflow_metadata.ttl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@ ro-crate:ScriptOrWorkflowImageTarget a sh:SPARQLTarget ;
WHERE {
?sw a schema:SoftwareSourceCode .
?sw schema:image ?this .
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
{ SELECT DISTINCT ?root WHERE {
?metadatafile schema:about ?root .
FILTER(contains(str(?metadatafile), "ro-crate-metadata.json"))
} }
}
""" .

Expand Down