Implement new version of FastRun

Author: LeMyst

README.md

@@ -869,7 +869,15 @@ for entrez_id, ensembl in raw_data.items():
 
 Note: Fastrun mode checks for equality of property/value pairs, qualifiers (not including qualifier attributes), labels,
 aliases and description, but it ignores references by default!
-References can be checked in fast run mode by setting `use_refs` to `True`.
+References can be checked in fast run mode by setting `use_references` to `True`.
+
+# Statistics #
+
+| Dataset                     | partial fastrun | fastrun without qualifiers/references | fastrun with qualifiers | fastrun with qualifiers/references |
+|:----------------------------|----------------:|--------------------------------------:|------------------------:|-----------------------------------:|
+| Communes (34990 elements)   |               ? |                                  7min |                     30s |                                60s |
+| Cantons (2042 elements)     |               ? |                                     ? |                       ? |                                  ? |
+| Départements (100 elements) |           70min |                                    1s |                     30s |                                60s |
 
 # Debugging #
 

pyproject.toml

@@ -120,6 +120,7 @@ disable = [
 
 [tool.pytest.ini_options]
 log_cli = true
+log_cli_level = 'DEBUG'
 testpaths = ["test"]
 markers = [
     "integration: tests running against a real Wikibase instance (deselected by default, see test/integration/README.md)"

test/test_datatypes.py

@@ -163,7 +163,7 @@ def test_all_datatypes(self):
             MonolingualText(text='xxx', language='fr', prop_nr='P7'),
             Quantity(amount=-5.04, prop_nr='P8'),
             Quantity(amount=5.06, upper_bound=9.99, lower_bound=-2.22, unit='Q11573', prop_nr='P8'),
-            CommonsMedia(value='xxx', prop_nr='P9'),
+            CommonsMedia(value='xxx.jpg', prop_nr='P9'),
             GlobeCoordinate(latitude=1.2345, longitude=-1.2345, precision=12, prop_nr='P10'),
             GeoShape(value='Data:xxx.map', prop_nr='P11'),
             Property(value='P123', prop_nr='P12'),

test/test_wbi_fastrun.py

@@ -11,6 +11,7 @@ class BaseDataType(Claim):
     The base class for all Wikibase data types, they inherit from it
     """
     DTYPE = 'base-data-type'
+    PTYPE = 'property-data-type'
     subclasses: list[type[BaseDataType]] = []
     sparql_query: str = '''
         SELECT * WHERE {{
@@ -28,7 +29,14 @@ def __init__(self, prop_nr: int | str | None = None, **kwargs: Any):
 
         super().__init__(**kwargs)
 
-        self.mainsnak.property_number = prop_nr or None
+        if isinstance(prop_nr, str):
+            pattern = re.compile(r'^([a-z][a-z\d+.-]*):([^][<>\"\x00-\x20\x7F])+$')
+            matches = pattern.match(str(prop_nr))
+
+            if matches:
+                prop_nr = prop_nr.rsplit('/', 1)[-1]
+
+        self.mainsnak.property_number = prop_nr
         # self.subclasses.append(self)
 
     # Allow registration of subclasses of BaseDataType into BaseDataType.subclasses
@@ -39,7 +47,7 @@ def __init_subclass__(cls, **kwargs):
     def set_value(self, value: Any | None = None):
         pass
 
-    def get_sparql_value(self) -> str:
+    def get_sparql_value(self, **kwargs: Any) -> str | None:
         return '"' + self.mainsnak.datavalue['value'] + '"'
 
     def parse_sparql_value(self, value, type='literal', unit='1') -> bool:
@@ -61,3 +69,6 @@ def parse_sparql_value(self, value, type='literal', unit='1') -> bool:
             raise ValueError
 
         return True
+
+    def from_sparql_value(self, sparql_value: dict) -> BaseDataType:  # type: ignore
+        pass

wikibaseintegrator/datatypes/basedatatype.py

@@ -1,17 +1,30 @@
 import re
 import urllib.parse
 
-from wikibaseintegrator.datatypes.string import String
+from wikibaseintegrator.datatypes.url import URL
 
 
-class CommonsMedia(String):
+class CommonsMedia(URL):
     """
     Implements the Wikibase data type for Wikimedia commons media files
     """
     DTYPE = 'commonsMedia'
+    PTYPE = 'http://wikiba.se/ontology#CommonsMedia'
 
-    def get_sparql_value(self) -> str:
-        return '<' + self.mainsnak.datavalue['value'] + '>'
+    def set_value(self, value: str | None = None):
+        assert isinstance(value, str) or value is None, f"Expected str, found {type(value)} ({value})"
+
+        if value:
+            pattern = re.compile(r'^.+\..+$')
+            matches = pattern.match(value)
+
+            if not matches:
+                raise ValueError(f"Invalid CommonsMedia {value}")
+
+            self.mainsnak.datavalue = {
+                'value': value,
+                'type': 'string'
+            }
 
     def parse_sparql_value(self, value, type='literal', unit='1') -> bool:
         pattern = re.compile(r'^<?.*?/?([^/]*?)>?$')

wikibaseintegrator/datatypes/commonsmedia.py

@@ -6,3 +6,4 @@ class ExternalID(String):
     Implements the Wikibase data type 'external-id'
     """
     DTYPE = 'external-id'
+    PTYPE = 'http://wikiba.se/ontology#ExternalId'

wikibaseintegrator/datatypes/externalid.py

@@ -2,13 +2,16 @@
 from typing import Any
 
 from wikibaseintegrator.datatypes.basedatatype import BaseDataType
+from wikibaseintegrator.wbi_config import config
+from wikibaseintegrator.wbi_enums import WikibaseSnakType
 
 
 class Form(BaseDataType):
     """
     Implements the Wikibase data type 'wikibase-form'
     """
     DTYPE = 'wikibase-form'
+    PTYPE = 'http://wikiba.se/ontology#WikibaseForm'
     sparql_query = '''
         SELECT * WHERE {{
           ?item_id <{wb_url}/prop/{pid}> ?s .
@@ -55,8 +58,14 @@ def set_value(self, value: str | None = None):
                 'type': 'wikibase-entityid'
             }
 
-    def get_sparql_value(self) -> str:
-        return self.mainsnak.datavalue['value']['id']
+    # TODO: add from_sparql_value()
+
+    def get_sparql_value(self, **kwargs: Any) -> str | None:
+        if self.mainsnak.snaktype == WikibaseSnakType.KNOWN_VALUE:
+            wikibase_url = str(kwargs['wikibase_url'] if 'wikibase_url' in kwargs else config['WIKIBASE_URL'])
+            return f'<{wikibase_url}/entity/' + self.mainsnak.datavalue['value']['id'] + '>'
+
+        return None
 
     def get_lexeme_id(self) -> str:
         """

wikibaseintegrator/datatypes/form.py

@@ -9,6 +9,7 @@ class GeoShape(BaseDataType):
     Implements the Wikibase data type 'geo-shape'
     """
     DTYPE = 'geo-shape'
+    PTYPE = 'http://wikiba.se/ontology#GeoShape'
     sparql_query = '''
         SELECT * WHERE {{
           ?item_id <{wb_url}/prop/{pid}> ?s .
@@ -53,3 +54,7 @@ def set_value(self, value: str | None = None):
                 'value': value,
                 'type': 'string'
             }
+
+    # TODO: Does GeoShape need a full URL to wikimedia commons?
+    def get_sparql_value(self, **kwargs: Any) -> str:
+        return '<' + self.mainsnak.datavalue['value'] + '>'

wikibaseintegrator/datatypes/geoshape.py

@@ -1,16 +1,20 @@
+from __future__ import annotations
+
 import re
 from typing import Any
 
 from wikibaseintegrator.datatypes.basedatatype import BaseDataType
 from wikibaseintegrator.models import Claim
 from wikibaseintegrator.wbi_config import config
+from wikibaseintegrator.wbi_enums import WikibaseSnakType
 
 
 class GlobeCoordinate(BaseDataType):
     """
     Implements the Wikibase data type for globe coordinates
     """
     DTYPE = 'globe-coordinate'
+    PTYPE = 'http://wikiba.se/ontology#GlobeCoordinate'
     sparql_query = '''
         SELECT * WHERE {{
           ?item_id <{wb_url}/prop/{pid}> ?s .
@@ -75,8 +79,37 @@ def rounded(datavalue: dict) -> dict:
 
         return super().__eq__(other)
 
-    def get_sparql_value(self) -> str:
-        return '"Point(' + str(self.mainsnak.datavalue['value']['longitude']) + ' ' + str(self.mainsnak.datavalue['value']['latitude']) + ')"'
+    def from_sparql_value(self, sparql_value: dict) -> GlobeCoordinate:
+        """
+        Parse data returned by a SPARQL endpoint and set the value to the object
+
+        :param sparql_value: A SPARQL value composed of datatype, type and value
+        :return: True if the parsing is successful
+        """
+        datatype = sparql_value['datatype']
+        type = sparql_value['type']
+        value = sparql_value['value']
+
+        if datatype != 'http://www.opengis.net/ont/geosparql#wktLiteral':
+            raise ValueError('Wrong SPARQL datatype')
+
+        if type != 'literal':
+            raise ValueError('Wrong SPARQL type')
+
+        if value.startswith('http://www.wikidata.org/.well-known/genid/'):
+            self.mainsnak.snaktype = WikibaseSnakType.UNKNOWN_VALUE
+        else:
+            pattern = re.compile(r'^Point\((.*) (.*)\)$')
+            matches = pattern.match(value)
+            if not matches:
+                raise ValueError('Invalid SPARQL value')
+
+            self.set_value(longitude=float(matches.group(1)), latitude=float(matches.group(2)))
+
+        return self
+
+    def get_sparql_value(self, **kwargs: Any) -> str:
+        return '"Point(' + str(self.mainsnak.datavalue['value']['longitude']) + ' ' + str(self.mainsnak.datavalue['value']['latitude']) + ')"^^geo:wktLiteral'
 
     def parse_sparql_value(self, value, type='literal', unit='1') -> bool:
         pattern = re.compile(r'^"?Point\((.*) (.*)\)"?(?:\^\^geo:wktLiteral)?$')