From 5a5879443db2a1718b38bad9142694fc83eda303 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 08:49:57 +0100 Subject: [PATCH 01/63] Update serialise_rdf_value.py --- .../models/serialise/serialise_rdf_value.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index 076f0b7570..b15d3544ea 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -34,6 +34,31 @@ def _get_generic_graph(self, val_bn: BNode) -> Graph: return g + +class RDF(ValueRDF): + value: Literal + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.)) + g.add((val_bn, KNORA_API., self.value)) + return g + + + +class ColorValueRDF(ValueRDF): + value: Literal + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.ColorValue)) + g.add((val_bn, KNORA_API.colorValueAsColor, self.value)) + return g + + + class BooleanValueRDF(ValueRDF): value: Literal From 8314eb37915cb0f71f32c64ae7f0d4868ae5e15d Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:00:05 +0100 Subject: [PATCH 02/63] Update serialise_rdf_value.py --- .../models/serialise/serialise_rdf_value.py | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index b15d3544ea..e4d65a444c 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -3,12 +3,18 @@ from dataclasses import dataclass from rdflib import RDF +from rdflib import XSD from rdflib import BNode from rdflib import Graph from rdflib import Literal from rdflib import Namespace from rdflib import URIRef +from dsp_tools.utils.date_util import Date +from dsp_tools.utils.date_util import DayMonthYearEra +from dsp_tools.utils.date_util import SingleDate +from dsp_tools.utils.date_util import StartEnd + KNORA_API = Namespace("http://api.knora.org/ontology/knora-api/v2#") @@ -34,19 +40,17 @@ def _get_generic_graph(self, val_bn: BNode) -> Graph: return g - -class RDF(ValueRDF): +class BooleanValueRDF(ValueRDF): value: Literal def as_graph(self) -> Graph: val_bn = BNode() g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.)) - g.add((val_bn, KNORA_API., self.value)) + g.add((val_bn, RDF.type, KNORA_API.BooleanValue)) + g.add((val_bn, KNORA_API.booleanValueAsBoolean, self.value)) return g - class ColorValueRDF(ValueRDF): value: Literal @@ -58,15 +62,33 @@ def as_graph(self) -> Graph: return g - -class BooleanValueRDF(ValueRDF): - value: Literal +class DateValueRDF(ValueRDF): + value: Date def as_graph(self) -> Graph: val_bn = BNode() g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.BooleanValue)) - g.add((val_bn, KNORA_API.booleanValueAsBoolean, self.value)) + g.add((val_bn, RDF.type, KNORA_API.DateValue)) + if cal := self.value.calendar.value: + g.add((val_bn, KNORA_API.dateValueHasCalendar, Literal(cal, datatype=XSD.string))) + g += self._get_one_date_graph(val_bn, self.value.start, StartEnd.START) + if self.value.end: + g += self._get_one_date_graph(val_bn, self.value.end, StartEnd.END) + return g + + def _get_one_date_graph(self, val_bn: BNode, date: SingleDate, start_end: StartEnd) -> Graph: + def get_prop(precision: DayMonthYearEra) -> URIRef: + return KNORA_API[f"dateValueHas{start_end.value}{precision.value}"] + + g = Graph() + if yr := date.year: + g.add((val_bn, get_prop(DayMonthYearEra.YEAR), Literal(yr, datatype=XSD.int))) + if mnt := date.month: + g.add((val_bn, get_prop(DayMonthYearEra.MONTH), Literal(mnt, datatype=XSD.int))) + if day := date.day: + g.add((val_bn, get_prop(DayMonthYearEra.DAY), Literal(day, datatype=XSD.int))) + if era := date.era: + g.add((val_bn, get_prop(DayMonthYearEra.ERA), Literal(era, datatype=XSD.string))) return g From eb2a81371317d93e8ef094722b51e7ffba9e0ee0 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:01:06 +0100 Subject: [PATCH 03/63] Update serialise_rdf_value.py --- .../models/serialise/serialise_rdf_value.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index e4d65a444c..bd74c52ad9 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -40,6 +40,19 @@ def _get_generic_graph(self, val_bn: BNode) -> Graph: return g + + +class RDF(ValueRDF): + value: Literal + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.)) + g.add((val_bn, KNORA_API., self.value)) + return g + + class BooleanValueRDF(ValueRDF): value: Literal @@ -92,6 +105,18 @@ def get_prop(precision: DayMonthYearEra) -> URIRef: return g + +class DecimalValueRDF(ValueRDF): + value: Literal + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.DecimalValue)) + g.add((val_bn, KNORA_API.decimalValueAsDecimal, self.value)) + return g + + class IntValueRDF(ValueRDF): value: Literal From 4cc604f533b3af33dd299926a06ab448d2393f50 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:02:07 +0100 Subject: [PATCH 04/63] Update serialise_rdf_value.py --- .../models/serialise/serialise_rdf_value.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index bd74c52ad9..ec86c5debd 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -117,6 +117,30 @@ def as_graph(self) -> Graph: return g +class GeomValueRDF(ValueRDF): + value: Literal + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.GeomValue)) + g.add((val_bn, KNORA_API.geometryValueAsGeometry, self.value)) + return g + + + + +class GeonameValueRDF(ValueRDF): + value: Literal + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.GeonameValue)) + g.add((val_bn, KNORA_API.geonameValueAsGeonameCode, self.value)) + return g + + class IntValueRDF(ValueRDF): value: Literal From af62e0b5761458b802ea2cf59b2de7ef9f050697 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:04:39 +0100 Subject: [PATCH 05/63] Update serialise_rdf_value.py --- .../models/serialise/serialise_rdf_value.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index ec86c5debd..c707794574 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from abc import ABC from abc import abstractmethod from dataclasses import dataclass @@ -150,3 +152,24 @@ def as_graph(self) -> Graph: g.add((val_bn, RDF.type, KNORA_API.IntValue)) g.add((val_bn, KNORA_API.intValueAsInt, self.value)) return g + + + +class IntervalValueValueRDF(ValueRDF): + """An IntervalValue to be serialised.""" + + value: Interval + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.IntervalValue)) + g.add((val_bn, KNORA_API.intervalValueHasStart, self.value.start)) + g.add((val_bn, KNORA_API.intervalValueHasEnd, self.value.end)) + return g + + +@dataclass +class Interval: + start: Literal + end: Literal \ No newline at end of file From 2299e2644193970df3507aa817bcdb3d2205e41b Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:07:52 +0100 Subject: [PATCH 06/63] Update serialise_rdf_value.py --- .../models/serialise/serialise_rdf_value.py | 86 +++++++++++++++---- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index c707794574..e5893c1e12 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -42,19 +42,6 @@ def _get_generic_graph(self, val_bn: BNode) -> Graph: return g - - -class RDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.)) - g.add((val_bn, KNORA_API., self.value)) - return g - - class BooleanValueRDF(ValueRDF): value: Literal @@ -107,7 +94,6 @@ def get_prop(precision: DayMonthYearEra) -> URIRef: return g - class DecimalValueRDF(ValueRDF): value: Literal @@ -130,8 +116,6 @@ def as_graph(self) -> Graph: return g - - class GeonameValueRDF(ValueRDF): value: Literal @@ -154,7 +138,6 @@ def as_graph(self) -> Graph: return g - class IntervalValueValueRDF(ValueRDF): """An IntervalValue to be serialised.""" @@ -172,4 +155,71 @@ def as_graph(self) -> Graph: @dataclass class Interval: start: Literal - end: Literal \ No newline at end of file + end: Literal + + +class ListValueRDF(ValueRDF): + value: URIRef + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.ListValue)) + g.add((val_bn, KNORA_API.listValueAsListNode, self.value)) + return g + + +class LinkValueRDF(ValueRDF): + value: URIRef + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.LinkValue)) + g.add((val_bn, KNORA_API.linkValueHasTargetIri, self.value)) + return g + + +class SimpletextRDF(ValueRDF): + value: Literal + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.TextValue)) + g.add((val_bn, KNORA_API.valueAsString, self.value)) + return g + + +class RichtextRDF(ValueRDF): + value: Literal + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.TextValue)) + g.add((val_bn, KNORA_API.textValueAsXml, self.value)) + g.add((val_bn, KNORA_API.textValueHasMapping, URIRef("http://rdfh.ch/standoff/mappings/StandardMapping"))) + return g + + +class TimeValueRDF(ValueRDF): + value: Literal + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.TimeValue)) + g.add((val_bn, KNORA_API.timeValueAsTimeStamp, self.value)) + return g + + +class UriValueRDF(ValueRDF): + value: Literal + + def as_graph(self) -> Graph: + val_bn = BNode() + g = self._get_generic_graph(val_bn) + g.add((val_bn, RDF.type, KNORA_API.UriValue)) + g.add((val_bn, KNORA_API.uriValueAsUri, self.value)) + return g From 2063da8bc7350b094ec08ef9c6e7b012287adfa1 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:26:38 +0100 Subject: [PATCH 07/63] Update serialise_rdf_value.py --- .../models/serialise/serialise_rdf_value.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index e5893c1e12..c88957ae7c 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -3,6 +3,8 @@ from abc import ABC from abc import abstractmethod from dataclasses import dataclass +from typing import Any +from typing import Callable from rdflib import RDF from rdflib import XSD @@ -12,6 +14,7 @@ from rdflib import Namespace from rdflib import URIRef +from dsp_tools.commands.xmlupload.value_transformers import transform_geometry, assert_is_string from dsp_tools.utils.date_util import Date from dsp_tools.utils.date_util import DayMonthYearEra from dsp_tools.utils.date_util import SingleDate @@ -20,6 +23,39 @@ KNORA_API = Namespace("http://api.knora.org/ontology/knora-api/v2#") +@dataclass +class RDFLiteralInfo: + knora_type: URIRef + knora_prop: URIRef + transformations: Callable[[Any], Literal] + + +def transform_xsd_string(): ... +def transform_xsd_decimal(): ... +def transform_xsd_boolean(): ... +def transform_xsd_integer(): ... +def transform_xsd_date_time(): ... +def transform_xsd_anyuri(): ... + + +rdf_mapper = { + "boolean": RDFLiteralInfo(KNORA_API.BooleanValue, KNORA_API.booleanValueAsBoolean, transform_xsd_boolean), + "color": RDFLiteralInfo(KNORA_API.ColorValue, KNORA_API.colorValueAsColor, transform_xsd_string), + "decimal": RDFLiteralInfo(KNORA_API.DecimalValue, KNORA_API.decimalValueAsDecimal, transform_xsd_decimal), + "geometry": RDFLiteralInfo(KNORA_API.GeomValue, KNORA_API.geometryValueAsGeometry, transform_geometry), + "geoname": RDFLiteralInfo(KNORA_API.GeonameValue, KNORA_API.geonameValueAsGeonameCode, transform_xsd_string), + "integer": RDFLiteralInfo(KNORA_API.IntValue, KNORA_API.intValueAsInt, transform_xsd_integer), + "time": RDFLiteralInfo(KNORA_API.TimeValue, KNORA_API.timeValueAsTimeStamp, transform_xsd_date_time), + "uri": RDFLiteralInfo(KNORA_API.UriValue, KNORA_API.uriValueAsUri, transform_xsd_anyuri), +} + +# date +# interval +# list +# resptr +# text + + @dataclass(frozen=True) class ValueRDF(ABC): resource_bn: BNode From 49c2f250b09c3c930e1ee332b83a62c892c4e50f Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:33:35 +0100 Subject: [PATCH 08/63] Update serialise_rdf_value.py --- .../models/serialise/serialise_rdf_value.py | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index c88957ae7c..63c32fc5e5 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from abc import ABC from abc import abstractmethod from dataclasses import dataclass @@ -14,7 +15,8 @@ from rdflib import Namespace from rdflib import URIRef -from dsp_tools.commands.xmlupload.value_transformers import transform_geometry, assert_is_string +from dsp_tools.commands.xmlupload.value_transformers import InputTypes +from dsp_tools.commands.xmlupload.value_transformers import assert_is_string from dsp_tools.utils.date_util import Date from dsp_tools.utils.date_util import DayMonthYearEra from dsp_tools.utils.date_util import SingleDate @@ -30,12 +32,40 @@ class RDFLiteralInfo: transformations: Callable[[Any], Literal] -def transform_xsd_string(): ... -def transform_xsd_decimal(): ... -def transform_xsd_boolean(): ... -def transform_xsd_integer(): ... -def transform_xsd_date_time(): ... -def transform_xsd_anyuri(): ... +def transform_xsd_string(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str_val, datatype=XSD.string) + + +def transform_xsd_decimal(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str_val, datatype=XSD.decimal) + + +def transform_xsd_boolean(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str_val, datatype=XSD.boolean) + + +def transform_xsd_integer(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str_val, datatype=XSD.integer) + + +def transform_xsd_date_time(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str_val, datatype=XSD.dateTimeStamp) + + +def transform_xsd_any_uri(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str_val, datatype=XSD.anyURI) + + +def transform_geometry(value: InputTypes): + str_val = assert_is_string(value) + str_val = json.dumps(json.loads(str_val)) + return Literal(str_val, datatype=XSD.string) rdf_mapper = { @@ -46,7 +76,7 @@ def transform_xsd_anyuri(): ... "geoname": RDFLiteralInfo(KNORA_API.GeonameValue, KNORA_API.geonameValueAsGeonameCode, transform_xsd_string), "integer": RDFLiteralInfo(KNORA_API.IntValue, KNORA_API.intValueAsInt, transform_xsd_integer), "time": RDFLiteralInfo(KNORA_API.TimeValue, KNORA_API.timeValueAsTimeStamp, transform_xsd_date_time), - "uri": RDFLiteralInfo(KNORA_API.UriValue, KNORA_API.uriValueAsUri, transform_xsd_anyuri), + "uri": RDFLiteralInfo(KNORA_API.UriValue, KNORA_API.uriValueAsUri, transform_xsd_any_uri), } # date From 45473718f56b3aa85d86b5dfadf609437e762453 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:34:43 +0100 Subject: [PATCH 09/63] Update serialise_rdf_value.py --- .../models/serialise/serialise_rdf_value.py | 79 +------------------ 1 file changed, 1 insertion(+), 78 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index 63c32fc5e5..d5cfe32733 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -68,7 +68,7 @@ def transform_geometry(value: InputTypes): return Literal(str_val, datatype=XSD.string) -rdf_mapper = { +rdf_literal_mapper = { "boolean": RDFLiteralInfo(KNORA_API.BooleanValue, KNORA_API.booleanValueAsBoolean, transform_xsd_boolean), "color": RDFLiteralInfo(KNORA_API.ColorValue, KNORA_API.colorValueAsColor, transform_xsd_string), "decimal": RDFLiteralInfo(KNORA_API.DecimalValue, KNORA_API.decimalValueAsDecimal, transform_xsd_decimal), @@ -119,17 +119,6 @@ def as_graph(self) -> Graph: return g -class ColorValueRDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.ColorValue)) - g.add((val_bn, KNORA_API.colorValueAsColor, self.value)) - return g - - class DateValueRDF(ValueRDF): value: Date @@ -160,50 +149,6 @@ def get_prop(precision: DayMonthYearEra) -> URIRef: return g -class DecimalValueRDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.DecimalValue)) - g.add((val_bn, KNORA_API.decimalValueAsDecimal, self.value)) - return g - - -class GeomValueRDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.GeomValue)) - g.add((val_bn, KNORA_API.geometryValueAsGeometry, self.value)) - return g - - -class GeonameValueRDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.GeonameValue)) - g.add((val_bn, KNORA_API.geonameValueAsGeonameCode, self.value)) - return g - - -class IntValueRDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.IntValue)) - g.add((val_bn, KNORA_API.intValueAsInt, self.value)) - return g - - class IntervalValueValueRDF(ValueRDF): """An IntervalValue to be serialised.""" @@ -267,25 +212,3 @@ def as_graph(self) -> Graph: g.add((val_bn, KNORA_API.textValueAsXml, self.value)) g.add((val_bn, KNORA_API.textValueHasMapping, URIRef("http://rdfh.ch/standoff/mappings/StandardMapping"))) return g - - -class TimeValueRDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.TimeValue)) - g.add((val_bn, KNORA_API.timeValueAsTimeStamp, self.value)) - return g - - -class UriValueRDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.UriValue)) - g.add((val_bn, KNORA_API.uriValueAsUri, self.value)) - return g From f336092a2cb1e01bbb2ce2980ec7ea40f734dd6b Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:52:12 +0100 Subject: [PATCH 10/63] change --- ...ransform_resource_and_values_for_upload.py | 128 ++++++++---------- .../xmlupload/test_resource_create_client.py | 1 + 2 files changed, 55 insertions(+), 74 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 15c54b1877..7b11800903 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -4,6 +4,7 @@ from typing import cast from rdflib import RDF +from rdflib import XSD from rdflib import BNode from rdflib import Graph from rdflib import Literal @@ -28,7 +29,8 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_file_value import SerialiseStillImageFileValue from dsp_tools.commands.xmlupload.models.serialise.serialise_file_value import SerialiseTextFileValue from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import BooleanValueRDF -from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import IntValueRDF +from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import RDFLiteralInfo +from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_literal_mapper from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseMigrationMetadata from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseResource from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseLink @@ -116,8 +118,15 @@ def get_link_prop_name(p: XMLProperty) -> str: for prop in resource.properties: match prop.valtype: + # serialised with rdflib + case "boolean" | "color" | "decimal" | "geometry" | "geoname" | "integer" | "time" | "uri" as val_type: + literal_info = rdf_literal_mapper[val_type] + prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) + properties_graph += _make_literal_prop(prop, res_bnode, prop_name, literal_info, lookup.permissions) + last_prop_name = prop_name + # serialised as dict - case "uri" | "color" | "geoname" | "time" | "decimal" | "geometry" | "date" | "interval" as val_type: + case "date" | "interval" as val_type: transformations = value_to_transformations_mapper[val_type] transformed_prop = _transform_into_prop_serialiser( prop=prop, @@ -148,17 +157,6 @@ def get_link_prop_name(p: XMLProperty) -> str: listnode_lookup=lookup.listnodes, ) properties_serialised.update(transformed_prop.serialise()) - # serialised with rdflib - case "integer": - int_prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) - int_graph = _make_integer_prop(prop, res_bnode, int_prop_name, lookup.permissions) - properties_graph += int_graph - last_prop_name = int_prop_name - case "boolean": - bool_prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) - bool_graph = _make_boolean_prop(prop, res_bnode, bool_prop_name, lookup.permissions) - properties_graph += bool_graph - last_prop_name = bool_prop_name case _: raise UserError(f"Unknown value type: {prop.valtype}") if resource.iiif_uri: @@ -200,22 +198,13 @@ def _transform_into_value_serialiser( return transformations.serialiser(transformed, permission_str, value.comment) -def _add_optional_permission_triple( - value: XMLValue | IIIFUriInfo, val_bn: BNode, g: Graph, permissions_lookup: dict[str, Permissions] -) -> None: - if value.permissions: - if not (per := permissions_lookup.get(value.permissions)): - raise PermissionNotExistsError(f"Could not find permissions for value: {value.permissions}") - g.add((val_bn, KNORA_API.hasPermissions, Literal(str(per)))) - - def _make_iiif_uri_value(iiif_uri: IIIFUriInfo, res_bnode: BNode, permissions_lookup: dict[str, Permissions]) -> Graph: g = Graph() iiif_bn = BNode() g.add((res_bnode, KNORA_API.hasStillImageFileValue, iiif_bn)) g.add((iiif_bn, RDF.type, KNORA_API.StillImageExternalFileValue)) g.add((iiif_bn, KNORA_API.fileValueHasExternalUrl, Literal(iiif_uri.value))) - _add_optional_permission_triple(iiif_uri, iiif_bn, g, permissions_lookup) + g += _add_optional_permission_triple(iiif_uri, iiif_bn, permissions_lookup) return g @@ -242,54 +231,57 @@ def _make_bitstream_file_value(bitstream_info: BitstreamInfo) -> dict[str, Any]: raise BaseError(f"Unknown file ending '{file_ending}' for file '{local_file}'") -def _make_boolean_prop( - prop: XMLProperty, res_bn: BNode, prop_name: URIRef, permissions_lookup: dict[str, Permissions] +def _make_literal_prop( + prop: XMLProperty, + res_bn: BNode, + prop_name: URIRef, + literal_info: RDFLiteralInfo, + permissions_lookup: dict[str, Permissions], ) -> Graph: g = Graph() - for value in prop.values: - boolean_value = _make_boolean_value(value, prop_name, res_bn, permissions_lookup) - g += boolean_value.as_graph() + for val in prop.values: + g += _make_literal_value(val, res_bn, prop_name, literal_info, permissions_lookup) return g -def _make_boolean_value( - value: XMLValue, prop_name: URIRef, res_bn: BNode, permissions_lookup: dict[str, Permissions] -) -> BooleanValueRDF: - s = assert_is_string(value.value) - as_bool = transform_boolean(s) - permission_literal = None - if permission_str := _get_permission_str(value.permissions, permissions_lookup): - permission_literal = Literal(permission_str) - return BooleanValueRDF( - resource_bn=res_bn, - prop_name=prop_name, - value=Literal(as_bool), - permissions=permission_literal, - comment=Literal(value.comment) if value.comment else None, - ) +def _make_literal_value( + val: XMLValue, + res_bn: BNode, + prop_name: URIRef, + literal_info: RDFLiteralInfo, + permissions_lookup: dict[str, Permissions], +) -> Graph(): + g = Graph() + transformed_val = literal_info.transformations(val.value) + val_bn = BNode() + g.add((res_bn, prop_name, val_bn)) + g.add((val_bn, RDF.type, literal_info.knora_type)) + g.add((val_bn, literal_info.knora_prop, transformed_val)) + g += _get_optional_triples(val, val_bn, permissions_lookup) + return g -def _make_integer_prop( - prop: XMLProperty, res_bn: BNode, prop_name: URIRef, permissions_lookup: dict[str, Permissions] -) -> Graph: +def _get_optional_triples(val: XMLValue, val_bn: BNode, permissions_lookup: dict[str, Permissions]) -> Graph: g = Graph() - for value in prop.values: - int_value = _make_integer_value(value, prop_name, res_bn, permissions_lookup) - g += int_value.as_graph() + g += _add_optional_permission_triple(val, val_bn, permissions_lookup) + if val.comment: + g.add((val_bn, KNORA_API.valueHasComment, Literal(val.comment, datatype=XSD.string))) return g -def _make_integer_value( +# TODO: delete +def _make_boolean_value( value: XMLValue, prop_name: URIRef, res_bn: BNode, permissions_lookup: dict[str, Permissions] -) -> IntValueRDF: +) -> BooleanValueRDF: s = assert_is_string(value.value) + as_bool = transform_boolean(s) permission_literal = None if permission_str := _get_permission_str(value.permissions, permissions_lookup): permission_literal = Literal(permission_str) - return IntValueRDF( + return BooleanValueRDF( resource_bn=res_bn, prop_name=prop_name, - value=Literal(int(s)), + value=Literal(as_bool), permissions=permission_literal, comment=Literal(value.comment) if value.comment else None, ) @@ -379,27 +371,15 @@ def _transform_into_richtext_value( return SerialiseRichtext(value=val_str, permissions=permission_str, comment=val.comment) -def _transform_into_serialise_prop( - prop: XMLProperty, - permissions_lookup: dict[str, Permissions], - transformer: TransformationSteps, -) -> SerialiseProperty: - serialised_values = [_transform_into_serialise_value(v, permissions_lookup, transformer) for v in prop.values] - prop_serialise = SerialiseProperty( - property_name=prop.name, - values=serialised_values, - ) - return prop_serialise - - -def _transform_into_serialise_value( - value: XMLValue, - permissions_lookup: dict[str, Permissions], - transformer: TransformationSteps, -) -> SerialiseValue: - permission_str = _get_permission_str(value.permissions, permissions_lookup) - value_str = cast(str, value.value) - return transformer.serialiser(value_str, permission_str, value.comment) +def _add_optional_permission_triple( + value: XMLValue | IIIFUriInfo, val_bn: BNode, permissions_lookup: dict[str, Permissions] +) -> Graph: + g = Graph() + if value.permissions: + if not (per := permissions_lookup.get(value.permissions)): + raise PermissionNotExistsError(f"Could not find permissions for value: {value.permissions}") + g.add((val_bn, KNORA_API.hasPermissions, Literal(str(per)))) + return g def _get_permission_str(permissions: str | None, permissions_lookup: dict[str, Permissions]) -> str | None: diff --git a/test/unittests/commands/xmlupload/test_resource_create_client.py b/test/unittests/commands/xmlupload/test_resource_create_client.py index 68a310b484..409f0e595d 100644 --- a/test/unittests/commands/xmlupload/test_resource_create_client.py +++ b/test/unittests/commands/xmlupload/test_resource_create_client.py @@ -442,6 +442,7 @@ def test_make_iiif_uri_value_serialised() -> None: assert serialised == expected +# TODO: change def test_make_boolean_value_with_permissions() -> None: permissions_lookup = {"open": Permissions({PermissionValue.CR: ["knora-admin:ProjectAdmin"]})} xml_str = """ From e64929f8ff4542c2c25a8ea80591702f2701cc60 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 09:53:51 +0100 Subject: [PATCH 11/63] remove redundant code --- .../models/serialise/serialise_value.py | 103 +----------------- .../commands/xmlupload/value_transformers.py | 25 ----- 2 files changed, 5 insertions(+), 123 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_value.py index 0fb23f2a1d..3489a38fac 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_value.py @@ -47,21 +47,7 @@ def _get_optionals(self) -> dict[str, str]: return optionals -class SerialiseColor(SerialiseValue): - """A ColorValue to be serialised.""" - - value: str - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:ColorValue", - "knora-api:colorValueAsColor": self.value, - } - serialised.update(self._get_optionals()) - return serialised - - -class SerialiseDate(SerialiseValue): +class SerialiseDate(SerialiseValue): # TODO """A DateValue to be serialised.""" value: Date @@ -89,52 +75,7 @@ def get_prop(precision: DayMonthYearEra) -> str: return date_dict -class SerialiseDecimal(SerialiseValue): - """A DecimalValue to be serialised.""" - - value: str - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:DecimalValue", - "knora-api:decimalValueAsDecimal": { - "@type": "xsd:decimal", - "@value": self.value, - }, - } - serialised.update(self._get_optionals()) - return serialised - - -class SerialiseGeometry(SerialiseValue): - """A GeomValue to be serialised.""" - - value: str - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:GeomValue", - "knora-api:geometryValueAsGeometry": self.value, - } - serialised.update(self._get_optionals()) - return serialised - - -class SerialiseGeoname(SerialiseValue): - """A GeonameValue to be serialised.""" - - value: str - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:GeonameValue", - "knora-api:geonameValueAsGeonameCode": self.value, - } - serialised.update(self._get_optionals()) - return serialised - - -class SerialiseInterval(SerialiseValue): +class SerialiseInterval(SerialiseValue): # TODO """An IntervalValue to be serialised.""" value: Interval @@ -150,7 +91,7 @@ def serialise(self) -> dict[str, Any]: @dataclass -class Interval: +class Interval: # TODO start: str end: str @@ -167,7 +108,7 @@ def _to_dict(self, interval_value: str) -> dict[str, str]: } -class SerialiseList(SerialiseValue): +class SerialiseList(SerialiseValue): # TODO """An ListValue to be serialised.""" value: str @@ -183,7 +124,7 @@ def serialise(self) -> dict[str, Any]: return serialised -class SerialiseLink(SerialiseValue): +class SerialiseLink(SerialiseValue): # TODO """A LinkValue to be serialised.""" value: str @@ -228,37 +169,3 @@ def serialise(self) -> dict[str, Any]: } serialised.update(self._get_optionals()) return serialised - - -class SerialiseTime(SerialiseValue): - """A TimeValue to be serialised.""" - - value: str - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:TimeValue", - "knora-api:timeValueAsTimeStamp": { - "@type": "xsd:dateTimeStamp", - "@value": self.value, - }, - } - serialised.update(self._get_optionals()) - return serialised - - -class SerialiseURI(SerialiseValue): - """A UriValue to be serialised.""" - - value: str - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:UriValue", - "knora-api:uriValueAsUri": { - "@type": "xsd:anyURI", - "@value": self.value, - }, - } - serialised.update(self._get_optionals()) - return serialised diff --git a/src/dsp_tools/commands/xmlupload/value_transformers.py b/src/dsp_tools/commands/xmlupload/value_transformers.py index 1f5a53dcc0..eb89dc2a3e 100644 --- a/src/dsp_tools/commands/xmlupload/value_transformers.py +++ b/src/dsp_tools/commands/xmlupload/value_transformers.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json from dataclasses import dataclass from typing import Callable from typing import TypeAlias @@ -9,14 +8,8 @@ from dsp_tools.commands.xmlupload.models.formatted_text_value import FormattedTextValue from dsp_tools.commands.xmlupload.models.serialise.serialise_value import Interval -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseColor from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseDate -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseDecimal -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseGeometry -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseGeoname from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseInterval -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseTime -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseURI from dsp_tools.commands.xmlupload.models.serialise.serialise_value import ValueSerialiser from dsp_tools.models.exceptions import BaseError from dsp_tools.utils.date_util import Date @@ -50,18 +43,6 @@ def transform_date(input_value: InputTypes) -> Date: return parse_date_string(val) -def transform_decimal(input_value: InputTypes) -> str: - """Transform an input into a decimal in string format.""" - val = assert_is_string(input_value) - return str(float(val)) - - -def transform_geometry(input_value: InputTypes) -> str: - """Transform a geometry input into a string.""" - val = assert_is_string(input_value) - return json.dumps(json.loads(val)) - - def transform_interval(input_value: InputTypes) -> Interval: """Transform a sting input into an interval object.""" val = assert_is_string(input_value) @@ -89,12 +70,6 @@ def assert_is_string(value: str | FormattedTextValue) -> str: value_to_transformations_mapper: dict[str, TransformationSteps] = { - "color": TransformationSteps(SerialiseColor, transform_string), "date": TransformationSteps(SerialiseDate, transform_date), - "decimal": TransformationSteps(SerialiseDecimal, transform_decimal), - "geometry": TransformationSteps(SerialiseGeometry, transform_geometry), - "geoname": TransformationSteps(SerialiseGeoname, transform_string), - "time": TransformationSteps(SerialiseTime, transform_string), - "uri": TransformationSteps(SerialiseURI, transform_string), "interval": TransformationSteps(SerialiseInterval, transform_interval), } From 7f6b350713c1aebaf7de88099123925d1149df2a Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:01:24 +0100 Subject: [PATCH 12/63] Update transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 7b11800903..168bc13031 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -103,14 +103,6 @@ def _make_resource( def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> dict[str, Any]: - def get_link_prop_name(p: XMLProperty) -> str: - if p.name == "knora-api:isSegmentOf" and resource.restype == "knora-api:VideoSegment": - return "knora-api:isVideoSegmentOfValue" - elif p.name == "knora-api:isSegmentOf" and resource.restype == "knora-api:AudioSegment": - return "knora-api:isAudioSegmentOfValue" - else: - return f"{p.name}Value" - properties_serialised = {} properties_graph = Graph() # To frame the json-ld correctly, we need one property used in the graph. It does not matter which. @@ -142,7 +134,7 @@ def get_link_prop_name(p: XMLProperty) -> str: ) properties_serialised.update(transformed_prop.serialise()) case "resptr": - prop_name = get_link_prop_name(prop) + prop_name = _get_link_prop_name(prop, resource.restype) transformed_prop = _transform_into_link_prop( prop=prop, prop_name=prop_name, @@ -168,6 +160,15 @@ def get_link_prop_name(p: XMLProperty) -> str: return properties_serialised +def _get_link_prop_name(p: XMLProperty, restype: str) -> str: + if p.name == "knora-api:isSegmentOf" and restype == "knora-api:VideoSegment": + return "knora-api:isVideoSegmentOfValue" + elif p.name == "knora-api:isSegmentOf" and restype == "knora-api:AudioSegment": + return "knora-api:isAudioSegmentOfValue" + else: + return f"{p.name}Value" + + def _get_absolute_prop_iri(prefixed_prop: str, namepsaces: dict[str, Namespace]) -> URIRef: prefix, prop = prefixed_prop.split(":", maxsplit=1) if not (namespace := namepsaces.get(prefix)): From 77c483c870c94d2b3d08aa550982fc6247bcdc62 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:36:42 +0100 Subject: [PATCH 13/63] link --- .../models/serialise/serialise_rdf_value.py | 44 ++++-- ...ransform_resource_and_values_for_upload.py | 134 +++++++++++++----- .../xmlupload/test_resource_create_client.py | 6 +- 3 files changed, 128 insertions(+), 56 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index d5cfe32733..86e87a1f29 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -4,8 +4,6 @@ from abc import ABC from abc import abstractmethod from dataclasses import dataclass -from typing import Any -from typing import Callable from rdflib import RDF from rdflib import XSD @@ -26,10 +24,17 @@ @dataclass -class RDFLiteralInfo: +class TransformedValue: + value: Literal | URIRef + prop_name: URIRef + permissions: str | None + comment: str | None + + +@dataclass +class RDFPropTypeInfo: knora_type: URIRef knora_prop: URIRef - transformations: Callable[[Any], Literal] def transform_xsd_string(value: InputTypes): @@ -68,15 +73,28 @@ def transform_geometry(value: InputTypes): return Literal(str_val, datatype=XSD.string) -rdf_literal_mapper = { - "boolean": RDFLiteralInfo(KNORA_API.BooleanValue, KNORA_API.booleanValueAsBoolean, transform_xsd_boolean), - "color": RDFLiteralInfo(KNORA_API.ColorValue, KNORA_API.colorValueAsColor, transform_xsd_string), - "decimal": RDFLiteralInfo(KNORA_API.DecimalValue, KNORA_API.decimalValueAsDecimal, transform_xsd_decimal), - "geometry": RDFLiteralInfo(KNORA_API.GeomValue, KNORA_API.geometryValueAsGeometry, transform_geometry), - "geoname": RDFLiteralInfo(KNORA_API.GeonameValue, KNORA_API.geonameValueAsGeonameCode, transform_xsd_string), - "integer": RDFLiteralInfo(KNORA_API.IntValue, KNORA_API.intValueAsInt, transform_xsd_integer), - "time": RDFLiteralInfo(KNORA_API.TimeValue, KNORA_API.timeValueAsTimeStamp, transform_xsd_date_time), - "uri": RDFLiteralInfo(KNORA_API.UriValue, KNORA_API.uriValueAsUri, transform_xsd_any_uri), +rdf_prop_type_mapper = { + "boolean": RDFPropTypeInfo(KNORA_API.BooleanValue, KNORA_API.booleanValueAsBoolean), + "color": RDFPropTypeInfo(KNORA_API.ColorValue, KNORA_API.colorValueAsColor), + "decimal": RDFPropTypeInfo(KNORA_API.DecimalValue, KNORA_API.decimalValueAsDecimal), + "geometry": RDFPropTypeInfo(KNORA_API.GeomValue, KNORA_API.geometryValueAsGeometry), + "geoname": RDFPropTypeInfo(KNORA_API.GeonameValue, KNORA_API.geonameValueAsGeonameCode), + "integer": RDFPropTypeInfo(KNORA_API.IntValue, KNORA_API.intValueAsInt), + "link": RDFPropTypeInfo(KNORA_API.LinkValue, KNORA_API.linkValueHasTargetIri), + "list": RDFPropTypeInfo(KNORA_API.ListValue, KNORA_API.listValueAsListNode), + "time": RDFPropTypeInfo(KNORA_API.TimeValue, KNORA_API.timeValueAsTimeStamp), + "uri": RDFPropTypeInfo(KNORA_API.UriValue, KNORA_API.uriValueAsUri), +} + +rdf_literal_transformer = { + "boolean": transform_xsd_boolean, + "color": transform_xsd_string, + "decimal": transform_xsd_decimal, + "geometry": transform_geometry, + "geoname": transform_xsd_string, + "integer": transform_xsd_integer, + "time": transform_xsd_date_time, + "uri": transform_xsd_any_uri, } # date diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 168bc13031..35183e229d 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -1,5 +1,6 @@ from pathlib import Path from typing import Any +from typing import Callable from typing import assert_never from typing import cast @@ -28,9 +29,9 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_file_value import SerialiseMovingImageFileValue from dsp_tools.commands.xmlupload.models.serialise.serialise_file_value import SerialiseStillImageFileValue from dsp_tools.commands.xmlupload.models.serialise.serialise_file_value import SerialiseTextFileValue -from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import BooleanValueRDF -from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import RDFLiteralInfo -from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_literal_mapper +from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import RDFPropTypeInfo +from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import TransformedValue +from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_prop_type_mapper from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseMigrationMetadata from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseResource from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseLink @@ -42,7 +43,6 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_value import ValueSerialiser from dsp_tools.commands.xmlupload.value_transformers import TransformationSteps from dsp_tools.commands.xmlupload.value_transformers import assert_is_string -from dsp_tools.commands.xmlupload.value_transformers import transform_boolean from dsp_tools.commands.xmlupload.value_transformers import transform_string from dsp_tools.commands.xmlupload.value_transformers import value_to_transformations_mapper from dsp_tools.models.exceptions import BaseError @@ -112,9 +112,11 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di match prop.valtype: # serialised with rdflib case "boolean" | "color" | "decimal" | "geometry" | "geoname" | "integer" | "time" | "uri" as val_type: - literal_info = rdf_literal_mapper[val_type] + literal_info = rdf_prop_type_mapper[val_type] prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) - properties_graph += _make_literal_prop(prop, res_bnode, prop_name, literal_info, lookup.permissions) + properties_graph += _make_simple_prop_graph( + prop, res_bnode, prop_name, literal_info, lookup.permissions + ) last_prop_name = prop_name # serialised as dict @@ -232,60 +234,114 @@ def _make_bitstream_file_value(bitstream_info: BitstreamInfo) -> dict[str, Any]: raise BaseError(f"Unknown file ending '{file_ending}' for file '{local_file}'") -def _make_literal_prop( +def _make_simple_prop_graph( prop: XMLProperty, res_bn: BNode, prop_name: URIRef, - literal_info: RDFLiteralInfo, + prop_type_info: RDFPropTypeInfo, + transformer: Callable[[Any], Any], permissions_lookup: dict[str, Permissions], ) -> Graph: g = Graph() for val in prop.values: - g += _make_literal_value(val, res_bn, prop_name, literal_info, permissions_lookup) + transformed_val = transformer(val.value) + per_str = _get_permission_str(val.permissions, permissions_lookup) + transformed = TransformedValue( + value=transformed_val, prop_name=prop_name, permissions=per_str, comment=val.comment + ) + g += _make_simple_value_graph(transformed, res_bn, prop_type_info) return g -def _make_literal_value( - val: XMLValue, +def _make_list_prop_graph( + prop: XMLProperty, res_bn: BNode, prop_name: URIRef, - literal_info: RDFLiteralInfo, permissions_lookup: dict[str, Permissions], -) -> Graph(): + listnode_lookup: dict[str, str], +) -> Graph: g = Graph() - transformed_val = literal_info.transformations(val.value) - val_bn = BNode() - g.add((res_bn, prop_name, val_bn)) - g.add((val_bn, RDF.type, literal_info.knora_type)) - g.add((val_bn, literal_info.knora_prop, transformed_val)) - g += _get_optional_triples(val, val_bn, permissions_lookup) + for val in prop.values: + s = assert_is_string(val.value) + if not (iri_str := listnode_lookup.get(s)): + msg = ( + f"Could not resolve list node ID '{s}' to IRI. " + f"This is probably because the list node '{s}' does not exist on the server." + ) + raise BaseError(msg) + + per_str = _get_permission_str(val.permissions, permissions_lookup) + transformed = TransformedValue( + value=URIRef(iri_str), + prop_name=prop_name, + permissions=per_str, + comment=val.comment, + ) + g += _make_simple_value_graph(transformed, res_bn, rdf_prop_type_mapper["list"]) return g -def _get_optional_triples(val: XMLValue, val_bn: BNode, permissions_lookup: dict[str, Permissions]) -> Graph: +def _make_link_prop_graph( + prop: XMLProperty, + res_bn: BNode, + prop_name: URIRef, + permissions_lookup: dict[str, Permissions], + iri_resolver: IriResolver, +) -> Graph: g = Graph() - g += _add_optional_permission_triple(val, val_bn, permissions_lookup) - if val.comment: - g.add((val_bn, KNORA_API.valueHasComment, Literal(val.comment, datatype=XSD.string))) + for val in prop.values: + s = assert_is_string(value.value) + if is_resource_iri(s): + iri_str = s + elif resolved_iri := iri_resolver.get(s): + iri_str = resolved_iri + else: + msg = ( + f"Could not find the ID {s} in the id2iri mapping. " + f"This is probably because the resource '{s}' could not be created. " + f"See {WARNINGS_SAVEPATH} for more information." + ) + raise BaseError(msg) + per_str = _get_permission_str(val.permissions, permissions_lookup) + transformed = TransformedValue( + value=URIRef(iri_str), + prop_name=prop_name, + permissions=per_str, + comment=val.comment, + ) + g += _make_simple_value_graph(transformed, res_bn, rdf_prop_type_mapper["link"]) return g -# TODO: delete -def _make_boolean_value( - value: XMLValue, prop_name: URIRef, res_bn: BNode, permissions_lookup: dict[str, Permissions] -) -> BooleanValueRDF: - s = assert_is_string(value.value) - as_bool = transform_boolean(s) - permission_literal = None - if permission_str := _get_permission_str(value.permissions, permissions_lookup): - permission_literal = Literal(permission_str) - return BooleanValueRDF( - resource_bn=res_bn, - prop_name=prop_name, - value=Literal(as_bool), - permissions=permission_literal, - comment=Literal(value.comment) if value.comment else None, - ) +def _make_simple_value_graph( + val: TransformedValue, + res_bn: BNode, + prop_type_info: RDFPropTypeInfo, +) -> Graph(): + val_bn = BNode() + g = _get_optional_triples(val_bn, val.permissions, val.comment) + g.add((res_bn, val.prop_name, val_bn)) + g.add((val_bn, RDF.type, prop_type_info.knora_type)) + g.add((val_bn, prop_type_info.knora_prop, val.value)) + return g + + +def _make_text_value_graph() -> Graph: ... + + +def _make_richtext_value_graph() -> Graph: ... + + +def _make_simpletext_value_graph() -> Graph: ... + + +def _get_optional_triples(val_bn: BNode, permissions: str | None, comment: str | None) -> Graph: + g = Graph() + if permissions: + g.add((val_bn, KNORA_API.hasPermissions, Literal(permissions, datatype=XSD.string))) + if comment: + g.add((val_bn, KNORA_API.valueHasComment, Literal(comment, datatype=XSD.string))) + return g def _transform_into_link_prop( diff --git a/test/unittests/commands/xmlupload/test_resource_create_client.py b/test/unittests/commands/xmlupload/test_resource_create_client.py index 409f0e595d..c96043bd27 100644 --- a/test/unittests/commands/xmlupload/test_resource_create_client.py +++ b/test/unittests/commands/xmlupload/test_resource_create_client.py @@ -4,6 +4,7 @@ from lxml import etree from rdflib import RDF from rdflib import BNode +from rdflib import Graph from rdflib import Literal from rdflib import Namespace from rdflib import URIRef @@ -17,7 +18,6 @@ from dsp_tools.commands.xmlupload.models.serialise.jsonld_serialiser import serialise_property_graph from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import KNORA_API from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import _make_bitstream_file_value -from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import _make_boolean_value from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import _make_iiif_uri_value from dsp_tools.models.exceptions import PermissionNotExistsError @@ -456,9 +456,7 @@ def test_make_boolean_value_with_permissions() -> None: test_val: XMLValue = xmlresource.properties[0].values[0] res_bn = BNode() prop_name = ONTO.isTrueOrFalse - bool_graph = _make_boolean_value( - value=test_val, prop_name=prop_name, res_bn=res_bn, permissions_lookup=permissions_lookup - ).as_graph() + bool_graph = Graph() number_of_triples = 4 assert len(bool_graph) == number_of_triples value_bn = next(bool_graph.objects(res_bn, prop_name)) From 8483a13ab1a6b3468396ff4bdc86786104cc6791 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:41:33 +0100 Subject: [PATCH 14/63] Update transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 48 +++++++++++-------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 35183e229d..f536b81f3e 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -31,6 +31,7 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_file_value import SerialiseTextFileValue from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import RDFPropTypeInfo from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import TransformedValue +from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_literal_transformer from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_prop_type_mapper from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseMigrationMetadata from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseResource @@ -109,13 +110,38 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di last_prop_name = None for prop in resource.properties: - match prop.valtype: - # serialised with rdflib + match prop.valtype: # serialised with rdflib case "boolean" | "color" | "decimal" | "geometry" | "geoname" | "integer" | "time" | "uri" as val_type: literal_info = rdf_prop_type_mapper[val_type] + transformer = rdf_literal_transformer[val_type] prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) properties_graph += _make_simple_prop_graph( - prop, res_bnode, prop_name, literal_info, lookup.permissions + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + prop_type_info=literal_info, + transformer=transformer, + permissions_lookup=lookup.permissions, + ) + last_prop_name = prop_name + case "list": + prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) + properties_graph += _make_list_prop_graph( + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + permissions_lookup=lookup.permissions, + listnode_lookup=lookup.listnodes, + ) + last_prop_name = prop_name + case "resptr": + prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) + properties_graph += _make_link_prop_graph( + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + permissions_lookup=lookup.permissions, + iri_resolver=lookup.id_to_iri, ) last_prop_name = prop_name @@ -135,22 +161,6 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di iri_resolver=lookup.id_to_iri, ) properties_serialised.update(transformed_prop.serialise()) - case "resptr": - prop_name = _get_link_prop_name(prop, resource.restype) - transformed_prop = _transform_into_link_prop( - prop=prop, - prop_name=prop_name, - permissions_lookup=lookup.permissions, - iri_resolver=lookup.id_to_iri, - ) - properties_serialised.update(transformed_prop.serialise()) - case "list": - transformed_prop = _transform_into_list_prop( - prop=prop, - permissions_lookup=lookup.permissions, - listnode_lookup=lookup.listnodes, - ) - properties_serialised.update(transformed_prop.serialise()) case _: raise UserError(f"Unknown value type: {prop.valtype}") if resource.iiif_uri: From d6fa0fed6809b7b12a6294d51aade12aeaa36893 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:43:50 +0100 Subject: [PATCH 15/63] - --- .../models/serialise/serialise_rdf_value.py | 23 +--------- ...ransform_resource_and_values_for_upload.py | 45 ++----------------- 2 files changed, 5 insertions(+), 63 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index 86e87a1f29..eba6dab188 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -82,6 +82,7 @@ def transform_geometry(value: InputTypes): "integer": RDFPropTypeInfo(KNORA_API.IntValue, KNORA_API.intValueAsInt), "link": RDFPropTypeInfo(KNORA_API.LinkValue, KNORA_API.linkValueHasTargetIri), "list": RDFPropTypeInfo(KNORA_API.ListValue, KNORA_API.listValueAsListNode), + "simpletext": RDFPropTypeInfo(KNORA_API.TextValue, KNORA_API.valueAsString), "time": RDFPropTypeInfo(KNORA_API.TimeValue, KNORA_API.timeValueAsTimeStamp), "uri": RDFPropTypeInfo(KNORA_API.UriValue, KNORA_API.uriValueAsUri), } @@ -198,28 +199,6 @@ def as_graph(self) -> Graph: return g -class LinkValueRDF(ValueRDF): - value: URIRef - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.LinkValue)) - g.add((val_bn, KNORA_API.linkValueHasTargetIri, self.value)) - return g - - -class SimpletextRDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.TextValue)) - g.add((val_bn, KNORA_API.valueAsString, self.value)) - return g - - class RichtextRDF(ValueRDF): value: Literal diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index f536b81f3e..8f21ba4ed1 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -35,13 +35,11 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_prop_type_mapper from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseMigrationMetadata from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseResource -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseLink from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseList from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseProperty from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseRichtext from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseSimpletext from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseValue -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import ValueSerialiser from dsp_tools.commands.xmlupload.value_transformers import TransformationSteps from dsp_tools.commands.xmlupload.value_transformers import assert_is_string from dsp_tools.commands.xmlupload.value_transformers import transform_string @@ -300,7 +298,7 @@ def _make_link_prop_graph( ) -> Graph: g = Graph() for val in prop.values: - s = assert_is_string(value.value) + s = assert_is_string(val.value) if is_resource_iri(s): iri_str = s elif resolved_iri := iri_resolver.get(s): @@ -323,6 +321,9 @@ def _make_link_prop_graph( return g +def _make_text_value_graph() -> Graph: ... + + def _make_simple_value_graph( val: TransformedValue, res_bn: BNode, @@ -336,15 +337,9 @@ def _make_simple_value_graph( return g -def _make_text_value_graph() -> Graph: ... - - def _make_richtext_value_graph() -> Graph: ... -def _make_simpletext_value_graph() -> Graph: ... - - def _get_optional_triples(val_bn: BNode, permissions: str | None, comment: str | None) -> Graph: g = Graph() if permissions: @@ -354,38 +349,6 @@ def _get_optional_triples(val_bn: BNode, permissions: str | None, comment: str | return g -def _transform_into_link_prop( - prop: XMLProperty, - prop_name: str, - permissions_lookup: dict[str, Permissions], - iri_resolver: IriResolver, -) -> SerialiseProperty: - vals = [_transform_into_link_value(v, permissions_lookup, SerialiseLink, iri_resolver) for v in prop.values] - return SerialiseProperty(property_name=prop_name, values=vals) - - -def _transform_into_link_value( - value: XMLValue, - permissions_lookup: dict[str, Permissions], - serialiser: ValueSerialiser, - iri_resolver: IriResolver, -) -> SerialiseValue: - s = assert_is_string(value.value) - if is_resource_iri(s): - iri = s - elif resolved_iri := iri_resolver.get(s): - iri = resolved_iri - else: - msg = ( - f"Could not find the ID {s} in the id2iri mapping. " - f"This is probably because the resource '{s}' could not be created. " - f"See {WARNINGS_SAVEPATH} for more information." - ) - raise BaseError(msg) - permission_str = _get_permission_str(value.permissions, permissions_lookup) - return serialiser(iri, permission_str, value.comment) - - def _transform_into_list_prop( prop: XMLProperty, permissions_lookup: dict[str, Permissions], listnode_lookup: dict[str, str] ) -> SerialiseProperty: From be0dc7b6d4256269509edba6393b02aca167bf65 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:04:08 +0100 Subject: [PATCH 16/63] Update transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 97 +++++++++++++------ 1 file changed, 66 insertions(+), 31 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 8f21ba4ed1..4902dad053 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -35,7 +35,6 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_prop_type_mapper from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseMigrationMetadata from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseResource -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseList from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseProperty from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseRichtext from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseSimpletext @@ -108,11 +107,12 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di last_prop_name = None for prop in resource.properties: - match prop.valtype: # serialised with rdflib + prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) + last_prop_name = prop_name + match prop.valtype: case "boolean" | "color" | "decimal" | "geometry" | "geoname" | "integer" | "time" | "uri" as val_type: literal_info = rdf_prop_type_mapper[val_type] transformer = rdf_literal_transformer[val_type] - prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) properties_graph += _make_simple_prop_graph( prop=prop, res_bn=res_bnode, @@ -121,9 +121,7 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di transformer=transformer, permissions_lookup=lookup.permissions, ) - last_prop_name = prop_name case "list": - prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) properties_graph += _make_list_prop_graph( prop=prop, res_bn=res_bnode, @@ -131,9 +129,7 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di permissions_lookup=lookup.permissions, listnode_lookup=lookup.listnodes, ) - last_prop_name = prop_name case "resptr": - prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) properties_graph += _make_link_prop_graph( prop=prop, res_bn=res_bnode, @@ -141,7 +137,15 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di permissions_lookup=lookup.permissions, iri_resolver=lookup.id_to_iri, ) - last_prop_name = prop_name + + case "text": + properties_graph += _make_text_prop_graph( + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + permissions_lookup=lookup.permissions, + iri_resolver=lookup.id_to_iri, + ) # serialised as dict case "date" | "interval" as val_type: @@ -321,7 +325,37 @@ def _make_link_prop_graph( return g -def _make_text_value_graph() -> Graph: ... +def _make_text_prop_graph( + prop: XMLProperty, + res_bn: BNode, + prop_name: URIRef, + permissions_lookup: dict[str, Permissions], + iri_resolver: IriResolver, +) -> Graph: + g = Graph() + for val in prop.values: + per_str = _get_permission_str(val.permissions, permissions_lookup) + match val.value: + case str(): + transformed = TransformedValue( + value=Literal(val.value, datatype=XSD.string), + prop_name=prop_name, + permissions=per_str, + comment=val.comment, + ) + g += _make_simple_value_graph( + val=transformed, res_bn=res_bn, prop_type_info=rdf_prop_type_mapper["simpletext"] + ) + case FormattedTextValue(): + g += _make_richtext_value_graph( + val=val, + prop_name=prop_name, + res_bn=res_bn, + permissions_lookup=permissions_lookup, + iri_resolver=iri_resolver, + ) + case _: + assert_never(val.value) def _make_simple_value_graph( @@ -337,7 +371,29 @@ def _make_simple_value_graph( return g -def _make_richtext_value_graph() -> Graph: ... +def _make_richtext_value_graph( + val: XMLValue, + prop_name: URIRef, + res_bn: BNode, + permissions_lookup: dict[str, Permissions], + iri_resolver: IriResolver, +) -> Graph: + val_bn = BNode() + per_str = _get_permission_str(val.permissions, permissions_lookup) + g = _get_optional_triples(val_bn, per_str, val.comment) + val_str = _get_richtext_string(val.value, iri_resolver) + literal_val = Literal(val_str, datatype=XSD.string) + g.add((res_bn, prop_name, val_bn)) + g.add((val_bn, RDF.type, KNORA_API.TextValue)) + g.add((val_bn, KNORA_API.textValueAsXml, literal_val)) + g.add((val_bn, KNORA_API.textValueHasMapping, URIRef("http://rdfh.ch/standoff/mappings/StandardMapping"))) + return g + + +def _get_richtext_string(value: FormattedTextValue, iri_resolver: IriResolver) -> str: + xml_val = cast(FormattedTextValue, value) + xml_with_iris = xml_val.with_iris(iri_resolver) + return xml_with_iris.as_xml() def _get_optional_triples(val_bn: BNode, permissions: str | None, comment: str | None) -> Graph: @@ -349,27 +405,6 @@ def _get_optional_triples(val_bn: BNode, permissions: str | None, comment: str | return g -def _transform_into_list_prop( - prop: XMLProperty, permissions_lookup: dict[str, Permissions], listnode_lookup: dict[str, str] -) -> SerialiseProperty: - vals = [_transform_into_list_value(v, permissions_lookup, listnode_lookup) for v in prop.values] - return SerialiseProperty(property_name=prop.name, values=vals) - - -def _transform_into_list_value( - value: XMLValue, permissions_lookup: dict[str, Permissions], listnode_lookup: dict[str, str] -) -> SerialiseValue: - s = assert_is_string(value.value) - if not (iri := listnode_lookup.get(s)): - msg = ( - f"Could not resolve list node ID '{s}' to IRI. " - f"This is probably because the list node '{s}' does not exist on the server." - ) - raise BaseError(msg) - permission_str = _get_permission_str(value.permissions, permissions_lookup) - return SerialiseList(iri, permission_str, value.comment) - - def _transform_text_prop( prop: XMLProperty, permissions_lookup: dict[str, Permissions], iri_resolver: IriResolver ) -> SerialiseProperty: From 11cf1c14889ddd1cead442fca9e7f8ec47912b96 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:10:10 +0100 Subject: [PATCH 17/63] Update transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 4902dad053..022481bc0a 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -108,7 +108,6 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di for prop in resource.properties: prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) - last_prop_name = prop_name match prop.valtype: case "boolean" | "color" | "decimal" | "geometry" | "geoname" | "integer" | "time" | "uri" as val_type: literal_info = rdf_prop_type_mapper[val_type] @@ -130,6 +129,7 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di listnode_lookup=lookup.listnodes, ) case "resptr": + prop_name = _get_link_prop_name(prop, resource.restype, lookup.namespaces) properties_graph += _make_link_prop_graph( prop=prop, res_bn=res_bnode, @@ -137,7 +137,6 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di permissions_lookup=lookup.permissions, iri_resolver=lookup.id_to_iri, ) - case "text": properties_graph += _make_text_prop_graph( prop=prop, @@ -146,7 +145,6 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di permissions_lookup=lookup.permissions, iri_resolver=lookup.id_to_iri, ) - # serialised as dict case "date" | "interval" as val_type: transformations = value_to_transformations_mapper[val_type] @@ -156,15 +154,10 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di transformations=transformations, ) properties_serialised.update(transformed_prop.serialise()) - case "text": - transformed_prop = _transform_text_prop( - prop=prop, - permissions_lookup=lookup.permissions, - iri_resolver=lookup.id_to_iri, - ) - properties_serialised.update(transformed_prop.serialise()) case _: raise UserError(f"Unknown value type: {prop.valtype}") + last_prop_name = prop_name + if resource.iiif_uri: properties_graph += _make_iiif_uri_value(resource.iiif_uri, res_bnode, lookup.permissions) last_prop_name = KNORA_API.hasStillImageFileValue @@ -174,13 +167,13 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di return properties_serialised -def _get_link_prop_name(p: XMLProperty, restype: str) -> str: +def _get_link_prop_name(p: XMLProperty, restype: str, namespaces: dict[str, Namespace]) -> URIRef: if p.name == "knora-api:isSegmentOf" and restype == "knora-api:VideoSegment": - return "knora-api:isVideoSegmentOfValue" + return KNORA_API.isVideoSegmentOfValue elif p.name == "knora-api:isSegmentOf" and restype == "knora-api:AudioSegment": - return "knora-api:isAudioSegmentOfValue" - else: - return f"{p.name}Value" + return KNORA_API.isAudioSegmentOfValue + prop = f"{p.name}Value" + return _get_absolute_prop_iri(prop, namespaces) def _get_absolute_prop_iri(prefixed_prop: str, namepsaces: dict[str, Namespace]) -> URIRef: From 88d18303c2544745b1f229b1e3e608ceb70dcae3 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:14:21 +0100 Subject: [PATCH 18/63] - --- .../models/serialise/serialise_rdf_value.py | 23 ------------ ...ransform_resource_and_values_for_upload.py | 35 +------------------ 2 files changed, 1 insertion(+), 57 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index eba6dab188..bdf2b87e74 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -186,26 +186,3 @@ def as_graph(self) -> Graph: class Interval: start: Literal end: Literal - - -class ListValueRDF(ValueRDF): - value: URIRef - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.ListValue)) - g.add((val_bn, KNORA_API.listValueAsListNode, self.value)) - return g - - -class RichtextRDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.TextValue)) - g.add((val_bn, KNORA_API.textValueAsXml, self.value)) - g.add((val_bn, KNORA_API.textValueHasMapping, URIRef("http://rdfh.ch/standoff/mappings/StandardMapping"))) - return g diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 022481bc0a..34c5bc1403 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -36,12 +36,9 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseMigrationMetadata from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseResource from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseProperty -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseRichtext -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseSimpletext from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseValue from dsp_tools.commands.xmlupload.value_transformers import TransformationSteps from dsp_tools.commands.xmlupload.value_transformers import assert_is_string -from dsp_tools.commands.xmlupload.value_transformers import transform_string from dsp_tools.commands.xmlupload.value_transformers import value_to_transformations_mapper from dsp_tools.models.exceptions import BaseError from dsp_tools.models.exceptions import InputError @@ -349,6 +346,7 @@ def _make_text_prop_graph( ) case _: assert_never(val.value) + return g def _make_simple_value_graph( @@ -398,37 +396,6 @@ def _get_optional_triples(val_bn: BNode, permissions: str | None, comment: str | return g -def _transform_text_prop( - prop: XMLProperty, permissions_lookup: dict[str, Permissions], iri_resolver: IriResolver -) -> SerialiseProperty: - values = [] - for val in prop.values: - match val.value: - case str(): - values.append( - _transform_into_value_serialiser( - value=val, - permissions_lookup=permissions_lookup, - transformations=TransformationSteps(SerialiseSimpletext, transform_string), - ) - ) - case FormattedTextValue(): - values.append(_transform_into_richtext_value(val, permissions_lookup, iri_resolver)) - case _: - assert_never(val.value) - return SerialiseProperty(property_name=prop.name, values=values) - - -def _transform_into_richtext_value( - val: XMLValue, permissions_lookup: dict[str, Permissions], iri_resolver: IriResolver -) -> SerialiseRichtext: - xml_val = cast(FormattedTextValue, val.value) - xml_with_iris = xml_val.with_iris(iri_resolver) - val_str = xml_with_iris.as_xml() - permission_str = _get_permission_str(val.permissions, permissions_lookup) - return SerialiseRichtext(value=val_str, permissions=permission_str, comment=val.comment) - - def _add_optional_permission_triple( value: XMLValue | IIIFUriInfo, val_bn: BNode, permissions_lookup: dict[str, Permissions] ) -> Graph: From 57f72dae528474a98460d4c47afc3fa40ad5ce06 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:15:58 +0100 Subject: [PATCH 19/63] Update transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 34c5bc1403..671a0864e4 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -292,18 +292,7 @@ def _make_link_prop_graph( ) -> Graph: g = Graph() for val in prop.values: - s = assert_is_string(val.value) - if is_resource_iri(s): - iri_str = s - elif resolved_iri := iri_resolver.get(s): - iri_str = resolved_iri - else: - msg = ( - f"Could not find the ID {s} in the id2iri mapping. " - f"This is probably because the resource '{s}' could not be created. " - f"See {WARNINGS_SAVEPATH} for more information." - ) - raise BaseError(msg) + iri_str = _resolve_id_to_iri(val.value, iri_resolver) per_str = _get_permission_str(val.permissions, permissions_lookup) transformed = TransformedValue( value=URIRef(iri_str), @@ -315,6 +304,22 @@ def _make_link_prop_graph( return g +def _resolve_id_to_iri(value: str, iri_resolver: IriResolver) -> str: + s = assert_is_string(value) + if is_resource_iri(s): + iri_str = s + elif resolved_iri := iri_resolver.get(s): + iri_str = resolved_iri + else: + msg = ( + f"Could not find the ID {s} in the id2iri mapping. " + f"This is probably because the resource '{s}' could not be created. " + f"See {WARNINGS_SAVEPATH} for more information." + ) + raise BaseError(msg) + return iri_str + + def _make_text_prop_graph( prop: XMLProperty, res_bn: BNode, From 8b0909fef4a82096ca5a7b2efade9d2b3886ae39 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:21:20 +0100 Subject: [PATCH 20/63] Update serialise_rdf_value.py --- .../models/serialise/serialise_rdf_value.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index bdf2b87e74..13a76e1e21 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -15,6 +15,7 @@ from dsp_tools.commands.xmlupload.value_transformers import InputTypes from dsp_tools.commands.xmlupload.value_transformers import assert_is_string +from dsp_tools.models.exceptions import BaseError from dsp_tools.utils.date_util import Date from dsp_tools.utils.date_util import DayMonthYearEra from dsp_tools.utils.date_util import SingleDate @@ -44,12 +45,17 @@ def transform_xsd_string(value: InputTypes): def transform_xsd_decimal(value: InputTypes): str_val = assert_is_string(value) - return Literal(str_val, datatype=XSD.decimal) + return Literal(str(float(str_val)), datatype=XSD.decimal) def transform_xsd_boolean(value: InputTypes): - str_val = assert_is_string(value) - return Literal(str_val, datatype=XSD.boolean) + match value: + case "True" | "true" | "1" | 1 | True: + return Literal(True, datatype=XSD.boolean) + case "False" | "false" | "0" | 0 | False: + return Literal(False, datatype=XSD.boolean) + case _: + raise BaseError(f"Could not parse boolean value: {value}") def transform_xsd_integer(value: InputTypes): From 2850879f5358fcedb31dd36acc5d7f9c8a3e99f5 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:22:36 +0100 Subject: [PATCH 21/63] remove boolean --- .../models/serialise/serialise_rdf_value.py | 68 ------------------- ...ransform_resource_and_values_for_upload.py | 2 +- .../commands/xmlupload/value_transformers.py | 57 ++++++++++++++++ 3 files changed, 58 insertions(+), 69 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index 13a76e1e21..6dfd892cc7 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json from abc import ABC from abc import abstractmethod from dataclasses import dataclass @@ -13,9 +12,6 @@ from rdflib import Namespace from rdflib import URIRef -from dsp_tools.commands.xmlupload.value_transformers import InputTypes -from dsp_tools.commands.xmlupload.value_transformers import assert_is_string -from dsp_tools.models.exceptions import BaseError from dsp_tools.utils.date_util import Date from dsp_tools.utils.date_util import DayMonthYearEra from dsp_tools.utils.date_util import SingleDate @@ -38,47 +34,6 @@ class RDFPropTypeInfo: knora_prop: URIRef -def transform_xsd_string(value: InputTypes): - str_val = assert_is_string(value) - return Literal(str_val, datatype=XSD.string) - - -def transform_xsd_decimal(value: InputTypes): - str_val = assert_is_string(value) - return Literal(str(float(str_val)), datatype=XSD.decimal) - - -def transform_xsd_boolean(value: InputTypes): - match value: - case "True" | "true" | "1" | 1 | True: - return Literal(True, datatype=XSD.boolean) - case "False" | "false" | "0" | 0 | False: - return Literal(False, datatype=XSD.boolean) - case _: - raise BaseError(f"Could not parse boolean value: {value}") - - -def transform_xsd_integer(value: InputTypes): - str_val = assert_is_string(value) - return Literal(str_val, datatype=XSD.integer) - - -def transform_xsd_date_time(value: InputTypes): - str_val = assert_is_string(value) - return Literal(str_val, datatype=XSD.dateTimeStamp) - - -def transform_xsd_any_uri(value: InputTypes): - str_val = assert_is_string(value) - return Literal(str_val, datatype=XSD.anyURI) - - -def transform_geometry(value: InputTypes): - str_val = assert_is_string(value) - str_val = json.dumps(json.loads(str_val)) - return Literal(str_val, datatype=XSD.string) - - rdf_prop_type_mapper = { "boolean": RDFPropTypeInfo(KNORA_API.BooleanValue, KNORA_API.booleanValueAsBoolean), "color": RDFPropTypeInfo(KNORA_API.ColorValue, KNORA_API.colorValueAsColor), @@ -92,18 +47,6 @@ def transform_geometry(value: InputTypes): "time": RDFPropTypeInfo(KNORA_API.TimeValue, KNORA_API.timeValueAsTimeStamp), "uri": RDFPropTypeInfo(KNORA_API.UriValue, KNORA_API.uriValueAsUri), } - -rdf_literal_transformer = { - "boolean": transform_xsd_boolean, - "color": transform_xsd_string, - "decimal": transform_xsd_decimal, - "geometry": transform_geometry, - "geoname": transform_xsd_string, - "integer": transform_xsd_integer, - "time": transform_xsd_date_time, - "uri": transform_xsd_any_uri, -} - # date # interval # list @@ -133,17 +76,6 @@ def _get_generic_graph(self, val_bn: BNode) -> Graph: return g -class BooleanValueRDF(ValueRDF): - value: Literal - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.BooleanValue)) - g.add((val_bn, KNORA_API.booleanValueAsBoolean, self.value)) - return g - - class DateValueRDF(ValueRDF): value: Date diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 671a0864e4..39a65aa284 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -31,7 +31,6 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_file_value import SerialiseTextFileValue from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import RDFPropTypeInfo from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import TransformedValue -from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_literal_transformer from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_prop_type_mapper from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseMigrationMetadata from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseResource @@ -39,6 +38,7 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseValue from dsp_tools.commands.xmlupload.value_transformers import TransformationSteps from dsp_tools.commands.xmlupload.value_transformers import assert_is_string +from dsp_tools.commands.xmlupload.value_transformers import rdf_literal_transformer from dsp_tools.commands.xmlupload.value_transformers import value_to_transformations_mapper from dsp_tools.models.exceptions import BaseError from dsp_tools.models.exceptions import InputError diff --git a/src/dsp_tools/commands/xmlupload/value_transformers.py b/src/dsp_tools/commands/xmlupload/value_transformers.py index eb89dc2a3e..070ba4b23d 100644 --- a/src/dsp_tools/commands/xmlupload/value_transformers.py +++ b/src/dsp_tools/commands/xmlupload/value_transformers.py @@ -1,11 +1,15 @@ from __future__ import annotations +import json from dataclasses import dataclass from typing import Callable from typing import TypeAlias from typing import Union from typing import assert_never +from rdflib import XSD +from rdflib import Literal + from dsp_tools.commands.xmlupload.models.formatted_text_value import FormattedTextValue from dsp_tools.commands.xmlupload.models.serialise.serialise_value import Interval from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseDate @@ -73,3 +77,56 @@ def assert_is_string(value: str | FormattedTextValue) -> str: "date": TransformationSteps(SerialiseDate, transform_date), "interval": TransformationSteps(SerialiseInterval, transform_interval), } + + +def transform_xsd_string(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str_val, datatype=XSD.string) + + +def transform_xsd_decimal(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str(float(str_val)), datatype=XSD.decimal) + + +def transform_xsd_boolean(value: InputTypes): + match value: + case "True" | "true" | "1" | 1 | True: + return Literal(True, datatype=XSD.boolean) + case "False" | "false" | "0" | 0 | False: + return Literal(False, datatype=XSD.boolean) + case _: + raise BaseError(f"Could not parse boolean value: {value}") + + +def transform_xsd_integer(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str_val, datatype=XSD.integer) + + +def transform_xsd_date_time(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str_val, datatype=XSD.dateTimeStamp) + + +def transform_xsd_any_uri(value: InputTypes): + str_val = assert_is_string(value) + return Literal(str_val, datatype=XSD.anyURI) + + +def transform_geometry(value: InputTypes): + str_val = assert_is_string(value) + str_val = json.dumps(json.loads(str_val)) + return Literal(str_val, datatype=XSD.string) + + +rdf_literal_transformer = { + "boolean": transform_xsd_boolean, + "color": transform_xsd_string, + "decimal": transform_xsd_decimal, + "geometry": transform_geometry, + "geoname": transform_xsd_string, + "integer": transform_xsd_integer, + "time": transform_xsd_date_time, + "uri": transform_xsd_any_uri, +} From 5048b936f87dc59e1cbd71c4719063b655d345e1 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:44:26 +0100 Subject: [PATCH 22/63] make interval rdf --- .../models/serialise/serialise_rdf_value.py | 94 +------------ ...ransform_resource_and_values_for_upload.py | 129 +++++++++++++++--- .../commands/xmlupload/value_transformers.py | 17 +-- 3 files changed, 115 insertions(+), 125 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py index 6dfd892cc7..58881820f0 100644 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py +++ b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_rdf_value.py @@ -1,22 +1,11 @@ from __future__ import annotations -from abc import ABC -from abc import abstractmethod from dataclasses import dataclass -from rdflib import RDF -from rdflib import XSD -from rdflib import BNode -from rdflib import Graph from rdflib import Literal from rdflib import Namespace from rdflib import URIRef -from dsp_tools.utils.date_util import Date -from dsp_tools.utils.date_util import DayMonthYearEra -from dsp_tools.utils.date_util import SingleDate -from dsp_tools.utils.date_util import StartEnd - KNORA_API = Namespace("http://api.knora.org/ontology/knora-api/v2#") @@ -34,6 +23,12 @@ class RDFPropTypeInfo: knora_prop: URIRef +@dataclass +class Interval: + start: Literal + end: Literal + + rdf_prop_type_mapper = { "boolean": RDFPropTypeInfo(KNORA_API.BooleanValue, KNORA_API.booleanValueAsBoolean), "color": RDFPropTypeInfo(KNORA_API.ColorValue, KNORA_API.colorValueAsColor), @@ -47,80 +42,3 @@ class RDFPropTypeInfo: "time": RDFPropTypeInfo(KNORA_API.TimeValue, KNORA_API.timeValueAsTimeStamp), "uri": RDFPropTypeInfo(KNORA_API.UriValue, KNORA_API.uriValueAsUri), } -# date -# interval -# list -# resptr -# text - - -@dataclass(frozen=True) -class ValueRDF(ABC): - resource_bn: BNode - prop_name: URIRef - value: Literal | URIRef - permissions: Literal | None - comment: Literal | None - - @abstractmethod - def as_graph(self) -> Graph: - """Creates the value as rdflib graph""" - - def _get_generic_graph(self, val_bn: BNode) -> Graph: - g = Graph() - g.add((self.resource_bn, self.prop_name, val_bn)) - if self.permissions: - g.add((val_bn, KNORA_API.hasPermissions, self.permissions)) - if self.comment: - g.add((val_bn, KNORA_API.valueHasComment, self.comment)) - return g - - -class DateValueRDF(ValueRDF): - value: Date - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.DateValue)) - if cal := self.value.calendar.value: - g.add((val_bn, KNORA_API.dateValueHasCalendar, Literal(cal, datatype=XSD.string))) - g += self._get_one_date_graph(val_bn, self.value.start, StartEnd.START) - if self.value.end: - g += self._get_one_date_graph(val_bn, self.value.end, StartEnd.END) - return g - - def _get_one_date_graph(self, val_bn: BNode, date: SingleDate, start_end: StartEnd) -> Graph: - def get_prop(precision: DayMonthYearEra) -> URIRef: - return KNORA_API[f"dateValueHas{start_end.value}{precision.value}"] - - g = Graph() - if yr := date.year: - g.add((val_bn, get_prop(DayMonthYearEra.YEAR), Literal(yr, datatype=XSD.int))) - if mnt := date.month: - g.add((val_bn, get_prop(DayMonthYearEra.MONTH), Literal(mnt, datatype=XSD.int))) - if day := date.day: - g.add((val_bn, get_prop(DayMonthYearEra.DAY), Literal(day, datatype=XSD.int))) - if era := date.era: - g.add((val_bn, get_prop(DayMonthYearEra.ERA), Literal(era, datatype=XSD.string))) - return g - - -class IntervalValueValueRDF(ValueRDF): - """An IntervalValue to be serialised.""" - - value: Interval - - def as_graph(self) -> Graph: - val_bn = BNode() - g = self._get_generic_graph(val_bn) - g.add((val_bn, RDF.type, KNORA_API.IntervalValue)) - g.add((val_bn, KNORA_API.intervalValueHasStart, self.value.start)) - g.add((val_bn, KNORA_API.intervalValueHasEnd, self.value.end)) - return g - - -@dataclass -class Interval: - start: Literal - end: Literal diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 39a65aa284..a23ad145fb 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -39,11 +39,15 @@ from dsp_tools.commands.xmlupload.value_transformers import TransformationSteps from dsp_tools.commands.xmlupload.value_transformers import assert_is_string from dsp_tools.commands.xmlupload.value_transformers import rdf_literal_transformer -from dsp_tools.commands.xmlupload.value_transformers import value_to_transformations_mapper +from dsp_tools.commands.xmlupload.value_transformers import transform_date +from dsp_tools.commands.xmlupload.value_transformers import transform_interval from dsp_tools.models.exceptions import BaseError from dsp_tools.models.exceptions import InputError from dsp_tools.models.exceptions import PermissionNotExistsError from dsp_tools.models.exceptions import UserError +from dsp_tools.utils.date_util import DayMonthYearEra +from dsp_tools.utils.date_util import SingleDate +from dsp_tools.utils.date_util import StartEnd from dsp_tools.utils.iri_util import is_resource_iri from dsp_tools.utils.logger_config import WARNINGS_SAVEPATH @@ -82,12 +86,12 @@ def _make_resource( res_iri = convert_ark_v0_to_resource_iri(resource.ark) if any([creation_date, res_iri]): migration_metadata = SerialiseMigrationMetadata(iri=res_iri, creation_date=resource.creation_date) - permission_str = _get_permission_str(resource.permissions, lookup.permissions) + resolved_permission = _resolve_permission(resource.permissions, lookup.permissions) serialise_resource = SerialiseResource( res_id=resource.res_id, res_type=resource.restype, label=resource.label, - permissions=permission_str, + permissions=resolved_permission, project_iri=lookup.project_iri, migration_metadata=migration_metadata, ) @@ -142,15 +146,20 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di permissions_lookup=lookup.permissions, iri_resolver=lookup.id_to_iri, ) - # serialised as dict - case "date" | "interval" as val_type: - transformations = value_to_transformations_mapper[val_type] - transformed_prop = _transform_into_prop_serialiser( + case "date": + properties_graph += _make_date_prop_graph( prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + permissions_lookup=lookup.permissions, + ) + case "interval": + properties_graph += _make_interval_prop_graph( + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, permissions_lookup=lookup.permissions, - transformations=transformations, ) - properties_serialised.update(transformed_prop.serialise()) case _: raise UserError(f"Unknown value type: {prop.valtype}") last_prop_name = prop_name @@ -199,8 +208,8 @@ def _transform_into_value_serialiser( transformations: TransformationSteps, ) -> SerialiseValue: transformed = transformations.transformer(value.value) - permission_str = _get_permission_str(value.permissions, permissions_lookup) - return transformations.serialiser(transformed, permission_str, value.comment) + resolved_permission = _resolve_permission(value.permissions, permissions_lookup) + return transformations.serialiser(transformed, resolved_permission, value.comment) def _make_iiif_uri_value(iiif_uri: IIIFUriInfo, res_bnode: BNode, permissions_lookup: dict[str, Permissions]) -> Graph: @@ -247,9 +256,9 @@ def _make_simple_prop_graph( g = Graph() for val in prop.values: transformed_val = transformer(val.value) - per_str = _get_permission_str(val.permissions, permissions_lookup) + resolved_permission = _resolve_permission(val.permissions, permissions_lookup) transformed = TransformedValue( - value=transformed_val, prop_name=prop_name, permissions=per_str, comment=val.comment + value=transformed_val, prop_name=prop_name, permissions=resolved_permission, comment=val.comment ) g += _make_simple_value_graph(transformed, res_bn, prop_type_info) return g @@ -272,17 +281,93 @@ def _make_list_prop_graph( ) raise BaseError(msg) - per_str = _get_permission_str(val.permissions, permissions_lookup) + resolved_permission = _resolve_permission(val.permissions, permissions_lookup) transformed = TransformedValue( value=URIRef(iri_str), prop_name=prop_name, - permissions=per_str, + permissions=resolved_permission, comment=val.comment, ) g += _make_simple_value_graph(transformed, res_bn, rdf_prop_type_mapper["list"]) return g +def _make_date_prop_graph( + prop: XMLProperty, + res_bn: BNode, + prop_name: URIRef, + permissions_lookup: dict[str, Permissions], +) -> Graph: + g = Graph() + for val in prop.values: + resolved_permission = _resolve_permission(val.permissions, permissions_lookup) + g += _make_date_value_graph(val, prop_name, resolved_permission, res_bn) + return g + + +def _make_date_value_graph( + val: XMLValue, + prop_name: URIRef, + resolved_permission: str | None, + res_bn: BNode, +) -> Graph: + date = transform_date(val.value) + val_bn = BNode() + g = _get_optional_triples(val_bn, resolved_permission, val.comment) + g.add((res_bn, prop_name, val_bn)) + g.add((val_bn, RDF.type, KNORA_API.DateValue)) + if cal := date.calendar.value: + g.add((val_bn, KNORA_API.dateValueHasCalendar, Literal(cal, datatype=XSD.string))) + g += _make_single_date_graph(val_bn, date.start, StartEnd.START) + if date.end: + g += _make_single_date_graph(val_bn, date.end, StartEnd.END) + return g + + +def _make_single_date_graph(val_bn: BNode, date: SingleDate, start_end: StartEnd) -> Graph: + def get_prop(precision: DayMonthYearEra) -> URIRef: + return KNORA_API[f"dateValueHas{start_end.value}{precision.value}"] + + g = Graph() + if yr := date.year: + g.add((val_bn, get_prop(DayMonthYearEra.YEAR), Literal(yr, datatype=XSD.int))) + if mnt := date.month: + g.add((val_bn, get_prop(DayMonthYearEra.MONTH), Literal(mnt, datatype=XSD.int))) + if day := date.day: + g.add((val_bn, get_prop(DayMonthYearEra.DAY), Literal(day, datatype=XSD.int))) + if era := date.era: + g.add((val_bn, get_prop(DayMonthYearEra.ERA), Literal(era, datatype=XSD.string))) + return g + + +def _make_interval_prop_graph( + prop: XMLProperty, + res_bn: BNode, + prop_name: URIRef, + permissions_lookup: dict[str, Permissions], +) -> Graph: + g = Graph() + for val in prop.values: + resolved_permission = _resolve_permission(val.permissions, permissions_lookup) + g += _make_interval_value_graph(val, prop_name, resolved_permission, res_bn) + return g + + +def _make_interval_value_graph( + val: XMLValue, + prop_name: URIRef, + resolved_permission: str | None, + res_bn: BNode, +) -> Graph: + interval = transform_interval(val.value) + val_bn = BNode() + g = _get_optional_triples(val_bn, resolved_permission, val.comment) + g.add((res_bn, prop_name, val_bn)) + g.add((val_bn, RDF.type, KNORA_API.IntervalValue)) + g.add((val_bn, KNORA_API.intervalValueHasStart, interval.start)) + g.add((val_bn, KNORA_API.intervalValueHasEnd, interval.end)) + + def _make_link_prop_graph( prop: XMLProperty, res_bn: BNode, @@ -293,11 +378,11 @@ def _make_link_prop_graph( g = Graph() for val in prop.values: iri_str = _resolve_id_to_iri(val.value, iri_resolver) - per_str = _get_permission_str(val.permissions, permissions_lookup) + resolved_permission = _resolve_permission(val.permissions, permissions_lookup) transformed = TransformedValue( value=URIRef(iri_str), prop_name=prop_name, - permissions=per_str, + permissions=resolved_permission, comment=val.comment, ) g += _make_simple_value_graph(transformed, res_bn, rdf_prop_type_mapper["link"]) @@ -329,13 +414,13 @@ def _make_text_prop_graph( ) -> Graph: g = Graph() for val in prop.values: - per_str = _get_permission_str(val.permissions, permissions_lookup) + resolved_permission = _resolve_permission(val.permissions, permissions_lookup) match val.value: case str(): transformed = TransformedValue( value=Literal(val.value, datatype=XSD.string), prop_name=prop_name, - permissions=per_str, + permissions=resolved_permission, comment=val.comment, ) g += _make_simple_value_graph( @@ -375,8 +460,8 @@ def _make_richtext_value_graph( iri_resolver: IriResolver, ) -> Graph: val_bn = BNode() - per_str = _get_permission_str(val.permissions, permissions_lookup) - g = _get_optional_triples(val_bn, per_str, val.comment) + resolved_permission = _resolve_permission(val.permissions, permissions_lookup) + g = _get_optional_triples(val_bn, resolved_permission, val.comment) val_str = _get_richtext_string(val.value, iri_resolver) literal_val = Literal(val_str, datatype=XSD.string) g.add((res_bn, prop_name, val_bn)) @@ -412,7 +497,7 @@ def _add_optional_permission_triple( return g -def _get_permission_str(permissions: str | None, permissions_lookup: dict[str, Permissions]) -> str | None: +def _resolve_permission(permissions: str | None, permissions_lookup: dict[str, Permissions]) -> str | None: if permissions: if not (per := permissions_lookup.get(permissions)): raise PermissionNotExistsError(f"Could not find permissions for value: {permissions}") diff --git a/src/dsp_tools/commands/xmlupload/value_transformers.py b/src/dsp_tools/commands/xmlupload/value_transformers.py index 070ba4b23d..1773bcaeda 100644 --- a/src/dsp_tools/commands/xmlupload/value_transformers.py +++ b/src/dsp_tools/commands/xmlupload/value_transformers.py @@ -11,9 +11,7 @@ from rdflib import Literal from dsp_tools.commands.xmlupload.models.formatted_text_value import FormattedTextValue -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import Interval -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseDate -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseInterval +from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import Interval from dsp_tools.commands.xmlupload.models.serialise.serialise_value import ValueSerialiser from dsp_tools.models.exceptions import BaseError from dsp_tools.utils.date_util import Date @@ -52,16 +50,11 @@ def transform_interval(input_value: InputTypes) -> Interval: val = assert_is_string(input_value) match val.split(":", 1): case [start, end]: - return Interval(start, end) + return Interval(transform_xsd_decimal(start), transform_xsd_decimal(end)) case _: raise BaseError(f"Could not parse interval value: {input_value}") -def transform_string(input_value: InputTypes) -> str: - """Assert that an input is of type string.""" - return assert_is_string(input_value) - - def assert_is_string(value: str | FormattedTextValue) -> str: """Assert a value is a string.""" match value: @@ -73,12 +66,6 @@ def assert_is_string(value: str | FormattedTextValue) -> str: assert_never(value) -value_to_transformations_mapper: dict[str, TransformationSteps] = { - "date": TransformationSteps(SerialiseDate, transform_date), - "interval": TransformationSteps(SerialiseInterval, transform_interval), -} - - def transform_xsd_string(value: InputTypes): str_val = assert_is_string(value) return Literal(str_val, datatype=XSD.string) From c2e021e44283214af314c318179ebf9a95f76c62 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:47:00 +0100 Subject: [PATCH 23/63] remove serialise value --- .../models/serialise/serialise_value.py | 171 ------------------ ...ransform_resource_and_values_for_upload.py | 26 --- .../commands/xmlupload/value_transformers.py | 8 - 3 files changed, 205 deletions(-) delete mode 100644 src/dsp_tools/commands/xmlupload/models/serialise/serialise_value.py diff --git a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_value.py b/src/dsp_tools/commands/xmlupload/models/serialise/serialise_value.py deleted file mode 100644 index 3489a38fac..0000000000 --- a/src/dsp_tools/commands/xmlupload/models/serialise/serialise_value.py +++ /dev/null @@ -1,171 +0,0 @@ -from __future__ import annotations - -from abc import ABC -from abc import abstractmethod -from collections.abc import Sequence -from dataclasses import dataclass -from typing import Any -from typing import Callable -from typing import TypeAlias -from typing import Union - -from dsp_tools.utils.date_util import Date -from dsp_tools.utils.date_util import DayMonthYearEra -from dsp_tools.utils.date_util import SingleDate -from dsp_tools.utils.date_util import StartEnd - -ValueTypes: TypeAlias = Union[str, Date, "Interval"] -ValueSerialiser: TypeAlias = Callable[[ValueTypes, str | None, str | None], "SerialiseValue"] - - -@dataclass(frozen=True) -class SerialiseProperty: - property_name: str - values: Sequence[SerialiseValue] - - def serialise(self) -> dict[str, Any]: - """Serialise the property and all its values.""" - return {self.property_name: [value.serialise() for value in self.values]} - - -@dataclass(frozen=True) -class SerialiseValue(ABC): - """A value to be serialised.""" - - value: ValueTypes - permissions: str | None - comment: str | None - - @abstractmethod - def serialise(self) -> dict[str, Any]: - """Serialise the value.""" - - def _get_optionals(self) -> dict[str, str]: - optionals = {"knora-api:valueHasComment": self.comment} if self.comment else {} - if self.permissions: - optionals["knora-api:hasPermissions"] = self.permissions - return optionals - - -class SerialiseDate(SerialiseValue): # TODO - """A DateValue to be serialised.""" - - value: Date - - def serialise(self) -> dict[str, Any]: - serialised = {"knora-api:dateValueHasCalendar": self.value.calendar.value} if self.value.calendar else {} - serialised["@type"] = "knora-api:DateValue" - serialised.update(self._get_one_date_dict(self.value.start, StartEnd.START)) - if self.value.end: - serialised.update(self._get_one_date_dict(self.value.end, StartEnd.END)) - serialised.update(self._get_optionals()) - return serialised - - def _get_one_date_dict(self, date: SingleDate, start_end: StartEnd) -> dict[str, Any]: - def get_prop(precision: DayMonthYearEra) -> str: - return f"knora-api:dateValueHas{start_end.value}{precision.value}" - - date_dict: dict[str, Any] = {get_prop(DayMonthYearEra.YEAR): date.year} if date.year else {} - if date.month: - date_dict[get_prop(DayMonthYearEra.MONTH)] = date.month - if date.day: - date_dict[get_prop(DayMonthYearEra.DAY)] = date.day - if date.era: - date_dict[get_prop(DayMonthYearEra.ERA)] = date.era.value - return date_dict - - -class SerialiseInterval(SerialiseValue): # TODO - """An IntervalValue to be serialised.""" - - value: Interval - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:IntervalValue", - "knora-api:intervalValueHasStart": self.value.interval_start(), - "knora-api:intervalValueHasEnd": self.value.interval_end(), - } - serialised.update(self._get_optionals()) - return serialised - - -@dataclass -class Interval: # TODO - start: str - end: str - - def interval_start(self) -> dict[str, str]: - return self._to_dict(self.start) - - def interval_end(self) -> dict[str, str]: - return self._to_dict(self.end) - - def _to_dict(self, interval_value: str) -> dict[str, str]: - return { - "@type": "xsd:decimal", - "@value": interval_value, - } - - -class SerialiseList(SerialiseValue): # TODO - """An ListValue to be serialised.""" - - value: str - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:ListValue", - "knora-api:listValueAsListNode": { - "@id": self.value, - }, - } - serialised.update(self._get_optionals()) - return serialised - - -class SerialiseLink(SerialiseValue): # TODO - """A LinkValue to be serialised.""" - - value: str - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:LinkValue", - "knora-api:linkValueHasTargetIri": { - "@id": self.value, - }, - } - serialised.update(self._get_optionals()) - return serialised - - -class SerialiseSimpletext(SerialiseValue): - """A Simpletext to be serialised.""" - - value: str - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:TextValue", - "knora-api:valueAsString": self.value, - } - serialised.update(self._get_optionals()) - return serialised - - -class SerialiseRichtext(SerialiseValue): - """A Richtext to be serialised.""" - - value: str - - def serialise(self) -> dict[str, Any]: - serialised = { - "@type": "knora-api:TextValue", - "knora-api:textValueAsXml": self.value, - "knora-api:textValueHasMapping": { - "@id": "http://rdfh.ch/standoff/mappings/StandardMapping", - }, - } - serialised.update(self._get_optionals()) - return serialised diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index a23ad145fb..dd310a7489 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -34,9 +34,6 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_prop_type_mapper from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseMigrationMetadata from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseResource -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseProperty -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import SerialiseValue -from dsp_tools.commands.xmlupload.value_transformers import TransformationSteps from dsp_tools.commands.xmlupload.value_transformers import assert_is_string from dsp_tools.commands.xmlupload.value_transformers import rdf_literal_transformer from dsp_tools.commands.xmlupload.value_transformers import transform_date @@ -189,29 +186,6 @@ def _get_absolute_prop_iri(prefixed_prop: str, namepsaces: dict[str, Namespace]) return namespace[prop] -def _transform_into_prop_serialiser( - prop: XMLProperty, - permissions_lookup: dict[str, Permissions], - transformations: TransformationSteps, -) -> SerialiseProperty: - serialised_values = [_transform_into_value_serialiser(v, permissions_lookup, transformations) for v in prop.values] - prop_serialise = SerialiseProperty( - property_name=prop.name, - values=serialised_values, - ) - return prop_serialise - - -def _transform_into_value_serialiser( - value: XMLValue, - permissions_lookup: dict[str, Permissions], - transformations: TransformationSteps, -) -> SerialiseValue: - transformed = transformations.transformer(value.value) - resolved_permission = _resolve_permission(value.permissions, permissions_lookup) - return transformations.serialiser(transformed, resolved_permission, value.comment) - - def _make_iiif_uri_value(iiif_uri: IIIFUriInfo, res_bnode: BNode, permissions_lookup: dict[str, Permissions]) -> Graph: g = Graph() iiif_bn = BNode() diff --git a/src/dsp_tools/commands/xmlupload/value_transformers.py b/src/dsp_tools/commands/xmlupload/value_transformers.py index 1773bcaeda..ad3dd77de3 100644 --- a/src/dsp_tools/commands/xmlupload/value_transformers.py +++ b/src/dsp_tools/commands/xmlupload/value_transformers.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -from dataclasses import dataclass from typing import Callable from typing import TypeAlias from typing import Union @@ -12,7 +11,6 @@ from dsp_tools.commands.xmlupload.models.formatted_text_value import FormattedTextValue from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import Interval -from dsp_tools.commands.xmlupload.models.serialise.serialise_value import ValueSerialiser from dsp_tools.models.exceptions import BaseError from dsp_tools.utils.date_util import Date from dsp_tools.utils.date_util import parse_date_string @@ -22,12 +20,6 @@ ValueTransformer: TypeAlias = Callable[[InputTypes], OutputTypes] -@dataclass -class TransformationSteps: - serialiser: ValueSerialiser - transformer: ValueTransformer - - def transform_boolean(s: str | int | bool) -> bool: """Takes an input value and transforms it into a boolean.""" match s: From 846d73a017dcdf42845bb173a5b37fe63d44f4ac Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:53:21 +0100 Subject: [PATCH 24/63] Update transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 117 ++++++++++-------- 1 file changed, 62 insertions(+), 55 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index dd310a7489..23ec0200f9 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -105,61 +105,10 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di last_prop_name = None for prop in resource.properties: - prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) - match prop.valtype: - case "boolean" | "color" | "decimal" | "geometry" | "geoname" | "integer" | "time" | "uri" as val_type: - literal_info = rdf_prop_type_mapper[val_type] - transformer = rdf_literal_transformer[val_type] - properties_graph += _make_simple_prop_graph( - prop=prop, - res_bn=res_bnode, - prop_name=prop_name, - prop_type_info=literal_info, - transformer=transformer, - permissions_lookup=lookup.permissions, - ) - case "list": - properties_graph += _make_list_prop_graph( - prop=prop, - res_bn=res_bnode, - prop_name=prop_name, - permissions_lookup=lookup.permissions, - listnode_lookup=lookup.listnodes, - ) - case "resptr": - prop_name = _get_link_prop_name(prop, resource.restype, lookup.namespaces) - properties_graph += _make_link_prop_graph( - prop=prop, - res_bn=res_bnode, - prop_name=prop_name, - permissions_lookup=lookup.permissions, - iri_resolver=lookup.id_to_iri, - ) - case "text": - properties_graph += _make_text_prop_graph( - prop=prop, - res_bn=res_bnode, - prop_name=prop_name, - permissions_lookup=lookup.permissions, - iri_resolver=lookup.id_to_iri, - ) - case "date": - properties_graph += _make_date_prop_graph( - prop=prop, - res_bn=res_bnode, - prop_name=prop_name, - permissions_lookup=lookup.permissions, - ) - case "interval": - properties_graph += _make_interval_prop_graph( - prop=prop, - res_bn=res_bnode, - prop_name=prop_name, - permissions_lookup=lookup.permissions, - ) - case _: - raise UserError(f"Unknown value type: {prop.valtype}") - last_prop_name = prop_name + single_prop_graph, last_prop_name = _make_one_prop_graph( + prop=prop, restype=resource.restype, res_bnode=res_bnode, lookup=lookup + ) + properties_graph += single_prop_graph if resource.iiif_uri: properties_graph += _make_iiif_uri_value(resource.iiif_uri, res_bnode, lookup.permissions) @@ -170,6 +119,64 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di return properties_serialised +def _make_one_prop_graph(prop: XMLProperty, restype: str, res_bnode: BNode, lookup: Lookups) -> tuple[Graph, URIRef]: + prop_name = _get_absolute_prop_iri(prop.name, lookup.namespaces) + match prop.valtype: + case "boolean" | "color" | "decimal" | "geometry" | "geoname" | "integer" | "time" | "uri" as val_type: + literal_info = rdf_prop_type_mapper[val_type] + transformer = rdf_literal_transformer[val_type] + properties_graph = _make_simple_prop_graph( + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + prop_type_info=literal_info, + transformer=transformer, + permissions_lookup=lookup.permissions, + ) + case "list": + properties_graph = _make_list_prop_graph( + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + permissions_lookup=lookup.permissions, + listnode_lookup=lookup.listnodes, + ) + case "resptr": + prop_name = _get_link_prop_name(prop, restype, lookup.namespaces) + properties_graph = _make_link_prop_graph( + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + permissions_lookup=lookup.permissions, + iri_resolver=lookup.id_to_iri, + ) + case "text": + properties_graph = _make_text_prop_graph( + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + permissions_lookup=lookup.permissions, + iri_resolver=lookup.id_to_iri, + ) + case "date": + properties_graph = _make_date_prop_graph( + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + permissions_lookup=lookup.permissions, + ) + case "interval": + properties_graph = _make_interval_prop_graph( + prop=prop, + res_bn=res_bnode, + prop_name=prop_name, + permissions_lookup=lookup.permissions, + ) + case _: + raise UserError(f"Unknown value type: {prop.valtype}") + return properties_graph, prop_name + + def _get_link_prop_name(p: XMLProperty, restype: str, namespaces: dict[str, Namespace]) -> URIRef: if p.name == "knora-api:isSegmentOf" and restype == "knora-api:VideoSegment": return KNORA_API.isVideoSegmentOfValue From f21b52cf9678c0dad755fea8d99399580e0aa11f Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:55:12 +0100 Subject: [PATCH 25/63] Update transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 23ec0200f9..af7c174222 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -245,6 +245,19 @@ def _make_simple_prop_graph( return g +def _make_simple_value_graph( + val: TransformedValue, + res_bn: BNode, + prop_type_info: RDFPropTypeInfo, +) -> Graph(): + val_bn = BNode() + g = _get_optional_triples(val_bn, val.permissions, val.comment) + g.add((res_bn, val.prop_name, val_bn)) + g.add((val_bn, RDF.type, prop_type_info.knora_type)) + g.add((val_bn, prop_type_info.knora_prop, val.value)) + return g + + def _make_list_prop_graph( prop: XMLProperty, res_bn: BNode, @@ -420,19 +433,6 @@ def _make_text_prop_graph( return g -def _make_simple_value_graph( - val: TransformedValue, - res_bn: BNode, - prop_type_info: RDFPropTypeInfo, -) -> Graph(): - val_bn = BNode() - g = _get_optional_triples(val_bn, val.permissions, val.comment) - g.add((res_bn, val.prop_name, val_bn)) - g.add((val_bn, RDF.type, prop_type_info.knora_type)) - g.add((val_bn, prop_type_info.knora_prop, val.value)) - return g - - def _make_richtext_value_graph( val: XMLValue, prop_name: URIRef, From ece186df28cc6b48a8753143ddcbe0691342f741 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:01:03 +0100 Subject: [PATCH 26/63] change value transformers --- .../commands/xmlupload/value_transformers.py | 50 +++++++++---------- .../xmlupload/test_value_transformers.py | 12 +++-- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/value_transformers.py b/src/dsp_tools/commands/xmlupload/value_transformers.py index ad3dd77de3..dffa8b41f4 100644 --- a/src/dsp_tools/commands/xmlupload/value_transformers.py +++ b/src/dsp_tools/commands/xmlupload/value_transformers.py @@ -20,15 +20,15 @@ ValueTransformer: TypeAlias = Callable[[InputTypes], OutputTypes] -def transform_boolean(s: str | int | bool) -> bool: - """Takes an input value and transforms it into a boolean.""" - match s: +def transform_xsd_boolean(value: InputTypes): + """Takes an input value and transforms it into an xsd:boolean.""" + match value: case "True" | "true" | "1" | 1 | True: - return True + return Literal(True, datatype=XSD.boolean) case "False" | "false" | "0" | 0 | False: - return False + return Literal(False, datatype=XSD.boolean) case _: - raise BaseError(f"Could not parse boolean value: {s}") + raise BaseError(f"Could not parse boolean value: {value}") def transform_date(input_value: InputTypes) -> Date: @@ -47,58 +47,54 @@ def transform_interval(input_value: InputTypes) -> Interval: raise BaseError(f"Could not parse interval value: {input_value}") -def assert_is_string(value: str | FormattedTextValue) -> str: - """Assert a value is a string.""" - match value: - case str() as s: - return s - case FormattedTextValue() as xml: - raise BaseError(f"Expected string value, but got XML value: {xml.as_xml()}") - case _: - assert_never(value) - - def transform_xsd_string(value: InputTypes): + """Transform a value into an rdflib Literal with datatype xsd:string""" str_val = assert_is_string(value) return Literal(str_val, datatype=XSD.string) def transform_xsd_decimal(value: InputTypes): + """Transform a value into an rdflib Literal with datatype xsd:decimal""" str_val = assert_is_string(value) return Literal(str(float(str_val)), datatype=XSD.decimal) -def transform_xsd_boolean(value: InputTypes): - match value: - case "True" | "true" | "1" | 1 | True: - return Literal(True, datatype=XSD.boolean) - case "False" | "false" | "0" | 0 | False: - return Literal(False, datatype=XSD.boolean) - case _: - raise BaseError(f"Could not parse boolean value: {value}") - - def transform_xsd_integer(value: InputTypes): + """Transform a value into an rdflib Literal with datatype xsd:integer""" str_val = assert_is_string(value) return Literal(str_val, datatype=XSD.integer) def transform_xsd_date_time(value: InputTypes): + """Transform a value into an rdflib Literal with datatype xsd:dateTimeStamp""" str_val = assert_is_string(value) return Literal(str_val, datatype=XSD.dateTimeStamp) def transform_xsd_any_uri(value: InputTypes): + """Transform a value into an rdflib Literal with datatype xsd:anyURI""" str_val = assert_is_string(value) return Literal(str_val, datatype=XSD.anyURI) def transform_geometry(value: InputTypes): + """Transform a value into a geometry string rdflib Literal with datatype xsd:string""" str_val = assert_is_string(value) str_val = json.dumps(json.loads(str_val)) return Literal(str_val, datatype=XSD.string) +def assert_is_string(value: str | FormattedTextValue) -> str: + """Assert a value is a string.""" + match value: + case str() as s: + return s + case FormattedTextValue() as xml: + raise BaseError(f"Expected string value, but got XML value: {xml.as_xml()}") + case _: + assert_never(value) + + rdf_literal_transformer = { "boolean": transform_xsd_boolean, "color": transform_xsd_string, diff --git a/test/unittests/commands/xmlupload/test_value_transformers.py b/test/unittests/commands/xmlupload/test_value_transformers.py index 24bb44738e..01a1427872 100644 --- a/test/unittests/commands/xmlupload/test_value_transformers.py +++ b/test/unittests/commands/xmlupload/test_value_transformers.py @@ -1,8 +1,10 @@ from typing import Any import pytest +from rdflib import XSD +from rdflib import Literal -from dsp_tools.commands.xmlupload.value_transformers import transform_boolean +from dsp_tools.commands.xmlupload.value_transformers import transform_xsd_boolean from dsp_tools.models.exceptions import BaseError @@ -22,15 +24,15 @@ ], ) def test_transform_boolean_success(input_bool: Any, success: bool) -> None: - result = transform_boolean(input_bool) - assert result == success + result = transform_xsd_boolean(input_bool) + assert result == Literal(success, XSD.boolean) def test_transform_boolean_raises_string() -> None: with pytest.raises(BaseError): - transform_boolean("foo") + transform_xsd_boolean("foo") def test_transform_boolean_raises_int() -> None: with pytest.raises(BaseError): - transform_boolean(2) + transform_xsd_boolean(2) From 2cb1213f68de7a521821012aa2d071d10926eb14 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:04:39 +0100 Subject: [PATCH 27/63] move tests --- ...lient.py => test_transform_file_values.py} | 31 --------------- ...ransform_resource_and_values_for_upload.py | 39 +++++++++++++++++++ 2 files changed, 39 insertions(+), 31 deletions(-) rename test/unittests/commands/xmlupload/{test_resource_create_client.py => test_transform_file_values.py} (92%) create mode 100644 test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py diff --git a/test/unittests/commands/xmlupload/test_resource_create_client.py b/test/unittests/commands/xmlupload/test_transform_file_values.py similarity index 92% rename from test/unittests/commands/xmlupload/test_resource_create_client.py rename to test/unittests/commands/xmlupload/test_transform_file_values.py index c96043bd27..c509a33b0d 100644 --- a/test/unittests/commands/xmlupload/test_resource_create_client.py +++ b/test/unittests/commands/xmlupload/test_transform_file_values.py @@ -2,21 +2,16 @@ import pytest from lxml import etree -from rdflib import RDF from rdflib import BNode -from rdflib import Graph -from rdflib import Literal from rdflib import Namespace from rdflib import URIRef from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import IIIFUriInfo -from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import XMLValue from dsp_tools.commands.xmlupload.models.deserialise.xmlresource import BitstreamInfo from dsp_tools.commands.xmlupload.models.deserialise.xmlresource import XMLResource from dsp_tools.commands.xmlupload.models.permission import Permissions from dsp_tools.commands.xmlupload.models.permission import PermissionValue from dsp_tools.commands.xmlupload.models.serialise.jsonld_serialiser import serialise_property_graph -from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import KNORA_API from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import _make_bitstream_file_value from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import _make_iiif_uri_value from dsp_tools.models.exceptions import PermissionNotExistsError @@ -442,31 +437,5 @@ def test_make_iiif_uri_value_serialised() -> None: assert serialised == expected -# TODO: change -def test_make_boolean_value_with_permissions() -> None: - permissions_lookup = {"open": Permissions({PermissionValue.CR: ["knora-admin:ProjectAdmin"]})} - xml_str = """ - - - true - - - """ - xmlresource = XMLResource.from_node(etree.fromstring(xml_str), "foo") - test_val: XMLValue = xmlresource.properties[0].values[0] - res_bn = BNode() - prop_name = ONTO.isTrueOrFalse - bool_graph = Graph() - number_of_triples = 4 - assert len(bool_graph) == number_of_triples - value_bn = next(bool_graph.objects(res_bn, prop_name)) - rdf_type = next(bool_graph.objects(value_bn, RDF.type)) - assert rdf_type == KNORA_API.BooleanValue - bool_val = next(bool_graph.objects(value_bn, KNORA_API.booleanValueAsBoolean)) - assert bool_val == Literal(True) - permissions = next(bool_graph.objects(value_bn, KNORA_API.hasPermissions)) - assert permissions == Literal(str(permissions_lookup.get("open"))) - - if __name__ == "__main__": pytest.main([__file__]) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py new file mode 100644 index 0000000000..d98e7cbf1d --- /dev/null +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -0,0 +1,39 @@ +from lxml import etree +from rdflib import RDF +from rdflib import BNode +from rdflib import Graph +from rdflib import Literal +from rdflib import Namespace + +from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import XMLValue +from dsp_tools.commands.xmlupload.models.deserialise.xmlresource import XMLResource +from dsp_tools.commands.xmlupload.models.permission import Permissions +from dsp_tools.commands.xmlupload.models.permission import PermissionValue +from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import KNORA_API + +ONTO = Namespace("http://0.0.0.0:3333/ontology/9999/onto/v2#") + + +def test_make_boolean_value_with_permissions() -> None: + permissions_lookup = {"open": Permissions({PermissionValue.CR: ["knora-admin:ProjectAdmin"]})} + xml_str = """ + + + true + + + """ + xmlresource = XMLResource.from_node(etree.fromstring(xml_str), "foo") + test_val: XMLValue = xmlresource.properties[0].values[0] + res_bn = BNode() + prop_name = ONTO.isTrueOrFalse + bool_graph = Graph() + number_of_triples = 4 + assert len(bool_graph) == number_of_triples + value_bn = next(bool_graph.objects(res_bn, prop_name)) + rdf_type = next(bool_graph.objects(value_bn, RDF.type)) + assert rdf_type == KNORA_API.BooleanValue + bool_val = next(bool_graph.objects(value_bn, KNORA_API.booleanValueAsBoolean)) + assert bool_val == Literal(True) + permissions = next(bool_graph.objects(value_bn, KNORA_API.hasPermissions)) + assert permissions == Literal(str(permissions_lookup.get("open"))) From 44fdec805e1fd65250aa2fb2699cd2ee282e5c47 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:06:37 +0100 Subject: [PATCH 28/63] Update test_value_transformers.py --- test/unittests/commands/xmlupload/test_value_transformers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unittests/commands/xmlupload/test_value_transformers.py b/test/unittests/commands/xmlupload/test_value_transformers.py index 01a1427872..50d015bd74 100644 --- a/test/unittests/commands/xmlupload/test_value_transformers.py +++ b/test/unittests/commands/xmlupload/test_value_transformers.py @@ -25,7 +25,7 @@ ) def test_transform_boolean_success(input_bool: Any, success: bool) -> None: result = transform_xsd_boolean(input_bool) - assert result == Literal(success, XSD.boolean) + assert result == Literal(success, datatype=XSD.boolean) def test_transform_boolean_raises_string() -> None: From a043d637df206bb60a0bb8f07ec3a81d4462acd8 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:21:26 +0100 Subject: [PATCH 29/63] linting --- ...ransform_resource_and_values_for_upload.py | 23 ++++++++++--------- .../commands/xmlupload/value_transformers.py | 17 ++++++-------- .../xmlupload/test_value_transformers.py | 2 +- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index af7c174222..ffb8851e50 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -249,7 +249,7 @@ def _make_simple_value_graph( val: TransformedValue, res_bn: BNode, prop_type_info: RDFPropTypeInfo, -) -> Graph(): +) -> Graph: val_bn = BNode() g = _get_optional_triples(val_bn, val.permissions, val.comment) g.add((res_bn, val.prop_name, val_bn)) @@ -360,6 +360,7 @@ def _make_interval_value_graph( g.add((val_bn, RDF.type, KNORA_API.IntervalValue)) g.add((val_bn, KNORA_API.intervalValueHasStart, interval.start)) g.add((val_bn, KNORA_API.intervalValueHasEnd, interval.end)) + return g def _make_link_prop_graph( @@ -371,7 +372,8 @@ def _make_link_prop_graph( ) -> Graph: g = Graph() for val in prop.values: - iri_str = _resolve_id_to_iri(val.value, iri_resolver) + str_val = assert_is_string(val.value) + iri_str = _resolve_id_to_iri(str_val, iri_resolver) resolved_permission = _resolve_permission(val.permissions, permissions_lookup) transformed = TransformedValue( value=URIRef(iri_str), @@ -384,15 +386,14 @@ def _make_link_prop_graph( def _resolve_id_to_iri(value: str, iri_resolver: IriResolver) -> str: - s = assert_is_string(value) - if is_resource_iri(s): - iri_str = s - elif resolved_iri := iri_resolver.get(s): + if is_resource_iri(value): + iri_str = value + elif resolved_iri := iri_resolver.get(value): iri_str = resolved_iri else: msg = ( - f"Could not find the ID {s} in the id2iri mapping. " - f"This is probably because the resource '{s}' could not be created. " + f"Could not find the ID {value} in the id2iri mapping. " + f"This is probably because the resource '{value}' could not be created. " f"See {WARNINGS_SAVEPATH} for more information." ) raise BaseError(msg) @@ -443,7 +444,8 @@ def _make_richtext_value_graph( val_bn = BNode() resolved_permission = _resolve_permission(val.permissions, permissions_lookup) g = _get_optional_triples(val_bn, resolved_permission, val.comment) - val_str = _get_richtext_string(val.value, iri_resolver) + xml_val = cast(FormattedTextValue, val.value) + val_str = _get_richtext_string(xml_val, iri_resolver) literal_val = Literal(val_str, datatype=XSD.string) g.add((res_bn, prop_name, val_bn)) g.add((val_bn, RDF.type, KNORA_API.TextValue)) @@ -452,8 +454,7 @@ def _make_richtext_value_graph( return g -def _get_richtext_string(value: FormattedTextValue, iri_resolver: IriResolver) -> str: - xml_val = cast(FormattedTextValue, value) +def _get_richtext_string(xml_val: FormattedTextValue, iri_resolver: IriResolver) -> str: xml_with_iris = xml_val.with_iris(iri_resolver) return xml_with_iris.as_xml() diff --git a/src/dsp_tools/commands/xmlupload/value_transformers.py b/src/dsp_tools/commands/xmlupload/value_transformers.py index dffa8b41f4..8e41300a94 100644 --- a/src/dsp_tools/commands/xmlupload/value_transformers.py +++ b/src/dsp_tools/commands/xmlupload/value_transformers.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -from typing import Callable from typing import TypeAlias from typing import Union from typing import assert_never @@ -16,11 +15,9 @@ from dsp_tools.utils.date_util import parse_date_string InputTypes: TypeAlias = Union[str, FormattedTextValue] -OutputTypes: TypeAlias = Union[str, Date, Interval] -ValueTransformer: TypeAlias = Callable[[InputTypes], OutputTypes] -def transform_xsd_boolean(value: InputTypes): +def transform_xsd_boolean(value: InputTypes) -> Literal: """Takes an input value and transforms it into an xsd:boolean.""" match value: case "True" | "true" | "1" | 1 | True: @@ -47,37 +44,37 @@ def transform_interval(input_value: InputTypes) -> Interval: raise BaseError(f"Could not parse interval value: {input_value}") -def transform_xsd_string(value: InputTypes): +def transform_xsd_string(value: InputTypes) -> Literal: """Transform a value into an rdflib Literal with datatype xsd:string""" str_val = assert_is_string(value) return Literal(str_val, datatype=XSD.string) -def transform_xsd_decimal(value: InputTypes): +def transform_xsd_decimal(value: InputTypes) -> Literal: """Transform a value into an rdflib Literal with datatype xsd:decimal""" str_val = assert_is_string(value) return Literal(str(float(str_val)), datatype=XSD.decimal) -def transform_xsd_integer(value: InputTypes): +def transform_xsd_integer(value: InputTypes) -> Literal: """Transform a value into an rdflib Literal with datatype xsd:integer""" str_val = assert_is_string(value) return Literal(str_val, datatype=XSD.integer) -def transform_xsd_date_time(value: InputTypes): +def transform_xsd_date_time(value: InputTypes) -> Literal: """Transform a value into an rdflib Literal with datatype xsd:dateTimeStamp""" str_val = assert_is_string(value) return Literal(str_val, datatype=XSD.dateTimeStamp) -def transform_xsd_any_uri(value: InputTypes): +def transform_xsd_any_uri(value: InputTypes) -> Literal: """Transform a value into an rdflib Literal with datatype xsd:anyURI""" str_val = assert_is_string(value) return Literal(str_val, datatype=XSD.anyURI) -def transform_geometry(value: InputTypes): +def transform_geometry(value: InputTypes) -> Literal: """Transform a value into a geometry string rdflib Literal with datatype xsd:string""" str_val = assert_is_string(value) str_val = json.dumps(json.loads(str_val)) diff --git a/test/unittests/commands/xmlupload/test_value_transformers.py b/test/unittests/commands/xmlupload/test_value_transformers.py index 50d015bd74..e03186b623 100644 --- a/test/unittests/commands/xmlupload/test_value_transformers.py +++ b/test/unittests/commands/xmlupload/test_value_transformers.py @@ -35,4 +35,4 @@ def test_transform_boolean_raises_string() -> None: def test_transform_boolean_raises_int() -> None: with pytest.raises(BaseError): - transform_xsd_boolean(2) + transform_xsd_boolean(2) # type: ignore[arg-type] From e528bc9d21bccd5aa5bfc57adf4fdd5f42127422 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:26:09 +0100 Subject: [PATCH 30/63] Update transform_resource_and_values_for_upload.py --- .../xmlupload/transform_resource_and_values_for_upload.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index ffb8851e50..42808ac9fd 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -99,7 +99,6 @@ def _make_resource( def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> dict[str, Any]: - properties_serialised = {} properties_graph = Graph() # To frame the json-ld correctly, we need one property used in the graph. It does not matter which. last_prop_name = None @@ -114,9 +113,8 @@ def _make_values(resource: XMLResource, res_bnode: BNode, lookup: Lookups) -> di properties_graph += _make_iiif_uri_value(resource.iiif_uri, res_bnode, lookup.permissions) last_prop_name = KNORA_API.hasStillImageFileValue if last_prop_name: - serialised_graph_props = serialise_property_graph(properties_graph, last_prop_name) - properties_serialised.update(serialised_graph_props) - return properties_serialised + return serialise_property_graph(properties_graph, last_prop_name) + return {} def _make_one_prop_graph(prop: XMLProperty, restype: str, res_bnode: BNode, lookup: Lookups) -> tuple[Graph, URIRef]: From 34eb34403bb287551271c68901104b154a957a8f Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:54:01 +0100 Subject: [PATCH 31/63] add test --- ...ransform_resource_and_values_for_upload.py | 58 +++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index d98e7cbf1d..fb6c1f591c 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -1,21 +1,69 @@ +import pytest from lxml import etree from rdflib import RDF from rdflib import BNode from rdflib import Graph from rdflib import Literal -from rdflib import Namespace +from rdflib import Namespace, XSD -from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import XMLValue +from dsp_tools.commands.xmlupload.iri_resolver import IriResolver +from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import XMLValue, XMLProperty from dsp_tools.commands.xmlupload.models.deserialise.xmlresource import XMLResource +from dsp_tools.commands.xmlupload.models.lookup_models import JSONLDContext +from dsp_tools.commands.xmlupload.models.lookup_models import Lookups from dsp_tools.commands.xmlupload.models.permission import Permissions from dsp_tools.commands.xmlupload.models.permission import PermissionValue -from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import KNORA_API +from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import KNORA_API, _make_one_prop_graph ONTO = Namespace("http://0.0.0.0:3333/ontology/9999/onto/v2#") +namespaces = {"onto": ONTO, "knora-api": KNORA_API} +PERMISSION_LITERAL = Literal("CR knora-admin:ProjectAdmin", datatype=XSD.string) -def test_make_boolean_value_with_permissions() -> None: - permissions_lookup = {"open": Permissions({PermissionValue.CR: ["knora-admin:ProjectAdmin"]})} +@pytest.fixture +def permissions_lookup() -> dict[str, Permissions]: + return {"open": Permissions({PermissionValue.CR: ["knora-admin:ProjectAdmin"]})} + + +@pytest.fixture +def lookups(permissions_lookup: dict[str, Permissions]) -> Lookups: + return Lookups( + project_iri="http://rdfh.ch/9999/project", + id_to_iri=IriResolver({"one": "1"}), + permissions=permissions_lookup, + listnodes={"node": "http://rdfh.ch/9999/node"}, + namespaces=namespaces, + jsonld_context=JSONLDContext({}), + ) + +@pytest.fixture +def res_info() -> tuple[BNode, str]: + return BNode(), "restype" + +class TestMakeOnePropGraph: + + def test_boolean_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + true + + """) + prop = XMLProperty.from_node(xml_prop, "boolean", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 4 + assert prop_name == ONTO.isTrueOrFalse + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API.BooleanValue + value = next(result.objects(val_bn, KNORA_API.booleanValueAsBoolean)) + assert value == Literal(True, datatype=XSD.boolean) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + + +def test_make_boolean_value_with_permissions(permissions_lookup: dict[str, Permissions]) -> None: xml_str = """ From 7a4af49e60cbf307f77dee24a24c046f63e8565a Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:54:45 +0100 Subject: [PATCH 32/63] Update test_transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 39 ++++--------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index fb6c1f591c..37d30b8c32 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -1,25 +1,26 @@ import pytest from lxml import etree from rdflib import RDF +from rdflib import XSD from rdflib import BNode -from rdflib import Graph from rdflib import Literal -from rdflib import Namespace, XSD +from rdflib import Namespace from dsp_tools.commands.xmlupload.iri_resolver import IriResolver -from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import XMLValue, XMLProperty -from dsp_tools.commands.xmlupload.models.deserialise.xmlresource import XMLResource +from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import XMLProperty from dsp_tools.commands.xmlupload.models.lookup_models import JSONLDContext from dsp_tools.commands.xmlupload.models.lookup_models import Lookups from dsp_tools.commands.xmlupload.models.permission import Permissions from dsp_tools.commands.xmlupload.models.permission import PermissionValue -from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import KNORA_API, _make_one_prop_graph +from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import KNORA_API +from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import _make_one_prop_graph ONTO = Namespace("http://0.0.0.0:3333/ontology/9999/onto/v2#") namespaces = {"onto": ONTO, "knora-api": KNORA_API} PERMISSION_LITERAL = Literal("CR knora-admin:ProjectAdmin", datatype=XSD.string) + @pytest.fixture def permissions_lookup() -> dict[str, Permissions]: return {"open": Permissions({PermissionValue.CR: ["knora-admin:ProjectAdmin"]})} @@ -36,12 +37,13 @@ def lookups(permissions_lookup: dict[str, Permissions]) -> Lookups: jsonld_context=JSONLDContext({}), ) + @pytest.fixture def res_info() -> tuple[BNode, str]: return BNode(), "restype" -class TestMakeOnePropGraph: +class TestMakeOnePropGraph: def test_boolean_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -60,28 +62,3 @@ def test_boolean_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> assert value == Literal(True, datatype=XSD.boolean) permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) assert permissions == PERMISSION_LITERAL - - - -def test_make_boolean_value_with_permissions(permissions_lookup: dict[str, Permissions]) -> None: - xml_str = """ - - - true - - - """ - xmlresource = XMLResource.from_node(etree.fromstring(xml_str), "foo") - test_val: XMLValue = xmlresource.properties[0].values[0] - res_bn = BNode() - prop_name = ONTO.isTrueOrFalse - bool_graph = Graph() - number_of_triples = 4 - assert len(bool_graph) == number_of_triples - value_bn = next(bool_graph.objects(res_bn, prop_name)) - rdf_type = next(bool_graph.objects(value_bn, RDF.type)) - assert rdf_type == KNORA_API.BooleanValue - bool_val = next(bool_graph.objects(value_bn, KNORA_API.booleanValueAsBoolean)) - assert bool_val == Literal(True) - permissions = next(bool_graph.objects(value_bn, KNORA_API.hasPermissions)) - assert permissions == Literal(str(permissions_lookup.get("open"))) From ef828cdc4b7d242e84564a60e721872ce7fac93c Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:08:30 +0100 Subject: [PATCH 33/63] add tests --- ...ransform_resource_and_values_for_upload.py | 291 +++++++++++++++++- 1 file changed, 290 insertions(+), 1 deletion(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 37d30b8c32..48d7d3dc59 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -30,7 +30,7 @@ def permissions_lookup() -> dict[str, Permissions]: def lookups(permissions_lookup: dict[str, Permissions]) -> Lookups: return Lookups( project_iri="http://rdfh.ch/9999/project", - id_to_iri=IriResolver({"one": "1"}), + id_to_iri=IriResolver({"res_one": "http://rdfh.ch/9999/res_one"}), permissions=permissions_lookup, listnodes={"node": "http://rdfh.ch/9999/node"}, namespaces=namespaces, @@ -62,3 +62,292 @@ def test_boolean_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> assert value == Literal(True, datatype=XSD.boolean) permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) assert permissions == PERMISSION_LITERAL + + def test_color_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + #5d1f1e + + """) + prop = XMLProperty.from_node(xml_prop, "color", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 3 + assert prop_name == ONTO.hasColor + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API.ColorValue + value = next(result.objects(val_bn, KNORA_API.colorValueAsColor)) + assert value == Literal("#5d1f1e", datatype=XSD.string) + + def test_decimal_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + 2.718281828459 + + """) + prop = XMLProperty.from_node(xml_prop, "decimal", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_geometry_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + + { + "status": "active", + "type": "polygon", + "lineWidth": 5, + "points": [{"x": 0.4, "y": 0.6}, + {"x": 0.5, "y": 0.9}, + {"x": 0.8, "y": 0.9}, + {"x": 0.7, "y": 0.6}] + } + + + """) + prop = XMLProperty.from_node(xml_prop, "geometry", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_geoname_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + 5416656 + + """) + prop = XMLProperty.from_node(xml_prop, "geoname", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_integer_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + 4711 + + """) + prop = XMLProperty.from_node(xml_prop, "integer", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_time_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + + + """) + prop = XMLProperty.from_node(xml_prop, "time", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_uri_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + https://dasch.swiss + + """) + prop = XMLProperty.from_node(xml_prop, "uri", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_list_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + node + + """) + prop = XMLProperty.from_node(xml_prop, "list", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_resptr_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + res_one + + """) + prop = XMLProperty.from_node(xml_prop, "resptr", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_simpletext_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + Text + + """) + prop = XMLProperty.from_node(xml_prop, "text", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_richtext_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + Text + + """) + prop = XMLProperty.from_node(xml_prop, "text", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_date_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + GREGORIAN:AD:0476-09-04:AD:0477 + + """) + prop = XMLProperty.from_node(xml_prop, "date", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_interval_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + """) + prop = XMLProperty.from_node(xml_prop, "interval", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_segment_of_video_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, _ = res_info + res_type = "knora-api:VideoSegment" + xml_prop = etree.fromstring(""" + res_one + """) + prop = XMLProperty.from_node(xml_prop, "resptr", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL + + def test_segment_of_audio_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, _ = res_info + res_type = "knora-api:AudioSegment" + xml_prop = etree.fromstring(""" + res_one + """) + prop = XMLProperty.from_node(xml_prop, "resptr", "onto") + result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) + assert len(result) == 0 + assert prop_name == ONTO + val_bn = next(result.objects(res_bn, prop_name)) + rdf_type = next(result.objects(val_bn, RDF.type)) + assert rdf_type == KNORA_API + value = next(result.objects(val_bn, KNORA_API)) + assert value == Literal("", datatype=XSD) + permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) + assert permissions == PERMISSION_LITERAL From af5375058c62475ffe9301b165e4e1039beab359 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:10:03 +0100 Subject: [PATCH 34/63] Update test_transform_resource_and_values_for_upload.py --- ...est_transform_resource_and_values_for_upload.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 48d7d3dc59..8d69e5309b 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -89,15 +89,15 @@ def test_decimal_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> """) prop = XMLProperty.from_node(xml_prop, "decimal", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 4 + assert prop_name == ONTO.hasDecimal val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.DecimalValue + value = next(result.objects(val_bn, KNORA_API.decimalValueAsDecimal)) + assert value == Literal("2.718281828459", datatype=XSD.decimal) + comment = next(result.objects(val_bn, KNORA_API.valueHasComment)) + assert comment == Literal("Eulersche Zahl", datatype=XSD.string) def test_geometry_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info From 5fbf0330eae0af524bbf53c85a4690e5c83026ee Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:13:18 +0100 Subject: [PATCH 35/63] Commit all changes --- ...t_transform_resource_and_values_for_upload.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 8d69e5309b..a9f978e629 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -102,8 +102,8 @@ def test_decimal_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> def test_geometry_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" - - + + { "status": "active", "type": "polygon", @@ -118,15 +118,13 @@ def test_geometry_success(self, lookups: Lookups, res_info: tuple[BNode, str]) - """) prop = XMLProperty.from_node(xml_prop, "geometry", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 3 + assert prop_name == ONTO.hasGeometry val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.GeomValue + value = next(result.objects(val_bn, KNORA_API.geometryValueAsGeometry)) + assert isinstance(value, Literal) def test_geoname_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info From 0e4ef8e61d464f572dc858d45f77dac9be9051ba Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:14:02 +0100 Subject: [PATCH 36/63] Commit all changes --- .../test_transform_resource_and_values_for_upload.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index a9f978e629..eb4ef33299 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -135,15 +135,13 @@ def test_geoname_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> """) prop = XMLProperty.from_node(xml_prop, "geoname", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 3 + assert prop_name == ONTO.hasGeoname val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.GeonameValue + value = next(result.objects(val_bn, KNORA_API.geonameValueAsGeonameCode)) + assert value == Literal("5416656", datatype=XSD.string) def test_integer_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info From 81b9c855a1f9bf0f93ffc1f4aa93a65495c79fc7 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:14:39 +0100 Subject: [PATCH 37/63] Commit all changes --- .../test_transform_resource_and_values_for_upload.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index eb4ef33299..fcbf1dfbb1 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -152,15 +152,13 @@ def test_integer_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> """) prop = XMLProperty.from_node(xml_prop, "integer", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 3 + assert prop_name == ONTO.hasInteger val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.IntValue + value = next(result.objects(val_bn, KNORA_API.intValueAsInt)) + assert value == Literal("4711", datatype=XSD.integer) def test_time_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info From 52bca42165cf0b4883021a7739ebc21eef945fef Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:15:39 +0100 Subject: [PATCH 38/63] Commit all changes --- .../test_transform_resource_and_values_for_upload.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index fcbf1dfbb1..4855aff83b 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -169,15 +169,13 @@ def test_time_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> No """) prop = XMLProperty.from_node(xml_prop, "time", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 3 + assert prop_name == ONTO.hasTime val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.TimeValue + value = next(result.objects(val_bn, KNORA_API.timeValueAsTimeStamp)) + assert value == Literal("2019-10-23T13:45:12.01-14:00", datatype=XSD.dateTimeStamp) def test_uri_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info From d40043930b5deb01e769311954ca110dbdf07547 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:17:26 +0100 Subject: [PATCH 39/63] Commit all changes --- ...ransform_resource_and_values_for_upload.py | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 4855aff83b..d62a543204 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -5,6 +5,7 @@ from rdflib import BNode from rdflib import Literal from rdflib import Namespace +from rdflib import URIRef from dsp_tools.commands.xmlupload.iri_resolver import IriResolver from dsp_tools.commands.xmlupload.models.deserialise.deserialise_value import XMLProperty @@ -186,15 +187,13 @@ def test_uri_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> Non """) prop = XMLProperty.from_node(xml_prop, "uri", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 3 + assert prop_name == ONTO.hasUri val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.UriValue + value = next(result.objects(val_bn, KNORA_API.uriValueAsUri)) + assert value == Literal("https://dasch.swiss", datatype=XSD.anyURI) def test_list_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info @@ -205,15 +204,13 @@ def test_list_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> No """) prop = XMLProperty.from_node(xml_prop, "list", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 3 + assert prop_name == ONTO.hasListItem val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.ListValue + value = next(result.objects(val_bn, KNORA_API.listValueAsListNode)) + assert value == URIRef("http://rdfh.ch/9999/node") def test_resptr_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info From 2143beee0ef42f56abd3442cc2af969646a0d09e Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:18:43 +0100 Subject: [PATCH 40/63] Commit all changes --- .../test_transform_resource_and_values_for_upload.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index d62a543204..724a915af7 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -221,15 +221,13 @@ def test_resptr_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> """) prop = XMLProperty.from_node(xml_prop, "resptr", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 3 + assert prop_name == ONTO.hasResourceValue val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.LinkValue + value = next(result.objects(val_bn, KNORA_API.linkValueHasTargetIri)) + assert value == URIRef("http://rdfh.ch/9999/res_one") def test_simpletext_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info From b36d95086211fa25f6f49c2affd026fdd50c75c1 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:21:11 +0100 Subject: [PATCH 41/63] Commit all changes --- ...ransform_resource_and_values_for_upload.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 724a915af7..8182f13e6d 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -238,32 +238,32 @@ def test_simpletext_success(self, lookups: Lookups, res_info: tuple[BNode, str]) """) prop = XMLProperty.from_node(xml_prop, "text", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 3 + assert prop_name == ONTO.hasSimpleText val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.TextValue + value = next(result.objects(val_bn, KNORA_API.valueAsString)) + assert value == Literal("Text", datatype=XSD.string) def test_richtext_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" - Text + Text """) prop = XMLProperty.from_node(xml_prop, "text", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 5 + assert prop_name == ONTO.hasRichtext val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) + assert rdf_type == KNORA_API.TextValue + value = next(result.objects(val_bn, KNORA_API.textValueAsXml)) + assert value == Literal('\nText', datatype=XSD.string) + mapping = next(result.objects(val_bn, KNORA_API.textValueHasMapping)) + assert mapping == URIRef("http://rdfh.ch/standoff/mappings/StandardMapping") permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) assert permissions == PERMISSION_LITERAL From eabbd6e55c69867e5590ff3cb36872a35c447215 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:24:27 +0100 Subject: [PATCH 42/63] Commit all changes --- ...ransform_resource_and_values_for_upload.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 8182f13e6d..4127650f14 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -20,6 +20,7 @@ namespaces = {"onto": ONTO, "knora-api": KNORA_API} PERMISSION_LITERAL = Literal("CR knora-admin:ProjectAdmin", datatype=XSD.string) +RES_ONE_URI = URIRef("http://rdfh.ch/9999/res_one") @pytest.fixture @@ -227,7 +228,7 @@ def test_resptr_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> rdf_type = next(result.objects(val_bn, RDF.type)) assert rdf_type == KNORA_API.LinkValue value = next(result.objects(val_bn, KNORA_API.linkValueHasTargetIri)) - assert value == URIRef("http://rdfh.ch/9999/res_one") + assert value == RES_ONE_URI def test_simpletext_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info @@ -311,15 +312,13 @@ def test_segment_of_video_success(self, lookups: Lookups, res_info: tuple[BNode, """) prop = XMLProperty.from_node(xml_prop, "resptr", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 3 + assert prop_name == KNORA_API.isVideoSegmentOfValue val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.LinkValue + value = next(result.objects(val_bn, KNORA_API.linkValueHasTargetIri)) + assert value == RES_ONE_URI def test_segment_of_audio_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, _ = res_info @@ -329,12 +328,10 @@ def test_segment_of_audio_success(self, lookups: Lookups, res_info: tuple[BNode, """) prop = XMLProperty.from_node(xml_prop, "resptr", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 3 + assert prop_name == KNORA_API.isAudioSegmentOfValue val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.LinkValue + value = next(result.objects(val_bn, KNORA_API.linkValueHasTargetIri)) + assert value == RES_ONE_URI From 453fe40fb4e25d9e0b011aed086d3e936ad9d2b8 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:26:09 +0100 Subject: [PATCH 43/63] Commit all changes --- ...est_transform_resource_and_values_for_upload.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 4127650f14..0ebb4f7ed3 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -294,15 +294,15 @@ def test_interval_success(self, lookups: Lookups, res_info: tuple[BNode, str]) - """) prop = XMLProperty.from_node(xml_prop, "interval", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 4 + assert prop_name == KNORA_API.hasSegmentBounds val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.IntervalValue + start = next(result.objects(val_bn, KNORA_API.intervalValueHasStart)) + assert start == Literal("0.1", datatype=XSD.float) + end = next(result.objects(val_bn, KNORA_API.intervalValueHasEnd)) + assert end == Literal("0.234", datatype=XSD.float) def test_segment_of_video_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, _ = res_info From 8f46b58404481acb9488068fe7d2e8350801c6f4 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:26:34 +0100 Subject: [PATCH 44/63] Commit all changes --- .../test_transform_resource_and_values_for_upload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 0ebb4f7ed3..bd095becfc 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -300,9 +300,9 @@ def test_interval_success(self, lookups: Lookups, res_info: tuple[BNode, str]) - rdf_type = next(result.objects(val_bn, RDF.type)) assert rdf_type == KNORA_API.IntervalValue start = next(result.objects(val_bn, KNORA_API.intervalValueHasStart)) - assert start == Literal("0.1", datatype=XSD.float) + assert start == Literal("0.1", datatype=XSD.decimal) end = next(result.objects(val_bn, KNORA_API.intervalValueHasEnd)) - assert end == Literal("0.234", datatype=XSD.float) + assert end == Literal("0.234", datatype=XSD.decimal) def test_segment_of_video_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, _ = res_info From d7802ec757d52cd6b7a3d35b0dc9cc57b5f7f96c Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:36:53 +0100 Subject: [PATCH 45/63] Commit all changes --- ...ransform_resource_and_values_for_upload.py | 8 +++--- ...ransform_resource_and_values_for_upload.py | 26 +++++++++++++------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 42808ac9fd..402bba01d7 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -322,13 +322,13 @@ def get_prop(precision: DayMonthYearEra) -> URIRef: g = Graph() if yr := date.year: - g.add((val_bn, get_prop(DayMonthYearEra.YEAR), Literal(yr, datatype=XSD.int))) + g.add((val_bn, get_prop(DayMonthYearEra.YEAR), Literal(yr, datatype=XSD.integer))) if mnt := date.month: - g.add((val_bn, get_prop(DayMonthYearEra.MONTH), Literal(mnt, datatype=XSD.int))) + g.add((val_bn, get_prop(DayMonthYearEra.MONTH), Literal(mnt, datatype=XSD.integer))) if day := date.day: - g.add((val_bn, get_prop(DayMonthYearEra.DAY), Literal(day, datatype=XSD.int))) + g.add((val_bn, get_prop(DayMonthYearEra.DAY), Literal(day, datatype=XSD.integer))) if era := date.era: - g.add((val_bn, get_prop(DayMonthYearEra.ERA), Literal(era, datatype=XSD.string))) + g.add((val_bn, get_prop(DayMonthYearEra.ERA), Literal(era.value, datatype=XSD.string))) return g diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index bd095becfc..ac1f8c26c5 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -272,20 +272,30 @@ def test_date_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> No res_bn, res_type = res_info xml_prop = etree.fromstring(""" - GREGORIAN:AD:0476-09-04:AD:0477 + GREGORIAN:AD:0476-09-04:0477 """) prop = XMLProperty.from_node(xml_prop, "date", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 0 - assert prop_name == ONTO + assert len(result) == 9 + assert prop_name == ONTO.hasDate val_bn = next(result.objects(res_bn, prop_name)) rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API - value = next(result.objects(val_bn, KNORA_API)) - assert value == Literal("", datatype=XSD) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) - assert permissions == PERMISSION_LITERAL + assert rdf_type == KNORA_API.DateValue + calendar = next(result.objects(val_bn, KNORA_API.dateValueHasCalendar)) + assert calendar == Literal("GREGORIAN", datatype=XSD.string) + start_day = next(result.objects(val_bn, KNORA_API.dateValueHasStartDay)) + assert start_day == Literal("4", datatype=XSD.integer) + start_month = next(result.objects(val_bn, KNORA_API.dateValueHasStartMonth)) + assert start_month == Literal("9", datatype=XSD.integer) + start_year = next(result.objects(val_bn, KNORA_API.dateValueHasStartYear)) + assert start_year == Literal("476", datatype=XSD.integer) + start_era = next(result.objects(val_bn, KNORA_API.dateValueHasStartEra)) + assert start_era == Literal("AD", datatype=XSD.string) + end_year = next(result.objects(val_bn, KNORA_API.dateValueHasEndYear)) + assert end_year == Literal("477", datatype=XSD.integer) + start_era = next(result.objects(val_bn, KNORA_API.dateValueHasEndEra)) + assert start_era == Literal("AD", datatype=XSD.string) def test_interval_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info From f952e08305c5dc3971ca2787ba393524f25d29c7 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:39:24 +0100 Subject: [PATCH 46/63] Commit all changes --- .../xmlupload/test_transform_resource_and_values_for_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index ac1f8c26c5..0e392b0fdd 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -34,7 +34,7 @@ def lookups(permissions_lookup: dict[str, Permissions]) -> Lookups: project_iri="http://rdfh.ch/9999/project", id_to_iri=IriResolver({"res_one": "http://rdfh.ch/9999/res_one"}), permissions=permissions_lookup, - listnodes={"node": "http://rdfh.ch/9999/node"}, + listnodes={"testlist:node": "http://rdfh.ch/9999/node"}, namespaces=namespaces, jsonld_context=JSONLDContext({}), ) From 438ba5ccf3152f3e8066445f7df28cb46b318653 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:40:45 +0100 Subject: [PATCH 47/63] Update test_transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 0e392b0fdd..d92eba74b8 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -45,8 +45,8 @@ def res_info() -> tuple[BNode, str]: return BNode(), "restype" -class TestMakeOnePropGraph: - def test_boolean_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: +class TestMakeOnePropGraphSuccess: + def test_boolean(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -65,7 +65,7 @@ def test_boolean_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) assert permissions == PERMISSION_LITERAL - def test_color_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_color(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -82,7 +82,7 @@ def test_color_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> N value = next(result.objects(val_bn, KNORA_API.colorValueAsColor)) assert value == Literal("#5d1f1e", datatype=XSD.string) - def test_decimal_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_decimal(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -101,7 +101,7 @@ def test_decimal_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> comment = next(result.objects(val_bn, KNORA_API.valueHasComment)) assert comment == Literal("Eulersche Zahl", datatype=XSD.string) - def test_geometry_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_geometry(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -128,7 +128,7 @@ def test_geometry_success(self, lookups: Lookups, res_info: tuple[BNode, str]) - value = next(result.objects(val_bn, KNORA_API.geometryValueAsGeometry)) assert isinstance(value, Literal) - def test_geoname_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_geoname(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -145,7 +145,7 @@ def test_geoname_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> value = next(result.objects(val_bn, KNORA_API.geonameValueAsGeonameCode)) assert value == Literal("5416656", datatype=XSD.string) - def test_integer_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_integer(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -162,7 +162,7 @@ def test_integer_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> value = next(result.objects(val_bn, KNORA_API.intValueAsInt)) assert value == Literal("4711", datatype=XSD.integer) - def test_time_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_time(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -179,7 +179,7 @@ def test_time_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> No value = next(result.objects(val_bn, KNORA_API.timeValueAsTimeStamp)) assert value == Literal("2019-10-23T13:45:12.01-14:00", datatype=XSD.dateTimeStamp) - def test_uri_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_uri(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -196,7 +196,7 @@ def test_uri_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> Non value = next(result.objects(val_bn, KNORA_API.uriValueAsUri)) assert value == Literal("https://dasch.swiss", datatype=XSD.anyURI) - def test_list_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_list(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -213,7 +213,7 @@ def test_list_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> No value = next(result.objects(val_bn, KNORA_API.listValueAsListNode)) assert value == URIRef("http://rdfh.ch/9999/node") - def test_resptr_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_resptr(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -230,7 +230,7 @@ def test_resptr_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> value = next(result.objects(val_bn, KNORA_API.linkValueHasTargetIri)) assert value == RES_ONE_URI - def test_simpletext_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_simpletext(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -247,7 +247,7 @@ def test_simpletext_success(self, lookups: Lookups, res_info: tuple[BNode, str]) value = next(result.objects(val_bn, KNORA_API.valueAsString)) assert value == Literal("Text", datatype=XSD.string) - def test_richtext_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_richtext(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -268,7 +268,7 @@ def test_richtext_success(self, lookups: Lookups, res_info: tuple[BNode, str]) - permissions = next(result.objects(val_bn, KNORA_API.hasPermissions)) assert permissions == PERMISSION_LITERAL - def test_date_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_date(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -297,7 +297,7 @@ def test_date_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> No start_era = next(result.objects(val_bn, KNORA_API.dateValueHasEndEra)) assert start_era == Literal("AD", datatype=XSD.string) - def test_interval_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_interval(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" @@ -314,7 +314,7 @@ def test_interval_success(self, lookups: Lookups, res_info: tuple[BNode, str]) - end = next(result.objects(val_bn, KNORA_API.intervalValueHasEnd)) assert end == Literal("0.234", datatype=XSD.decimal) - def test_segment_of_video_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_segment_of_video(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, _ = res_info res_type = "knora-api:VideoSegment" xml_prop = etree.fromstring(""" @@ -330,7 +330,7 @@ def test_segment_of_video_success(self, lookups: Lookups, res_info: tuple[BNode, value = next(result.objects(val_bn, KNORA_API.linkValueHasTargetIri)) assert value == RES_ONE_URI - def test_segment_of_audio_success(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + def test_segment_of_audio(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, _ = res_info res_type = "knora-api:AudioSegment" xml_prop = etree.fromstring(""" From f17dd88f518044f39c117a2741c1753726e1a690 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:51:47 +0100 Subject: [PATCH 48/63] Update test_transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index d92eba74b8..76ad9ae425 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -1,4 +1,5 @@ import pytest +import regex from lxml import etree from rdflib import RDF from rdflib import XSD @@ -15,6 +16,10 @@ from dsp_tools.commands.xmlupload.models.permission import PermissionValue from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import KNORA_API from dsp_tools.commands.xmlupload.transform_resource_and_values_for_upload import _make_one_prop_graph +from dsp_tools.models.exceptions import BaseError +from dsp_tools.models.exceptions import InputError +from dsp_tools.models.exceptions import PermissionNotExistsError +from dsp_tools.models.exceptions import UserError ONTO = Namespace("http://0.0.0.0:3333/ontology/9999/onto/v2#") namespaces = {"onto": ONTO, "knora-api": KNORA_API} @@ -345,3 +350,76 @@ def test_segment_of_audio(self, lookups: Lookups, res_info: tuple[BNode, str]) - assert rdf_type == KNORA_API.LinkValue value = next(result.objects(val_bn, KNORA_API.linkValueHasTargetIri)) assert value == RES_ONE_URI + + +class TestMakeOnePropGraphRaises: + def test_permissions(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + 4711 + + """) + prop = XMLProperty.from_node(xml_prop, "integer", "onto") + err_str = regex.escape("Could not find permissions for value: nonExistent") + with pytest.raises(PermissionNotExistsError, match=err_str): + _make_one_prop_graph(prop, res_type, res_bn, lookups) + + def test_unknown_type(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + 4711 + + """) + prop = XMLProperty.from_node(xml_prop, "other", "onto") + err_str = regex.escape("Unknown value type: other") + with pytest.raises(UserError, match=err_str): + _make_one_prop_graph(prop, res_type, res_bn, lookups) + + def test_unknown_prefix(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + 4711 + + """) + prop = XMLProperty.from_node(xml_prop, "integer", "onto") + err_str = regex.escape("Could not find namespace for prefix: other") + with pytest.raises(InputError, match=err_str): + _make_one_prop_graph(prop, res_type, res_bn, lookups) + + def test_link_traget_not_found(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + non_existing + + """) + prop = XMLProperty.from_node(xml_prop, "resptr", "onto") + err_str = regex.escape( + ( + "Could not find the ID non_existing in the id2iri mapping. " + "This is probably because the resource 'non_existing' could not be created. " + "See warnings.log for more information." + ) + ) + with pytest.raises(BaseError, match=err_str): + _make_one_prop_graph(prop, res_type, res_bn, lookups) + + def test_list_node_not_found(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: + res_bn, res_type = res_info + xml_prop = etree.fromstring(""" + + other + + """) + prop = XMLProperty.from_node(xml_prop, "list", "onto") + err_str = regex.escape( + ( + "Could not resolve list node ID 'testlist:other' to IRI. " + "This is probably because the list node 'testlist:other' does not exist on the server." + ) + ) + with pytest.raises(BaseError, match=err_str): + _make_one_prop_graph(prop, res_type, res_bn, lookups) From 340c3d414398d0fb823264d08c54cb361bea1502 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:02:59 +0100 Subject: [PATCH 49/63] Update test_value_transformers.py --- .../xmlupload/test_value_transformers.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_value_transformers.py b/test/unittests/commands/xmlupload/test_value_transformers.py index e03186b623..edd1d0f45b 100644 --- a/test/unittests/commands/xmlupload/test_value_transformers.py +++ b/test/unittests/commands/xmlupload/test_value_transformers.py @@ -4,12 +4,13 @@ from rdflib import XSD from rdflib import Literal -from dsp_tools.commands.xmlupload.value_transformers import transform_xsd_boolean +from dsp_tools.commands.xmlupload.value_transformers import transform_xsd_boolean, transform_interval from dsp_tools.models.exceptions import BaseError +from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import Interval @pytest.mark.parametrize( - ("input_bool", "success"), + ("input_bool", "out_bool"), [ ("true", True), ("True", True), @@ -23,9 +24,9 @@ (False, False), ], ) -def test_transform_boolean_success(input_bool: Any, success: bool) -> None: +def test_transform_boolean_success(input_bool: Any, out_bool: bool) -> None: result = transform_xsd_boolean(input_bool) - assert result == Literal(success, datatype=XSD.boolean) + assert result == Literal(out_bool, datatype=XSD.boolean) def test_transform_boolean_raises_string() -> None: @@ -36,3 +37,16 @@ def test_transform_boolean_raises_string() -> None: def test_transform_boolean_raises_int() -> None: with pytest.raises(BaseError): transform_xsd_boolean(2) # type: ignore[arg-type] + + +def test_transform_interval_success() -> None: + result = transform_interval("1:2") + assert isinstance(result, Interval) + assert result.start==Literal("1.0", datatype=XSD.decimal) + assert result.end==Literal("2.0", datatype=XSD.decimal) + +def test_transform_interval_success_with_space() -> None: + result = transform_interval("1:2.1 ") + assert isinstance(result, Interval) + assert result.start==Literal("1.0", datatype=XSD.decimal) + assert result.end==Literal("2.1", datatype=XSD.decimal) From dcaccc90c475da08bbd7e32cd59e1ec4898613ce Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:09:46 +0100 Subject: [PATCH 50/63] - --- .../commands/xmlupload/value_transformers.py | 7 ++--- .../xmlupload/test_value_transformers.py | 31 +++++++++++++++---- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/value_transformers.py b/src/dsp_tools/commands/xmlupload/value_transformers.py index 8e41300a94..f99d791598 100644 --- a/src/dsp_tools/commands/xmlupload/value_transformers.py +++ b/src/dsp_tools/commands/xmlupload/value_transformers.py @@ -37,11 +37,8 @@ def transform_date(input_value: InputTypes) -> Date: def transform_interval(input_value: InputTypes) -> Interval: """Transform a sting input into an interval object.""" val = assert_is_string(input_value) - match val.split(":", 1): - case [start, end]: - return Interval(transform_xsd_decimal(start), transform_xsd_decimal(end)) - case _: - raise BaseError(f"Could not parse interval value: {input_value}") + start, end = val.split(":", 1) + return Interval(transform_xsd_decimal(start.strip()), transform_xsd_decimal(end.strip())) def transform_xsd_string(value: InputTypes) -> Literal: diff --git a/test/unittests/commands/xmlupload/test_value_transformers.py b/test/unittests/commands/xmlupload/test_value_transformers.py index edd1d0f45b..d7ab9c36c9 100644 --- a/test/unittests/commands/xmlupload/test_value_transformers.py +++ b/test/unittests/commands/xmlupload/test_value_transformers.py @@ -4,10 +4,13 @@ from rdflib import XSD from rdflib import Literal -from dsp_tools.commands.xmlupload.value_transformers import transform_xsd_boolean, transform_interval +from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import Interval +from dsp_tools.commands.xmlupload.value_transformers import transform_interval +from dsp_tools.commands.xmlupload.value_transformers import transform_xsd_boolean +from dsp_tools.commands.xmlupload.value_transformers import transform_xsd_decimal +from dsp_tools.commands.xmlupload.value_transformers import transform_xsd_integer from dsp_tools.models.exceptions import BaseError -from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import Interval @pytest.mark.parametrize( ("input_bool", "out_bool"), @@ -39,14 +42,30 @@ def test_transform_boolean_raises_int() -> None: transform_xsd_boolean(2) # type: ignore[arg-type] +def test_transform_xsd_decimal_int() -> None: + result = transform_xsd_decimal("1") + assert result == Literal("1.0", datatype=XSD.decimal) + + +def test_transform_xsd_decimal_decimal() -> None: + result = transform_xsd_decimal("1.1") + assert result == Literal("1.1", datatype=XSD.decimal) + + def test_transform_interval_success() -> None: result = transform_interval("1:2") assert isinstance(result, Interval) - assert result.start==Literal("1.0", datatype=XSD.decimal) - assert result.end==Literal("2.0", datatype=XSD.decimal) + assert result.start == Literal("1.0", datatype=XSD.decimal) + assert result.end == Literal("2.0", datatype=XSD.decimal) + def test_transform_interval_success_with_space() -> None: result = transform_interval("1:2.1 ") assert isinstance(result, Interval) - assert result.start==Literal("1.0", datatype=XSD.decimal) - assert result.end==Literal("2.1", datatype=XSD.decimal) + assert result.start == Literal("1.0", datatype=XSD.decimal) + assert result.end == Literal("2.1", datatype=XSD.decimal) + + +def test_transform_xsd_integer_success() -> None: + result = transform_xsd_integer("1") + assert result == Literal(1, datatype=XSD.integer) From 1fb50463dc8f86c144add5d10e0ed95f8619f734 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:14:12 +0100 Subject: [PATCH 51/63] Update test_value_transformers.py --- .../commands/xmlupload/test_value_transformers.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/unittests/commands/xmlupload/test_value_transformers.py b/test/unittests/commands/xmlupload/test_value_transformers.py index d7ab9c36c9..9c7440f1af 100644 --- a/test/unittests/commands/xmlupload/test_value_transformers.py +++ b/test/unittests/commands/xmlupload/test_value_transformers.py @@ -5,6 +5,7 @@ from rdflib import Literal from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import Interval +from dsp_tools.commands.xmlupload.value_transformers import transform_geometry from dsp_tools.commands.xmlupload.value_transformers import transform_interval from dsp_tools.commands.xmlupload.value_transformers import transform_xsd_boolean from dsp_tools.commands.xmlupload.value_transformers import transform_xsd_decimal @@ -69,3 +70,17 @@ def test_transform_interval_success_with_space() -> None: def test_transform_xsd_integer_success() -> None: result = transform_xsd_integer("1") assert result == Literal(1, datatype=XSD.integer) + + +def test_transform_geometry() -> None: + test_geom = """ + { + "status": "active", + "type": "polygon", + "lineWidth": 5, + "points": [] + } + """ + result = transform_geometry(test_geom) + expected = '{"status": "active", "type": "polygon", "lineWidth": 5, "points": []}' + assert result == Literal(expected, datatype=XSD.string) From 2ecbcc23259dd19c2884538110516e3c1ae25576 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:18:06 +0100 Subject: [PATCH 52/63] Update transform_resource_and_values_for_upload.py --- .../xmlupload/transform_resource_and_values_for_upload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 402bba01d7..089f74de7e 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -34,6 +34,7 @@ from dsp_tools.commands.xmlupload.models.serialise.serialise_rdf_value import rdf_prop_type_mapper from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseMigrationMetadata from dsp_tools.commands.xmlupload.models.serialise.serialise_resource import SerialiseResource +from dsp_tools.commands.xmlupload.value_transformers import InputTypes from dsp_tools.commands.xmlupload.value_transformers import assert_is_string from dsp_tools.commands.xmlupload.value_transformers import rdf_literal_transformer from dsp_tools.commands.xmlupload.value_transformers import transform_date @@ -229,7 +230,7 @@ def _make_simple_prop_graph( res_bn: BNode, prop_name: URIRef, prop_type_info: RDFPropTypeInfo, - transformer: Callable[[Any], Any], + transformer: Callable[[InputTypes], Literal], permissions_lookup: dict[str, Permissions], ) -> Graph: g = Graph() From 633fd72ebdc56a68ddf2efd2a2c13ae4cb95813f Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:24:47 +0100 Subject: [PATCH 53/63] Update transform_resource_and_values_for_upload.py --- .../transform_resource_and_values_for_upload.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 089f74de7e..5daa780451 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -250,7 +250,7 @@ def _make_simple_value_graph( prop_type_info: RDFPropTypeInfo, ) -> Graph: val_bn = BNode() - g = _get_optional_triples(val_bn, val.permissions, val.comment) + g = _add_optional_triples(val_bn, val.permissions, val.comment) g.add((res_bn, val.prop_name, val_bn)) g.add((val_bn, RDF.type, prop_type_info.knora_type)) g.add((val_bn, prop_type_info.knora_prop, val.value)) @@ -306,7 +306,7 @@ def _make_date_value_graph( ) -> Graph: date = transform_date(val.value) val_bn = BNode() - g = _get_optional_triples(val_bn, resolved_permission, val.comment) + g = _add_optional_triples(val_bn, resolved_permission, val.comment) g.add((res_bn, prop_name, val_bn)) g.add((val_bn, RDF.type, KNORA_API.DateValue)) if cal := date.calendar.value: @@ -354,7 +354,7 @@ def _make_interval_value_graph( ) -> Graph: interval = transform_interval(val.value) val_bn = BNode() - g = _get_optional_triples(val_bn, resolved_permission, val.comment) + g = _add_optional_triples(val_bn, resolved_permission, val.comment) g.add((res_bn, prop_name, val_bn)) g.add((val_bn, RDF.type, KNORA_API.IntervalValue)) g.add((val_bn, KNORA_API.intervalValueHasStart, interval.start)) @@ -442,7 +442,7 @@ def _make_richtext_value_graph( ) -> Graph: val_bn = BNode() resolved_permission = _resolve_permission(val.permissions, permissions_lookup) - g = _get_optional_triples(val_bn, resolved_permission, val.comment) + g = _add_optional_triples(val_bn, resolved_permission, val.comment) xml_val = cast(FormattedTextValue, val.value) val_str = _get_richtext_string(xml_val, iri_resolver) literal_val = Literal(val_str, datatype=XSD.string) @@ -458,7 +458,7 @@ def _get_richtext_string(xml_val: FormattedTextValue, iri_resolver: IriResolver) return xml_with_iris.as_xml() -def _get_optional_triples(val_bn: BNode, permissions: str | None, comment: str | None) -> Graph: +def _add_optional_triples(val_bn: BNode, permissions: str | None, comment: str | None) -> Graph: g = Graph() if permissions: g.add((val_bn, KNORA_API.hasPermissions, Literal(permissions, datatype=XSD.string))) @@ -471,10 +471,8 @@ def _add_optional_permission_triple( value: XMLValue | IIIFUriInfo, val_bn: BNode, permissions_lookup: dict[str, Permissions] ) -> Graph: g = Graph() - if value.permissions: - if not (per := permissions_lookup.get(value.permissions)): - raise PermissionNotExistsError(f"Could not find permissions for value: {value.permissions}") - g.add((val_bn, KNORA_API.hasPermissions, Literal(str(per)))) + if per_str := _resolve_permission(value.permissions, permissions_lookup): + g.add((val_bn, KNORA_API.hasPermissions, Literal(per_str, datatype=XSD.string))) return g From c75c0dbd31d66330039927e9eb75714fd7f7bf4e Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:25:21 +0100 Subject: [PATCH 54/63] Update transform_resource_and_values_for_upload.py --- .../xmlupload/transform_resource_and_values_for_upload.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 5daa780451..35805e9479 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -445,10 +445,9 @@ def _make_richtext_value_graph( g = _add_optional_triples(val_bn, resolved_permission, val.comment) xml_val = cast(FormattedTextValue, val.value) val_str = _get_richtext_string(xml_val, iri_resolver) - literal_val = Literal(val_str, datatype=XSD.string) g.add((res_bn, prop_name, val_bn)) g.add((val_bn, RDF.type, KNORA_API.TextValue)) - g.add((val_bn, KNORA_API.textValueAsXml, literal_val)) + g.add((val_bn, KNORA_API.textValueAsXml, Literal(val_str, datatype=XSD.string))) g.add((val_bn, KNORA_API.textValueHasMapping, URIRef("http://rdfh.ch/standoff/mappings/StandardMapping"))) return g From 7593e0713e878e290bb95fca1f69a1a6b56a630f Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:47:52 +0100 Subject: [PATCH 55/63] Update test_resource_creation.py --- .../xmlupload/test_resource_creation.py | 65 +++++++++---------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/test/integration/commands/xmlupload/test_resource_creation.py b/test/integration/commands/xmlupload/test_resource_creation.py index 0f58a18bdf..12533862b1 100644 --- a/test/integration/commands/xmlupload/test_resource_creation.py +++ b/test/integration/commands/xmlupload/test_resource_creation.py @@ -52,7 +52,7 @@ def get_ontology_iris(self) -> list[str]: raise NotImplementedError("get_project_iri not implemented") def get_ontology_name_dict(self) -> dict[str, str]: - return {} + return {"my_onto": "http://0.0.0.0:3333/ontology/9999/onto/v2"} def get_ontology_iri_dict(self) -> dict[str, str]: raise NotImplementedError("get_project_iri not implemented") @@ -72,27 +72,28 @@ def test_one_resource_without_links(ingest_client_mock: AssetClient) -> None: post_responses = [{"@id": "foo_1_iri", "rdfs:label": "foo_1_label"}] con.post = Mock(side_effect=post_responses) project_client = ProjectClientStub(con, "1234", None) + clients = dsp_tools.commands.xmlupload.models.upload_clients.UploadClients( ingest_client_mock, project_client, ListClientMock() ) - xmlupload._upload_resources(clients, upload_state) assert len(con.post.call_args_list) == len(post_responses) - match con.post.call_args_list[0].kwargs: - case { - "route": "/v2/resources", - "data": { - "@type": "my_onto:foo_1_type", - "rdfs:label": "foo_1_label", - "knora-api:attachedToProject": {"@id": "https://admin.test.dasch.swiss/project/MsOaiQkcQ7-QPxsYBKckfQ"}, - "@context": dict(), - "my_onto:hasSimpleText": [{"@type": "knora-api:TextValue", "knora-api:valueAsString": "foo_1 text"}], - }, - }: - assert True - case _: - pytest.fail("POST request was not sent correctly") + post_call_args = con.post.call_args_list[0].kwargs + assert len(post_call_args) == 3 + assert post_call_args["route"] == "/v2/resources" + assert not post_call_args["headers"] + data = post_call_args["data"] + text_val = { + "@type": "http://api.knora.org/ontology/knora-api/v2#TextValue", + "http://api.knora.org/ontology/knora-api/v2#valueAsString": "foo_1 text", + } + assert data["http://0.0.0.0:3333/ontology/9999/onto/v2#hasSimpleText"] == text_val + assert data["knora-api:attachedToProject"] == { + "@id": "https://admin.test.dasch.swiss/project/MsOaiQkcQ7-QPxsYBKckfQ" + } + assert data["rdfs:label"] == "foo_1_label" + assert data["@type"] == "my_onto:foo_1_type" assert not upload_state.pending_resources assert not upload_state.failed_uploads assert upload_state.iri_resolver.lookup == {"foo_1_id": "foo_1_iri"} @@ -116,26 +117,24 @@ def test_one_resource_with_link_to_existing_resource(ingest_client_mock: AssetCl clients = dsp_tools.commands.xmlupload.models.upload_clients.UploadClients( ingest_client_mock, project_client, ListClientMock() ) - xmlupload._upload_resources(clients, upload_state) assert len(con.post.call_args_list) == len(post_responses) - match con.post.call_args_list[0].kwargs: - case { - "route": "/v2/resources", - "data": { - "@type": "my_onto:foo_1_type", - "rdfs:label": "foo_1_label", - "knora-api:attachedToProject": {"@id": "https://admin.test.dasch.swiss/project/MsOaiQkcQ7-QPxsYBKckfQ"}, - "@context": dict(), - "my_onto:hasCustomLinkValue": [ - {"@type": "knora-api:LinkValue", "knora-api:linkValueHasTargetIri": {"@id": "foo_2_iri"}} - ], - }, - }: - assert True - case _: - pytest.fail("POST request was not sent correctly") + post_call_args = con.post.call_args_list[0].kwargs + assert len(post_call_args) == 3 + assert post_call_args["route"] == "/v2/resources" + assert not post_call_args["headers"] + data = post_call_args["data"] + link_val = { + "@type": "http://api.knora.org/ontology/knora-api/v2#LinkValue", + "http://api.knora.org/ontology/knora-api/v2#linkValueHasTargetIri": {"@id": "foo_2_iri"}, + } + assert data["http://0.0.0.0:3333/ontology/9999/onto/v2#hasCustomLinkValue"] == link_val + assert data["knora-api:attachedToProject"] == { + "@id": "https://admin.test.dasch.swiss/project/MsOaiQkcQ7-QPxsYBKckfQ" + } + assert data["rdfs:label"] == "foo_1_label" + assert data["@type"] == "my_onto:foo_1_type" assert not upload_state.pending_resources assert not upload_state.failed_uploads assert upload_state.iri_resolver.lookup == {"foo_1_id": "foo_1_iri", "foo_2_id": "foo_2_iri"} From 02a3ac03967b46027ebd68686df97515827c5cb6 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:08:33 +0100 Subject: [PATCH 56/63] add error handling --- src/dsp_tools/commands/xmlupload/value_transformers.py | 8 ++++++-- .../commands/xmlupload/test_value_transformers.py | 10 +++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/value_transformers.py b/src/dsp_tools/commands/xmlupload/value_transformers.py index f99d791598..a1563d04a5 100644 --- a/src/dsp_tools/commands/xmlupload/value_transformers.py +++ b/src/dsp_tools/commands/xmlupload/value_transformers.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from json.decoder import JSONDecodeError from typing import TypeAlias from typing import Union from typing import assert_never @@ -74,8 +75,11 @@ def transform_xsd_any_uri(value: InputTypes) -> Literal: def transform_geometry(value: InputTypes) -> Literal: """Transform a value into a geometry string rdflib Literal with datatype xsd:string""" str_val = assert_is_string(value) - str_val = json.dumps(json.loads(str_val)) - return Literal(str_val, datatype=XSD.string) + try: + str_val = json.dumps(json.loads(str_val)) + return Literal(str_val, datatype=XSD.string) + except JSONDecodeError: + raise BaseError(f"Could not parse json value: {value}") from None def assert_is_string(value: str | FormattedTextValue) -> str: diff --git a/test/unittests/commands/xmlupload/test_value_transformers.py b/test/unittests/commands/xmlupload/test_value_transformers.py index 9c7440f1af..33f9d26db0 100644 --- a/test/unittests/commands/xmlupload/test_value_transformers.py +++ b/test/unittests/commands/xmlupload/test_value_transformers.py @@ -1,6 +1,7 @@ from typing import Any import pytest +import regex from rdflib import XSD from rdflib import Literal @@ -72,7 +73,7 @@ def test_transform_xsd_integer_success() -> None: assert result == Literal(1, datatype=XSD.integer) -def test_transform_geometry() -> None: +def test_transform_geometry_success() -> None: test_geom = """ { "status": "active", @@ -84,3 +85,10 @@ def test_transform_geometry() -> None: result = transform_geometry(test_geom) expected = '{"status": "active", "type": "polygon", "lineWidth": 5, "points": []}' assert result == Literal(expected, datatype=XSD.string) + + +def test_transform_geometry_raises() -> None: + value = ' { "status": "active",' + msg = regex.escape(f"Could not parse json value: {value}") + with pytest.raises(BaseError, match=msg): + transform_geometry(value) From fc719567368df82b02f60eb65ba08ac5c8ef17d2 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:08:44 +0100 Subject: [PATCH 57/63] change tests --- .../xmlupload/test_resource_creation.py | 66 ++++++++++++------- .../xmlupload/test_resource_creation.py | 4 +- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/test/integration/commands/xmlupload/test_resource_creation.py b/test/integration/commands/xmlupload/test_resource_creation.py index 12533862b1..7554ce1004 100644 --- a/test/integration/commands/xmlupload/test_resource_creation.py +++ b/test/integration/commands/xmlupload/test_resource_creation.py @@ -81,19 +81,28 @@ def test_one_resource_without_links(ingest_client_mock: AssetClient) -> None: assert len(con.post.call_args_list) == len(post_responses) post_call_args = con.post.call_args_list[0].kwargs assert len(post_call_args) == 3 - assert post_call_args["route"] == "/v2/resources" - assert not post_call_args["headers"] - data = post_call_args["data"] - text_val = { - "@type": "http://api.knora.org/ontology/knora-api/v2#TextValue", - "http://api.knora.org/ontology/knora-api/v2#valueAsString": "foo_1 text", - } - assert data["http://0.0.0.0:3333/ontology/9999/onto/v2#hasSimpleText"] == text_val - assert data["knora-api:attachedToProject"] == { - "@id": "https://admin.test.dasch.swiss/project/MsOaiQkcQ7-QPxsYBKckfQ" + + prop_name = "http://0.0.0.0:3333/ontology/9999/onto/v2#hasSimpleText" + + expected = { + "route": "/v2/resources", + "data": { + "@type": "my_onto:foo_1_type", + "rdfs:label": "foo_1_label", + "knora-api:attachedToProject": {"@id": "https://admin.test.dasch.swiss/project/MsOaiQkcQ7-QPxsYBKckfQ"}, + "@context": dict(), + prop_name: { + "@type": "http://api.knora.org/ontology/knora-api/v2#TextValue", + "http://api.knora.org/ontology/knora-api/v2#valueAsString": "foo_1 text", + }, + }, } - assert data["rdfs:label"] == "foo_1_label" - assert data["@type"] == "my_onto:foo_1_type" + assert post_call_args["route"] == expected["route"] + assert not post_call_args["headers"] + assert post_call_args["data"][prop_name] == expected["data"][prop_name] + assert post_call_args["data"]["knora-api:attachedToProject"] == expected["data"]["knora-api:attachedToProject"] + assert post_call_args["data"]["rdfs:label"] == expected["data"]["rdfs:label"] + assert post_call_args["data"]["@type"] == expected["data"]["@type"] assert not upload_state.pending_resources assert not upload_state.failed_uploads assert upload_state.iri_resolver.lookup == {"foo_1_id": "foo_1_iri"} @@ -122,19 +131,28 @@ def test_one_resource_with_link_to_existing_resource(ingest_client_mock: AssetCl assert len(con.post.call_args_list) == len(post_responses) post_call_args = con.post.call_args_list[0].kwargs assert len(post_call_args) == 3 - assert post_call_args["route"] == "/v2/resources" - assert not post_call_args["headers"] - data = post_call_args["data"] - link_val = { - "@type": "http://api.knora.org/ontology/knora-api/v2#LinkValue", - "http://api.knora.org/ontology/knora-api/v2#linkValueHasTargetIri": {"@id": "foo_2_iri"}, - } - assert data["http://0.0.0.0:3333/ontology/9999/onto/v2#hasCustomLinkValue"] == link_val - assert data["knora-api:attachedToProject"] == { - "@id": "https://admin.test.dasch.swiss/project/MsOaiQkcQ7-QPxsYBKckfQ" + + prop_name = "http://0.0.0.0:3333/ontology/9999/onto/v2#hasCustomLinkValue" + + expected = { + "route": "/v2/resources", + "data": { + "@type": "my_onto:foo_1_type", + "rdfs:label": "foo_1_label", + "knora-api:attachedToProject": {"@id": "https://admin.test.dasch.swiss/project/MsOaiQkcQ7-QPxsYBKckfQ"}, + "@context": dict(), + prop_name: { + "@type": "http://api.knora.org/ontology/knora-api/v2#LinkValue", + "http://api.knora.org/ontology/knora-api/v2#linkValueHasTargetIri": {"@id": "foo_2_iri"}, + }, + }, } - assert data["rdfs:label"] == "foo_1_label" - assert data["@type"] == "my_onto:foo_1_type" + assert post_call_args["route"] == expected["route"] + assert not post_call_args["headers"] + assert post_call_args["data"][prop_name] == expected["data"][prop_name] + assert post_call_args["data"]["knora-api:attachedToProject"] == expected["data"]["knora-api:attachedToProject"] + assert post_call_args["data"]["rdfs:label"] == expected["data"]["rdfs:label"] + assert post_call_args["data"]["@type"] == expected["data"]["@type"] assert not upload_state.pending_resources assert not upload_state.failed_uploads assert upload_state.iri_resolver.lookup == {"foo_1_id": "foo_1_iri", "foo_2_id": "foo_2_iri"} diff --git a/test/unittests/commands/xmlupload/test_resource_creation.py b/test/unittests/commands/xmlupload/test_resource_creation.py index 9d9d6c434d..cc4f64806a 100644 --- a/test/unittests/commands/xmlupload/test_resource_creation.py +++ b/test/unittests/commands/xmlupload/test_resource_creation.py @@ -19,7 +19,7 @@ def test_idempotency_on_success() -> None: """, ] - xml_resources = [XMLResource.from_node(etree.fromstring(xml_str), "my_onto") for xml_str in xml_strings] + xml_resources = [XMLResource.from_node(etree.fromstring(xml_str), "onto") for xml_str in xml_strings] upload_state = UploadState(xml_resources.copy(), None, UploadConfig(), {}) for _ in range(3): _tidy_up_resource_creation_idempotent(upload_state, "foo_1_iri", xml_resources[0]) @@ -42,7 +42,7 @@ def test_idempotency_on_failure() -> None: """, ] - xml_resources = [XMLResource.from_node(etree.fromstring(xml_str), "my_onto") for xml_str in xml_strings] + xml_resources = [XMLResource.from_node(etree.fromstring(xml_str), "onto") for xml_str in xml_strings] upload_state = UploadState(xml_resources.copy(), None, UploadConfig(), {}) for _ in range(3): _tidy_up_resource_creation_idempotent(upload_state, None, xml_resources[0]) From e6fdb66a4015d80a08c8decbb6fcb64e2fedf0a6 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:09:13 +0100 Subject: [PATCH 58/63] Update test_resource_creation.py --- test/integration/commands/xmlupload/test_resource_creation.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/integration/commands/xmlupload/test_resource_creation.py b/test/integration/commands/xmlupload/test_resource_creation.py index 7554ce1004..c23fcde1fc 100644 --- a/test/integration/commands/xmlupload/test_resource_creation.py +++ b/test/integration/commands/xmlupload/test_resource_creation.py @@ -81,9 +81,7 @@ def test_one_resource_without_links(ingest_client_mock: AssetClient) -> None: assert len(con.post.call_args_list) == len(post_responses) post_call_args = con.post.call_args_list[0].kwargs assert len(post_call_args) == 3 - prop_name = "http://0.0.0.0:3333/ontology/9999/onto/v2#hasSimpleText" - expected = { "route": "/v2/resources", "data": { @@ -131,9 +129,7 @@ def test_one_resource_with_link_to_existing_resource(ingest_client_mock: AssetCl assert len(con.post.call_args_list) == len(post_responses) post_call_args = con.post.call_args_list[0].kwargs assert len(post_call_args) == 3 - prop_name = "http://0.0.0.0:3333/ontology/9999/onto/v2#hasCustomLinkValue" - expected = { "route": "/v2/resources", "data": { From 72b2c3f9c51c2e7695fed124b50844b418ab403e Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:11:51 +0100 Subject: [PATCH 59/63] Update test_transform_resource_and_values_for_upload.py --- .../test_transform_resource_and_values_for_upload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 76ad9ae425..aad5e3682c 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -369,7 +369,7 @@ def test_unknown_type(self, lookups: Lookups, res_info: tuple[BNode, str]) -> No res_bn, res_type = res_info xml_prop = etree.fromstring(""" - 4711 + 4711 """) prop = XMLProperty.from_node(xml_prop, "other", "onto") @@ -381,7 +381,7 @@ def test_unknown_prefix(self, lookups: Lookups, res_info: tuple[BNode, str]) -> res_bn, res_type = res_info xml_prop = etree.fromstring(""" - 4711 + 4711 """) prop = XMLProperty.from_node(xml_prop, "integer", "onto") From 9fda32bcf79d2b7ee303271c8853c6f66d80cf1b Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:17:55 +0100 Subject: [PATCH 60/63] Update test_transform_resource_and_values_for_upload.py --- ...ransform_resource_and_values_for_upload.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index aad5e3682c..35152c654e 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -154,18 +154,24 @@ def test_integer(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info xml_prop = etree.fromstring(""" - 4711 + 1 + 2 """) prop = XMLProperty.from_node(xml_prop, "integer", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) - assert len(result) == 3 + assert len(result) == 7 assert prop_name == ONTO.hasInteger - val_bn = next(result.objects(res_bn, prop_name)) - rdf_type = next(result.objects(val_bn, RDF.type)) - assert rdf_type == KNORA_API.IntValue - value = next(result.objects(val_bn, KNORA_API.intValueAsInt)) - assert value == Literal("4711", datatype=XSD.integer) + + val_one = next(result.subjects(KNORA_API.intValueAsInt, Literal("1", datatype=XSD.integer))) + assert next(result.objects(val_one, RDF.type)) == KNORA_API.IntValue + assert next(result.subjects(prop_name, val_one)) == res_bn + comment = next(result.objects(val_one, KNORA_API.valueHasComment)) + assert comment == Literal("comment", datatype=XSD.string) + + val_two = next(result.subjects(KNORA_API.intValueAsInt, Literal("2", datatype=XSD.integer))) + assert next(result.objects(val_two, RDF.type)) == KNORA_API.IntValue + assert next(result.subjects(prop_name, val_two)) == res_bn def test_time(self, lookups: Lookups, res_info: tuple[BNode, str]) -> None: res_bn, res_type = res_info From 24b5ba19b154764972cf8dfccd6e1f0d5a924f12 Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:18:17 +0100 Subject: [PATCH 61/63] Update test_transform_resource_and_values_for_upload.py --- .../test_transform_resource_and_values_for_upload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py index 35152c654e..748296a8d0 100644 --- a/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py +++ b/test/unittests/commands/xmlupload/test_transform_resource_and_values_for_upload.py @@ -329,7 +329,7 @@ def test_segment_of_video(self, lookups: Lookups, res_info: tuple[BNode, str]) - res_bn, _ = res_info res_type = "knora-api:VideoSegment" xml_prop = etree.fromstring(""" - res_one + res_one """) prop = XMLProperty.from_node(xml_prop, "resptr", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) @@ -345,7 +345,7 @@ def test_segment_of_audio(self, lookups: Lookups, res_info: tuple[BNode, str]) - res_bn, _ = res_info res_type = "knora-api:AudioSegment" xml_prop = etree.fromstring(""" - res_one + res_one """) prop = XMLProperty.from_node(xml_prop, "resptr", "onto") result, prop_name = _make_one_prop_graph(prop, res_type, res_bn, lookups) From 030f1df02e6e314b126644928c06ab1e335ae1fd Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:22:22 +0100 Subject: [PATCH 62/63] Update transform_resource_and_values_for_upload.py --- .../xmlupload/transform_resource_and_values_for_upload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py index 35805e9479..01e8dd9919 100644 --- a/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py +++ b/src/dsp_tools/commands/xmlupload/transform_resource_and_values_for_upload.py @@ -185,9 +185,9 @@ def _get_link_prop_name(p: XMLProperty, restype: str, namespaces: dict[str, Name return _get_absolute_prop_iri(prop, namespaces) -def _get_absolute_prop_iri(prefixed_prop: str, namepsaces: dict[str, Namespace]) -> URIRef: +def _get_absolute_prop_iri(prefixed_prop: str, namespaces: dict[str, Namespace]) -> URIRef: prefix, prop = prefixed_prop.split(":", maxsplit=1) - if not (namespace := namepsaces.get(prefix)): + if not (namespace := namespaces.get(prefix)): raise InputError(f"Could not find namespace for prefix: {prefix}") return namespace[prop] From c323380d68ecf41ee368c0efd7b8ab3bca407c8f Mon Sep 17 00:00:00 2001 From: Nora-Olivia-Ammann <103038637+Nora-Olivia-Ammann@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:32:45 +0100 Subject: [PATCH 63/63] typing --- .../xmlupload/test_resource_creation.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/integration/commands/xmlupload/test_resource_creation.py b/test/integration/commands/xmlupload/test_resource_creation.py index c23fcde1fc..7a5166018d 100644 --- a/test/integration/commands/xmlupload/test_resource_creation.py +++ b/test/integration/commands/xmlupload/test_resource_creation.py @@ -1,5 +1,7 @@ from copy import deepcopy from dataclasses import dataclass +from typing import Any +from typing import cast from unittest.mock import Mock import pytest @@ -79,7 +81,7 @@ def test_one_resource_without_links(ingest_client_mock: AssetClient) -> None: xmlupload._upload_resources(clients, upload_state) assert len(con.post.call_args_list) == len(post_responses) - post_call_args = con.post.call_args_list[0].kwargs + post_call_args = cast(dict[str, Any], con.post.call_args_list[0].kwargs) assert len(post_call_args) == 3 prop_name = "http://0.0.0.0:3333/ontology/9999/onto/v2#hasSimpleText" expected = { @@ -88,7 +90,7 @@ def test_one_resource_without_links(ingest_client_mock: AssetClient) -> None: "@type": "my_onto:foo_1_type", "rdfs:label": "foo_1_label", "knora-api:attachedToProject": {"@id": "https://admin.test.dasch.swiss/project/MsOaiQkcQ7-QPxsYBKckfQ"}, - "@context": dict(), + "@context": {}, prop_name: { "@type": "http://api.knora.org/ontology/knora-api/v2#TextValue", "http://api.knora.org/ontology/knora-api/v2#valueAsString": "foo_1 text", @@ -97,10 +99,10 @@ def test_one_resource_without_links(ingest_client_mock: AssetClient) -> None: } assert post_call_args["route"] == expected["route"] assert not post_call_args["headers"] - assert post_call_args["data"][prop_name] == expected["data"][prop_name] - assert post_call_args["data"]["knora-api:attachedToProject"] == expected["data"]["knora-api:attachedToProject"] - assert post_call_args["data"]["rdfs:label"] == expected["data"]["rdfs:label"] - assert post_call_args["data"]["@type"] == expected["data"]["@type"] + assert post_call_args["data"][prop_name] == expected["data"][prop_name] # type: ignore[index] + assert post_call_args["data"]["knora-api:attachedToProject"] == expected["data"]["knora-api:attachedToProject"] # type: ignore[index] + assert post_call_args["data"]["rdfs:label"] == expected["data"]["rdfs:label"] # type: ignore[index] + assert post_call_args["data"]["@type"] == expected["data"]["@type"] # type: ignore[index] assert not upload_state.pending_resources assert not upload_state.failed_uploads assert upload_state.iri_resolver.lookup == {"foo_1_id": "foo_1_iri"} @@ -127,7 +129,7 @@ def test_one_resource_with_link_to_existing_resource(ingest_client_mock: AssetCl xmlupload._upload_resources(clients, upload_state) assert len(con.post.call_args_list) == len(post_responses) - post_call_args = con.post.call_args_list[0].kwargs + post_call_args = cast(dict[str, Any], con.post.call_args_list[0].kwargs) assert len(post_call_args) == 3 prop_name = "http://0.0.0.0:3333/ontology/9999/onto/v2#hasCustomLinkValue" expected = { @@ -136,7 +138,7 @@ def test_one_resource_with_link_to_existing_resource(ingest_client_mock: AssetCl "@type": "my_onto:foo_1_type", "rdfs:label": "foo_1_label", "knora-api:attachedToProject": {"@id": "https://admin.test.dasch.swiss/project/MsOaiQkcQ7-QPxsYBKckfQ"}, - "@context": dict(), + "@context": {}, prop_name: { "@type": "http://api.knora.org/ontology/knora-api/v2#LinkValue", "http://api.knora.org/ontology/knora-api/v2#linkValueHasTargetIri": {"@id": "foo_2_iri"}, @@ -145,10 +147,10 @@ def test_one_resource_with_link_to_existing_resource(ingest_client_mock: AssetCl } assert post_call_args["route"] == expected["route"] assert not post_call_args["headers"] - assert post_call_args["data"][prop_name] == expected["data"][prop_name] - assert post_call_args["data"]["knora-api:attachedToProject"] == expected["data"]["knora-api:attachedToProject"] - assert post_call_args["data"]["rdfs:label"] == expected["data"]["rdfs:label"] - assert post_call_args["data"]["@type"] == expected["data"]["@type"] + assert post_call_args["data"][prop_name] == expected["data"][prop_name] # type: ignore[index] + assert post_call_args["data"]["knora-api:attachedToProject"] == expected["data"]["knora-api:attachedToProject"] # type: ignore[index] + assert post_call_args["data"]["rdfs:label"] == expected["data"]["rdfs:label"] # type: ignore[index] + assert post_call_args["data"]["@type"] == expected["data"]["@type"] # type: ignore[index] assert not upload_state.pending_resources assert not upload_state.failed_uploads assert upload_state.iri_resolver.lookup == {"foo_1_id": "foo_1_iri", "foo_2_id": "foo_2_iri"}