From fab363c16732a4f654f61b9e2fd5fabd9b92b2ef Mon Sep 17 00:00:00 2001 From: Enrico Risa Date: Tue, 12 Nov 2024 11:56:33 +0100 Subject: [PATCH] fix: sub-catalogs serialization (#4588) * fix: sub-catalogs serialization * chore: fixes tests --- .../JsonObjectFromCatalogTransformer.java | 17 ++++++- ...JsonObjectFromCatalogV2024Transformer.java | 17 ++++++- .../JsonObjectFromCatalogTransformerTest.java | 51 ++++++++++++++++++- ...ObjectFromCatalogV2024TransformerTest.java | 51 ++++++++++++++++++- .../edc/jsonld/spi/PropertyAndTypeNames.java | 1 + .../managementapi/CatalogApiEndToEndTest.java | 13 +++-- 6 files changed, 137 insertions(+), 13 deletions(-) diff --git a/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformer.java b/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformer.java index 63d4d326edc..f94558e6844 100644 --- a/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformer.java +++ b/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformer.java @@ -18,6 +18,7 @@ import jakarta.json.JsonBuilderFactory; import jakarta.json.JsonObject; import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog; +import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset; import org.eclipse.edc.jsonld.spi.JsonLdNamespace; import org.eclipse.edc.jsonld.spi.transformer.AbstractNamespaceAwareJsonLdTransformer; import org.eclipse.edc.participant.spi.ParticipantIdMapper; @@ -25,10 +26,14 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; +import java.util.stream.Collectors; + import static jakarta.json.stream.JsonCollectors.toJsonArray; import static java.util.Optional.ofNullable; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; +import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_CATALOG_ATTRIBUTE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_CATALOG_TYPE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATASET_ATTRIBUTE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATA_SERVICE_ATTRIBUTE; @@ -57,7 +62,15 @@ public JsonObjectFromCatalogTransformer(JsonBuilderFactory jsonFactory, ObjectMa @Override public @Nullable JsonObject transform(@NotNull Catalog catalog, @NotNull TransformerContext context) { - var datasets = catalog.getDatasets().stream() + var partitions = catalog.getDatasets().stream().collect(Collectors.groupingBy(Dataset::getClass)); + + var datasets = ofNullable(partitions.get(Dataset.class)).orElseGet(ArrayList::new) + .stream() + .map(offer -> context.transform(offer, JsonObject.class)) + .collect(toJsonArray()); + + var subCatalogs = ofNullable(partitions.get(Catalog.class)).orElseGet(ArrayList::new) + .stream() .map(offer -> context.transform(offer, JsonObject.class)) .collect(toJsonArray()); @@ -73,6 +86,7 @@ public JsonObjectFromCatalogTransformer(JsonBuilderFactory jsonFactory, ObjectMa .add(ID, catalog.getId()) .add(TYPE, DCAT_CATALOG_TYPE) .add(DCAT_DATASET_ATTRIBUTE, datasets) + .add(DCAT_CATALOG_ATTRIBUTE, subCatalogs) .add(DCAT_DISTRIBUTION_ATTRIBUTE, distributions) .add(DCAT_DATA_SERVICE_ATTRIBUTE, dataServices); @@ -82,4 +96,5 @@ public JsonObjectFromCatalogTransformer(JsonBuilderFactory jsonFactory, ObjectMa return objectBuilder.build(); } + } diff --git a/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/v2024/from/JsonObjectFromCatalogV2024Transformer.java b/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/v2024/from/JsonObjectFromCatalogV2024Transformer.java index 0fa5c252c60..6db6120fbe8 100644 --- a/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/v2024/from/JsonObjectFromCatalogV2024Transformer.java +++ b/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/v2024/from/JsonObjectFromCatalogV2024Transformer.java @@ -18,6 +18,7 @@ import jakarta.json.JsonBuilderFactory; import jakarta.json.JsonObject; import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog; +import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset; import org.eclipse.edc.jsonld.spi.JsonLdNamespace; import org.eclipse.edc.jsonld.spi.transformer.AbstractNamespaceAwareJsonLdTransformer; import org.eclipse.edc.participant.spi.ParticipantIdMapper; @@ -25,10 +26,14 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; +import java.util.stream.Collectors; + import static jakarta.json.stream.JsonCollectors.toJsonArray; import static java.util.Optional.ofNullable; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; +import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_CATALOG_ATTRIBUTE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_CATALOG_TYPE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATASET_ATTRIBUTE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATA_SERVICE_ATTRIBUTE; @@ -57,7 +62,15 @@ public JsonObjectFromCatalogV2024Transformer(JsonBuilderFactory jsonFactory, Obj @Override public @Nullable JsonObject transform(@NotNull Catalog catalog, @NotNull TransformerContext context) { - var datasets = catalog.getDatasets().stream() + var partitions = catalog.getDatasets().stream().collect(Collectors.groupingBy(Dataset::getClass)); + + var datasets = ofNullable(partitions.get(Dataset.class)).orElseGet(ArrayList::new) + .stream() + .map(offer -> context.transform(offer, JsonObject.class)) + .collect(toJsonArray()); + + var subCatalogs = ofNullable(partitions.get(Catalog.class)).orElseGet(ArrayList::new) + .stream() .map(offer -> context.transform(offer, JsonObject.class)) .collect(toJsonArray()); @@ -73,6 +86,7 @@ public JsonObjectFromCatalogV2024Transformer(JsonBuilderFactory jsonFactory, Obj .add(ID, catalog.getId()) .add(TYPE, DCAT_CATALOG_TYPE) .add(DCAT_DATASET_ATTRIBUTE, datasets) + .add(DCAT_CATALOG_ATTRIBUTE, subCatalogs) .add(DCAT_DISTRIBUTION_ATTRIBUTE, distributions) .add(DCAT_DATA_SERVICE_ATTRIBUTE, dataServices); @@ -82,4 +96,5 @@ public JsonObjectFromCatalogV2024Transformer(JsonBuilderFactory jsonFactory, Obj return objectBuilder.build(); } + } diff --git a/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/test/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformerTest.java b/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/test/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformerTest.java index 44055b0ef92..fcb52c90d6f 100644 --- a/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/test/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformerTest.java +++ b/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/test/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformerTest.java @@ -36,6 +36,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; +import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_CATALOG_ATTRIBUTE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_CATALOG_TYPE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATASET_ATTRIBUTE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATA_SERVICE_ATTRIBUTE; @@ -61,15 +62,18 @@ class JsonObjectFromCatalogTransformerTest { private final JsonObjectFromCatalogTransformer transformer = new JsonObjectFromCatalogTransformer(jsonFactory, mapper, participantIdMapper); private JsonObject datasetJson; + private JsonObject catalogJson; private JsonObject dataServiceJson; @BeforeEach void setUp() { datasetJson = getJsonObject("dataset"); + catalogJson = getJsonObject("Catalog"); dataServiceJson = getJsonObject("dataService"); when(context.transform(isA(Dataset.class), eq(JsonObject.class))).thenReturn(datasetJson); + when(context.transform(isA(Catalog.class), eq(JsonObject.class))).thenReturn(catalogJson); when(context.transform(isA(DataService.class), eq(JsonObject.class))).thenReturn(dataServiceJson); when(context.problem()).thenReturn(new ProblemBuilder(context)); when(participantIdMapper.toIri(any())).thenReturn("urn:namespace:participantId"); @@ -104,6 +108,41 @@ void transform_returnJsonObject() { verify(context, times(1)).transform(catalog.getDataServices().get(0), JsonObject.class); } + @Test + void transform_SubCatalogs_returnJsonObject() { + when(mapper.convertValue(any(), eq(JsonValue.class))).thenReturn(Json.createValue("value")); + var catalog = getCatalogWithSubCatalog(); + + var result = transformer.transform(catalog, context); + + assertThat(result).isNotNull(); + assertThat(result.getJsonString(ID).getString()).isEqualTo(catalog.getId()); + assertThat(result.getJsonString(TYPE).getString()).isEqualTo(DCAT_CATALOG_TYPE); + + assertThat(result.get(DCAT_DATASET_ATTRIBUTE)) + .isNotNull() + .isInstanceOf(JsonArray.class) + .matches(v -> v.asJsonArray().size() == 1) + .matches(v -> v.asJsonArray().get(0).equals(datasetJson)); + + assertThat(result.get(DCAT_CATALOG_ATTRIBUTE)) + .isNotNull() + .isInstanceOf(JsonArray.class) + .matches(v -> v.asJsonArray().size() == 1) + .matches(v -> v.asJsonArray().get(0).equals(catalogJson)); + + assertThat(result.get(DCAT_DATA_SERVICE_ATTRIBUTE)) + .isNotNull() + .isInstanceOf(JsonArray.class) + .matches(v -> v.asJsonArray().size() == 1) + .matches(v -> v.asJsonArray().get(0).equals(dataServiceJson)); + assertThat(result.getString(DSPACE_PROPERTY_PARTICIPANT_ID_IRI)).isEqualTo("urn:namespace:participantId"); + assertThat(result.get(CATALOG_PROPERTY)).isNotNull(); + + verify(context, times(1)).transform(catalog.getDatasets().get(0), JsonObject.class); + verify(context, times(1)).transform(catalog.getDataServices().get(0), JsonObject.class); + } + @Test void transform_mappingPropertyFails_reportProblem() { when(mapper.convertValue(any(), eq(JsonValue.class))).thenThrow(IllegalArgumentException.class); @@ -118,6 +157,15 @@ void transform_mappingPropertyFails_reportProblem() { } private Catalog getCatalog() { + return getCatalogBuilder().build(); + } + + private Catalog getCatalogWithSubCatalog() { + return getCatalogBuilder() + .dataset(Catalog.Builder.newInstance().build()).build(); + } + + private Catalog.Builder getCatalogBuilder() { return Catalog.Builder.newInstance() .id("catalog") .dataset(Dataset.Builder.newInstance() @@ -129,8 +177,7 @@ private Catalog getCatalog() { .build()) .dataService(DataService.Builder.newInstance().build()) .participantId("participantId") - .property(CATALOG_PROPERTY, "value") - .build(); + .property(CATALOG_PROPERTY, "value"); } private JsonObject getJsonObject(String type) { diff --git a/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/test/java/org/eclipse/edc/protocol/dsp/catalog/transform/v2024/from/JsonObjectFromCatalogV2024TransformerTest.java b/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/test/java/org/eclipse/edc/protocol/dsp/catalog/transform/v2024/from/JsonObjectFromCatalogV2024TransformerTest.java index e1cffa57946..568b8169bec 100644 --- a/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/test/java/org/eclipse/edc/protocol/dsp/catalog/transform/v2024/from/JsonObjectFromCatalogV2024TransformerTest.java +++ b/data-protocols/dsp/dsp-catalog/lib/dsp-catalog-transform-lib/src/test/java/org/eclipse/edc/protocol/dsp/catalog/transform/v2024/from/JsonObjectFromCatalogV2024TransformerTest.java @@ -36,6 +36,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; +import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_CATALOG_ATTRIBUTE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_CATALOG_TYPE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATASET_ATTRIBUTE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATA_SERVICE_ATTRIBUTE; @@ -62,15 +63,18 @@ class JsonObjectFromCatalogV2024TransformerTest { private final JsonObjectFromCatalogV2024Transformer transformer = new JsonObjectFromCatalogV2024Transformer(jsonFactory, mapper, participantIdMapper); private JsonObject datasetJson; + private JsonObject catalogJson; private JsonObject dataServiceJson; @BeforeEach void setUp() { datasetJson = getJsonObject("dataset"); + catalogJson = getJsonObject("Catalog"); dataServiceJson = getJsonObject("dataService"); when(context.transform(isA(Dataset.class), eq(JsonObject.class))).thenReturn(datasetJson); + when(context.transform(isA(Catalog.class), eq(JsonObject.class))).thenReturn(catalogJson); when(context.transform(isA(DataService.class), eq(JsonObject.class))).thenReturn(dataServiceJson); when(context.problem()).thenReturn(new ProblemBuilder(context)); when(participantIdMapper.toIri(any())).thenReturn("urn:namespace:participantId"); @@ -105,6 +109,41 @@ void transform_returnJsonObject() { verify(context, times(1)).transform(catalog.getDataServices().get(0), JsonObject.class); } + @Test + void transform_SubCatalogs_returnJsonObject() { + when(mapper.convertValue(any(), eq(JsonValue.class))).thenReturn(Json.createValue("value")); + var catalog = getCatalogWithSubCatalog(); + + var result = transformer.transform(catalog, context); + + assertThat(result).isNotNull(); + assertThat(result.getJsonString(ID).getString()).isEqualTo(catalog.getId()); + assertThat(result.getJsonString(TYPE).getString()).isEqualTo(DCAT_CATALOG_TYPE); + + assertThat(result.get(DCAT_DATASET_ATTRIBUTE)) + .isNotNull() + .isInstanceOf(JsonArray.class) + .matches(v -> v.asJsonArray().size() == 1) + .matches(v -> v.asJsonArray().get(0).equals(datasetJson)); + + assertThat(result.get(DCAT_CATALOG_ATTRIBUTE)) + .isNotNull() + .isInstanceOf(JsonArray.class) + .matches(v -> v.asJsonArray().size() == 1) + .matches(v -> v.asJsonArray().get(0).equals(catalogJson)); + + assertThat(result.get(DCAT_DATA_SERVICE_ATTRIBUTE)) + .isNotNull() + .isInstanceOf(JsonArray.class) + .matches(v -> v.asJsonArray().size() == 1) + .matches(v -> v.asJsonArray().get(0).equals(dataServiceJson)); + assertThat(result.getJsonObject(DSP_NAMESPACE_V_2024_1.toIri(DSPACE_PROPERTY_PARTICIPANT_ID_TERM)).getString(ID)).isEqualTo("urn:namespace:participantId"); + assertThat(result.get(CATALOG_PROPERTY)).isNotNull(); + + verify(context, times(1)).transform(catalog.getDatasets().get(0), JsonObject.class); + verify(context, times(1)).transform(catalog.getDataServices().get(0), JsonObject.class); + } + @Test void transform_mappingPropertyFails_reportProblem() { when(mapper.convertValue(any(), eq(JsonValue.class))).thenThrow(IllegalArgumentException.class); @@ -119,6 +158,15 @@ void transform_mappingPropertyFails_reportProblem() { } private Catalog getCatalog() { + return getCatalogBuilder().build(); + } + + private Catalog getCatalogWithSubCatalog() { + return getCatalogBuilder() + .dataset(Catalog.Builder.newInstance().build()).build(); + } + + private Catalog.Builder getCatalogBuilder() { return Catalog.Builder.newInstance() .id("catalog") .dataset(Dataset.Builder.newInstance() @@ -130,8 +178,7 @@ private Catalog getCatalog() { .build()) .dataService(DataService.Builder.newInstance().build()) .participantId("participantId") - .property(CATALOG_PROPERTY, "value") - .build(); + .property(CATALOG_PROPERTY, "value"); } private JsonObject getJsonObject(String type) { diff --git a/spi/common/json-ld-spi/src/main/java/org/eclipse/edc/jsonld/spi/PropertyAndTypeNames.java b/spi/common/json-ld-spi/src/main/java/org/eclipse/edc/jsonld/spi/PropertyAndTypeNames.java index 46e7c713005..74bea24f597 100644 --- a/spi/common/json-ld-spi/src/main/java/org/eclipse/edc/jsonld/spi/PropertyAndTypeNames.java +++ b/spi/common/json-ld-spi/src/main/java/org/eclipse/edc/jsonld/spi/PropertyAndTypeNames.java @@ -31,6 +31,7 @@ public interface PropertyAndTypeNames { String DCAT_DATA_SERVICE_TYPE = DCAT_SCHEMA + "DataService"; String DCAT_DATA_SERVICE_ATTRIBUTE = DCAT_SCHEMA + "service"; String DCAT_DATASET_ATTRIBUTE = DCAT_SCHEMA + "dataset"; + String DCAT_CATALOG_ATTRIBUTE = DCAT_SCHEMA + "catalog"; String DCAT_DISTRIBUTION_ATTRIBUTE = DCAT_SCHEMA + "distribution"; String DCAT_ACCESS_SERVICE_ATTRIBUTE = DCAT_SCHEMA + "accessService"; @Deprecated(since = "0.10.0") diff --git a/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java b/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java index f1b08b6d71e..70bbc3f7e47 100644 --- a/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java +++ b/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java @@ -44,8 +44,7 @@ import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; import static org.eclipse.edc.spi.constants.CoreConstants.EDC_PREFIX; import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; public class CatalogApiEndToEndTest { @@ -175,11 +174,11 @@ void requestCatalog_whenAssetIsCatalogAsset_shouldReturnCatalogOfCatalogs(Manage .body(TYPE, is("dcat:Catalog")) .body("'dcat:service'", notNullValue()) // findAll is the restAssured way to express JSON Path filters - .body("'dcat:dataset'", hasSize(2)) - .body("'dcat:dataset'.findAll { it -> it.'@type' == 'dcat:Catalog' }.isCatalog", contains(true)) - .body("'dcat:dataset'.findAll { it -> it.'@type' == 'dcat:Catalog' }.'@id'", contains(catalogAssetId)) - .body("'dcat:dataset'.findAll { it -> it.'@type' == 'dcat:Catalog' }.'dcat:service'.'dcat:endpointUrl'", contains("http://quizzqua.zz/buzz")) - .body("'dcat:dataset'.findAll { it -> it.'@type' == 'dcat:Catalog' }.'dcat:distribution'.'dcat:accessService'.'@id'", contains(Base64.getUrlEncoder().encodeToString(catalogAssetId.getBytes()))); + .body("'dcat:catalog'.'@type'", equalTo("dcat:Catalog")) + .body("'dcat:catalog'.isCatalog", equalTo(true)) + .body("'dcat:catalog'.'@id'", equalTo(catalogAssetId)) + .body("'dcat:catalog'.'dcat:service'.'dcat:endpointUrl'", equalTo("http://quizzqua.zz/buzz")) + .body("'dcat:catalog'.'dcat:distribution'.'dcat:accessService'.'@id'", equalTo(Base64.getUrlEncoder().encodeToString(catalogAssetId.getBytes()))); } @Test