From 84375d25c6e736b929c2182cdf71bea799ff3b7f Mon Sep 17 00:00:00 2001 From: Kevin Chun Date: Fri, 19 Jul 2024 13:00:23 -0700 Subject: [PATCH 1/6] add a structured type to batchGet in OpenAPI V3 spec --- .../openapi/v3/OpenAPIV3Generator.java | 31 +++++++++++-------- .../openapi/v3/OpenAPIV3GeneratorTest.java | 13 ++++++++ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index 3a93eb304b8f8..f423be82d6e8d 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -82,6 +82,20 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { "SystemMetadata", new Schema().type(TYPE_OBJECT).additionalProperties(true)); components.addSchemas("SortOrder", new Schema()._enum(List.of("ASCENDING", "DESCENDING"))); components.addSchemas("AspectPatch", buildAspectPatchSchema()); + components.addSchemas( + "BatchGetRequestBody", + new Schema<>() + .type(TYPE_OBJECT) + .description("Request body for batch get aspects.") + .properties( + Map.of( + "headers", + new Schema<>() + .type(TYPE_OBJECT) + .additionalProperties(new Schema<>().type(TYPE_STRING)) + .description("System headers for the operation.") + .nullable(true))) + .nullable(true)); entityRegistry .getAspectSpecs() .values() @@ -645,28 +659,19 @@ private static Schema buildEntityScrollSchema(final EntitySpec entity) { private static Schema buildEntityBatchGetRequestSchema( final EntitySpec entity, Set aspectNames) { - final Schema stringTypeSchema = new Schema<>(); - stringTypeSchema.setType(TYPE_STRING); - final Map headers = - Map.of( - "headers", - new Schema<>() - .type(TYPE_OBJECT) - .additionalProperties(stringTypeSchema) - .description("System headers for the operation.") - .nullable(true)); - final Map properties = entity.getAspectSpecMap().entrySet().stream() .filter(a -> aspectNames.contains(a.getValue().getName())) .collect( Collectors.toMap( - Map.Entry::getKey, a -> new Schema().type(TYPE_OBJECT).properties(headers))); + Map.Entry::getKey, + a -> new Schema().$ref("#/components/schemas/BatchGetRequestBody"))); properties.put( PROPERTY_URN, new Schema<>().type(TYPE_STRING).description("Unique id for " + entity.getName())); - properties.put(entity.getKeyAspectName(), new Schema().type(TYPE_OBJECT).properties(headers)); + properties.put( + entity.getKeyAspectName(), new Schema().$ref("#/components/schemas/BatchGetRequestBody")); return new Schema<>() .type(TYPE_OBJECT) diff --git a/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java b/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java index 10b75fd7faed3..b0fbbce05a0f8 100644 --- a/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java +++ b/metadata-service/openapi-servlet/src/test/java/io/datahubproject/openapi/v3/OpenAPIV3GeneratorTest.java @@ -88,5 +88,18 @@ public void testOpenApiSpecBuilder() throws Exception { Schema fabricType = openAPI.getComponents().getSchemas().get("FabricType"); assertEquals("string", fabricType.getType()); assertFalse(fabricType.getEnum().isEmpty()); + + Map batchProperties = + openAPI + .getComponents() + .getSchemas() + .get("BatchGetContainerEntityRequest_v3") + .getProperties(); + batchProperties.entrySet().stream() + .filter(entry -> !entry.getKey().equals("urn")) + .forEach( + entry -> + assertEquals( + "#/components/schemas/BatchGetRequestBody", entry.getValue().get$ref())); } } From e184bcdc2e3bef0cf20b5ce98ba20b4dbfe1fab8 Mon Sep 17 00:00:00 2001 From: Kevin Chun Date: Fri, 19 Jul 2024 20:06:29 -0700 Subject: [PATCH 2/6] fix bug with aspect POST request body - expected request body is the aspect itself, not the request wrapper --- .../java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index f423be82d6e8d..670b74d1a3926 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -827,9 +827,7 @@ private static PathItem buildSingleEntityAspectPath( .schema( new Schema() .$ref( - String.format( - "#/components/schemas/%s%s", - upperFirstAspect, ASPECT_REQUEST_SUFFIX))))); + String.format("#/components/schemas/%s", upperFirstAspect))))); final Operation postOperation = new Operation() .summary(String.format("Create aspect %s on %s ", aspect, upperFirstEntity)) From 73fb5e97b88a8ca78b00db4c8f130f8d5d9b5945 Mon Sep 17 00:00:00 2001 From: Kevin Chun Date: Mon, 22 Jul 2024 10:50:01 -0700 Subject: [PATCH 3/6] update default --- .../openapi/controller/GenericEntitiesController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java index c91c8ac987e5c..6f9f1a81f1483 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -508,7 +508,7 @@ public ResponseEntity createAspect( @PathVariable("aspectName") String aspectName, @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") Boolean withSystemMetadata, - @RequestParam(value = "createIfNotExists", required = false, defaultValue = "false") + @RequestParam(value = "createIfNotExists", required = false, defaultValue = "true") Boolean createIfNotExists, @RequestBody @Nonnull String jsonAspect) throws URISyntaxException { From 143554d6072e41d91e75233abd342e4f5d688778 Mon Sep 17 00:00:00 2001 From: Kevin Chun Date: Mon, 22 Jul 2024 10:52:03 -0700 Subject: [PATCH 4/6] lint --- .../java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index 670b74d1a3926..22fee7d6bc974 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -827,7 +827,8 @@ private static PathItem buildSingleEntityAspectPath( .schema( new Schema() .$ref( - String.format("#/components/schemas/%s", upperFirstAspect))))); + String.format( + "#/components/schemas/%s", upperFirstAspect))))); final Operation postOperation = new Operation() .summary(String.format("Create aspect %s on %s ", aspect, upperFirstEntity)) From a26f3bb1851e9215d77797804ad7f03f3c4c3f03 Mon Sep 17 00:00:00 2001 From: Kevin Chun Date: Mon, 22 Jul 2024 13:57:58 -0700 Subject: [PATCH 5/6] add @Hidden to V3 Entity controller as all endpoints are registered via OpenAPIV3Generator --- .../datahubproject/openapi/v3/controller/EntityController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java index d6feb6cc460c9..9ca34934e4c65 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java @@ -37,6 +37,7 @@ import io.datahubproject.openapi.v3.models.GenericAspectV3; import io.datahubproject.openapi.v3.models.GenericEntityScrollResultV3; import io.datahubproject.openapi.v3.models.GenericEntityV3; +import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.servlet.http.HttpServletRequest; @@ -67,6 +68,7 @@ @RequiredArgsConstructor @RequestMapping("/v3/entity") @Slf4j +@Hidden public class EntityController extends GenericEntitiesController< GenericAspectV3, GenericEntityV3, GenericEntityScrollResultV3> { From e377b438f338e3279e093970b8f45dd3a8392f9f Mon Sep 17 00:00:00 2001 From: Kevin Chun Date: Mon, 22 Jul 2024 15:37:57 -0700 Subject: [PATCH 6/6] extract the aspect json from the of the request body. Revert change to request body declaration in open api --- .../openapi/controller/GenericEntitiesController.java | 11 +++++++---- .../datahubproject/openapi/v3/OpenAPIV3Generator.java | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java index 6f9f1a81f1483..de5d2ae1118d4 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -13,6 +13,7 @@ import com.datahub.authorization.AuthorizerChain; import com.datahub.util.RecordUtils; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; @@ -511,7 +512,7 @@ public ResponseEntity createAspect( @RequestParam(value = "createIfNotExists", required = false, defaultValue = "true") Boolean createIfNotExists, @RequestBody @Nonnull String jsonAspect) - throws URISyntaxException { + throws URISyntaxException, JsonProcessingException { Urn urn = validatedUrn(entityUrn); EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); @@ -649,8 +650,8 @@ protected Boolean exists( * fixes) * * @param requestedAspectNames requested aspects - * @return updated map * @param map values + * @return updated map */ protected LinkedHashMap> resolveAspectNames( LinkedHashMap> requestedAspectNames, @Nonnull T defaultValue) { @@ -732,7 +733,9 @@ protected ChangeMCP toUpsertItem( Boolean createIfNotExists, String jsonAspect, Actor actor) - throws URISyntaxException { + throws JsonProcessingException { + JsonNode jsonNode = objectMapper.readTree(jsonAspect); + String aspectJson = jsonNode.get("value").toString(); return ChangeItemImpl.builder() .urn(entityUrn) .aspectName(aspectSpec.getName()) @@ -740,7 +743,7 @@ protected ChangeMCP toUpsertItem( .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr())) .recordTemplate( GenericRecordUtils.deserializeAspect( - ByteString.copyString(jsonAspect, StandardCharsets.UTF_8), + ByteString.copyString(aspectJson, StandardCharsets.UTF_8), GenericRecordUtils.JSON, aspectSpec)) .build(aspectRetriever); diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index 22fee7d6bc974..f423be82d6e8d 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -828,7 +828,8 @@ private static PathItem buildSingleEntityAspectPath( new Schema() .$ref( String.format( - "#/components/schemas/%s", upperFirstAspect))))); + "#/components/schemas/%s%s", + upperFirstAspect, ASPECT_REQUEST_SUFFIX))))); final Operation postOperation = new Operation() .summary(String.format("Create aspect %s on %s ", aspect, upperFirstEntity))