Skip to content

Commit

Permalink
Port json schema dereferencing (#5312)
Browse files Browse the repository at this point in the history
  • Loading branch information
carlesarnal authored Oct 8, 2024
1 parent 7685240 commit 4b5e37d
Show file tree
Hide file tree
Showing 71 changed files with 1,356 additions and 428 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import io.apicurio.registry.storage.error.ArtifactNotFoundException;
import io.apicurio.registry.storage.error.RuleNotFoundException;
import io.apicurio.registry.storage.error.VersionNotFoundException;
import io.apicurio.registry.storage.impl.sql.RegistryContentUtils;
import io.apicurio.registry.types.ArtifactType;
import io.apicurio.registry.types.ContentTypes;
import io.apicurio.registry.types.Current;
Expand Down Expand Up @@ -104,7 +105,8 @@ protected ArtifactVersionMetaDataDto createOrUpdateArtifact(String artifactId, S
.map(dto -> ArtifactReference.builder().name(dto.getName()).groupId(dto.getGroupId())
.artifactId(dto.getArtifactId()).version(dto.getVersion()).build())
.collect(Collectors.toList());
final Map<String, TypedContent> resolvedReferences = storage.resolveReferences(parsedReferences);
final Map<String, TypedContent> resolvedReferences = RegistryContentUtils
.recursivelyResolveReferences(parsedReferences, storage::getContentByReference);
try {
ContentHandle schemaContent;
schemaContent = ContentHandle.create(schema);
Expand Down Expand Up @@ -175,8 +177,9 @@ protected ArtifactVersionMetaDataDto lookupSchema(String groupId, String artifac
.getArtifactVersionContent(groupId, artifactId, version);
TypedContent typedArtifactVersion = TypedContent
.create(artifactVersion.getContent(), artifactVersion.getContentType());
Map<String, TypedContent> artifactVersionReferences = storage
.resolveReferences(artifactVersion.getReferences());
Map<String, TypedContent> artifactVersionReferences = RegistryContentUtils
.recursivelyResolveReferences(artifactVersion.getReferences(),
storage::getContentByReference);
String dereferencedExistingContentSha = DigestUtils
.sha256Hex(artifactTypeProvider.getContentDereferencer()
.dereference(typedArtifactVersion, artifactVersionReferences)
Expand Down Expand Up @@ -215,7 +218,8 @@ protected Map<String, TypedContent> resolveReferences(List<SchemaReference> refe
return artifactReferenceDto;
}).collect(Collectors.toList());

resolvedReferences = storage.resolveReferences(referencesAsDtos);
resolvedReferences = RegistryContentUtils.recursivelyResolveReferences(referencesAsDtos,
storage::getContentByReference);

if (references.size() > resolvedReferences.size()) {
// There are unresolvable references, which is not allowed.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import io.apicurio.registry.storage.dto.ArtifactVersionMetaDataDto;
import io.apicurio.registry.storage.dto.ContentWrapperDto;
import io.apicurio.registry.storage.dto.StoredArtifactVersionDto;
import io.apicurio.registry.storage.impl.sql.RegistryContentUtils;
import io.apicurio.registry.types.ArtifactType;
import io.apicurio.registry.types.VersionState;
import io.apicurio.registry.util.ArtifactTypeUtil;
Expand Down Expand Up @@ -47,8 +48,10 @@ public SchemaInfo getSchema(int id, String subject, String groupId) {
references = contentWrapper.getReferences();
}
TypedContent typedContent = TypedContent.create(contentHandle, contentType);
return converter.convert(contentHandle, ArtifactTypeUtil.determineArtifactType(typedContent, null,
storage.resolveReferences(references), factory), references);
return converter.convert(contentHandle,
ArtifactTypeUtil.determineArtifactType(typedContent, null, RegistryContentUtils
.recursivelyResolveReferences(references, storage::getContentByReference), factory),
references);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -1,143 +1,10 @@
package io.apicurio.registry.rest.v2;

import io.apicurio.common.apps.config.Info;
import io.apicurio.registry.content.TypedContent;
import io.apicurio.registry.content.dereference.ContentDereferencer;
import io.apicurio.registry.content.refs.JsonPointerExternalReference;
import io.apicurio.registry.storage.RegistryStorage;
import io.apicurio.registry.storage.dto.ArtifactReferenceDto;
import io.apicurio.registry.types.Current;
import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider;
import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory;
import io.apicurio.registry.utils.StringUtil;
import jakarta.inject.Inject;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.ws.rs.core.Context;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.slf4j.Logger;

import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public abstract class AbstractResourceImpl {

@Inject
Logger log;

@Inject
@Current
RegistryStorage storage;

@Inject
ArtifactTypeUtilProviderFactory factory;

@Context
HttpServletRequest request;

@ConfigProperty(name = "apicurio.apis.v2.base-href", defaultValue = "_")
@Info(category = "api", description = "API base href (URI)", availableSince = "2.5.0.Final")
String apiBaseHref;

/**
* Handle the content references based on the value of "dereference" - this can mean we need to fully
* dereference the content.
*
* @param dereference
* @param content
*/
protected TypedContent handleContentReferences(boolean dereference, String artifactType,
TypedContent content, List<ArtifactReferenceDto> references) {
// Dereference or rewrite references
if (!references.isEmpty() && dereference) {
ArtifactTypeUtilProvider artifactTypeProvider = factory.getArtifactTypeProvider(artifactType);
ContentDereferencer contentDereferencer = artifactTypeProvider.getContentDereferencer();
Map<String, TypedContent> resolvedReferences = storage.resolveReferences(references);
content = contentDereferencer.dereference(content, resolvedReferences);
}
return content;
}

/**
* Convert the list of references into a list of REST API URLs that point to the content. This means that
* we generate a REST API URL from the GAV (groupId, artifactId, version) information found in each
* reference.
*
* @param references
*/
protected Map<String, String> resolveReferenceUrls(List<ArtifactReferenceDto> references) {
Map<String, String> rval = new HashMap<>();
for (ArtifactReferenceDto reference : references) {
String resolvedReferenceUrl = resolveReferenceUrl(reference);
if (reference.getName().contains("#")) {
JsonPointerExternalReference jpRef = new JsonPointerExternalReference(reference.getName());
resolvedReferenceUrl = resolvedReferenceUrl + jpRef.getComponent();
}
if (resolvedReferenceUrl != null) {
rval.put(reference.getName(), resolvedReferenceUrl);
}
}
return rval;
}

/**
* Convert a single artifact reference to a REST API URL. This means that we generate a REST API URL from
* the GAV (groupId, artifactId, version) information found in the reference.
*
* @param reference
*/
protected String resolveReferenceUrl(ArtifactReferenceDto reference) {
URI baseHref = null;
try {
if (!"_".equals(apiBaseHref)) {
baseHref = new URI(apiBaseHref);
} else {
baseHref = getApiBaseHrefFromXForwarded(request);
if (baseHref == null) {
baseHref = getApiBaseHrefFromRequest(request);
}
}
} catch (URISyntaxException e) {
this.log.error("Error trying to determine the baseHref of the REST API.", e);
return null;
}

if (baseHref == null) {
this.log.warn("Failed to determine baseHref for the REST API.");
return null;
}

String path = String.format("/apis/registry/v2/groups/%s/artifacts/%s/versions/%s?references=REWRITE",
URLEncoder.encode(reference.getGroupId(), StandardCharsets.UTF_8),
URLEncoder.encode(reference.getArtifactId(), StandardCharsets.UTF_8),
URLEncoder.encode(reference.getVersion(), StandardCharsets.UTF_8));
return baseHref.resolve(path).toString();
}

/**
* Resolves a host name from the information found in X-Forwarded-Host and X-Forwarded-Proto.
*/
private static URI getApiBaseHrefFromXForwarded(HttpServletRequest request) throws URISyntaxException {
String fproto = request.getHeader("X-Forwarded-Proto");
String fhost = request.getHeader("X-Forwarded-Host");
if (!StringUtil.isEmpty(fproto) && !StringUtil.isEmpty(fhost)) {
return new URI(fproto + "://" + fhost);
} else {
return null;
}
}

/**
* Resolves a host name from the request information.
*/
private static URI getApiBaseHrefFromRequest(HttpServletRequest request) throws URISyntaxException {
String requestUrl = request.getRequestURL().toString();
URI requestUri = new URI(requestUrl);
return requestUri.resolve("/");
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@
import io.apicurio.registry.rest.v2.beans.UpdateState;
import io.apicurio.registry.rest.v2.beans.VersionMetaData;
import io.apicurio.registry.rest.v2.beans.VersionSearchResults;
import io.apicurio.registry.rest.v2.shared.CommonResourceOperations;
import io.apicurio.registry.rules.RuleApplicationType;
import io.apicurio.registry.rules.RulesService;
import io.apicurio.registry.storage.RegistryStorage;
import io.apicurio.registry.storage.RegistryStorage.RetrievalBehavior;
import io.apicurio.registry.storage.dto.ArtifactMetaDataDto;
import io.apicurio.registry.storage.dto.ArtifactReferenceDto;
Expand All @@ -63,12 +63,15 @@
import io.apicurio.registry.storage.error.InvalidArtifactIdException;
import io.apicurio.registry.storage.error.InvalidGroupIdException;
import io.apicurio.registry.storage.error.VersionNotFoundException;
import io.apicurio.registry.storage.impl.sql.RegistryContentUtils;
import io.apicurio.registry.types.ArtifactState;
import io.apicurio.registry.types.ContentTypes;
import io.apicurio.registry.types.Current;
import io.apicurio.registry.types.ReferenceType;
import io.apicurio.registry.types.RuleType;
import io.apicurio.registry.types.VersionState;
import io.apicurio.registry.types.provider.ArtifactTypeUtilProvider;
import io.apicurio.registry.types.provider.ArtifactTypeUtilProviderFactory;
import io.apicurio.registry.util.ArtifactIdGenerator;
import io.apicurio.registry.util.ArtifactTypeUtil;
import io.apicurio.registry.utils.ArtifactIdValidator;
Expand All @@ -78,10 +81,12 @@
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import jakarta.interceptor.Interceptors;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.ws.rs.BadRequestException;
import jakarta.ws.rs.HttpMethod;
import jakarta.ws.rs.NotAllowedException;
import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.Response;
import org.apache.commons.lang3.tuple.Pair;
import org.jose4j.base64url.Base64;
Expand Down Expand Up @@ -130,7 +135,7 @@
@ApplicationScoped
@Interceptors({ ResponseErrorLivenessCheck.class, ResponseTimeoutReadinessCheck.class })
@Logged
public class GroupsResourceImpl extends AbstractResourceImpl implements GroupsResource {
public class GroupsResourceImpl implements GroupsResource {

private static final String EMPTY_CONTENT_ERROR_MESSAGE = "Empty content is not allowed.";
@SuppressWarnings("unused")
Expand All @@ -149,11 +154,18 @@ public class GroupsResourceImpl extends AbstractResourceImpl implements GroupsRe
SecurityIdentity securityIdentity;

@Inject
CommonResourceOperations common;
@Current
RegistryStorage storage;

@Inject
ArtifactTypeUtilProviderFactory factory;

@Inject
io.apicurio.registry.rest.v3.GroupsResourceImpl v3;

@Context
HttpServletRequest request;

/**
* @see io.apicurio.registry.rest.v2.GroupsResource#getLatestArtifact(java.lang.String, java.lang.String,
* Boolean)
Expand All @@ -179,8 +191,25 @@ public Response getLatestArtifact(String groupId, String artifactId, Boolean der

TypedContent contentToReturn = TypedContent.create(artifact.getContent(),
artifact.getContentType());
contentToReturn = handleContentReferences(dereference, metaData.getArtifactType(),
contentToReturn, artifact.getReferences());

ArtifactTypeUtilProvider artifactTypeProvider = factory
.getArtifactTypeProvider(metaData.getArtifactType());

if (dereference && !artifact.getReferences().isEmpty()) {
if (artifactTypeProvider.supportsReferencesWithContext()) {
RegistryContentUtils.RewrittenContentHolder rewrittenContent = RegistryContentUtils
.recursivelyResolveReferencesWithContext(contentToReturn,
metaData.getArtifactType(), artifact.getReferences(),
storage::getContentByReference);

contentToReturn = artifactTypeProvider.getContentDereferencer().dereference(
rewrittenContent.getRewrittenContent(), rewrittenContent.getResolvedReferences());
} else {
contentToReturn = artifactTypeProvider.getContentDereferencer()
.dereference(contentToReturn, RegistryContentUtils.recursivelyResolveReferences(
artifact.getReferences(), storage::getContentByReference));
}
}

Response.ResponseBuilder builder = Response.ok(contentToReturn.getContent(),
contentToReturn.getContentType());
Expand Down Expand Up @@ -642,8 +671,24 @@ public Response getArtifactVersion(String groupId, String artifactId, String ver
artifactId, version);

TypedContent contentToReturn = TypedContent.create(artifact.getContent(), artifact.getContentType());
contentToReturn = handleContentReferences(dereference, metaData.getArtifactType(), contentToReturn,
artifact.getReferences());

ArtifactTypeUtilProvider artifactTypeProvider = factory
.getArtifactTypeProvider(metaData.getArtifactType());

if (dereference && !artifact.getReferences().isEmpty()) {
if (artifactTypeProvider.supportsReferencesWithContext()) {
RegistryContentUtils.RewrittenContentHolder rewrittenContent = RegistryContentUtils
.recursivelyResolveReferencesWithContext(contentToReturn, metaData.getArtifactType(),
artifact.getReferences(), storage::getContentByReference);

contentToReturn = artifactTypeProvider.getContentDereferencer().dereference(
rewrittenContent.getRewrittenContent(), rewrittenContent.getResolvedReferences());
} else {
contentToReturn = artifactTypeProvider.getContentDereferencer().dereference(contentToReturn,
RegistryContentUtils.recursivelyResolveReferences(artifact.getReferences(),
storage::getContentByReference));
}
}

Response.ResponseBuilder builder = Response.ok(contentToReturn.getContent(),
contentToReturn.getContentType());
Expand Down Expand Up @@ -1064,7 +1109,8 @@ private ArtifactMetaData createArtifactWithRefs(String groupId, String xRegistry
final List<ArtifactReferenceDto> referencesAsDtos = toReferenceDtos(references);

// Try to resolve the new artifact references and the nested ones (if any)
final Map<String, TypedContent> resolvedReferences = storage.resolveReferences(referencesAsDtos);
final Map<String, TypedContent> resolvedReferences = RegistryContentUtils
.recursivelyResolveReferences(referencesAsDtos, storage::getContentByReference);

rulesService.applyRules(defaultGroupIdToNull(groupId), artifactId, artifactType, typedContent,
RuleApplicationType.CREATE, toV3Refs(references), resolvedReferences);
Expand Down Expand Up @@ -1200,7 +1246,8 @@ private VersionMetaData createArtifactVersionWithRefs(String groupId, String art
final List<ArtifactReferenceDto> referencesAsDtos = toReferenceDtos(references);

// Try to resolve the new artifact references and the nested ones (if any)
final Map<String, TypedContent> resolvedReferences = storage.resolveReferences(referencesAsDtos);
final Map<String, TypedContent> resolvedReferences = RegistryContentUtils
.recursivelyResolveReferences(referencesAsDtos, storage::getContentByReference);

String artifactType = lookupArtifactType(groupId, artifactId);
TypedContent typedContent = TypedContent.create(content, ct);
Expand Down Expand Up @@ -1325,7 +1372,8 @@ private ArtifactMetaData updateArtifactInternal(String groupId, String artifactI
// passed references does not exist.
final List<ArtifactReferenceDto> referencesAsDtos = toReferenceDtos(references);

final Map<String, TypedContent> resolvedReferences = storage.resolveReferences(referencesAsDtos);
final Map<String, TypedContent> resolvedReferences = RegistryContentUtils
.recursivelyResolveReferences(referencesAsDtos, storage::getContentByReference);

TypedContent typedContent = TypedContent.create(content, contentType);
rulesService.applyRules(defaultGroupIdToNull(groupId), artifactId, artifactType, typedContent,
Expand Down
Loading

0 comments on commit 4b5e37d

Please sign in to comment.