Skip to content

Commit

Permalink
eclipse-ditto#2072: add configuration option to configure pre-defined…
Browse files Browse the repository at this point in the history
… "extraFields"
  • Loading branch information
thjaeckle committed Dec 16, 2024
1 parent f8e96f1 commit 75b666f
Show file tree
Hide file tree
Showing 21 changed files with 767 additions and 56 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,41 @@ public enum DittoHeaderDefinition implements HeaderDefinition {
JsonObject.class,
false,
true,
HeaderValueValidators.getJsonObjectValidator()),

/**
* Internal header containing the pre-defined configured {@code extraFields} as list of jsonPointers for the
* emitted thing event.
*
* @since 3.7.0
*/
PRE_DEFINED_EXTRA_FIELDS("ditto-pre-defined-extra-fields",
JsonArray.class,
false,
false,
HeaderValueValidators.getJsonArrayValidator()),

/**
* Internal header containing the pre-defined configured {@code extraFields} as keys and the allowed "read subjects"
* as array of stings - defining which "auth subjects" are allowed to read which pre-defined extra field.
*
* @since 3.7.0
*/
PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT("ditto-pre-defined-extra-fields-read-grant",
JsonObject.class,
false,
false,
HeaderValueValidators.getJsonObjectValidator()),

/**
* Internal header containing pre-defined {@code extraFields} as JSON object sent along for emitted thing event.
*
* @since 3.7.0
*/
PRE_DEFINED_EXTRA_FIELDS_OBJECT("ditto-pre-defined-extra-fields-object",
JsonObject.class,
false,
false,
HeaderValueValidators.getJsonObjectValidator());

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,16 @@ public final class ImmutableDittoHeadersTest {
.set(DittoHeaderDefinition.ORIGINATOR.getKey(), "foo:bar")
.build();

private static final JsonArray KNOWN_PRE_DEFINED_EXTRA_FIELDS = JsonArray.newBuilder()
.add("foo:bar:123")
.build();
private static final JsonObject KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT = JsonObject.newBuilder()
.set("/definition", "known:subject")
.build();
private static final JsonObject KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT = JsonObject.newBuilder()
.set("definition", "foo:bar:123")
.build();


static {
KNOWN_METADATA_HEADERS = MetadataHeaders.newInstance();
Expand Down Expand Up @@ -205,6 +215,12 @@ public void settingAllKnownHeadersWorksAsExpected() {
.putHeader(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION))
.putHeader(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP))
.putHeader(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString())
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS.formatAsString())
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.formatAsString())
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT.formatAsString())
.build();

assertThat(underTest).isEqualTo(expectedHeaderMap);
Expand Down Expand Up @@ -535,6 +551,11 @@ public void toJsonReturnsExpected() {
.set(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), KNOWN_AT_HISTORICAL_REVISION)
.set(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), KNOWN_AT_HISTORICAL_TIMESTAMP.toString())
.set(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS)
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(), KNOWN_PRE_DEFINED_EXTRA_FIELDS)
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT)
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT)
.build();

final Map<String, String> allKnownHeaders = createMapContainingAllKnownHeaders();
Expand Down Expand Up @@ -774,6 +795,12 @@ private static Map<String, String> createMapContainingAllKnownHeaders() {
result.put(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION));
result.put(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP));
result.put(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString());
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS.formatAsString());
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.formatAsString());
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT.formatAsString());

return result;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,17 @@

import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.Executor;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import javax.annotation.Nullable;

Expand All @@ -38,8 +41,10 @@
import org.eclipse.ditto.internal.utils.pekko.logging.ThreadSafeDittoLogger;
import org.eclipse.ditto.internal.utils.tracing.DittoTracing;
import org.eclipse.ditto.internal.utils.tracing.span.SpanOperationName;
import org.eclipse.ditto.json.JsonArray;
import org.eclipse.ditto.json.JsonFactory;
import org.eclipse.ditto.json.JsonFieldSelector;
import org.eclipse.ditto.json.JsonKey;
import org.eclipse.ditto.json.JsonObject;
import org.eclipse.ditto.json.JsonObjectBuilder;
import org.eclipse.ditto.json.JsonPointer;
Expand Down Expand Up @@ -160,6 +165,16 @@ public CompletionStage<JsonObject> retrievePartialThing(final ThingId thingId,
(concernedSignal instanceof ThingEvent) && !(ProtocolAdapter.isLiveSignal(concernedSignal)) ?
List.of((ThingEvent<?>) concernedSignal) : List.of();

final DittoHeaders signalHeaders = Optional.ofNullable(concernedSignal)
.map(Signal::getDittoHeaders)
.orElseGet(DittoHeaders::empty);
if (jsonFieldSelector != null &&
signalHeaders.containsKey(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey())
) {
return performPreDefinedExtraFieldsOptimization(
thingId, jsonFieldSelector, dittoHeaders, signalHeaders, thingEvents
);
}
// as second step only return what was originally requested as fields:
final var cachingParameters =
new CachingParameters(jsonFieldSelector, thingEvents, true, 0);
Expand Down Expand Up @@ -199,6 +214,80 @@ public CompletionStage<JsonObject> retrievePartialThing(final EntityId thingId,
.thenApply(jsonObject -> applyJsonFieldSelector(jsonObject, jsonFieldSelector));
}

private CompletionStage<JsonObject> performPreDefinedExtraFieldsOptimization(final ThingId thingId,
final JsonFieldSelector jsonFieldSelector,
final DittoHeaders dittoHeaders,
final DittoHeaders signalHeaders,
final List<ThingEvent<?>> thingEvents
) {
final JsonArray configuredPredefinedExtraFields =
JsonArray.of(signalHeaders.get(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey()));
final Set<JsonPointer> allConfiguredPredefinedExtraFields = configuredPredefinedExtraFields.stream()
.filter(JsonValue::isString)
.map(JsonValue::asString)
.map(JsonPointer::of)
.collect(Collectors.toSet());

final JsonObject preDefinedExtraFields =
JsonObject.of(signalHeaders.get(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey()));
final CompletionStage<JsonObject> filteredPreDefinedExtraFieldsReadGranted =
filterPreDefinedExtraReadGrantedObject(jsonFieldSelector, dittoHeaders, signalHeaders,
preDefinedExtraFields);

final boolean allExtraFieldsPresent =
allConfiguredPredefinedExtraFields.containsAll(jsonFieldSelector.getPointers());
if (allExtraFieldsPresent) {
return filteredPreDefinedExtraFieldsReadGranted;
} else {
// optimization to only fetch extra fields which were not pre-defined
final List<JsonPointer> missingFieldsPointers = new ArrayList<>(jsonFieldSelector.getPointers());
missingFieldsPointers.removeAll(allConfiguredPredefinedExtraFields);
final JsonFieldSelector missingFieldsSelector = JsonFactory.newFieldSelector(missingFieldsPointers);
final var cachingParameters =
new CachingParameters(missingFieldsSelector, thingEvents, true, 0);

return doRetrievePartialThing(thingId, dittoHeaders, null, cachingParameters)
.thenCompose(jsonObject -> filteredPreDefinedExtraFieldsReadGranted
.thenApply(preDefinedObject ->
preDefinedObject.toBuilder()
.setAll(applyJsonFieldSelector(jsonObject, missingFieldsSelector))
.build()
)
);
}
}

private static CompletionStage<JsonObject> filterPreDefinedExtraReadGrantedObject(
final JsonFieldSelector jsonFieldSelector,
final DittoHeaders dittoHeaders, final DittoHeaders signalHeaders, final JsonObject preDefinedExtraFields) {
final JsonObject preDefinedExtraFieldsReadGrant = JsonObject.of(
signalHeaders.get(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey())
);
final JsonFieldSelector grantedReadJsonFieldSelector = filterAskedForFieldSelectorToGrantedFields(
jsonFieldSelector, preDefinedExtraFieldsReadGrant,
dittoHeaders.getAuthorizationContext().getAuthorizationSubjectIds()
);
return CompletableFuture.completedStage(preDefinedExtraFields.get(grantedReadJsonFieldSelector));
}

private static JsonFieldSelector filterAskedForFieldSelectorToGrantedFields(
final JsonFieldSelector jsonFieldSelector,
final JsonObject preDefinedExtraFieldsReadGrant,
final List<String> authorizationSubjectIds)
{
final List<JsonPointer> allowedPointers = StreamSupport.stream(jsonFieldSelector.spliterator(), false)
.filter(pointer -> preDefinedExtraFieldsReadGrant.getValue(JsonKey.of(pointer.toString()))
.filter(JsonValue::isArray)
.map(JsonValue::asArray)
.filter(readGrantArray -> readGrantArray.stream()
.filter(JsonValue::isString)
.map(JsonValue::asString)
.anyMatch(authorizationSubjectIds::contains)
).isPresent()
).toList();
return JsonFactory.newFieldSelector(allowedPointers);
}

protected CompletionStage<JsonObject> doRetrievePartialThing(final EntityId thingId,
final DittoHeaders dittoHeaders,
@Nullable final DittoHeaders dittoHeadersNotAddedToCacheKey,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public interface EventConfig {

/**
* An enumeration of the known config path expressions and their associated default values for
* {@code SnapshotConfig}.
* {@code EventConfig}.
*/
enum EventConfigValue implements KnownConfigValue {

Expand Down Expand Up @@ -65,7 +65,6 @@ public Object getDefaultValue() {
public String getConfigPath() {
return path;
}

}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -585,8 +585,6 @@ private record PersistEventAsync<
E extends EventsourcedEvent<? extends E>,
S extends Jsonifiable.WithFieldSelectorAndPredicate<JsonField>>(E event, BiConsumer<E, S> handler) {}

;

/**
* Persist an event, modify actor state by the event strategy, then invoke the handler.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import org.eclipse.ditto.internal.utils.config.KnownConfigValue;

/**
* Provides configuration settings for Concierge entity creation behaviour.
* Provides configuration settings for entity creation behaviour.
*/
@Immutable
public interface EntityCreationConfig {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
* Copyright (c) 2024 Contributors to the Eclipse Foundation
*
* See the NOTICE file(s) distributed with this work for additional
* information regarding copyright ownership.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipse.ditto.things.service.common.config;

import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Pattern;

import javax.annotation.Nullable;
import javax.annotation.concurrent.Immutable;

import org.eclipse.ditto.base.model.common.LikeHelper;
import org.eclipse.ditto.internal.utils.config.ConfigWithFallback;
import org.eclipse.ditto.json.JsonFieldSelector;

import com.typesafe.config.Config;

/**
* This class implements {@link PreDefinedExtraFieldsConfig}.
*/
@Immutable
public final class DefaultPreDefinedExtraFieldsConfig implements PreDefinedExtraFieldsConfig {

private final List<Pattern> namespacePatterns;
@Nullable private final String rqlCondition;
private final JsonFieldSelector extraFields;

private DefaultPreDefinedExtraFieldsConfig(final ConfigWithFallback config) {
this.namespacePatterns = compile(List.copyOf(config.getStringList(
PreDefinedExtraFieldsConfig.ConfigValues.NAMESPACES.getConfigPath())
));
this.rqlCondition = config.getStringOrNull(ConfigValues.CONDITION);
final List<String> configuredExtraFields = config.getStringList(ConfigValues.EXTRA_FIELDS.getConfigPath());
this.extraFields = JsonFieldSelector.newInstance(
configuredExtraFields.getFirst(),
configuredExtraFields.subList(1, configuredExtraFields.size()).toArray(CharSequence[]::new)
);
}

/**
* Returns an instance of {@code CreationRestrictionConfig} based on the settings of the specified Config.
*
* @param config is supposed to provide the settings of the restriction config.
* @return the instance.
* @throws org.eclipse.ditto.internal.utils.config.DittoConfigError if {@code config} is invalid.
*/
public static DefaultPreDefinedExtraFieldsConfig of(final Config config) {
return new DefaultPreDefinedExtraFieldsConfig(ConfigWithFallback.newInstance(config,
PreDefinedExtraFieldsConfig.ConfigValues.values()));
}

private static List<Pattern> compile(final List<String> patterns) {
return patterns.stream()
.map(LikeHelper::convertToRegexSyntax)
.filter(Objects::nonNull)
.map(Pattern::compile)
.toList();
}

@Override
public List<Pattern> getNamespace() {
return namespacePatterns;
}

@Override
public Optional<String> getCondition() {
return Optional.ofNullable(rqlCondition);
}

@Override
public JsonFieldSelector getExtraFields() {
return extraFields;
}

@Override
public boolean equals(final Object o) {
if (!(o instanceof final DefaultPreDefinedExtraFieldsConfig that)) {
return false;
}
return Objects.equals(namespacePatterns, that.namespacePatterns) &&
Objects.equals(rqlCondition, that.rqlCondition) &&
Objects.equals(extraFields, that.extraFields);
}

@Override
public int hashCode() {
return Objects.hash(namespacePatterns, rqlCondition, extraFields);
}

@Override
public String toString() {
return getClass().getSimpleName() + "[" +
"namespacePatterns=" + namespacePatterns +
", rqlCondition='" + rqlCondition + '\'' +
", extraFields=" + extraFields +
"]";
}
}
Loading

0 comments on commit 75b666f

Please sign in to comment.