Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#2072: add configuration option to configure pre-defined "extraFields" #2076

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

#2072: add configuration option to configure pre-defined "extraFields" #2076

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e3574cb
#2072: add configuration option to configure pre-defined "extraFields"
thjaeckle Dec 16, 2024
2d4f2a2
#2072: add unit test for caching signal enrichment with pre-defined e…
thjaeckle Jan 2, 2025
859cee8
#2072: fixed license year of new files
thjaeckle Jan 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions base/model/src/main/java/org/eclipse/ditto/base/model/headers/DittoHeaderDefinition.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,41 @@ public enum DittoHeaderDefinition implements HeaderDefinition {
JsonObject.class,
false,
true,
HeaderValueValidators.getJsonObjectValidator()),

/**
* Internal header containing the pre-defined configured {@code extraFields} as list of jsonPointers for the
* emitted thing event.
*
* @since 3.7.0
*/
PRE_DEFINED_EXTRA_FIELDS("ditto-pre-defined-extra-fields",
JsonArray.class,
false,
false,
HeaderValueValidators.getJsonArrayValidator()),

/**
* Internal header containing the pre-defined configured {@code extraFields} as keys and the allowed "read subjects"
* as array of stings - defining which "auth subjects" are allowed to read which pre-defined extra field.
*
* @since 3.7.0
*/
PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT("ditto-pre-defined-extra-fields-read-grant",
JsonObject.class,
false,
false,
HeaderValueValidators.getJsonObjectValidator()),

/**
* Internal header containing pre-defined {@code extraFields} as JSON object sent along for emitted thing event.
*
* @since 3.7.0
*/
PRE_DEFINED_EXTRA_FIELDS_OBJECT("ditto-pre-defined-extra-fields-object",
JsonObject.class,
false,
false,
HeaderValueValidators.getJsonObjectValidator());

/**
Expand Down
27 changes: 27 additions & 0 deletions base/model/src/test/java/org/eclipse/ditto/base/model/headers/ImmutableDittoHeadersTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,16 @@ public final class ImmutableDittoHeadersTest {
.set(DittoHeaderDefinition.ORIGINATOR.getKey(), "foo:bar")
.build();

private static final JsonArray KNOWN_PRE_DEFINED_EXTRA_FIELDS = JsonArray.newBuilder()
.add("foo:bar:123")
.build();
private static final JsonObject KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT = JsonObject.newBuilder()
.set("/definition", "known:subject")
.build();
private static final JsonObject KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT = JsonObject.newBuilder()
.set("definition", "foo:bar:123")
.build();


static {
KNOWN_METADATA_HEADERS = MetadataHeaders.newInstance();
Expand Down Expand Up @@ -205,6 +215,12 @@ public void settingAllKnownHeadersWorksAsExpected() {
.putHeader(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION))
.putHeader(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP))
.putHeader(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString())
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS.formatAsString())
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.formatAsString())
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT.formatAsString())
.build();

assertThat(underTest).isEqualTo(expectedHeaderMap);
Expand Down Expand Up @@ -535,6 +551,11 @@ public void toJsonReturnsExpected() {
.set(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), KNOWN_AT_HISTORICAL_REVISION)
.set(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), KNOWN_AT_HISTORICAL_TIMESTAMP.toString())
.set(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS)
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(), KNOWN_PRE_DEFINED_EXTRA_FIELDS)
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT)
.set(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT)
.build();

final Map<String, String> allKnownHeaders = createMapContainingAllKnownHeaders();
Expand Down Expand Up @@ -774,6 +795,12 @@ private static Map<String, String> createMapContainingAllKnownHeaders() {
result.put(DittoHeaderDefinition.AT_HISTORICAL_REVISION.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_REVISION));
result.put(DittoHeaderDefinition.AT_HISTORICAL_TIMESTAMP.getKey(), String.valueOf(KNOWN_AT_HISTORICAL_TIMESTAMP));
result.put(DittoHeaderDefinition.HISTORICAL_HEADERS.getKey(), KNOWN_HISTORICAL_HEADERS.formatAsString());
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS.formatAsString());
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.formatAsString());
result.put(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
KNOWN_PRE_DEFINED_EXTRA_FIELDS_OBJECT.formatAsString());

return result;
}
Expand Down
93 changes: 91 additions & 2 deletions ...rg/eclipse/ditto/internal/models/signalenrichment/DittoCachingSignalEnrichmentFacade.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,17 @@

import static org.eclipse.ditto.base.model.common.ConditionChecker.checkNotNull;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.Executor;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import javax.annotation.Nullable;

Expand All @@ -38,8 +41,10 @@
import org.eclipse.ditto.internal.utils.pekko.logging.ThreadSafeDittoLogger;
import org.eclipse.ditto.internal.utils.tracing.DittoTracing;
import org.eclipse.ditto.internal.utils.tracing.span.SpanOperationName;
import org.eclipse.ditto.json.JsonArray;
import org.eclipse.ditto.json.JsonFactory;
import org.eclipse.ditto.json.JsonFieldSelector;
import org.eclipse.ditto.json.JsonKey;
import org.eclipse.ditto.json.JsonObject;
import org.eclipse.ditto.json.JsonObjectBuilder;
import org.eclipse.ditto.json.JsonPointer;
Expand Down Expand Up @@ -160,6 +165,16 @@ public CompletionStage<JsonObject> retrievePartialThing(final ThingId thingId,
(concernedSignal instanceof ThingEvent) && !(ProtocolAdapter.isLiveSignal(concernedSignal)) ?
List.of((ThingEvent<?>) concernedSignal) : List.of();

final DittoHeaders signalHeaders = Optional.ofNullable(concernedSignal)
.map(Signal::getDittoHeaders)
.orElseGet(DittoHeaders::empty);
if (jsonFieldSelector != null &&
signalHeaders.containsKey(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey())
) {
return performPreDefinedExtraFieldsOptimization(
thingId, jsonFieldSelector, dittoHeaders, signalHeaders, thingEvents
);
}
// as second step only return what was originally requested as fields:
final var cachingParameters =
new CachingParameters(jsonFieldSelector, thingEvents, true, 0);
Expand Down Expand Up @@ -199,6 +214,80 @@ public CompletionStage<JsonObject> retrievePartialThing(final EntityId thingId,
.thenApply(jsonObject -> applyJsonFieldSelector(jsonObject, jsonFieldSelector));
}

private CompletionStage<JsonObject> performPreDefinedExtraFieldsOptimization(final ThingId thingId,
final JsonFieldSelector jsonFieldSelector,
final DittoHeaders dittoHeaders,
final DittoHeaders signalHeaders,
final List<ThingEvent<?>> thingEvents
) {
final JsonArray configuredPredefinedExtraFields =
JsonArray.of(signalHeaders.get(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey()));
final Set<JsonPointer> allConfiguredPredefinedExtraFields = configuredPredefinedExtraFields.stream()
.filter(JsonValue::isString)
.map(JsonValue::asString)
.map(JsonPointer::of)
.collect(Collectors.toSet());

final JsonObject preDefinedExtraFields =
JsonObject.of(signalHeaders.get(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey()));
final CompletionStage<JsonObject> filteredPreDefinedExtraFieldsReadGranted =
filterPreDefinedExtraReadGrantedObject(jsonFieldSelector, dittoHeaders, signalHeaders,
preDefinedExtraFields);

final boolean allExtraFieldsPresent =
allConfiguredPredefinedExtraFields.containsAll(jsonFieldSelector.getPointers());
if (allExtraFieldsPresent) {
return filteredPreDefinedExtraFieldsReadGranted;
} else {
// optimization to only fetch extra fields which were not pre-defined
final List<JsonPointer> missingFieldsPointers = new ArrayList<>(jsonFieldSelector.getPointers());
missingFieldsPointers.removeAll(allConfiguredPredefinedExtraFields);
final JsonFieldSelector missingFieldsSelector = JsonFactory.newFieldSelector(missingFieldsPointers);
final var cachingParameters =
new CachingParameters(missingFieldsSelector, thingEvents, true, 0);

return doRetrievePartialThing(thingId, dittoHeaders, null, cachingParameters)
.thenCompose(jsonObject -> filteredPreDefinedExtraFieldsReadGranted
.thenApply(preDefinedObject ->
preDefinedObject.toBuilder()
.setAll(applyJsonFieldSelector(jsonObject, missingFieldsSelector))
.build()
)
);
}
}

private static CompletionStage<JsonObject> filterPreDefinedExtraReadGrantedObject(
final JsonFieldSelector jsonFieldSelector,
final DittoHeaders dittoHeaders, final DittoHeaders signalHeaders, final JsonObject preDefinedExtraFields) {
final JsonObject preDefinedExtraFieldsReadGrant = JsonObject.of(
signalHeaders.get(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey())
);
final JsonFieldSelector grantedReadJsonFieldSelector = filterAskedForFieldSelectorToGrantedFields(
jsonFieldSelector, preDefinedExtraFieldsReadGrant,
dittoHeaders.getAuthorizationContext().getAuthorizationSubjectIds()
);
return CompletableFuture.completedStage(preDefinedExtraFields.get(grantedReadJsonFieldSelector));
}

private static JsonFieldSelector filterAskedForFieldSelectorToGrantedFields(
final JsonFieldSelector jsonFieldSelector,
final JsonObject preDefinedExtraFieldsReadGrant,
final List<String> authorizationSubjectIds)
{
final List<JsonPointer> allowedPointers = StreamSupport.stream(jsonFieldSelector.spliterator(), false)
.filter(pointer -> preDefinedExtraFieldsReadGrant.getValue(JsonKey.of(pointer.toString()))
.filter(JsonValue::isArray)
.map(JsonValue::asArray)
.filter(readGrantArray -> readGrantArray.stream()
.filter(JsonValue::isString)
.map(JsonValue::asString)
.anyMatch(authorizationSubjectIds::contains)
).isPresent()
).toList();
return JsonFactory.newFieldSelector(allowedPointers);
}

protected CompletionStage<JsonObject> doRetrievePartialThing(final EntityId thingId,
final DittoHeaders dittoHeaders,
@Nullable final DittoHeaders dittoHeadersNotAddedToCacheKey,
Expand Down Expand Up @@ -369,11 +458,11 @@ private CompletionStage<JsonObject> doSmartUpdateCachedObject(final SignalEnrich
}

private static <T> T getLast(final List<T> list) {
return list.get(list.size() - 1);
return list.getLast();
}

private static <T> T getFirst(final List<T> list) {
return list.get(0);
return list.getFirst();
}

private CompletionStage<JsonObject> handleNextExpectedThingEvents(final SignalEnrichmentCacheKey cacheKey,
Expand Down
2 changes: 1 addition & 1 deletion ...pse/ditto/internal/models/signalenrichment/AbstractCachingSignalEnrichmentFacadeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
*/
abstract class AbstractCachingSignalEnrichmentFacadeTest extends AbstractSignalEnrichmentFacadeTest {

private static final String ISSUER_PREFIX = "test:";
protected static final String ISSUER_PREFIX = "test:";
private static final String CACHE_CONFIG_KEY = "my-cache";
private static final String CACHE_CONFIG = CACHE_CONFIG_KEY + """
{
Expand Down
76 changes: 75 additions & 1 deletion ...clipse/ditto/internal/models/signalenrichment/DittoCachingSignalEnrichmentFacadeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,26 @@
*/
package org.eclipse.ditto.internal.models.signalenrichment;

import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.CompletionStage;

import org.apache.pekko.testkit.javadsl.TestKit;
import org.eclipse.ditto.base.model.auth.AuthorizationContext;
import org.eclipse.ditto.base.model.auth.AuthorizationSubject;
import org.eclipse.ditto.base.model.auth.DittoAuthorizationContextType;
import org.eclipse.ditto.base.model.entity.metadata.MetadataModelFactory;
import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition;
import org.eclipse.ditto.base.model.headers.DittoHeaders;
import org.eclipse.ditto.base.model.signals.DittoTestSystem;
import org.eclipse.ditto.internal.utils.cache.config.CacheConfig;
import org.eclipse.ditto.json.JsonFieldSelector;
import org.eclipse.ditto.json.JsonObject;
import org.eclipse.ditto.json.JsonPointer;
import org.eclipse.ditto.json.JsonValue;
import org.eclipse.ditto.things.model.ThingId;
import org.eclipse.ditto.things.model.signals.events.AttributeModified;
import org.junit.Test;

/**
* Unit tests for {@link DittoCachingSignalEnrichmentFacade}.
Expand All @@ -27,7 +44,37 @@ public final class DittoCachingSignalEnrichmentFacadeTest extends AbstractCachin
"attributes": {"x": 5},
"features": {"y": {"properties": {"z": true}}},
"_metadata": {"attributes": {"x": {"type": "x attribute"}}}
}""");
}"""
);

private static final JsonObject EXPECTED_THING_JSON_PRE_DEFINED_EXTRA = JsonObject.of("""
{
"definition": "some:cool:definition",
"attributes": {"x": 5, "pre": {"bar": [1,2,3]}, "pre2": {"some": 41, "secret": true}}
}"""
);

private static final AttributeModified THING_EVENT_PRE_DEFINED_EXTRA_FIELDS = AttributeModified.of(
ThingId.generateRandom("org.eclipse.test"),
JsonPointer.of("x"),
JsonValue.of(42),
4L,
Instant.EPOCH,
DittoHeaders.newBuilder()
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS.getKey(),
"[\"/definition\",\"/attributes/pre\",\"/attributes/pre2\"]")
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_READ_GRANT_OBJECT.getKey(),
"{\"/definition\":[\"test:user\"],\"/attributes/pre\":[\"test:user\"]}")
.putHeader(DittoHeaderDefinition.PRE_DEFINED_EXTRA_FIELDS_OBJECT.getKey(),
"{\"definition\":\"some:cool:definition\",\"attributes\":{\"pre\":{\"bar\": [1,2,3]}}}")
.build(),
MetadataModelFactory.newMetadataBuilder()
.set("type", "x attribute")
.build());

private static final JsonFieldSelector SELECTOR_PRE_DEFINED_EXTRA_FIELDS =
JsonFieldSelector.newInstance("definition", "attributes/pre", "attributes/pre2");


@Override
protected CachingSignalEnrichmentFacade createCachingSignalEnrichmentFacade(final TestKit kit,
Expand All @@ -44,5 +91,32 @@ protected JsonObject getExpectedThingJson() {
return EXPECTED_THING_JSON;
}

@Test
public void enrichedEventWithPreDefinedExtraFieldsDoesNotLeadToCacheLookup() {
DittoTestSystem.run(this, kit -> {
final SignalEnrichmentFacade underTest =
createSignalEnrichmentFacadeUnderTest(kit, Duration.ofSeconds(10L));
final ThingId thingId = ThingId.generateRandom();
final String userId = ISSUER_PREFIX + "user";
final DittoHeaders headers = DittoHeaders.newBuilder()
.authorizationContext(AuthorizationContext.newInstance(DittoAuthorizationContextType.UNSPECIFIED,
AuthorizationSubject.newInstance(userId)))
.randomCorrelationId()
.build();
final CompletionStage<JsonObject> askResult =
underTest.retrievePartialThing(thingId, SELECTOR_PRE_DEFINED_EXTRA_FIELDS, headers,
THING_EVENT_PRE_DEFINED_EXTRA_FIELDS);

// THEN: no cache lookup should be done
kit.expectNoMessage(Duration.ofSeconds(1));
askResult.toCompletableFuture().join();
// AND: the resulting thing JSON includes the with the updated value:
final JsonObject expectedThingJson = EXPECTED_THING_JSON_PRE_DEFINED_EXTRA.toBuilder()
.remove("attributes/x") // x was not asked for in extra fields
.remove("attributes/pre2") // we don't have the read grant for this field
.build();
softly.assertThat(askResult).isCompletedWithValue(expectedThingJson);
});
}

}
5 changes: 2 additions & 3 deletions .../src/main/java/org/eclipse/ditto/internal/utils/persistence/mongo/config/EventConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public interface EventConfig {

/**
* An enumeration of the known config path expressions and their associated default values for
* {@code SnapshotConfig}.
* {@code EventConfig}.
*/
enum EventConfigValue implements KnownConfigValue {

Expand Down Expand Up @@ -65,7 +65,6 @@ public Object getDefaultValue() {
public String getConfigPath() {
return path;
}

}
}

}
2 changes: 0 additions & 2 deletions ...main/java/org/eclipse/ditto/internal/utils/persistentactors/AbstractPersistenceActor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -585,8 +585,6 @@ private record PersistEventAsync<
E extends EventsourcedEvent<? extends E>,
S extends Jsonifiable.WithFieldSelectorAndPredicate<JsonField>>(E event, BiConsumer<E, S> handler) {}

;

/**
* Persist an event, modify actor state by the event strategy, then invoke the handler.
*
Expand Down
2 changes: 1 addition & 1 deletion ...ent/src/main/java/org/eclipse/ditto/policies/enforcement/config/EntityCreationConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import org.eclipse.ditto.internal.utils.config.KnownConfigValue;

/**
* Provides configuration settings for Concierge entity creation behaviour.
* Provides configuration settings for entity creation behaviour.
*/
@Immutable
public interface EntityCreationConfig {
Expand Down
Loading
Loading