Merge pull request #392 from bsinno/feature/unify-things-search

Unify implementations of things-search across API versions

documentation/src/main/resources/pages/ditto/architecture-services-things-search.md

@@ -34,5 +34,36 @@ Other services can communicate with the things-search service via:
 
 ## Persistence
 
-The things-search service maintains its own persistence in which it stores `Things` in an optimized way in order to 
+The Things-Search service maintains its own persistence in which it stores `Things` in an optimized way in order to 
 provide a full search on arbitrary `Thing` data. 
+
+Things-Search creates the following MongoDB collections:
+
+* `searchThings`: The search index.
+* `searchThingsSyncThings`: A single-document capped collection containing the instant until which `Thing` events are
+indexed for sure; expected to be 30 minutes before the current time.
+* `searchThingsSyncPolicies`: A single-document capped collection containing the instant until which `Policy` events
+are indexed for sure; expected to be 30 minutes before the current time.
+
+## Migration from Ditto 0.9.0-M1
+
+The index schema has changed since Ditto version 0.9.0-M1. Data migration is obligatory to upgrade an existing
+installation running Ditto version 0.9.0-M1 or earlier. Expected duration of data migration is 1/60th of the lifetime
+of the Ditto installation.
+
+1. *After* stopping the cluster of Ditto 0.9.0-M1, drop unnecessary collections:
+```javascript
+db.getCollection('thingEntities').drop();
+db.getCollection('policyBasedSearchIndex').drop();
+db.getCollection('thingsSearchSyncStatePolicies').drop();
+```
+
+2. *Before* starting the upgraded Ditto cluster, write into `searchThingsSyncThings` the timestamp when the Ditto cluster started for the first time:
+```javascript
+var startingTimestamp = new Date(TIMESTAMP-WHEN-DITTO-CLUSTER-STARTED-FOR-THE-FIRST-TIME); // e.g. new Date('2019-01-01T00:00:00.000Z')
+db.getCollection('thingsSearchSyncStateThings').renameCollection('searchThingsSyncThings');
+db.getCollection('searchThingsSyncThings').insert({'ts':startingTimestamp});
+```
+
+3. Start the upgraded Ditto cluster. All `Thing` events persisted after the timestamp in `searchThingsSyncThings` 
+will be indexed in the background.

model/query/pom.xml

@@ -62,6 +62,10 @@
             <plugin>
                 <groupId>com.github.siom79.japicmp</groupId>
                 <artifactId>japicmp-maven-plugin</artifactId>
+                <configuration>
+                    <!-- TODO enable after release -->
+                    <skip>true</skip>
+                </configuration>
             </plugin>
         </plugins>
     </build>

model/query/src/main/java/org/eclipse/ditto/model/query/expression/AttributeExpressionImpl.java

@@ -14,17 +14,17 @@
 
 import static java.util.Objects.requireNonNull;
 
+import java.util.Objects;
+
 import org.eclipse.ditto.model.query.expression.visitors.ExistsFieldExpressionVisitor;
 import org.eclipse.ditto.model.query.expression.visitors.FieldExpressionVisitor;
 import org.eclipse.ditto.model.query.expression.visitors.FilterFieldExpressionVisitor;
-import org.eclipse.ditto.model.query.expression.visitors.PolicyRestrictedFieldExpressionVisitor;
 import org.eclipse.ditto.model.query.expression.visitors.SortFieldExpressionVisitor;
 
 /**
  * Expression for an attribute.
  */
-public class AttributeExpressionImpl implements FilterFieldExpression, SortFieldExpression, ExistsFieldExpression,
-        PolicyRestrictedFieldExpression {
+public class AttributeExpressionImpl implements FilterFieldExpression, SortFieldExpression, ExistsFieldExpression {
 
     private final String key;
 
@@ -64,18 +64,9 @@ public <T> T accept(final FieldExpressionVisitor<T> visitor) {
         return visitor.visitAttribute(key);
     }
 
-    @Override
-    public <T> T acceptPolicyRestrictedVisitor(final PolicyRestrictedFieldExpressionVisitor<T> visitor) {
-        return visitor.visitAttribute(key);
-    }
-
-    @SuppressWarnings("squid:S109")
     @Override
     public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = (prime * result) + ((key == null) ? 0 : key.hashCode());
-        return result;
+        return Objects.hash(key);
     }
 
     @SuppressWarnings("squid:MethodCyclomaticComplexity")

model/query/src/main/java/org/eclipse/ditto/model/query/expression/FeatureExpressionImpl.java

@@ -18,14 +18,11 @@
 
 import org.eclipse.ditto.model.query.expression.visitors.ExistsFieldExpressionVisitor;
 import org.eclipse.ditto.model.query.expression.visitors.FieldExpressionVisitor;
-import org.eclipse.ditto.model.query.expression.visitors.PolicyRestrictedFieldExpressionVisitor;
 
 /**
  * Field expression for features itself with a given feature id.
  */
-public class FeatureExpressionImpl implements ExistsFieldExpression, PolicyRestrictedFieldExpression {
-
-    private static final String DOT = ".";
+public class FeatureExpressionImpl implements ExistsFieldExpression {
 
     private final String featureId;
 
@@ -43,11 +40,6 @@ public <T> T acceptExistsVisitor(final ExistsFieldExpressionVisitor<T> visitor)
         return visitor.visitFeature(featureId);
     }
 
-    @Override
-    public <T> T acceptPolicyRestrictedVisitor(final PolicyRestrictedFieldExpressionVisitor<T> visitor) {
-        return visitor.visitFeature(featureId);
-    }
-
     @Override
     public <T> T accept(final FieldExpressionVisitor<T> visitor) {
         return visitor.visitFeature(featureId);

model/query/src/main/java/org/eclipse/ditto/model/query/expression/FeatureIdPropertyExpressionImpl.java

@@ -19,14 +19,13 @@
 import org.eclipse.ditto.model.query.expression.visitors.ExistsFieldExpressionVisitor;
 import org.eclipse.ditto.model.query.expression.visitors.FieldExpressionVisitor;
 import org.eclipse.ditto.model.query.expression.visitors.FilterFieldExpressionVisitor;
-import org.eclipse.ditto.model.query.expression.visitors.PolicyRestrictedFieldExpressionVisitor;
 import org.eclipse.ditto.model.query.expression.visitors.SortFieldExpressionVisitor;
 
 /**
  * Field expression for feature properties with a given feature id.
  */
-public class FeatureIdPropertyExpressionImpl implements SortFieldExpression, FilterFieldExpression,
-        ExistsFieldExpression, PolicyRestrictedFieldExpression {
+public class FeatureIdPropertyExpressionImpl
+        implements SortFieldExpression, FilterFieldExpression, ExistsFieldExpression {
 
     private final String property;
     private final String featureId;
@@ -42,11 +41,6 @@ public FeatureIdPropertyExpressionImpl(final String featureId, final String prop
         this.featureId = requireNonNull(featureId);
     }
 
-    @Override
-    public <T> T acceptPolicyRestrictedVisitor(final PolicyRestrictedFieldExpressionVisitor<T> visitor) {
-        return visitor.visitFeatureIdProperty(featureId, property);
-    }
-
     @Override
     public <T> T acceptSortVisitor(final SortFieldExpressionVisitor<T> visitor) {
         return visitor.visitFeatureIdProperty(featureId, property);

model/query/src/main/java/org/eclipse/ditto/model/query/expression/FieldExpression.java

@@ -12,7 +12,6 @@
  */
 package org.eclipse.ditto.model.query.expression;
 
-
 import org.eclipse.ditto.model.query.criteria.CriteriaFactory;
 import org.eclipse.ditto.model.query.expression.visitors.FieldExpressionVisitor;
 

model/query/src/main/java/org/eclipse/ditto/model/query/expression/FieldExpressionUtil.java

@@ -33,11 +33,6 @@ public final class FieldExpressionUtil {
      */
     public static final String FIELD_NAME_THING_ID = "thingId";
 
-    /**
-     * Owner field name.
-     */
-    public static final String FIELD_NAME_OWNER = "owner";
-
     /**
      * namespace field to be saved in field expressions.
      */
@@ -112,14 +107,11 @@ public static String addAttributesPrefix(final String fieldName) {
      */
     public static final class FeatureField {
 
-        private static final Pattern FIELD_NAME_FEATURE_PATTERN1 = Pattern.compile("^features/" +
-                "(?<featureId>[^\\*][^/]*)/properties/(?<property>.+)");
+        private static final Pattern FIELD_NAME_FEATURE_PATTERN1 =
+                Pattern.compile("^features/(?<featureId>[^/]++)/properties/(?<property>.+)");
 
         private static final Pattern FIELD_NAME_FEATURE_PATTERN2 =
-                Pattern.compile("^features/(?<featureId>[^\\*][^/]*)");
-
-        private static final Pattern FIELD_NAME_FEATURE_PATTERN3 =
-                Pattern.compile("^features/\\*/properties/(?<property>.+)");
+                Pattern.compile("^features/(?<featureId>[^/]++)");
 
         private final boolean matches;
         private final String featureId;
@@ -140,17 +132,9 @@ private FeatureField(final String fieldName) {
                     this.featureId = matcher.group("featureId");
                     this.property = null;
                 } else {
-                    matcher = FIELD_NAME_FEATURE_PATTERN3.matcher(fieldName);
-
-                    if (matcher.matches()) {
-                        this.matches = true;
-                        this.featureId = null;
-                        this.property = matcher.group("property");
-                    } else {
-                        this.matches = false;
-                        this.featureId = null;
-                        this.property = null;
-                    }
+                    this.matches = false;
+                    this.featureId = null;
+                    this.property = null;
                 }
             }
         }