diff --git a/api/src/main/java/run/halo/app/extension/ListOptions.java b/api/src/main/java/run/halo/app/extension/ListOptions.java index efb0a286f2..c52222d305 100644 --- a/api/src/main/java/run/halo/app/extension/ListOptions.java +++ b/api/src/main/java/run/halo/app/extension/ListOptions.java @@ -10,4 +10,19 @@ public class ListOptions { private LabelSelector labelSelector; private FieldSelector fieldSelector; -} \ No newline at end of file + + @Override + public String toString() { + var sb = new StringBuilder(); + if (fieldSelector != null) { + sb.append("fieldSelector: ").append(fieldSelector.query()); + } + if (labelSelector != null) { + if (!sb.isEmpty()) { + sb.append(", "); + } + sb.append("labelSelector: ").append(labelSelector); + } + return sb.toString(); + } +} diff --git a/api/src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java b/api/src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java index 5792a37291..bc5e14ab29 100644 --- a/api/src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java +++ b/api/src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java @@ -2,6 +2,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Sort; import run.halo.app.extension.Extension; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.ExtensionMatcher; @@ -58,7 +59,7 @@ public void start() { listOptions.setFieldSelector(listMatcher.getFieldSelector()); listOptions.setLabelSelector(listMatcher.getLabelSelector()); } - indexedQueryEngine.retrieveAll(type, listOptions) + indexedQueryEngine.retrieveAll(type, listOptions, Sort.by("metadata.creationTimestamp")) .forEach(name -> watcher.onAdd(new Request(name))); } client.watch(this.watcher); diff --git a/application/src/main/java/run/halo/app/extension/index/IndexDescriptor.java b/api/src/main/java/run/halo/app/extension/index/IndexDescriptor.java similarity index 83% rename from application/src/main/java/run/halo/app/extension/index/IndexDescriptor.java rename to api/src/main/java/run/halo/app/extension/index/IndexDescriptor.java index 497eea145c..086ffe8d23 100644 --- a/application/src/main/java/run/halo/app/extension/index/IndexDescriptor.java +++ b/api/src/main/java/run/halo/app/extension/index/IndexDescriptor.java @@ -10,7 +10,7 @@ public class IndexDescriptor { private final IndexSpec spec; /** - * Record whether the index is ready, managed by {@link IndexBuilder}. + * Record whether the index is ready, managed by {@code IndexBuilder}. */ private boolean ready; diff --git a/application/src/main/java/run/halo/app/extension/index/IndexEntry.java b/api/src/main/java/run/halo/app/extension/index/IndexEntry.java similarity index 78% rename from application/src/main/java/run/halo/app/extension/index/IndexEntry.java rename to api/src/main/java/run/halo/app/extension/index/IndexEntry.java index ececd000a3..d6ec03e5cf 100644 --- a/application/src/main/java/run/halo/app/extension/index/IndexEntry.java +++ b/api/src/main/java/run/halo/app/extension/index/IndexEntry.java @@ -3,7 +3,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.NavigableSet; import run.halo.app.extension.Metadata; /** @@ -34,7 +34,7 @@ public interface IndexEntry { /** - * Acquires the read lock for reading such as {@link #getByIndexKey(String)}, + * Acquires the read lock for reading such as {@link #getObjectNamesBy(String)}, * {@link #entries()}, {@link #indexedKeys()}, because the returned result set of these * methods is not immutable. */ @@ -87,7 +87,7 @@ public interface IndexEntry { * * @return distinct indexed keys of this entry. */ - Set<String> indexedKeys(); + NavigableSet<String> indexedKeys(); /** * <p>Returns the entries of this entry in order.</p> @@ -99,19 +99,34 @@ public interface IndexEntry { Collection<Map.Entry<String, String>> entries(); /** - * Returns the immutable entries of this entry in order, it is safe to modify the returned - * result, but extra cost is made. - * - * @return immutable entries of this entry. + * <p>Returns the position of the object name in the indexed attribute value mapping for + * sorting.</p> + * For example: + * <pre> + * metadata.name | field1 + * ------------- | ------ + * foo | 1 + * bar | 2 + * baz | 2 + * </pre> + * "field1" is the indexed attribute, and the position of the object name in the indexed + * attribute + * value mapping for sorting is: + * <pre> + * foo -> 0 + * bar -> 1 + * baz -> 1 + * </pre> + * "bar" and "baz" have the same value, so they have the same position. */ - Collection<Map.Entry<String, String>> immutableEntries(); + Map<String, Integer> getIdPositionMap(); /** * Returns the object names of this entry in order. * * @return object names of this entry. */ - List<String> getByIndexKey(String indexKey); + List<String> getObjectNamesBy(String indexKey); void clear(); } diff --git a/api/src/main/java/run/halo/app/extension/index/IndexEntryOperator.java b/api/src/main/java/run/halo/app/extension/index/IndexEntryOperator.java new file mode 100644 index 0000000000..5f3360b024 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/IndexEntryOperator.java @@ -0,0 +1,55 @@ +package run.halo.app.extension.index; + +import java.util.Collection; +import java.util.NavigableSet; +import java.util.Set; + +public interface IndexEntryOperator { + + /** + * Search all values that key less than the target key. + * + * @param key target key + * @param orEqual whether to include the value of the target key + * @return object names that key less than the target key + */ + NavigableSet<String> lessThan(String key, boolean orEqual); + + /** + * Search all values that key greater than the target key. + * + * @param key target key + * @param orEqual whether to include the value of the target key + * @return object names that key greater than the target key + */ + NavigableSet<String> greaterThan(String key, boolean orEqual); + + /** + * Search all values that key in the range of [start, end]. + * + * @param start start key + * @param end end key + * @param startInclusive whether to include the value of the start key + * @param endInclusive whether to include the value of the end key + * @return object names that key in the range of [start, end] + */ + NavigableSet<String> range(String start, String end, boolean startInclusive, + boolean endInclusive); + + /** + * Find all values that key equals to the target key. + * + * @param key target key + * @return object names that key equals to the target key + */ + NavigableSet<String> find(String key); + + NavigableSet<String> findIn(Collection<String> keys); + + /** + * Get all values in the index entry. + * + * @return a set of all object names + */ + Set<String> getValues(); +} diff --git a/api/src/main/java/run/halo/app/extension/index/IndexEntryOperatorImpl.java b/api/src/main/java/run/halo/app/extension/index/IndexEntryOperatorImpl.java new file mode 100644 index 0000000000..8570e31120 --- /dev/null +++ b/api/src/main/java/run/halo/app/extension/index/IndexEntryOperatorImpl.java @@ -0,0 +1,112 @@ +package run.halo.app.extension.index; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; +import java.util.TreeSet; +import org.springframework.util.Assert; + +public class IndexEntryOperatorImpl implements IndexEntryOperator { + private final IndexEntry indexEntry; + + public IndexEntryOperatorImpl(IndexEntry indexEntry) { + this.indexEntry = indexEntry; + } + + private static NavigableSet<String> createNavigableSet() { + return new TreeSet<>(KeyComparator.INSTANCE); + } + + @Override + public NavigableSet<String> lessThan(String key, boolean orEqual) { + Assert.notNull(key, "Key must not be null."); + indexEntry.acquireReadLock(); + try { + var navigableIndexedKeys = indexEntry.indexedKeys(); + var headSetKeys = navigableIndexedKeys.headSet(key, orEqual); + return findIn(headSetKeys); + } finally { + indexEntry.releaseReadLock(); + } + } + + @Override + public NavigableSet<String> greaterThan(String key, boolean orEqual) { + Assert.notNull(key, "Key must not be null."); + indexEntry.acquireReadLock(); + try { + var navigableIndexedKeys = indexEntry.indexedKeys(); + var tailSetKeys = navigableIndexedKeys.tailSet(key, orEqual); + return findIn(tailSetKeys); + } finally { + indexEntry.releaseReadLock(); + } + } + + @Override + public NavigableSet<String> range(String start, String end, boolean startInclusive, + boolean endInclusive) { + Assert.notNull(start, "The start must not be null."); + Assert.notNull(end, "The end must not be null."); + indexEntry.acquireReadLock(); + try { + var navigableIndexedKeys = indexEntry.indexedKeys(); + var tailSetKeys = navigableIndexedKeys.subSet(start, startInclusive, end, endInclusive); + return findIn(tailSetKeys); + } finally { + indexEntry.releaseReadLock(); + } + } + + @Override + public NavigableSet<String> find(String key) { + Assert.notNull(key, "The key must not be null."); + indexEntry.acquireReadLock(); + try { + var resultSet = createNavigableSet(); + var result = indexEntry.getObjectNamesBy(key); + if (result != null) { + resultSet.addAll(result); + } + return resultSet; + } finally { + indexEntry.releaseReadLock(); + } + } + + @Override + public NavigableSet<String> findIn(Collection<String> keys) { + if (keys == null || keys.isEmpty()) { + return createNavigableSet(); + } + indexEntry.acquireReadLock(); + try { + var keysToSearch = new HashSet<>(keys); + var resultSet = createNavigableSet(); + for (var entry : indexEntry.entries()) { + if (keysToSearch.contains(entry.getKey())) { + resultSet.add(entry.getValue()); + } + } + return resultSet; + } finally { + indexEntry.releaseReadLock(); + } + } + + @Override + public Set<String> getValues() { + indexEntry.acquireReadLock(); + try { + Set<String> uniqueValues = new HashSet<>(); + for (Map.Entry<String, String> entry : indexEntry.entries()) { + uniqueValues.add(entry.getValue()); + } + return uniqueValues; + } finally { + indexEntry.releaseReadLock(); + } + } +} diff --git a/api/src/main/java/run/halo/app/extension/index/IndexedQueryEngine.java b/api/src/main/java/run/halo/app/extension/index/IndexedQueryEngine.java index 54ac43b98e..b6c46239a3 100644 --- a/api/src/main/java/run/halo/app/extension/index/IndexedQueryEngine.java +++ b/api/src/main/java/run/halo/app/extension/index/IndexedQueryEngine.java @@ -1,6 +1,7 @@ package run.halo.app.extension.index; import java.util.List; +import org.springframework.data.domain.Sort; import run.halo.app.extension.GroupVersionKind; import run.halo.app.extension.ListOptions; import run.halo.app.extension.ListResult; @@ -36,7 +37,8 @@ public interface IndexedQueryEngine { * * @param type the type of the object must exist in {@link run.halo.app.extension.SchemeManager} * @param options the list options to use for retrieving the object records + * @param sort the sort to use for retrieving the object records * @return a collection of {@link Metadata#getName()} */ - List<String> retrieveAll(GroupVersionKind type, ListOptions options); + List<String> retrieveAll(GroupVersionKind type, ListOptions options, Sort sort); } diff --git a/application/src/main/java/run/halo/app/extension/index/Indexer.java b/api/src/main/java/run/halo/app/extension/index/Indexer.java similarity index 84% rename from application/src/main/java/run/halo/app/extension/index/Indexer.java rename to api/src/main/java/run/halo/app/extension/index/Indexer.java index 78702f3c75..e92f3360de 100644 --- a/application/src/main/java/run/halo/app/extension/index/Indexer.java +++ b/api/src/main/java/run/halo/app/extension/index/Indexer.java @@ -2,6 +2,7 @@ import java.util.Iterator; import java.util.function.Function; +import org.springframework.lang.NonNull; import run.halo.app.extension.Extension; /** @@ -20,7 +21,7 @@ public interface Indexer { /** * <p>Index the specified {@link Extension} by {@link IndexDescriptor}s.</p> * <p>First, the {@link Indexer} will index the {@link Extension} by the - * {@link IndexDescriptor}s and record the index entries to {@link IndexerTransaction} and + * {@link IndexDescriptor}s and record the index entries to {@code IndexerTransaction} and * commit the transaction, if any error occurs, the transaction will be rollback to keep the * {@link Indexer} consistent.</p> * @@ -33,7 +34,7 @@ public interface Indexer { * <p>Update indexes for the specified {@link Extension} by {@link IndexDescriptor}s.</p> * <p>First, the {@link Indexer} will remove the index entries of the {@link Extension} by * the old {@link IndexDescriptor}s and reindex the {@link Extension} to generate change logs - * to {@link IndexerTransaction} and commit the transaction, if any error occurs, the + * to {@code IndexerTransaction} and commit the transaction, if any error occurs, the * transaction will be rollback to keep the {@link Indexer} consistent.</p> * * @param extension the {@link Extension} to be updated @@ -73,11 +74,21 @@ public interface Indexer { */ void removeIndexRecords(Function<IndexDescriptor, Boolean> matchFn); + /** + * <p>Get the {@link IndexEntry} by index name if found and ready.</p> + * + * @param name an index name + * @return the {@link IndexEntry} if found + * @throws IllegalArgumentException if the index name is not found or the index is not ready + */ + @NonNull + IndexEntry getIndexEntry(String name); + /** * <p>Gets an iterator over all the ready {@link IndexEntry}s, in no particular order.</p> * * @return an iterator over all the ready {@link IndexEntry}s - * @link {@link IndexDescriptor#isReady()} + * @see IndexDescriptor#isReady() */ Iterator<IndexEntry> readyIndexesIterator(); @@ -85,7 +96,11 @@ public interface Indexer { * <p>Gets an iterator over all the {@link IndexEntry}s, in no particular order.</p> * * @return an iterator over all the {@link IndexEntry}s - * @link {@link IndexDescriptor#isReady()} + * @see IndexDescriptor#isReady() */ Iterator<IndexEntry> allIndexesIterator(); + + void acquireReadLock(); + + void releaseReadLock(); } diff --git a/api/src/main/java/run/halo/app/extension/index/query/All.java b/api/src/main/java/run/halo/app/extension/index/query/All.java index 541d730207..46b47b413f 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/All.java +++ b/api/src/main/java/run/halo/app/extension/index/query/All.java @@ -10,6 +10,11 @@ public All(String fieldName) { @Override public NavigableSet<String> matches(QueryIndexView indexView) { - return indexView.getAllIdsForField(fieldName); + return indexView.getIdsForField(fieldName); + } + + @Override + public String toString() { + return fieldName + " != null"; } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/And.java b/api/src/main/java/run/halo/app/extension/index/query/And.java index 8a80b319f2..b47e8f0c87 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/And.java +++ b/api/src/main/java/run/halo/app/extension/index/query/And.java @@ -3,6 +3,7 @@ import com.google.common.collect.Sets; import java.util.Collection; import java.util.NavigableSet; +import java.util.stream.Collectors; public class And extends LogicalQuery { @@ -33,4 +34,10 @@ public NavigableSet<String> matches(QueryIndexView indexView) { } return resultSet == null ? Sets.newTreeSet() : resultSet; } + + @Override + public String toString() { + return "(" + childQueries.stream().map(Query::toString) + .collect(Collectors.joining(" AND ")) + ")"; + } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/Between.java b/api/src/main/java/run/halo/app/extension/index/query/Between.java index 829770b5d8..e512a3f009 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/Between.java +++ b/api/src/main/java/run/halo/app/extension/index/query/Between.java @@ -1,6 +1,5 @@ package run.halo.app.extension.index.query; -import com.google.common.collect.Sets; import java.util.NavigableSet; public class Between extends SimpleQuery { @@ -19,17 +18,14 @@ public Between(String fieldName, String lowerValue, boolean lowerInclusive, this.upperInclusive = upperInclusive; } - @Override public NavigableSet<String> matches(QueryIndexView indexView) { - NavigableSet<String> allValues = indexView.getAllValuesForField(fieldName); - // get all values in the specified range - var subSet = allValues.subSet(lowerValue, lowerInclusive, upperValue, upperInclusive); + return indexView.between(fieldName, lowerValue, lowerInclusive, upperValue, upperInclusive); + } - var resultSet = Sets.<String>newTreeSet(); - for (String val : subSet) { - resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); - } - return resultSet; + @Override + public String toString() { + return fieldName + " BETWEEN " + (lowerInclusive ? "[" : "(") + lowerValue + ", " + + upperValue + (upperInclusive ? "]" : ")"); } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/EqualQuery.java b/api/src/main/java/run/halo/app/extension/index/query/EqualQuery.java index 6f23f901b9..0ce826adc8 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/EqualQuery.java +++ b/api/src/main/java/run/halo/app/extension/index/query/EqualQuery.java @@ -17,16 +17,16 @@ public EqualQuery(String fieldName, String value, boolean isFieldRef) { @Override public NavigableSet<String> matches(QueryIndexView indexView) { if (isFieldRef) { - return resultSetForRefValue(indexView); + return indexView.findMatchingIdsWithEqualValues(fieldName, value); } - return resultSetForExactValue(indexView); + return indexView.findIds(fieldName, value); } - private NavigableSet<String> resultSetForRefValue(QueryIndexView indexView) { - return indexView.findIdsForFieldValueEqual(fieldName, value); - } - - private NavigableSet<String> resultSetForExactValue(QueryIndexView indexView) { - return indexView.getIdsForFieldValue(fieldName, value); + @Override + public String toString() { + if (isFieldRef) { + return fieldName + " = " + value; + } + return fieldName + " = '" + value + "'"; } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/GreaterThanQuery.java b/api/src/main/java/run/halo/app/extension/index/query/GreaterThanQuery.java index b8670ee4d4..101286d168 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/GreaterThanQuery.java +++ b/api/src/main/java/run/halo/app/extension/index/query/GreaterThanQuery.java @@ -1,6 +1,5 @@ package run.halo.app.extension.index.query; -import com.google.common.collect.Sets; import java.util.NavigableSet; public class GreaterThanQuery extends SimpleQuery { @@ -18,24 +17,15 @@ public GreaterThanQuery(String fieldName, String value, boolean orEqual, boolean @Override public NavigableSet<String> matches(QueryIndexView indexView) { if (isFieldRef) { - return resultSetForRefValue(indexView); + return indexView.findMatchingIdsWithGreaterValues(fieldName, value, orEqual); } - return resultSetForExtractValue(indexView); + return indexView.findIdsGreaterThan(fieldName, value, orEqual); } - private NavigableSet<String> resultSetForRefValue(QueryIndexView indexView) { - return indexView.findIdsForFieldValueGreaterThan(fieldName, value, orEqual); - } - - private NavigableSet<String> resultSetForExtractValue(QueryIndexView indexView) { - var resultSet = Sets.<String>newTreeSet(); - var allValues = indexView.getAllValuesForField(fieldName); - NavigableSet<String> tailSet = - orEqual ? allValues.tailSet(value, true) : allValues.tailSet(value, false); - - for (String val : tailSet) { - resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); - } - return resultSet; + @Override + public String toString() { + return fieldName + + (orEqual ? " >= " : " > ") + + (isFieldRef ? value : "'" + value + "'"); } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/InQuery.java b/api/src/main/java/run/halo/app/extension/index/query/InQuery.java index 5c743190e5..b9aa026830 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/InQuery.java +++ b/api/src/main/java/run/halo/app/extension/index/query/InQuery.java @@ -1,8 +1,9 @@ package run.halo.app.extension.index.query; -import com.google.common.collect.Sets; import java.util.NavigableSet; import java.util.Set; +import java.util.stream.Collectors; +import run.halo.app.extension.index.IndexEntryOperatorImpl; public class InQuery extends SimpleQuery { private final Set<String> values; @@ -14,10 +15,15 @@ public InQuery(String columnName, Set<String> values) { @Override public NavigableSet<String> matches(QueryIndexView indexView) { - NavigableSet<String> resultSet = Sets.newTreeSet(); - for (String val : values) { - resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); - } - return resultSet; + var indexEntry = indexView.getIndexEntry(fieldName); + var operator = new IndexEntryOperatorImpl(indexEntry); + return operator.findIn(values); + } + + @Override + public String toString() { + return fieldName + " IN (" + values.stream() + .map(value -> "'" + value + "'") + .collect(Collectors.joining(", ")) + ")"; } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/IsNotNull.java b/api/src/main/java/run/halo/app/extension/index/query/IsNotNull.java index 114da9562f..1c66133d5b 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/IsNotNull.java +++ b/api/src/main/java/run/halo/app/extension/index/query/IsNotNull.java @@ -10,6 +10,11 @@ protected IsNotNull(String fieldName) { @Override public NavigableSet<String> matches(QueryIndexView indexView) { - return indexView.getAllIdsForField(fieldName); + return indexView.getIdsForField(fieldName); + } + + @Override + public String toString() { + return fieldName + " IS NOT NULL"; } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/IsNull.java b/api/src/main/java/run/halo/app/extension/index/query/IsNull.java index 5b09e8f8e2..d74040c9f8 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/IsNull.java +++ b/api/src/main/java/run/halo/app/extension/index/query/IsNull.java @@ -10,9 +10,19 @@ protected IsNull(String fieldName) { @Override public NavigableSet<String> matches(QueryIndexView indexView) { - var allIds = indexView.getAllIds(); - var idsForField = indexView.getAllIdsForField(fieldName); - allIds.removeAll(idsForField); - return allIds; + indexView.acquireReadLock(); + try { + var allIds = indexView.getAllIds(); + var idsForNonNullValue = indexView.getIdsForField(fieldName); + allIds.removeAll(idsForNonNullValue); + return allIds; + } finally { + indexView.releaseReadLock(); + } + } + + @Override + public String toString() { + return fieldName + " IS NULL"; } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/LessThanQuery.java b/api/src/main/java/run/halo/app/extension/index/query/LessThanQuery.java index 4f882cb648..d7168e8aed 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/LessThanQuery.java +++ b/api/src/main/java/run/halo/app/extension/index/query/LessThanQuery.java @@ -1,6 +1,5 @@ package run.halo.app.extension.index.query; -import com.google.common.collect.Sets; import java.util.NavigableSet; public class LessThanQuery extends SimpleQuery { @@ -18,24 +17,15 @@ public LessThanQuery(String fieldName, String value, boolean orEqual, boolean is @Override public NavigableSet<String> matches(QueryIndexView indexView) { if (isFieldRef) { - return resultSetForRefValue(indexView); + return indexView.findMatchingIdsWithSmallerValues(fieldName, value, orEqual); } - return resultSetForExactValue(indexView); + return indexView.findIdsLessThan(fieldName, value, orEqual); } - private NavigableSet<String> resultSetForRefValue(QueryIndexView indexView) { - return indexView.findIdsForFieldValueLessThan(fieldName, value, orEqual); - } - - private NavigableSet<String> resultSetForExactValue(QueryIndexView indexView) { - var resultSet = Sets.<String>newTreeSet(); - var allValues = indexView.getAllValuesForField(fieldName); - var headSet = orEqual ? allValues.headSet(value, true) - : allValues.headSet(value, false); - - for (String val : headSet) { - resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); - } - return resultSet; + @Override + public String toString() { + return fieldName + + (orEqual ? " <= " : " < ") + + (isFieldRef ? value : "'" + value + "'"); } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/Not.java b/api/src/main/java/run/halo/app/extension/index/query/Not.java index 908507ff9c..98d0e29fcd 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/Not.java +++ b/api/src/main/java/run/halo/app/extension/index/query/Not.java @@ -24,4 +24,9 @@ public NavigableSet<String> matches(QueryIndexView indexView) { allIds.removeAll(negatedResult); return allIds; } + + @Override + public String toString() { + return "NOT (" + negatedQuery + ")"; + } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/NotEqual.java b/api/src/main/java/run/halo/app/extension/index/query/NotEqual.java index baa5f2ed7a..3ffa33ff03 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/NotEqual.java +++ b/api/src/main/java/run/halo/app/extension/index/query/NotEqual.java @@ -1,6 +1,5 @@ package run.halo.app.extension.index.query; -import com.google.common.collect.Sets; import java.util.NavigableSet; import org.springframework.util.Assert; @@ -19,15 +18,19 @@ public NotEqual(String fieldName, String value, boolean isFieldRef) { @Override public NavigableSet<String> matches(QueryIndexView indexView) { - var names = equalQuery.matches(indexView); - var allNames = indexView.getAllIdsForField(fieldName); - - var resultSet = Sets.<String>newTreeSet(); - for (String name : allNames) { - if (!names.contains(name)) { - resultSet.add(name); - } + indexView.acquireReadLock(); + try { + NavigableSet<String> equalNames = equalQuery.matches(indexView); + NavigableSet<String> allNames = indexView.getIdsForField(fieldName); + allNames.removeAll(equalNames); + return allNames; + } finally { + indexView.releaseReadLock(); } - return resultSet; + } + + @Override + public String toString() { + return fieldName + " != " + (isFieldRef ? value : "'" + value + "'"); } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/Or.java b/api/src/main/java/run/halo/app/extension/index/query/Or.java index ec79270ca9..c8579c54fa 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/Or.java +++ b/api/src/main/java/run/halo/app/extension/index/query/Or.java @@ -3,6 +3,7 @@ import com.google.common.collect.Sets; import java.util.Collection; import java.util.NavigableSet; +import java.util.stream.Collectors; public class Or extends LogicalQuery { @@ -18,4 +19,10 @@ public NavigableSet<String> matches(QueryIndexView indexView) { } return resultSet; } + + @Override + public String toString() { + return "(" + childQueries.stream().map(Query::toString) + .collect(Collectors.joining(" OR ")) + ")"; + } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/QueryIndexView.java b/api/src/main/java/run/halo/app/extension/index/query/QueryIndexView.java index 18a8a8fc3d..eb6a915426 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/QueryIndexView.java +++ b/api/src/main/java/run/halo/app/extension/index/query/QueryIndexView.java @@ -4,6 +4,7 @@ import java.util.NavigableSet; import org.springframework.data.domain.Sort; import run.halo.app.extension.Metadata; +import run.halo.app.extension.index.IndexEntry; import run.halo.app.extension.index.IndexSpec; /** @@ -19,6 +20,7 @@ * @since 2.12.0 */ public interface QueryIndexView { + /** * Gets all object ids for a given field name and field value. * @@ -27,16 +29,7 @@ public interface QueryIndexView { * @return all indexed object ids associated with the given field name and field value * @throws IllegalArgumentException if the field name is not indexed */ - NavigableSet<String> getIdsForFieldValue(String fieldName, String fieldValue); - - /** - * Gets all field values for a given field name. - * - * @param fieldName the field name - * @return all field values for the given field name - * @throws IllegalArgumentException if the field name is not indexed - */ - NavigableSet<String> getAllValuesForField(String fieldName); + NavigableSet<String> findIds(String fieldName, String fieldValue); /** * Gets all object ids for a given field name without null cells. @@ -45,7 +38,7 @@ public interface QueryIndexView { * @return all indexed object ids for the given field name * @throws IllegalArgumentException if the field name is not indexed */ - NavigableSet<String> getAllIdsForField(String fieldName); + NavigableSet<String> getIdsForField(String fieldName); /** * Gets all object ids in this view. @@ -54,15 +47,97 @@ public interface QueryIndexView { */ NavigableSet<String> getAllIds(); - NavigableSet<String> findIdsForFieldValueEqual(String fieldName1, String fieldName2); + /** + * <p>Finds and returns a set of unique identifiers (metadata.name) for entries that have + * matching values across two fields and where the values are equal.</p> + * For example: + * <pre> + * metadata.name | field1 | field2 + * ------------- | ------ | ------ + * foo | 1 | 1 + * bar | 2 | 3 + * baz | 3 | 3 + * </pre> + * <code>findMatchingIdsWithEqualValues("field1", "field2")</code> would return ["foo","baz"] + * + * @see #findMatchingIdsWithGreaterValues(String, String, boolean) + * @see #findMatchingIdsWithSmallerValues(String, String, boolean) + */ + NavigableSet<String> findMatchingIdsWithEqualValues(String fieldName1, String fieldName2); - NavigableSet<String> findIdsForFieldValueGreaterThan(String fieldName1, String fieldName2, + /** + * <p>Finds and returns a set of unique identifiers (metadata.name) for entries that have + * matching values across two fields, but where the value associated with fieldName1 is + * greater than the value associated with fieldName2.</p> + * For example: + * <pre> + * metadata.name | field1 | field2 + * ------------- | ------ | ------ + * foo | 1 | 1 + * bar | 2 | 3 + * baz | 3 | 3 + * qux | 4 | 2 + * </pre> + * <p><code>findMatchingIdsWithGreaterValues("field1", "field2")</code>would return ["qux"]</p> + * <p><code>findMatchingIdsWithGreaterValues("field2", "field1")</code>would return ["bar"]</p> + * <p><code>findMatchingIdsWithGreaterValues("field1", "field2", true)</code>would return + * ["foo","baz","qux"]</p> + * + * @param fieldName1 The field name whose values are compared as the larger values. + * @param fieldName2 The field name whose values are compared as the smaller values. + * @param orEqual whether to include equal values + * @return A result set of ids where the entries in fieldName1 have greater values than those + * in fieldName2 for entries that have the same id across both fields + */ + NavigableSet<String> findMatchingIdsWithGreaterValues(String fieldName1, String fieldName2, boolean orEqual); - NavigableSet<String> findIdsForFieldValueLessThan(String fieldName1, String fieldName2, + NavigableSet<String> findIdsGreaterThan(String fieldName, String fieldValue, boolean orEqual); + + /** + * <p>Finds and returns a set of unique identifiers (metadata.name) for entries that have + * matching values across two fields, but where the value associated with fieldName1 is + * less than the value associated with fieldName2.</p> + * For example: + * <pre> + * metadata.name | field1 | field2 + * ------------- | ------ | ------ + * foo | 1 | 1 + * bar | 2 | 3 + * baz | 3 | 3 + * qux | 4 | 2 + * </pre> + * <p><code>findMatchingIdsWithSmallerValues("field1", "field2")</code> would return ["bar"]</p> + * <p><code>findMatchingIdsWithSmallerValues("field2", "field1")</code> would return ["qux"]</p> + * <p><code>findMatchingIdsWithSmallerValues("field1", "field2", true)</code> would return + * ["foo","bar","baz"]</p> + * + * @param fieldName1 The field name whose values are compared as the smaller values. + * @param fieldName2 The field name whose values are compared as the larger values. + * @param orEqual whether to include equal values + * @return A result set of ids where the entries in fieldName1 have smaller values than those + * in fieldName2 for entries that have the same id across both fields + */ + NavigableSet<String> findMatchingIdsWithSmallerValues(String fieldName1, String fieldName2, boolean orEqual); - void removeByIdNotIn(NavigableSet<String> ids); + NavigableSet<String> findIdsLessThan(String fieldName, String fieldValue, boolean orEqual); + + NavigableSet<String> between(String fieldName, String lowerValue, boolean lowerInclusive, + String upperValue, boolean upperInclusive); + + List<String> sortBy(NavigableSet<String> resultSet, Sort sort); + + IndexEntry getIndexEntry(String fieldName); + + /** + * Acquire a read lock on the indexer. + * if you need to operate on more than one {@code IndexEntry} at the same time, you need to + * lock first. + * + * @see #getIndexEntry(String) + */ + void acquireReadLock(); - List<String> sortBy(Sort sort); + void releaseReadLock(); } diff --git a/api/src/main/java/run/halo/app/extension/index/query/QueryIndexViewImpl.java b/api/src/main/java/run/halo/app/extension/index/query/QueryIndexViewImpl.java index d85ce935ff..e5456a9b7c 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/QueryIndexViewImpl.java +++ b/api/src/main/java/run/halo/app/extension/index/query/QueryIndexViewImpl.java @@ -1,352 +1,230 @@ package run.halo.app.extension.index.query; -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Sets; -import com.google.common.collect.Table; import java.util.ArrayList; -import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.NavigableSet; -import java.util.Optional; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; +import java.util.function.BiPredicate; import org.springframework.data.domain.Sort; -import org.springframework.lang.Nullable; -import org.springframework.util.CollectionUtils; +import run.halo.app.extension.index.IndexEntry; +import run.halo.app.extension.index.IndexEntryOperator; +import run.halo.app.extension.index.IndexEntryOperatorImpl; +import run.halo.app.extension.index.Indexer; import run.halo.app.extension.index.KeyComparator; /** * A default implementation for {@link run.halo.app.extension.index.query.QueryIndexView}. * * @author guqing - * @since 2.12.0 + * @since 2.17.0 */ public class QueryIndexViewImpl implements QueryIndexView { - private final Lock lock = new ReentrantLock(); - private final Set<String> fieldNames; - private final Table<String, String, NavigableSet<String>> orderedMatches; + + public static final String PRIMARY_INDEX_NAME = "metadata.name"; + + private final Indexer indexer; /** - * Creates a new {@link QueryIndexViewImpl} for the given {@link Map} of index entries. + * Construct a new {@link QueryIndexViewImpl} with the given {@link Indexer}. * - * @param indexEntries index entries from indexer to create the view for. + * @throws IllegalArgumentException if the primary index does not exist */ - public QueryIndexViewImpl(Map<String, Collection<Map.Entry<String, String>>> indexEntries) { - this.fieldNames = new HashSet<>(); - this.orderedMatches = HashBasedTable.create(); - for (var entry : indexEntries.entrySet()) { - String fieldName = entry.getKey(); - this.fieldNames.add(fieldName); - for (var fieldEntry : entry.getValue()) { - var id = fieldEntry.getValue(); - var fieldValue = fieldEntry.getKey(); - var columnValue = this.orderedMatches.get(id, fieldName); - if (columnValue == null) { - columnValue = Sets.newTreeSet(); - this.orderedMatches.put(id, fieldName, columnValue); - } - columnValue.add(fieldValue); - } - } + public QueryIndexViewImpl(Indexer indexer) { + // check if primary index exists + indexer.getIndexEntry(PRIMARY_INDEX_NAME); + this.indexer = indexer; } @Override - public NavigableSet<String> getIdsForFieldValue(String fieldName, String fieldValue) { - lock.lock(); - try { - checkFieldNameIndexed(fieldName); - var result = new TreeSet<String>(); - for (var cell : orderedMatches.cellSet()) { - if (cell.getColumnKey().equals(fieldName) && cell.getValue().contains(fieldValue)) { - result.add(cell.getRowKey()); - } - } - return result; - } finally { - lock.unlock(); - } + public NavigableSet<String> findIds(String fieldName, String fieldValue) { + var operator = getEntryOperator(fieldName); + return operator.find(fieldValue); } @Override - public NavigableSet<String> getAllValuesForField(String fieldName) { - lock.lock(); - try { - checkFieldNameIndexed(fieldName); - var result = Sets.<String>newTreeSet(); - for (var cell : orderedMatches.cellSet()) { - if (cell.getColumnKey().equals(fieldName)) { - result.addAll(cell.getValue()); - } - } - return result; - } finally { - lock.unlock(); - } + public NavigableSet<String> getIdsForField(String fieldName) { + var operator = getEntryOperator(fieldName); + return new TreeSet<>(operator.getValues()); } @Override - public NavigableSet<String> getAllIdsForField(String fieldName) { - lock.lock(); - try { - checkFieldNameIndexed(fieldName); - NavigableSet<String> ids = new TreeSet<>(); - // iterate over the table and collect all IDs associated with the given field name - for (var cell : orderedMatches.cellSet()) { - if (cell.getColumnKey().equals(fieldName)) { - ids.add(cell.getRowKey()); - } - } - return ids; - } finally { - lock.unlock(); - } + public NavigableSet<String> getAllIds() { + return new TreeSet<>(allIds()); } @Override - public NavigableSet<String> getAllIds() { - lock.lock(); + public NavigableSet<String> findMatchingIdsWithEqualValues(String fieldName1, + String fieldName2) { + indexer.acquireReadLock(); try { - return new TreeSet<>(orderedMatches.rowKeySet()); + return findIdsWithKeyComparator(fieldName1, fieldName2, (k1, k2) -> { + var compare = KeyComparator.INSTANCE.compare(k1, k2); + return compare == 0; + }); } finally { - lock.unlock(); + indexer.releaseReadLock(); } } @Override - public NavigableSet<String> findIdsForFieldValueEqual(String fieldName1, String fieldName2) { - lock.lock(); + public NavigableSet<String> findMatchingIdsWithGreaterValues(String fieldName1, + String fieldName2, boolean orEqual) { + indexer.acquireReadLock(); try { - checkFieldNameIndexed(fieldName1); - checkFieldNameIndexed(fieldName2); - - NavigableSet<String> result = new TreeSet<>(); - - // obtain all values for both fields and their corresponding IDs - var field1ValuesToIds = getColumnValuesToIdsMap(fieldName1); - var field2ValuesToIds = getColumnValuesToIdsMap(fieldName2); - - // iterate over each value of the first field - for (Map.Entry<String, NavigableSet<String>> entry : field1ValuesToIds.entrySet()) { - String fieldValue = entry.getKey(); - NavigableSet<String> idsForFieldValue = entry.getValue(); - - // if the second field contains the same value, add all matching IDs - if (field2ValuesToIds.containsKey(fieldValue)) { - NavigableSet<String> matchingIds = field2ValuesToIds.get(fieldValue); - for (String id : idsForFieldValue) { - if (matchingIds.contains(id)) { - result.add(id); - } - } - } - } - return result; + return findIdsWithKeyComparator(fieldName1, fieldName2, (k1, k2) -> { + var compare = KeyComparator.INSTANCE.compare(k1, k2); + return orEqual ? compare <= 0 : compare < 0; + }); } finally { - lock.unlock(); + indexer.releaseReadLock(); } } - private Map<String, NavigableSet<String>> getColumnValuesToIdsMap(String fieldName) { - var valuesToIdsMap = new HashMap<String, NavigableSet<String>>(); - for (var cell : orderedMatches.cellSet()) { - if (cell.getColumnKey().equals(fieldName)) { - var celValues = cell.getValue(); - if (CollectionUtils.isEmpty(celValues)) { - continue; - } - if (celValues.size() != 1) { - throw new IllegalArgumentException( - "Unsupported multi cell values to join with other field for: " + fieldName - + " with values: " + celValues); - } - String fieldValue = cell.getValue().first(); - if (!valuesToIdsMap.containsKey(fieldValue)) { - valuesToIdsMap.put(fieldValue, new TreeSet<>()); - } - valuesToIdsMap.get(fieldValue).add(cell.getRowKey()); - } - } - return valuesToIdsMap; + @Override + public NavigableSet<String> findIdsGreaterThan(String fieldName, String fieldValue, + boolean orEqual) { + var operator = getEntryOperator(fieldName); + return operator.greaterThan(fieldValue, orEqual); } @Override - public NavigableSet<String> findIdsForFieldValueGreaterThan(String fieldName1, + public NavigableSet<String> findMatchingIdsWithSmallerValues(String fieldName1, String fieldName2, boolean orEqual) { - lock.lock(); + indexer.acquireReadLock(); try { - checkFieldNameIndexed(fieldName1); - checkFieldNameIndexed(fieldName2); - - NavigableSet<String> result = new TreeSet<>(); - - // obtain all values for both fields and their corresponding IDs - var field1ValuesToIds = getColumnValuesToIdsMap(fieldName1); - var field2ValuesToIds = getColumnValuesToIdsMap(fieldName2); - - // iterate over each value of the first field - for (var entryField1 : field1ValuesToIds.entrySet()) { - String fieldValue1 = entryField1.getKey(); - - // iterate over each value of the second field - for (var entryField2 : field2ValuesToIds.entrySet()) { - String fieldValue2 = entryField2.getKey(); - - int comparison = fieldValue1.compareTo(fieldValue2); - if (orEqual ? comparison >= 0 : comparison > 0) { - // if the second field contains the same value, add all matching IDs - for (String id : entryField1.getValue()) { - if (field2ValuesToIds.get(fieldValue2).contains(id)) { - result.add(id); - } - } - } - } - } - return result; + return findIdsWithKeyComparator(fieldName1, fieldName2, (k1, k2) -> { + var compare = KeyComparator.INSTANCE.compare(k1, k2); + return orEqual ? compare >= 0 : compare > 0; + }); } finally { - lock.unlock(); + indexer.releaseReadLock(); } } @Override - public NavigableSet<String> findIdsForFieldValueLessThan(String fieldName1, String fieldName2, + public NavigableSet<String> findIdsLessThan(String fieldName, String fieldValue, boolean orEqual) { - lock.lock(); - try { - checkFieldNameIndexed(fieldName1); - checkFieldNameIndexed(fieldName2); - - NavigableSet<String> result = new TreeSet<>(); - - // obtain all values for both fields and their corresponding IDs - var field1ValuesToIds = getColumnValuesToIdsMap(fieldName1); - var field2ValuesToIds = getColumnValuesToIdsMap(fieldName2); - - // iterate over each value of the first field - for (var entryField1 : field1ValuesToIds.entrySet()) { - String fieldValue1 = entryField1.getKey(); + var operator = getEntryOperator(fieldName); + return operator.lessThan(fieldValue, orEqual); + } - // iterate over each value of the second field - for (var entryField2 : field2ValuesToIds.entrySet()) { - String fieldValue2 = entryField2.getKey(); + @Override + public NavigableSet<String> between(String fieldName, String lowerValue, boolean lowerInclusive, + String upperValue, boolean upperInclusive) { + var operator = getEntryOperator(fieldName); + return operator.range(lowerValue, upperValue, lowerInclusive, upperInclusive); + } - int comparison = fieldValue1.compareTo(fieldValue2); - if (orEqual ? comparison <= 0 : comparison < 0) { - // if the second field contains the same value, add all matching IDs - for (String id : entryField1.getValue()) { - if (field2ValuesToIds.get(fieldValue2).contains(id)) { - result.add(id); - } - } - } - } - } - return result; + @Override + public List<String> sortBy(NavigableSet<String> ids, Sort sort) { + if (sort.isUnsorted()) { + return new ArrayList<>(ids); + } + indexer.acquireReadLock(); + try { + var combinedComparator = sort.stream() + .map(this::comparatorFrom) + .reduce(Comparator::thenComparing) + .orElseThrow(); + return ids.stream() + .sorted(combinedComparator) + .toList(); } finally { - lock.unlock(); + indexer.releaseReadLock(); } } - @Override - public void removeByIdNotIn(NavigableSet<String> ids) { - lock.lock(); - try { - Set<String> idsToRemove = new HashSet<>(); - // check each row key if it is not in the given ids set - for (String rowKey : orderedMatches.rowKeySet()) { - if (!ids.contains(rowKey)) { - idsToRemove.add(rowKey); - } + Comparator<String> comparatorFrom(Sort.Order order) { + var indexEntry = getIndexEntry(order.getProperty()); + var idPositionMap = indexEntry.getIdPositionMap(); + var isDesc = order.isDescending(); + // This sort algorithm works leveraging on that the idPositionMap is a map of id -> position + // if the id is not in the map, it means that it is not indexed, and it will be placed at + // the end + return (a, b) -> { + var indexOfA = idPositionMap.get(a); + var indexOfB = idPositionMap.get(b); + + var isAIndexed = indexOfA != null; + var isBIndexed = indexOfB != null; + + if (!isAIndexed && !isBIndexed) { + return 0; } - - // remove all rows that are not in the given ids set - for (String idToRemove : idsToRemove) { - orderedMatches.row(idToRemove).clear(); + // un-indexed item are always at the end + if (!isAIndexed) { + return isDesc ? -1 : 1; } - } finally { - lock.unlock(); - } + if (!isBIndexed) { + return isDesc ? 1 : -1; + } + return isDesc ? Integer.compare(indexOfB, indexOfA) + : Integer.compare(indexOfA, indexOfB); + }; } @Override - public List<String> sortBy(Sort sort) { - lock.lock(); - try { - for (Sort.Order order : sort) { - String fieldName = order.getProperty(); - checkFieldNameIndexed(fieldName); - } + public IndexEntry getIndexEntry(String fieldName) { + return indexer.getIndexEntry(fieldName); + } - // obtain all row keys (IDs) - Set<String> allRowKeys = orderedMatches.rowKeySet(); + @Override + public void acquireReadLock() { + indexer.acquireReadLock(); + } - // convert row keys to list for sorting - List<String> sortedRowKeys = new ArrayList<>(allRowKeys); - if (sort.isUnsorted()) { - return sortedRowKeys; - } + @Override + public void releaseReadLock() { + indexer.releaseReadLock(); + } - // sort row keys according to sort criteria in a Sort object - sortedRowKeys.sort((id1, id2) -> { - for (Sort.Order order : sort) { - String fieldName = order.getProperty(); + private IndexEntryOperator getEntryOperator(String fieldName) { + var indexEntry = getIndexEntry(fieldName); + return createIndexEntryOperator(indexEntry); + } - // compare the values of the two rows on the field - int comparison = compareRowValue(id1, id2, fieldName, order.isAscending()); - if (comparison != 0) { - return comparison; - } - } - // if all sort criteria are equal, return 0 - return 0; - }); + private IndexEntryOperator createIndexEntryOperator(IndexEntry entry) { + return new IndexEntryOperatorImpl(entry); + } - return sortedRowKeys; - } finally { - lock.unlock(); - } + private Set<String> allIds() { + var indexEntry = getIndexEntry(PRIMARY_INDEX_NAME); + return createIndexEntryOperator(indexEntry).getValues(); } - private int compareRowValue(String id1, String id2, String fieldName, boolean isAscending) { - var value1 = getSingleFieldValueForSort(id1, fieldName); - var value2 = getSingleFieldValueForSort(id2, fieldName); - // nulls are less than everything whatever the sort order is - // do not simply the following code for null check,it's different from KeyComparator - if (value1 == null && value2 == null) { - return 0; - } else if (value1 == null) { - return 1; - } else if (value2 == null) { - return -1; + /** + * Must lock the indexer before calling this method. + */ + private NavigableSet<String> findIdsWithKeyComparator(String fieldName1, String fieldName2, + BiPredicate<String, String> keyComparator) { + // get entries from indexer for fieldName1 + var entriesA = getIndexEntry(fieldName1).entries(); + + Map<String, List<String>> keyMap = new HashMap<>(); + for (Map.Entry<String, String> entry : entriesA) { + keyMap.computeIfAbsent(entry.getValue(), v -> new ArrayList<>()).add(entry.getKey()); } - return isAscending ? KeyComparator.INSTANCE.compare(value1, value2) - : KeyComparator.INSTANCE.compare(value2, value1); - } - @Nullable - String getSingleFieldValueForSort(String rowKey, String fieldName) { - return Optional.ofNullable(orderedMatches.get(rowKey, fieldName)) - .filter(values -> !CollectionUtils.isEmpty(values)) - .map(values -> { - if (values.size() != 1) { - throw new IllegalArgumentException( - "Unsupported multi field values to sort for: " + fieldName - + " with values: " + values); + NavigableSet<String> result = new TreeSet<>(); + + // get entries from indexer for fieldName2 + var entriesB = getIndexEntry(fieldName2).entries(); + for (Map.Entry<String, String> entry : entriesB) { + List<String> matchedKeys = keyMap.get(entry.getValue()); + if (matchedKeys != null) { + for (String key : matchedKeys) { + if (keyComparator.test(entry.getKey(), key)) { + result.add(entry.getValue()); + // found one match, no need to continue + break; + } } - return values.first(); - }) - .orElse(null); - } - - private void checkFieldNameIndexed(String fieldName) { - if (!fieldNames.contains(fieldName)) { - throw new IllegalArgumentException("Field name " + fieldName - + " is not indexed, please ensure it added to the index spec before querying"); + } } + return result; } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/StringContains.java b/api/src/main/java/run/halo/app/extension/index/query/StringContains.java index 069ba90bb8..f6e7dc5b84 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/StringContains.java +++ b/api/src/main/java/run/halo/app/extension/index/query/StringContains.java @@ -1,6 +1,7 @@ package run.halo.app.extension.index.query; import com.google.common.collect.Sets; +import java.util.Map; import java.util.NavigableSet; import org.apache.commons.lang3.StringUtils; @@ -12,12 +13,24 @@ public StringContains(String fieldName, String value) { @Override public NavigableSet<String> matches(QueryIndexView indexView) { var resultSet = Sets.<String>newTreeSet(); - var fieldValues = indexView.getAllValuesForField(fieldName); - for (String val : fieldValues) { - if (StringUtils.containsIgnoreCase(val, value)) { - resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); + var indexEntry = indexView.getIndexEntry(fieldName); + + indexEntry.acquireReadLock(); + try { + for (Map.Entry<String, String> entry : indexEntry.entries()) { + var fieldValue = entry.getKey(); + if (StringUtils.containsIgnoreCase(fieldValue, value)) { + resultSet.add(entry.getValue()); + } } + return resultSet; + } finally { + indexEntry.releaseReadLock(); } - return resultSet; + } + + @Override + public String toString() { + return "contains(" + fieldName + ", '" + value + "')"; } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/StringEndsWith.java b/api/src/main/java/run/halo/app/extension/index/query/StringEndsWith.java index b6e2bed00f..51853be7a6 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/StringEndsWith.java +++ b/api/src/main/java/run/halo/app/extension/index/query/StringEndsWith.java @@ -1,7 +1,9 @@ package run.halo.app.extension.index.query; import com.google.common.collect.Sets; +import java.util.Map; import java.util.NavigableSet; +import org.apache.commons.lang3.StringUtils; public class StringEndsWith extends SimpleQuery { public StringEndsWith(String fieldName, String value) { @@ -11,12 +13,24 @@ public StringEndsWith(String fieldName, String value) { @Override public NavigableSet<String> matches(QueryIndexView indexView) { var resultSet = Sets.<String>newTreeSet(); - var fieldValues = indexView.getAllValuesForField(fieldName); - for (String val : fieldValues) { - if (val.endsWith(value)) { - resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); + var indexEntry = indexView.getIndexEntry(fieldName); + + indexEntry.acquireReadLock(); + try { + for (Map.Entry<String, String> entry : indexEntry.entries()) { + var fieldValue = entry.getKey(); + if (StringUtils.endsWith(fieldValue, value)) { + resultSet.add(entry.getValue()); + } } + return resultSet; + } finally { + indexEntry.releaseReadLock(); } - return resultSet; + } + + @Override + public String toString() { + return "endsWith(" + fieldName + ", '" + value + "')"; } } diff --git a/api/src/main/java/run/halo/app/extension/index/query/StringStartsWith.java b/api/src/main/java/run/halo/app/extension/index/query/StringStartsWith.java index 5d0fd5a46e..bad07d806c 100644 --- a/api/src/main/java/run/halo/app/extension/index/query/StringStartsWith.java +++ b/api/src/main/java/run/halo/app/extension/index/query/StringStartsWith.java @@ -1,7 +1,9 @@ package run.halo.app.extension.index.query; import com.google.common.collect.Sets; +import java.util.Map; import java.util.NavigableSet; +import org.apache.commons.lang3.StringUtils; public class StringStartsWith extends SimpleQuery { public StringStartsWith(String fieldName, String value) { @@ -11,13 +13,24 @@ public StringStartsWith(String fieldName, String value) { @Override public NavigableSet<String> matches(QueryIndexView indexView) { var resultSet = Sets.<String>newTreeSet(); - var allValues = indexView.getAllValuesForField(fieldName); + var indexEntry = indexView.getIndexEntry(fieldName); - for (String val : allValues) { - if (val.startsWith(value)) { - resultSet.addAll(indexView.getIdsForFieldValue(fieldName, val)); + indexEntry.acquireReadLock(); + try { + for (Map.Entry<String, String> entry : indexEntry.entries()) { + var fieldValue = entry.getKey(); + if (StringUtils.startsWith(fieldValue, value)) { + resultSet.add(entry.getValue()); + } } + return resultSet; + } finally { + indexEntry.releaseReadLock(); } - return resultSet; + } + + @Override + public String toString() { + return "startsWith(" + fieldName + ", '" + value + "')"; } } diff --git a/api/src/test/java/run/halo/app/extension/controller/RequestSynchronizerTest.java b/api/src/test/java/run/halo/app/extension/controller/RequestSynchronizerTest.java index dc6234eec1..b8368c65f5 100644 --- a/api/src/test/java/run/halo/app/extension/controller/RequestSynchronizerTest.java +++ b/api/src/test/java/run/halo/app/extension/controller/RequestSynchronizerTest.java @@ -16,6 +16,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.domain.Sort; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.ExtensionMatcher; import run.halo.app.extension.FakeExtension; @@ -53,7 +54,7 @@ void setUp() { @Test void shouldStartCorrectlyWhenSyncingAllOnStart() { var type = GroupVersionKind.fromExtension(FakeExtension.class); - when(indexedQueryEngine.retrieveAll(eq(type), isA(ListOptions.class))) + when(indexedQueryEngine.retrieveAll(eq(type), isA(ListOptions.class), any(Sort.class))) .thenReturn(List.of("fake-01", "fake-02")); synchronizer.start(); @@ -62,7 +63,7 @@ void shouldStartCorrectlyWhenSyncingAllOnStart() { assertFalse(synchronizer.isDisposed()); verify(indexedQueryEngine, times(1)).retrieveAll(eq(type), - isA(ListOptions.class)); + isA(ListOptions.class), isA(Sort.class)); verify(watcher, times(2)).onAdd(isA(Reconciler.Request.class)); verify(client, times(1)).watch(same(watcher)); } diff --git a/api/src/test/java/run/halo/app/extension/index/query/InQueryTest.java b/api/src/test/java/run/halo/app/extension/index/query/InQueryTest.java new file mode 100644 index 0000000000..c7e8d961b4 --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/index/query/InQueryTest.java @@ -0,0 +1,24 @@ +package run.halo.app.extension.index.query; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.LinkedHashSet; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link InQuery}. + * + * @author guqing + * @since 2.17.0 + */ +class InQueryTest { + + @Test + void testToString() { + var values = new LinkedHashSet<String>(); + values.add("Alice"); + values.add("Bob"); + var inQuery = new InQuery("name", values); + assertThat(inQuery.toString()).isEqualTo("name IN ('Alice', 'Bob')"); + } +} \ No newline at end of file diff --git a/api/src/test/java/run/halo/app/extension/index/query/IndexViewDataSet.java b/api/src/test/java/run/halo/app/extension/index/query/IndexViewDataSet.java deleted file mode 100644 index c618e63e06..0000000000 --- a/api/src/test/java/run/halo/app/extension/index/query/IndexViewDataSet.java +++ /dev/null @@ -1,176 +0,0 @@ -package run.halo.app.extension.index.query; - -import java.util.Collection; -import java.util.List; -import java.util.Map; - -public class IndexViewDataSet { - - /** - * Create a {@link QueryIndexView} for employee to test. - * - * @return a {@link QueryIndexView} for employee to test - */ - public static QueryIndexView createEmployeeIndexView() { - /* - * id firstName lastName email hireDate salary managerId departmentId - * 100 Pat Fay p 17 2600 101 50 - * 101 Lee Day l 17 2400 102 40 - * 102 William Jay w 19 2200 102 50 - * 103 Mary Day p 17 2000 103 50 - * 104 John Fay j 17 1800 103 50 - * 105 Gon Fay p 18 1900 101 40 - */ - Collection<Map.Entry<String, String>> idEntry = List.of( - Map.entry("100", "100"), - Map.entry("101", "101"), - Map.entry("102", "102"), - Map.entry("103", "103"), - Map.entry("104", "104"), - Map.entry("105", "105") - ); - Collection<Map.Entry<String, String>> firstNameEntry = List.of( - Map.entry("Pat", "100"), - Map.entry("Lee", "101"), - Map.entry("William", "102"), - Map.entry("Mary", "103"), - Map.entry("John", "104"), - Map.entry("Gon", "105") - ); - Collection<Map.Entry<String, String>> lastNameEntry = List.of( - Map.entry("Fay", "100"), - Map.entry("Day", "101"), - Map.entry("Jay", "102"), - Map.entry("Day", "103"), - Map.entry("Fay", "104"), - Map.entry("Fay", "105") - ); - Collection<Map.Entry<String, String>> emailEntry = List.of( - Map.entry("p", "100"), - Map.entry("l", "101"), - Map.entry("w", "102"), - Map.entry("p", "103"), - Map.entry("j", "104"), - Map.entry("p", "105") - ); - Collection<Map.Entry<String, String>> hireDateEntry = List.of( - Map.entry("17", "100"), - Map.entry("17", "101"), - Map.entry("19", "102"), - Map.entry("17", "103"), - Map.entry("17", "104"), - Map.entry("18", "105") - ); - Collection<Map.Entry<String, String>> salaryEntry = List.of( - Map.entry("2600", "100"), - Map.entry("2400", "101"), - Map.entry("2200", "102"), - Map.entry("2000", "103"), - Map.entry("1800", "104"), - Map.entry("1900", "105") - ); - Collection<Map.Entry<String, String>> managerIdEntry = List.of( - Map.entry("101", "100"), - Map.entry("102", "101"), - Map.entry("102", "102"), - Map.entry("103", "103"), - Map.entry("103", "104"), - Map.entry("101", "105") - ); - Collection<Map.Entry<String, String>> departmentIdEntry = List.of( - Map.entry("50", "100"), - Map.entry("40", "101"), - Map.entry("50", "102"), - Map.entry("50", "103"), - Map.entry("50", "104"), - Map.entry("40", "105") - ); - var entries = Map.of("id", idEntry, - "firstName", firstNameEntry, - "lastName", lastNameEntry, - "email", emailEntry, - "hireDate", hireDateEntry, - "salary", salaryEntry, - "managerId", managerIdEntry, - "departmentId", departmentIdEntry); - return new QueryIndexViewImpl(entries); - } - - /** - * Create a {@link QueryIndexView} for post to test. - * - * @return a {@link QueryIndexView} for post to test - */ - public static QueryIndexView createPostIndexViewWithNullCell() { - /* - * id title published publishTime owner - * 100 title1 true 2024-01-01T00:00:00 jack - * 101 title2 true 2024-01-02T00:00:00 rose - * 102 title3 false null smith - * 103 title4 false null peter - * 104 title5 false null john - * 105 title6 true 2024-01-05 00:00:00 tom - * 106 title7 true 2024-01-05 13:00:00 jerry - * 107 title8 true 2024-01-05 12:00:00 jerry - * 108 title9 false null jerry - */ - Collection<Map.Entry<String, String>> idEntry = List.of( - Map.entry("100", "100"), - Map.entry("101", "101"), - Map.entry("102", "102"), - Map.entry("103", "103"), - Map.entry("104", "104"), - Map.entry("105", "105"), - Map.entry("106", "106"), - Map.entry("107", "107"), - Map.entry("108", "108") - ); - Collection<Map.Entry<String, String>> titleEntry = List.of( - Map.entry("title1", "100"), - Map.entry("title2", "101"), - Map.entry("title3", "102"), - Map.entry("title4", "103"), - Map.entry("title5", "104"), - Map.entry("title6", "105"), - Map.entry("title7", "106"), - Map.entry("title8", "107"), - Map.entry("title9", "108") - ); - Collection<Map.Entry<String, String>> publishedEntry = List.of( - Map.entry("true", "100"), - Map.entry("true", "101"), - Map.entry("false", "102"), - Map.entry("false", "103"), - Map.entry("false", "104"), - Map.entry("true", "105"), - Map.entry("true", "106"), - Map.entry("true", "107"), - Map.entry("false", "108") - ); - Collection<Map.Entry<String, String>> publishTimeEntry = List.of( - Map.entry("2024-01-01T00:00:00", "100"), - Map.entry("2024-01-02T00:00:00", "101"), - Map.entry("2024-01-05 00:00:00", "105"), - Map.entry("2024-01-05 13:00:00", "106"), - Map.entry("2024-01-05 12:00:00", "107") - ); - - Collection<Map.Entry<String, String>> ownerEntry = List.of( - Map.entry("jack", "100"), - Map.entry("rose", "101"), - Map.entry("smith", "102"), - Map.entry("peter", "103"), - Map.entry("john", "104"), - Map.entry("tom", "105"), - Map.entry("jerry", "106"), - Map.entry("jerry", "107"), - Map.entry("jerry", "108") - ); - var entries = Map.of("id", idEntry, - "title", titleEntry, - "published", publishedEntry, - "publishTime", publishTimeEntry, - "owner", ownerEntry); - return new QueryIndexViewImpl(entries); - } -} diff --git a/api/src/test/java/run/halo/app/extension/index/query/IsNotNullTest.java b/api/src/test/java/run/halo/app/extension/index/query/IsNotNullTest.java new file mode 100644 index 0000000000..be6b708f93 --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/index/query/IsNotNullTest.java @@ -0,0 +1,20 @@ +package run.halo.app.extension.index.query; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link IsNotNull}. + * + * @author guqing + * @since 2.17.0 + */ +class IsNotNullTest { + + @Test + void testToString() { + var isNotNull = new IsNotNull("name"); + assertThat(isNotNull.toString()).isEqualTo("name IS NOT NULL"); + } +} \ No newline at end of file diff --git a/api/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java b/api/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java deleted file mode 100644 index 86f61db7c6..0000000000 --- a/api/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java +++ /dev/null @@ -1,218 +0,0 @@ -package run.halo.app.extension.index.query; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.springframework.data.domain.Sort; - -/** - * Tests for {@link QueryIndexViewImpl}. - * - * @author guqing - * @since 2.12.0 - */ -class QueryIndexViewImplTest { - - @Test - void getAllIdsForFieldTest() { - var indexView = IndexViewDataSet.createPostIndexViewWithNullCell(); - var resultSet = indexView.getAllIdsForField("title"); - assertThat(resultSet).containsExactlyInAnyOrder( - "100", "101", "102", "103", "104", "105", "106", "107", "108" - ); - - resultSet = indexView.getAllIdsForField("publishTime"); - assertThat(resultSet).containsExactlyInAnyOrder( - "100", "101", "105", "106", "107" - ); - } - - @Test - void findIdsForFieldValueEqualTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = indexView.findIdsForFieldValueEqual("managerId", "id"); - assertThat(resultSet).containsExactlyInAnyOrder( - "102", "103" - ); - } - - @Test - void findIdsForFieldValueGreaterThanTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = indexView.findIdsForFieldValueGreaterThan("id", "managerId", false); - assertThat(resultSet).containsExactlyInAnyOrder( - "104", "105" - ); - - indexView = IndexViewDataSet.createEmployeeIndexView(); - resultSet = indexView.findIdsForFieldValueGreaterThan("id", "managerId", true); - assertThat(resultSet).containsExactlyInAnyOrder( - "103", "102", "104", "105" - ); - } - - @Test - void findIdsForFieldValueGreaterThanTest2() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = indexView.findIdsForFieldValueGreaterThan("managerId", "id", false); - assertThat(resultSet).containsExactlyInAnyOrder( - "100", "101" - ); - - indexView = IndexViewDataSet.createEmployeeIndexView(); - resultSet = indexView.findIdsForFieldValueGreaterThan("managerId", "id", true); - assertThat(resultSet).containsExactlyInAnyOrder( - "100", "101", "102", "103" - ); - } - - @Test - void findIdsForFieldValueLessThanTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = indexView.findIdsForFieldValueLessThan("id", "managerId", false); - assertThat(resultSet).containsExactlyInAnyOrder( - "100", "101" - ); - - indexView = IndexViewDataSet.createEmployeeIndexView(); - resultSet = indexView.findIdsForFieldValueLessThan("id", "managerId", true); - assertThat(resultSet).containsExactlyInAnyOrder( - "100", "101", "102", "103" - ); - } - - @Test - void findIdsForFieldValueLessThanTest2() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = indexView.findIdsForFieldValueLessThan("managerId", "id", false); - assertThat(resultSet).containsExactlyInAnyOrder( - "104", "105" - ); - - indexView = IndexViewDataSet.createEmployeeIndexView(); - resultSet = indexView.findIdsForFieldValueLessThan("managerId", "id", true); - assertThat(resultSet).containsExactlyInAnyOrder( - "103", "102", "104", "105" - ); - } - - @Nested - class SortTest { - @Test - void testSortByUnsorted() { - Collection<Map.Entry<String, String>> entries = List.of( - Map.entry("Item1", "Item1"), - Map.entry("Item2", "Item2") - ); - var indexView = new QueryIndexViewImpl(Map.of("field1", entries)); - var sort = Sort.unsorted(); - - List<String> sortedList = indexView.sortBy(sort); - assertThat(sortedList).isEqualTo(List.of("Item1", "Item2")); - } - - @Test - void testSortBySortedAscending() { - var indexEntries = new HashMap<String, Collection<Map.Entry<String, String>>>(); - indexEntries.put("field1", - List.of(Map.entry("key2", "Item2"), Map.entry("key1", "Item1"))); - var indexView = new QueryIndexViewImpl(indexEntries); - var sort = Sort.by(Sort.Order.asc("field1")); - - List<String> sortedList = indexView.sortBy(sort); - - assertThat(sortedList).containsExactly("Item1", "Item2"); - } - - @Test - void testSortBySortedDescending() { - var indexEntries = new HashMap<String, Collection<Map.Entry<String, String>>>(); - indexEntries.put("field1", - List.of(Map.entry("key1", "Item1"), Map.entry("key2", "Item2"))); - var indexView = new QueryIndexViewImpl(indexEntries); - var sort = Sort.by(Sort.Order.desc("field1")); - - List<String> sortedList = indexView.sortBy(sort); - - assertThat(sortedList).containsExactly("Item2", "Item1"); - } - - @Test - void testSortByMultipleFields() { - var indexEntries = new LinkedHashMap<String, Collection<Map.Entry<String, String>>>(); - indexEntries.put("field1", List.of(Map.entry("k3", "Item3"), Map.entry("k2", "Item2"))); - indexEntries.put("field2", List.of(Map.entry("k1", "Item1"), Map.entry("k3", "Item3"))); - var indexView = new QueryIndexViewImpl(indexEntries); - var sort = Sort.by(Sort.Order.asc("field1"), Sort.Order.desc("field2")); - - List<String> sortedList = indexView.sortBy(sort); - - assertThat(sortedList).containsExactly("Item2", "Item3", "Item1"); - } - - @Test - void testSortByWithMissingFieldInMap() { - var indexEntries = new LinkedHashMap<String, Collection<Map.Entry<String, String>>>(); - var indexView = new QueryIndexViewImpl(indexEntries); - var sort = Sort.by(Sort.Order.asc("missingField")); - - assertThatThrownBy(() -> indexView.sortBy(sort)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Field name missingField is not indexed"); - } - - @Test - void testSortByMultipleFields2() { - var indexEntries = new LinkedHashMap<String, Collection<Map.Entry<String, String>>>(); - - var entry1 = List.of(Map.entry("John", "John"), - Map.entry("Bob", "Bob"), - Map.entry("Alice", "Alice") - ); - var entry2 = List.of(Map.entry("David", "David"), - Map.entry("Eva", "Eva"), - Map.entry("Frank", "Frank") - ); - var entry3 = List.of(Map.entry("George", "George"), - Map.entry("Helen", "Helen"), - Map.entry("Ivy", "Ivy") - ); - - indexEntries.put("field1", entry1); - indexEntries.put("field2", entry2); - indexEntries.put("field3", entry3); - - /* - * <pre> - * Row Key | field1 | field2 | field3 - * -------|-------|-------|------- - * John | John | | - * Bob | Bob | | - * Alice | Alice | | - * David | | David | - * Eva | | Eva | - * Frank | | Frank | - * George | | | George - * Helen | | | Helen - * Ivy | | | Ivy - * </pre> - */ - var indexView = new QueryIndexViewImpl(indexEntries); - // "John", "Bob", "Alice", "David", "Eva", "Frank", "George", "Helen", "Ivy" - var sort = Sort.by(Sort.Order.desc("field1"), Sort.Order.asc("field2"), - Sort.Order.asc("field3")); - - List<String> sortedList = indexView.sortBy(sort); - - assertThat(sortedList).containsSequence("John", "Bob", "Alice", "David", "Eva", "Frank", - "George", "Helen", "Ivy"); - } - } -} diff --git a/api/src/test/java/run/halo/app/extension/index/query/StringContainsTest.java b/api/src/test/java/run/halo/app/extension/index/query/StringContainsTest.java new file mode 100644 index 0000000000..f55cdd6806 --- /dev/null +++ b/api/src/test/java/run/halo/app/extension/index/query/StringContainsTest.java @@ -0,0 +1,20 @@ +package run.halo.app.extension.index.query; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link StringContains}. + * + * @author guqing + * @since 2.17.0 + */ +class StringContainsTest { + + @Test + void testToString() { + var stringContains = new StringContains("name", "Alice"); + assertThat(stringContains.toString()).isEqualTo("contains(name, 'Alice')"); + } +} \ No newline at end of file diff --git a/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java b/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java index e6f11098d8..0515aa730a 100644 --- a/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java +++ b/application/src/main/java/run/halo/app/extension/ReactiveExtensionClientImpl.java @@ -112,8 +112,30 @@ public <E extends Extension> Mono<ListResult<E>> list(Class<E> type, Predicate<E @Override public <E extends Extension> Flux<E> listAll(Class<E> type, ListOptions options, Sort sort) { - return listBy(type, options, PageRequestImpl.ofSize(0).withSort(sort)) - .flatMapIterable(ListResult::getItems); + var scheme = schemeManager.get(type); + return Mono.fromSupplier( + () -> indexedQueryEngine.retrieveAll(scheme.groupVersionKind(), options, sort)) + .doOnSuccess(objectKeys -> { + if (log.isDebugEnabled()) { + if (objectKeys.size() > 500) { + log.warn("The number of objects retrieved by listAll is too large ({}) " + + "and it is recommended to use paging query.", + objectKeys.size()); + } + } + }) + .flatMapMany(objectKeys -> { + var storeNames = objectKeys.stream() + .map(objectKey -> ExtensionStoreUtil.buildStoreName(scheme, objectKey)) + .toList(); + final long startTimeMs = System.currentTimeMillis(); + return client.listByNames(storeNames) + .map(extensionStore -> converter.convertFrom(type, extensionStore)) + .doOnNext(s -> { + log.debug("Successfully retrieved all by names from db for {} in {}ms", + scheme.groupVersionKind(), System.currentTimeMillis() - startTimeMs); + }); + }); } @Override @@ -130,7 +152,7 @@ public <E extends Extension> Mono<ListResult<E>> listBy(Class<E> type, ListOptio final long startTimeMs = System.currentTimeMillis(); return client.listByNames(storeNames) .map(extensionStore -> converter.convertFrom(type, extensionStore)) - .doFinally(s -> { + .doOnNext(s -> { log.debug("Successfully retrieved by names from db for {} in {}ms", scheme.groupVersionKind(), System.currentTimeMillis() - startTimeMs); }) diff --git a/application/src/main/java/run/halo/app/extension/index/DefaultIndexer.java b/application/src/main/java/run/halo/app/extension/index/DefaultIndexer.java index bc4030401e..10c570978b 100644 --- a/application/src/main/java/run/halo/app/extension/index/DefaultIndexer.java +++ b/application/src/main/java/run/halo/app/extension/index/DefaultIndexer.java @@ -11,6 +11,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import org.apache.commons.lang3.BooleanUtils; +import org.springframework.lang.NonNull; import run.halo.app.extension.Extension; /** @@ -172,6 +173,28 @@ public void removeIndexRecords(Function<IndexDescriptor, Boolean> matchFn) { } } + @Override + @NonNull + public IndexEntry getIndexEntry(String name) { + readLock.lock(); + try { + var indexDescriptor = findIndexByName(name); + if (indexDescriptor == null) { + throw new IllegalArgumentException( + "No index found for fieldPath [" + name + "], " + + "make sure you have created an index for this field."); + } + if (!indexDescriptor.isReady()) { + throw new IllegalStateException( + "Index [" + name + "] is not ready, " + + "Please wait for more time or check the index status."); + } + return indexEntries.get(indexDescriptor); + } finally { + readLock.unlock(); + } + } + @Override public Iterator<IndexEntry> readyIndexesIterator() { readLock.lock(); @@ -197,4 +220,14 @@ public Iterator<IndexEntry> allIndexesIterator() { readLock.unlock(); } } + + @Override + public void acquireReadLock() { + readLock.lock(); + } + + @Override + public void releaseReadLock() { + readLock.unlock(); + } } diff --git a/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java index cc204c5bd7..b677d6e2a5 100644 --- a/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java +++ b/application/src/main/java/run/halo/app/extension/index/IndexEntryImpl.java @@ -4,9 +4,11 @@ import com.google.common.collect.MultimapBuilder; import java.util.Collection; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.NavigableSet; +import java.util.TreeSet; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -94,10 +96,13 @@ public void remove(String objectName) { } @Override - public Set<String> indexedKeys() { + public NavigableSet<String> indexedKeys() { readLock.lock(); try { - return indexKeyObjectNamesMap.keySet(); + var keys = indexKeyObjectNamesMap.keySet(); + var resultSet = new TreeSet<>(keyComparator()); + resultSet.addAll(keys); + return resultSet; } finally { readLock.unlock(); } @@ -114,20 +119,32 @@ public Collection<Map.Entry<String, String>> entries() { } @Override - public Collection<Map.Entry<String, String>> immutableEntries() { + public Map<String, Integer> getIdPositionMap() { readLock.lock(); try { - // Copy to a new list to avoid ConcurrentModificationException - return indexKeyObjectNamesMap.entries().stream() - .map(entry -> Map.entry(entry.getKey(), entry.getValue())) - .toList(); + // asMap is sorted by key + var keyObjectMap = getKeyObjectMap(); + int i = 0; + var idPositionMap = new HashMap<String, Integer>(); + for (var valueIdsEntry : keyObjectMap.entrySet()) { + var ids = valueIdsEntry.getValue(); + for (String id : ids) { + idPositionMap.put(id, i); + } + i++; + } + return idPositionMap; } finally { readLock.unlock(); } } + protected Map<String, Collection<String>> getKeyObjectMap() { + return indexKeyObjectNamesMap.asMap(); + } + @Override - public List<String> getByIndexKey(String indexKey) { + public List<String> getObjectNamesBy(String indexKey) { readLock.lock(); try { return indexKeyObjectNamesMap.get(indexKey); diff --git a/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java b/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java index 1e30d05d8a..809cea401d 100644 --- a/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java +++ b/application/src/main/java/run/halo/app/extension/index/IndexedQueryEngineImpl.java @@ -1,15 +1,14 @@ package run.halo.app.extension.index; -import java.util.ArrayList; -import java.util.Collection; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.NavigableSet; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -23,6 +22,7 @@ import run.halo.app.extension.ListResult; import run.halo.app.extension.PageRequest; import run.halo.app.extension.index.query.QueryFactory; +import run.halo.app.extension.index.query.QueryIndexView; import run.halo.app.extension.index.query.QueryIndexViewImpl; import run.halo.app.extension.router.selector.FieldSelector; import run.halo.app.extension.router.selector.LabelSelector; @@ -41,155 +41,108 @@ public class IndexedQueryEngineImpl implements IndexedQueryEngine { private final IndexerFactory indexerFactory; - private static Map<String, IndexEntry> fieldPathIndexEntryMap(Indexer indexer) { - // O(n) time complexity - Map<String, IndexEntry> indexEntryMap = new HashMap<>(); - var iterator = indexer.readyIndexesIterator(); - while (iterator.hasNext()) { - var indexEntry = iterator.next(); - var descriptor = indexEntry.getIndexDescriptor(); - var indexedFieldPath = descriptor.getSpec().getName(); - indexEntryMap.put(indexedFieldPath, indexEntry); - } - return indexEntryMap; - } - - static IndexEntry getIndexEntry(String fieldPath, Map<String, IndexEntry> fieldPathEntryMap) { - if (!fieldPathEntryMap.containsKey(fieldPath)) { - throwNotIndexedException(fieldPath); - } - return fieldPathEntryMap.get(fieldPath); - } - @Override public ListResult<String> retrieve(GroupVersionKind type, ListOptions options, PageRequest page) { - var indexer = indexerFactory.getIndexer(type); - var allMatchedResult = doRetrieve(indexer, options, page.getSort()); + var allMatchedResult = doRetrieve(type, options, page.getSort()); var list = ListResult.subList(allMatchedResult, page.getPageNumber(), page.getPageSize()); return new ListResult<>(page.getPageNumber(), page.getPageSize(), allMatchedResult.size(), list); } @Override - public List<String> retrieveAll(GroupVersionKind type, ListOptions options) { - var indexer = indexerFactory.getIndexer(type); - return doRetrieve(indexer, options, Sort.unsorted()); - } - - static <T> List<T> intersection(List<T> list1, List<T> list2) { - Set<T> set = new LinkedHashSet<>(list1); - List<T> intersection = new ArrayList<>(); - for (T item : list2) { - if (set.contains(item) && !intersection.contains(item)) { - intersection.add(item); - } - } - return intersection; - } - - static void throwNotIndexedException(String fieldPath) { - throw new IllegalArgumentException( - "No index found for fieldPath: " + fieldPath - + ", make sure you have created an index for this field."); + public List<String> retrieveAll(GroupVersionKind type, ListOptions options, Sort sort) { + return doRetrieve(type, options, sort); } - List<String> retrieveForLabelMatchers(List<SelectorMatcher> labelMatchers, - Map<String, IndexEntry> fieldPathEntryMap, List<String> allMetadataNames) { - var indexEntry = getIndexEntry(LabelIndexSpecUtils.LABEL_PATH, fieldPathEntryMap); - // O(m) time complexity, m is the number of labelMatchers - var labelKeysToQuery = labelMatchers.stream() - .sorted(Comparator.comparing(SelectorMatcher::getKey)) - .map(SelectorMatcher::getKey) - .collect(Collectors.toSet()); - - Map<String, Map<String, String>> objectNameLabelsMap = new HashMap<>(); - indexEntry.acquireReadLock(); - try { - indexEntry.entries().forEach(entry -> { - // key is labelKey=labelValue, value is objectName - var labelPair = LabelIndexSpecUtils.labelKeyValuePair(entry.getKey()); - if (!labelKeysToQuery.contains(labelPair.getFirst())) { - return; - } - objectNameLabelsMap.computeIfAbsent(entry.getValue(), k -> new HashMap<>()) - .put(labelPair.getFirst(), labelPair.getSecond()); - }); - } finally { - indexEntry.releaseReadLock(); - } - // O(p * m) time complexity, p is the number of allMetadataNames - return allMetadataNames.stream() - .filter(objectName -> { - var labels = objectNameLabelsMap.getOrDefault(objectName, Map.of()); + NavigableSet<String> retrieveForLabelMatchers(Indexer indexer, + List<SelectorMatcher> labelMatchers) { + var objectLabelMap = ObjectLabelMap.buildFrom(indexer, labelMatchers); + // O(k×m) time complexity, k is the number of keys, m is the number of labelMatchers + return objectLabelMap.objectIdLabelsMap() + .entrySet() + .stream() + .filter(entry -> { + var labels = entry.getValue(); // object match all labels will be returned return labelMatchers.stream() .allMatch(matcher -> matcher.test(labels.get(matcher.getKey()))); }) - .toList(); + .map(Map.Entry::getKey) + .collect(Collectors.toCollection(TreeSet::new)); } - List<String> doRetrieve(Indexer indexer, ListOptions options, Sort sort) { - StopWatch stopWatch = new StopWatch(); - stopWatch.start("build index entry map"); - var fieldPathEntryMap = fieldPathIndexEntryMap(indexer); - var primaryEntry = getIndexEntry(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME, fieldPathEntryMap); - stopWatch.stop(); + NavigableSet<String> evaluateSelectorsForIndex(Indexer indexer, QueryIndexView indexView, + ListOptions options) { + final var hasLabelSelector = hasLabelSelector(options.getLabelSelector()); + final var hasFieldSelector = hasFieldSelector(options.getFieldSelector()); - // O(n) time complexity - stopWatch.start("retrieve all metadata names"); - var allMetadataNames = new ArrayList<String>(); - primaryEntry.acquireReadLock(); - try { - allMetadataNames.addAll(primaryEntry.indexedKeys()); - } finally { - primaryEntry.releaseReadLock(); + if (!hasLabelSelector && !hasFieldSelector) { + return QueryFactory.all().matches(indexView); } - stopWatch.stop(); - stopWatch.start("build index view"); - var fieldNamesUsedInQuery = getFieldNamesUsedInListOptions(options, sort); - var indexViewMap = new HashMap<String, Collection<Map.Entry<String, String>>>(); - for (Map.Entry<String, IndexEntry> entry : fieldPathEntryMap.entrySet()) { - if (!fieldNamesUsedInQuery.contains(entry.getKey())) { - continue; - } - indexViewMap.put(entry.getKey(), entry.getValue().immutableEntries()); + // only label selector + if (hasLabelSelector && !hasFieldSelector) { + return retrieveForLabelMatchers(indexer, options.getLabelSelector().getMatchers()); } - // TODO optimize build indexView time - var indexView = new QueryIndexViewImpl(indexViewMap); - stopWatch.stop(); - stopWatch.start("retrieve matched metadata names"); - if (hasLabelSelector(options.getLabelSelector())) { - var matchedByLabels = retrieveForLabelMatchers(options.getLabelSelector().getMatchers(), - fieldPathEntryMap, allMetadataNames); - if (allMetadataNames.size() != matchedByLabels.size()) { - indexView.removeByIdNotIn(new TreeSet<>(matchedByLabels)); - } + // only field selector + if (!hasLabelSelector) { + var fieldSelector = options.getFieldSelector(); + return fieldSelector.query().matches(indexView); } + + // both label and field selector + var fieldSelector = options.getFieldSelector(); + var forField = fieldSelector.query().matches(indexView); + var forLabel = + retrieveForLabelMatchers(indexer, options.getLabelSelector().getMatchers()); + + // determine the optimal retainAll direction based on the size of the collection + var resultSet = (forField.size() <= forLabel.size()) ? forField : forLabel; + resultSet.retainAll((resultSet == forField) ? forLabel : forField); + return resultSet; + } + + List<String> doRetrieve(GroupVersionKind type, ListOptions options, Sort sort) { + var indexer = indexerFactory.getIndexer(type); + + StopWatch stopWatch = new StopWatch(type.toString()); + + stopWatch.start("Check index status to ensure all indexes are ready"); + var fieldNamesUsedInQuery = getFieldNamesUsedInListOptions(options, sort); + checkIndexForNames(indexer, fieldNamesUsedInQuery); stopWatch.stop(); - stopWatch.start("retrieve matched metadata names by fields"); - final var hasFieldSelector = hasFieldSelector(options.getFieldSelector()); - if (hasFieldSelector) { - var fieldSelector = options.getFieldSelector(); - var query = fieldSelector.query(); - var resultSet = query.matches(indexView); - indexView.removeByIdNotIn(resultSet); - } + var indexView = new QueryIndexViewImpl(indexer); + + stopWatch.start("Evaluate selectors for index"); + var resultSet = evaluateSelectorsForIndex(indexer, indexView, options); stopWatch.stop(); - stopWatch.start("sort result"); - var result = indexView.sortBy(sort); + stopWatch.start("Sort result set by sort order"); + var result = indexView.sortBy(resultSet, sort); stopWatch.stop(); if (log.isTraceEnabled()) { - log.trace("Retrieve result from indexer, {}", stopWatch.prettyPrint()); + log.trace("Retrieve result from indexer by query [{}],\n {}", options, + stopWatch.prettyPrint(TimeUnit.MILLISECONDS)); } return result; } + void checkIndexForNames(Indexer indexer, Set<String> indexNames) { + indexer.acquireReadLock(); + try { + for (String indexName : indexNames) { + // get index entry will throw exception if index not found + indexer.getIndexEntry(indexName); + } + } finally { + indexer.releaseReadLock(); + } + } + @NonNull private Set<String> getFieldNamesUsedInListOptions(ListOptions options, Sort sort) { var fieldNamesUsedInQuery = new HashSet<String>(); @@ -213,4 +166,46 @@ boolean hasLabelSelector(LabelSelector labelSelector) { boolean hasFieldSelector(FieldSelector fieldSelector) { return fieldSelector != null; } + + record ObjectLabelMap(Map<String, Map<String, String>> objectIdLabelsMap) { + + public static ObjectLabelMap buildFrom(Indexer indexer, + List<SelectorMatcher> labelMatchers) { + indexer.acquireReadLock(); + try { + final var objectNameLabelsMap = new HashMap<String, Map<String, String>>(); + final var labelIndexEntry = indexer.getIndexEntry(LabelIndexSpecUtils.LABEL_PATH); + // O(m) time complexity, m is the number of labelMatchers + final var labelKeysToQuery = labelMatchers.stream() + .sorted(Comparator.comparing(SelectorMatcher::getKey)) + .map(SelectorMatcher::getKey) + .collect(Collectors.toSet()); + + labelIndexEntry.entries().forEach(entry -> { + // key is labelKey=labelValue, value is objectName + var labelPair = LabelIndexSpecUtils.labelKeyValuePair(entry.getKey()); + if (!labelKeysToQuery.contains(labelPair.getFirst())) { + return; + } + objectNameLabelsMap.computeIfAbsent(entry.getValue(), k -> new HashMap<>()) + .put(labelPair.getFirst(), labelPair.getSecond()); + }); + + var nameIndexOperator = new IndexEntryOperatorImpl( + indexer.getIndexEntry(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME) + ); + var allIndexedObjectNames = nameIndexOperator.getValues(); + + // remove all object names that exist labels,O(n) time complexity + allIndexedObjectNames.removeAll(objectNameLabelsMap.keySet()); + // add absent object names to object labels map + for (String name : allIndexedObjectNames) { + objectNameLabelsMap.put(name, new HashMap<>()); + } + return new ObjectLabelMap(objectNameLabelsMap); + } finally { + indexer.releaseReadLock(); + } + } + } } diff --git a/application/src/test/java/run/halo/app/content/PostQueryTest.java b/application/src/test/java/run/halo/app/content/PostQueryTest.java deleted file mode 100644 index d8b50bfc47..0000000000 --- a/application/src/test/java/run/halo/app/content/PostQueryTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package run.halo.app.content; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.mock.web.reactive.function.server.MockServerRequest; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.server.ServerWebExchange; -import run.halo.app.extension.index.query.QueryIndexViewImpl; - -/** - * Tests for {@link PostQuery}. - * - * @author guqing - * @since 2.6.0 - */ -@ExtendWith(MockitoExtension.class) -class PostQueryTest { - - @Test - void userScopedQueryTest() { - MultiValueMap<String, String> multiValueMap = new LinkedMultiValueMap<>(); - MockServerRequest request = MockServerRequest.builder() - .queryParams(multiValueMap) - .exchange(mock(ServerWebExchange.class)) - .build(); - - PostQuery postQuery = new PostQuery(request, "faker"); - - var listOptions = postQuery.toListOptions(); - assertThat(listOptions).isNotNull(); - assertThat(listOptions.getFieldSelector()).isNotNull(); - var nameEntry = - (Collection<Map.Entry<String, String>>) List.of(Map.entry("metadata.name", "faker")); - var entry = (Collection<Map.Entry<String, String>>) List.of(Map.entry("faker", "faker")); - var indexView = - new QueryIndexViewImpl(Map.of("spec.owner", entry, "metadata.name", nameEntry)); - assertThat(listOptions.getFieldSelector().query().matches(indexView)) - .containsExactly("faker"); - - entry = List.of(Map.entry("another-faker", "user1")); - indexView = new QueryIndexViewImpl(Map.of("spec.owner", entry, "metadata.name", nameEntry)); - assertThat(listOptions.getFieldSelector().query().matches(indexView)).isEmpty(); - } -} diff --git a/application/src/test/java/run/halo/app/extension/index/DefaultIndexerTest.java b/application/src/test/java/run/halo/app/extension/index/DefaultIndexerTest.java index ca6ebf133d..a1d6010b19 100644 --- a/application/src/test/java/run/halo/app/extension/index/DefaultIndexerTest.java +++ b/application/src/test/java/run/halo/app/extension/index/DefaultIndexerTest.java @@ -24,6 +24,28 @@ @ExtendWith(MockitoExtension.class) class DefaultIndexerTest { + private static FakeExtension createFakeExtension() { + var fake = new FakeExtension(); + fake.setMetadata(new Metadata()); + fake.getMetadata().setName("fake-extension"); + fake.setEmail("fake-email"); + return fake; + } + + private static IndexSpec getNameIndexSpec() { + return getIndexSpec("metadata.name", true, + IndexAttributeFactory.simpleAttribute(FakeExtension.class, + e -> e.getMetadata().getName())); + } + + private static IndexSpec getIndexSpec(String name, boolean unique, IndexAttribute attribute) { + return new IndexSpec() + .setName(name) + .setOrder(IndexSpec.OrderType.ASC) + .setUnique(unique) + .setIndexFunc(attribute); + } + @Test void constructor() { var spec = getNameIndexSpec(); @@ -48,20 +70,29 @@ void constructorWithException() { .hasMessage("Index entry not found for: metadata.name"); } + @Test + void getIndexEntryTest() { + var spec = getNameIndexSpec(); + var descriptor = new IndexDescriptor(spec); + descriptor.setReady(true); + var indexContainer = new IndexEntryContainer(); + indexContainer.add(new IndexEntryImpl(descriptor)); + + var defaultIndexer = new DefaultIndexer(List.of(descriptor), indexContainer); + assertThatThrownBy(() -> defaultIndexer.getIndexEntry("not-exist")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("No index found for fieldPath [not-exist], " + + "make sure you have created an index for this field."); + + assertThat(defaultIndexer.getIndexEntry("metadata.name")).isNotNull(); + } + @Test void getObjectKey() { var fake = createFakeExtension(); assertThat(DefaultIndexer.getObjectKey(fake)).isEqualTo("fake-extension"); } - private static FakeExtension createFakeExtension() { - var fake = new FakeExtension(); - fake.setMetadata(new Metadata()); - fake.getMetadata().setName("fake-extension"); - fake.setEmail("fake-email"); - return fake; - } - @Test void indexRecord() { var nameIndex = getNameIndexSpec(); @@ -81,12 +112,6 @@ void indexRecord() { assertThat(entries).contains(Map.entry("fake-extension", "fake-extension")); } - private static IndexSpec getNameIndexSpec() { - return getIndexSpec("metadata.name", true, - IndexAttributeFactory.simpleAttribute(FakeExtension.class, - e -> e.getMetadata().getName())); - } - @Test void indexRecordWithExceptionShouldRollback() { var indexContainer = new IndexEntryContainer(); @@ -242,14 +267,6 @@ void readyIndexesIterator() { assertThat(iterator.hasNext()).isFalse(); } - private static IndexSpec getIndexSpec(String name, boolean unique, IndexAttribute attribute) { - return new IndexSpec() - .setName(name) - .setOrder(IndexSpec.OrderType.ASC) - .setUnique(unique) - .setIndexFunc(attribute); - } - @Data @EqualsAndHashCode(callSuper = true) @GVK(group = "test", version = "v1", kind = "FakeExtension", plural = "fakeextensions", diff --git a/application/src/test/java/run/halo/app/extension/index/IndexEntryImplTest.java b/application/src/test/java/run/halo/app/extension/index/IndexEntryImplTest.java index 479b9cf313..fbadfedff9 100644 --- a/application/src/test/java/run/halo/app/extension/index/IndexEntryImplTest.java +++ b/application/src/test/java/run/halo/app/extension/index/IndexEntryImplTest.java @@ -1,12 +1,19 @@ package run.halo.app.extension.index; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static run.halo.app.extension.index.query.IndexViewDataSet.createCommentIndexView; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import run.halo.app.extension.index.query.IndexViewDataSet; +import run.halo.app.extension.index.query.QueryIndexView; /** * Tests for {@link IndexEntryImpl}. @@ -58,7 +65,7 @@ void removeByIndex() { } @Test - void getByIndexKey() { + void getObjectIdsTest() { var spec = PrimaryKeySpecUtils.primaryKeyIndexSpec(IndexEntryContainerTest.FakeExtension.class); var descriptor = new IndexDescriptor(spec); @@ -67,7 +74,7 @@ void getByIndexKey() { assertThat(entry.indexedKeys()).containsExactly("slug-1", "slug-2"); assertThat(entry.entries()).hasSize(2); - assertThat(entry.getByIndexKey("slug-1")).isEqualTo(List.of("fake-name-1")); + assertThat(entry.getObjectNamesBy("slug-1")).isEqualTo(List.of("fake-name-1")); } @Test @@ -91,7 +98,6 @@ void keyOrder() { assertThat(entry.indexedKeys()).containsSequence("slug-4", "slug-3", "slug-2", "slug-1"); - spec.setOrder(IndexSpec.OrderType.ASC); var descriptor2 = new IndexDescriptor(spec); var entry2 = new IndexEntryImpl(descriptor2); @@ -105,4 +111,40 @@ void keyOrder() { Map.entry("slug-4", "fake-name-3")); assertThat(entry2.indexedKeys()).containsSequence("slug-1", "slug-2", "slug-3", "slug-4"); } + + @Test + void getIdPositionMapTest() { + var indexView = createCommentIndexView(); + var topIndexEntry = prepareForPositionMapTest(indexView, "spec.top"); + var topIndexEntryFromView = indexView.getIndexEntry("spec.top"); + assertThat(topIndexEntry.getIdPositionMap()) + .isEqualTo(topIndexEntryFromView.getIdPositionMap()); + + var creationTimeIndexEntry = prepareForPositionMapTest(indexView, "spec.creationTime"); + var creationTimeIndexEntryFromView = indexView.getIndexEntry("spec.creationTime"); + assertThat(creationTimeIndexEntry.getIdPositionMap()) + .isEqualTo(creationTimeIndexEntryFromView.getIdPositionMap()); + + var priorityIndexEntry = prepareForPositionMapTest(indexView, "spec.priority"); + var priorityIndexEntryFromView = indexView.getIndexEntry("spec.priority"); + assertThat(priorityIndexEntry.getIdPositionMap()) + .isEqualTo(priorityIndexEntryFromView.getIdPositionMap()); + } + + IndexEntry prepareForPositionMapTest(QueryIndexView indexView, String property) { + var indexSpec = mock(IndexSpec.class); + var descriptor = mock(IndexDescriptor.class); + when(descriptor.getSpec()).thenReturn(indexSpec); + var indexEntry = new IndexEntryImpl(descriptor); + + var indexEntryFromView = indexView.getIndexEntry(property); + var sortedEntries = IndexViewDataSet.sortEntries(indexEntryFromView.entries()); + + var spyIndexEntry = spy(indexEntry); + + doReturn(IndexViewDataSet.toKeyObjectMap(sortedEntries)) + .when(spyIndexEntry).getKeyObjectMap(); + + return spyIndexEntry; + } } diff --git a/application/src/test/java/run/halo/app/extension/index/IndexEntryOperatorImplTest.java b/application/src/test/java/run/halo/app/extension/index/IndexEntryOperatorImplTest.java new file mode 100644 index 0000000000..15d0f15d41 --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/IndexEntryOperatorImplTest.java @@ -0,0 +1,177 @@ +package run.halo.app.extension.index; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import run.halo.app.extension.index.query.IndexViewDataSet; + +/** + * Tests for {@link IndexEntryOperatorImpl}. + * + * @author guqing + * @since 2.17.0 + */ +@ExtendWith(MockitoExtension.class) +class IndexEntryOperatorImplTest { + + @Mock + private IndexEntry indexEntry; + + @InjectMocks + private IndexEntryOperatorImpl indexEntryOperator; + + private LinkedHashMap<String, List<String>> createIndexedMapAndPile() { + var entries = new ArrayList<Map.Entry<String, String>>(); + entries.add(Map.entry("apple", "A")); + entries.add(Map.entry("banana", "B")); + entries.add(Map.entry("cherry", "C")); + entries.add(Map.entry("date", "D")); + entries.add(Map.entry("egg", "E")); + entries.add(Map.entry("f", "F")); + + var indexedMap = IndexViewDataSet.toKeyObjectMap(entries); + lenient().when(indexEntry.indexedKeys()).thenReturn(new TreeSet<>(indexedMap.keySet())); + lenient().when(indexEntry.getObjectNamesBy(anyString())).thenAnswer(invocation -> { + var key = (String) invocation.getArgument(0); + return indexedMap.get(key); + }); + lenient().when(indexEntry.entries()).thenReturn(entries); + return indexedMap; + } + + @Test + void lessThan() { + final var indexedMap = createIndexedMapAndPile(); + + var result = indexEntryOperator.lessThan("banana", false); + assertThat(result).containsExactly("A"); + + result = indexEntryOperator.lessThan("banana", true); + assertThat(result).containsExactly("A", "B"); + + result = indexEntryOperator.lessThan("cherry", true); + assertThat(result).containsExactly("A", "B", "C"); + + // does not exist key + result = indexEntryOperator.lessThan("z", false); + var objectIds = indexedMap.values().stream() + .flatMap(Collection::stream) + .toArray(String[]::new); + assertThat(result).contains(objectIds); + + result = indexEntryOperator.lessThan("a", false); + assertThat(result).isEmpty(); + } + + @Test + void greaterThan() { + createIndexedMapAndPile(); + + var result = indexEntryOperator.greaterThan("banana", false); + assertThat(result).containsExactly("C", "D", "E", "F"); + + result = indexEntryOperator.greaterThan("banana", true); + assertThat(result).containsExactly("B", "C", "D", "E", "F"); + + result = indexEntryOperator.greaterThan("cherry", true); + assertThat(result).containsExactly("C", "D", "E", "F"); + + result = indexEntryOperator.greaterThan("cherry", false); + assertThat(result).containsExactly("D", "E", "F"); + + // does not exist key + result = indexEntryOperator.greaterThan("z", false); + assertThat(result).isEmpty(); + } + + @Test + void greaterThanForNumberString() { + var entries = List.of( + Map.entry("100", "1"), + Map.entry("101", "2"), + Map.entry("102", "3"), + Map.entry("103", "4"), + Map.entry("110", "5"), + Map.entry("111", "6"), + Map.entry("112", "7"), + Map.entry("120", "8") + ); + var indexedMap = IndexViewDataSet.toKeyObjectMap(entries); + when(indexEntry.indexedKeys()).thenReturn(new TreeSet<>(indexedMap.keySet())); + lenient().when(indexEntry.getObjectNamesBy(anyString())).thenAnswer(invocation -> { + var key = (String) invocation.getArgument(0); + return indexedMap.get(key); + }); + when(indexEntry.entries()).thenReturn(entries); + + var result = indexEntryOperator.greaterThan("102", false); + assertThat(result).containsExactly("4", "5", "6", "7", "8"); + + result = indexEntryOperator.greaterThan("110", false); + assertThat(result).containsExactly("6", "7", "8"); + } + + @Test + void range() { + createIndexedMapAndPile(); + + var result = indexEntryOperator.range("banana", "date", true, false); + assertThat(result).containsExactly("B", "C"); + + result = indexEntryOperator.range("banana", "date", false, false); + assertThat(result).containsExactly("C"); + + result = indexEntryOperator.range("banana", "date", true, true); + assertThat(result).containsExactly("B", "C", "D"); + + result = indexEntryOperator.range("apple", "egg", false, true); + assertThat(result).containsExactly("B", "C", "D", "E"); + + // end not exist + result = indexEntryOperator.range("d", "z", false, false); + assertThat(result).containsExactly("D", "E", "F"); + + // start key > end key + assertThatThrownBy(() -> indexEntryOperator.range("z", "f", false, false)) + .isInstanceOf(IllegalArgumentException.class); + + // both not exist + result = indexEntryOperator.range("z", "zz", false, false); + assertThat(result).isEmpty(); + } + + @Test + void findTest() { + createIndexedMapAndPile(); + + var result = indexEntryOperator.find("banana"); + assertThat(result).containsExactly("B"); + + result = indexEntryOperator.find("date"); + assertThat(result).containsExactly("D"); + + result = indexEntryOperator.find("z"); + assertThat(result).isEmpty(); + } + + @Test + void findInTest() { + createIndexedMapAndPile(); + var result = indexEntryOperator.findIn(List.of("banana", "date")); + assertThat(result).containsExactly("B", "D"); + } +} \ No newline at end of file diff --git a/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java b/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java index 753acbf298..ce7d19dba2 100644 --- a/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java +++ b/application/src/test/java/run/halo/app/extension/index/IndexedQueryEngineImplTest.java @@ -1,7 +1,6 @@ package run.halo.app.extension.index; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; @@ -10,13 +9,12 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static run.halo.app.extension.index.query.IndexViewDataSet.pileForIndexer; import static run.halo.app.extension.index.query.QueryFactory.equal; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import lombok.Data; import lombok.EqualsAndHashCode; import org.junit.jupiter.api.Nested; @@ -51,16 +49,6 @@ class IndexedQueryEngineImplTest { @InjectMocks private IndexedQueryEngineImpl indexedQueryEngine; - @Test - void getIndexEntry() { - Map<String, IndexEntry> indexMap = new HashMap<>(); - assertThatThrownBy(() -> IndexedQueryEngineImpl.getIndexEntry("field1", indexMap)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "No index found for fieldPath: field1, make sure you have created an index for " - + "this field."); - } - @Test void retrieve() { var spyIndexedQueryEngine = spy(indexedQueryEngine); @@ -69,8 +57,6 @@ void retrieve() { var gvk = GroupVersionKind.fromExtension(DemoExtension.class); - when(indexerFactory.getIndexer(eq(gvk))).thenReturn(mock(Indexer.class)); - var pageRequest = mock(PageRequest.class); when(pageRequest.getPageNumber()).thenReturn(1); when(pageRequest.getPageSize()).thenReturn(2); @@ -80,8 +66,7 @@ void retrieve() { assertThat(result.getItems()).containsExactly("object1", "object2"); assertThat(result.getTotal()).isEqualTo(3); - verify(spyIndexedQueryEngine).doRetrieve(any(), any(), eq(Sort.unsorted())); - verify(indexerFactory).getIndexer(eq(gvk)); + verify(spyIndexedQueryEngine).doRetrieve(eq(gvk), any(), eq(Sort.unsorted())); verify(pageRequest, times(2)).getPageNumber(); verify(pageRequest, times(2)).getPageSize(); verify(pageRequest).getSort(); @@ -95,85 +80,53 @@ void retrieveAll() { var gvk = GroupVersionKind.fromExtension(DemoExtension.class); - when(indexerFactory.getIndexer(eq(gvk))).thenReturn(mock(Indexer.class)); - - var result = spyIndexedQueryEngine.retrieveAll(gvk, new ListOptions()); + var result = spyIndexedQueryEngine.retrieveAll(gvk, new ListOptions(), Sort.unsorted()); assertThat(result).isEmpty(); - verify(spyIndexedQueryEngine).doRetrieve(any(), any(), eq(Sort.unsorted())); - verify(indexerFactory).getIndexer(eq(gvk)); + verify(spyIndexedQueryEngine).doRetrieve(eq(gvk), any(), eq(Sort.unsorted())); } @Test void doRetrieve() { var indexer = mock(Indexer.class); - var labelEntry = mock(IndexEntry.class); - var fieldSlugEntry = mock(IndexEntry.class); - var nameEntry = mock(IndexEntry.class); - when(indexer.readyIndexesIterator()).thenReturn( - List.of(labelEntry, nameEntry, fieldSlugEntry).iterator()); - - when(nameEntry.getIndexDescriptor()) - .thenReturn(new IndexDescriptor( - PrimaryKeySpecUtils.primaryKeyIndexSpec(DemoExtension.class))); - when(nameEntry.indexedKeys()).thenReturn(Set.of("object1", "object2", "object3")); - - when(fieldSlugEntry.getIndexDescriptor()) - .thenReturn(new IndexDescriptor(new IndexSpec() - .setName("slug") - .setOrder(IndexSpec.OrderType.ASC))); - when((fieldSlugEntry.immutableEntries())).thenReturn( - List.of(Map.entry("slug1", "object1"), Map.entry("slug2", "object2"))); - - when(labelEntry.getIndexDescriptor()) - .thenReturn( - new IndexDescriptor(LabelIndexSpecUtils.labelIndexSpec(DemoExtension.class))); - when(labelEntry.entries()).thenReturn(List.of( + + var gvk = GroupVersionKind.fromExtension(DemoExtension.class); + + when(indexerFactory.getIndexer(eq(gvk))).thenReturn(indexer); + + pileForIndexer(indexer, PrimaryKeySpecUtils.PRIMARY_INDEX_NAME, List.of( + Map.entry("object1", "object1"), + Map.entry("object2", "object2"), + Map.entry("object3", "object3") + )); + + pileForIndexer(indexer, LabelIndexSpecUtils.LABEL_PATH, List.of( Map.entry("key1=value1", "object1"), Map.entry("key2=value2", "object1"), Map.entry("key1=value1", "object2"), Map.entry("key2=value2", "object2"), Map.entry("key1=value1", "object3") )); + + pileForIndexer(indexer, "slug", List.of( + Map.entry("slug1", "object1"), + Map.entry("slug2", "object2") + )); + var listOptions = new ListOptions(); listOptions.setLabelSelector(LabelSelector.builder() .eq("key1", "value1").build()); listOptions.setFieldSelector(FieldSelector.of(equal("slug", "slug1"))); - var result = indexedQueryEngine.doRetrieve(indexer, listOptions, Sort.unsorted()); + + var result = indexedQueryEngine.doRetrieve(gvk, listOptions, Sort.unsorted()); assertThat(result).containsExactly("object1"); } - @Test - void intersection() { - var list1 = Arrays.asList(1, 2, 3, 4); - var list2 = Arrays.asList(3, 4, 5, 6); - var expected = Arrays.asList(3, 4); - assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); - - list1 = Arrays.asList(1, 2, 3); - list2 = Arrays.asList(4, 5, 6); - expected = List.of(); - assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); - - list1 = List.of(); - list2 = Arrays.asList(1, 2, 3); - expected = List.of(); - assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); - - list1 = Arrays.asList(1, 2, 3); - list2 = List.of(); - expected = List.of(); - assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); - - list1 = List.of(); - list2 = List.of(); - expected = List.of(); - assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); - - list1 = Arrays.asList(1, 2, 2, 3); - list2 = Arrays.asList(2, 3, 3, 4); - expected = Arrays.asList(2, 3); - assertThat(IndexedQueryEngineImpl.intersection(list1, list2)).isEqualTo(expected); + @Data + @EqualsAndHashCode(callSuper = true) + @GVK(group = "test", version = "v1", kind = "demo", plural = "demos", singular = "demo") + static class DemoExtension extends AbstractExtension { + } @Nested @@ -186,10 +139,6 @@ class LabelMatcherTest { void testRetrieveForLabelMatchers() { // Setup mocks IndexEntry indexEntryMock = mock(IndexEntry.class); - Map<String, IndexEntry> fieldPathEntryMap = - Map.of(LabelIndexSpecUtils.LABEL_PATH, indexEntryMock); - List<String> allMetadataNames = Arrays.asList("object1", "object2", "object3"); - // Setup mock behavior when(indexEntryMock.entries()) .thenReturn(List.of(Map.entry("key1=value1", "object1"), @@ -203,23 +152,22 @@ void testRetrieveForLabelMatchers() { List<SelectorMatcher> labelMatchers = Arrays.asList(matcher1, matcher2); - // Expected results - List<String> expected = Arrays.asList("object1", "object2"); - + var indexer = mock(Indexer.class); + when(indexer.getIndexEntry(eq(LabelIndexSpecUtils.LABEL_PATH))) + .thenReturn(indexEntryMock); + var nameIndexEntry = mock(IndexEntry.class); + when(indexer.getIndexEntry(eq(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME))) + .thenReturn(nameIndexEntry); + when(nameIndexEntry.entries()).thenReturn(List.of(Map.entry("object1", "object1"), + Map.entry("object2", "object2"), Map.entry("object3", "object3"))); // Test - assertThat(indexedQueryEngine.retrieveForLabelMatchers(labelMatchers, fieldPathEntryMap, - allMetadataNames)) - .isEqualTo(expected); + assertThat(indexedQueryEngine.retrieveForLabelMatchers(indexer, labelMatchers)) + .containsSequence("object1", "object2"); } @Test void testRetrieveForLabelMatchersNoMatch() { - // Setup mocks IndexEntry indexEntryMock = mock(IndexEntry.class); - Map<String, IndexEntry> fieldPathEntryMap = - Map.of(LabelIndexSpecUtils.LABEL_PATH, indexEntryMock); - List<String> allMetadataNames = Arrays.asList("object1", "object2", "object3"); - // Setup mock behavior when(indexEntryMock.entries()) .thenReturn(List.of(Map.entry("key1=value1", "object1"), @@ -230,20 +178,17 @@ void testRetrieveForLabelMatchersNoMatch() { var matcher1 = EqualityMatcher.equal("key3", "value3"); List<SelectorMatcher> labelMatchers = List.of(matcher1); - // Expected results - List<String> expected = List.of(); - + var indexer = mock(Indexer.class); + when(indexer.getIndexEntry(eq(LabelIndexSpecUtils.LABEL_PATH))) + .thenReturn(indexEntryMock); + var nameIndexEntry = mock(IndexEntry.class); + when(indexer.getIndexEntry(eq(PrimaryKeySpecUtils.PRIMARY_INDEX_NAME))) + .thenReturn(nameIndexEntry); + when(nameIndexEntry.entries()).thenReturn(List.of(Map.entry("object1", "object1"), + Map.entry("object2", "object2"), Map.entry("object3", "object3"))); // Test - assertThat(indexedQueryEngine.retrieveForLabelMatchers(labelMatchers, fieldPathEntryMap, - allMetadataNames)) - .isEqualTo(expected); + assertThat( + indexedQueryEngine.retrieveForLabelMatchers(indexer, labelMatchers)).isEmpty(); } } - - @Data - @EqualsAndHashCode(callSuper = true) - @GVK(group = "test", version = "v1", kind = "demo", plural = "demos", singular = "demo") - static class DemoExtension extends AbstractExtension { - - } } diff --git a/api/src/test/java/run/halo/app/extension/index/query/AndTest.java b/application/src/test/java/run/halo/app/extension/index/query/AndTest.java similarity index 81% rename from api/src/test/java/run/halo/app/extension/index/query/AndTest.java rename to application/src/test/java/run/halo/app/extension/index/query/AndTest.java index d96fcf070e..0c29a8570f 100644 --- a/api/src/test/java/run/halo/app/extension/index/query/AndTest.java +++ b/application/src/test/java/run/halo/app/extension/index/query/AndTest.java @@ -1,15 +1,17 @@ package run.halo.app.extension.index.query; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static run.halo.app.extension.index.query.IndexViewDataSet.pileForIndexer; import static run.halo.app.extension.index.query.QueryFactory.and; import static run.halo.app.extension.index.query.QueryFactory.equal; import static run.halo.app.extension.index.query.QueryFactory.greaterThan; import static run.halo.app.extension.index.query.QueryFactory.or; -import java.util.Collection; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; +import run.halo.app.extension.index.Indexer; /** * Tests for the {@link And} query. @@ -21,33 +23,45 @@ public class AndTest { @Test void testMatches() { - Collection<Map.Entry<String, String>> deptEntry = List.of(Map.entry("A", "guqing"), + var deptEntry = List.of(Map.entry("A", "guqing"), Map.entry("A", "halo"), Map.entry("B", "lisi"), Map.entry("B", "zhangsan"), Map.entry("C", "ryanwang"), Map.entry("C", "johnniang") ); - Collection<Map.Entry<String, String>> ageEntry = List.of(Map.entry("19", "halo"), + var ageEntry = List.of(Map.entry("19", "halo"), Map.entry("19", "guqing"), Map.entry("18", "zhangsan"), Map.entry("17", "lisi"), Map.entry("17", "ryanwang"), Map.entry("17", "johnniang") ); - var entries = Map.of("dept", deptEntry, "age", ageEntry); - var indexView = new QueryIndexViewImpl(entries); + var indexer = mock(Indexer.class); + + pileForIndexer(indexer, QueryIndexViewImpl.PRIMARY_INDEX_NAME, List.of( + Map.entry("guqing", "guqing"), + Map.entry("halo", "halo"), + Map.entry("lisi", "lisi"), + Map.entry("zhangsan", "zhangsan"), + Map.entry("ryanwang", "ryanwang"), + Map.entry("johnniang", "johnniang") + )); + + pileForIndexer(indexer, "dept", deptEntry); + + pileForIndexer(indexer, "age", ageEntry); + + var indexView = new QueryIndexViewImpl(indexer); var query = and(equal("dept", "B"), equal("age", "18")); var resultSet = query.matches(indexView); assertThat(resultSet).containsExactly("zhangsan"); - indexView = new QueryIndexViewImpl(entries); query = and(equal("dept", "C"), equal("age", "18")); resultSet = query.matches(indexView); assertThat(resultSet).isEmpty(); - indexView = new QueryIndexViewImpl(entries); query = and( // guqing, halo, lisi, zhangsan or(equal("dept", "A"), equal("dept", "B")), @@ -57,7 +71,6 @@ void testMatches() { resultSet = query.matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder("guqing", "halo", "zhangsan"); - indexView = new QueryIndexViewImpl(entries); query = and( // guqing, halo, lisi, zhangsan or(equal("dept", "A"), equal("dept", "B")), @@ -67,7 +80,6 @@ void testMatches() { resultSet = query.matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder("guqing", "halo", "zhangsan"); - indexView = new QueryIndexViewImpl(entries); query = and( // guqing, halo, lisi, zhangsan or(equal("dept", "A"), equal("dept", "C")), diff --git a/application/src/test/java/run/halo/app/extension/index/query/IndexViewDataSet.java b/application/src/test/java/run/halo/app/extension/index/query/IndexViewDataSet.java new file mode 100644 index 0000000000..5768ea8256 --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/query/IndexViewDataSet.java @@ -0,0 +1,340 @@ +package run.halo.app.extension.index.query; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; + +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import java.util.stream.Collectors; +import run.halo.app.extension.index.IndexEntry; +import run.halo.app.extension.index.Indexer; +import run.halo.app.extension.index.KeyComparator; + +public class IndexViewDataSet { + + /** + * Create a {@link QueryIndexView} for employee to test. + * <pre> + * | id | firstName | lastName | email | hireDate | salary | managerId | departmentId | + * |----|-----------|----------|-------|----------|--------|-----------|--------------| + * | 100| Pat | Fay | p | 17 | 2600 | 101 | 50 | + * | 101| Lee | Day | l | 17 | 2400 | 102 | 40 | + * | 102| William | Jay | w | 19 | 2200 | 102 | 50 | + * | 103| Mary | Day | p | 17 | 2000 | 103 | 50 | + * | 104| John | Fay | j | 17 | 1800 | 103 | 50 | + * | 105| Gon | Fay | p | 18 | 1900 | 101 | 40 | + * </pre> + * + * @return a {@link QueryIndexView} for employee to test + */ + public static QueryIndexView createEmployeeIndexView() { + final var idEntry = List.of( + Map.entry("100", "100"), + Map.entry("101", "101"), + Map.entry("102", "102"), + Map.entry("103", "103"), + Map.entry("104", "104"), + Map.entry("105", "105") + ); + final var firstNameEntry = List.of( + Map.entry("Pat", "100"), + Map.entry("Lee", "101"), + Map.entry("William", "102"), + Map.entry("Mary", "103"), + Map.entry("John", "104"), + Map.entry("Gon", "105") + ); + final var lastNameEntry = List.of( + Map.entry("Fay", "100"), + Map.entry("Day", "101"), + Map.entry("Jay", "102"), + Map.entry("Day", "103"), + Map.entry("Fay", "104"), + Map.entry("Fay", "105") + ); + final var emailEntry = List.of( + Map.entry("p", "100"), + Map.entry("l", "101"), + Map.entry("w", "102"), + Map.entry("p", "103"), + Map.entry("j", "104"), + Map.entry("p", "105") + ); + final var hireDateEntry = List.of( + Map.entry("17", "100"), + Map.entry("17", "101"), + Map.entry("19", "102"), + Map.entry("17", "103"), + Map.entry("17", "104"), + Map.entry("18", "105") + ); + final var salaryEntry = List.of( + Map.entry("2600", "100"), + Map.entry("2400", "101"), + Map.entry("2200", "102"), + Map.entry("2000", "103"), + Map.entry("1800", "104"), + Map.entry("1900", "105") + ); + final var managerIdEntry = List.of( + Map.entry("101", "100"), + Map.entry("102", "101"), + Map.entry("102", "102"), + Map.entry("103", "103"), + Map.entry("103", "104"), + Map.entry("101", "105") + ); + final var departmentIdEntry = List.of( + Map.entry("50", "100"), + Map.entry("40", "101"), + Map.entry("50", "102"), + Map.entry("50", "103"), + Map.entry("50", "104"), + Map.entry("40", "105") + ); + + var indexer = mock(Indexer.class); + + pileForIndexer(indexer, QueryIndexViewImpl.PRIMARY_INDEX_NAME, idEntry); + pileForIndexer(indexer, "firstName", firstNameEntry); + pileForIndexer(indexer, "lastName", lastNameEntry); + pileForIndexer(indexer, "email", emailEntry); + pileForIndexer(indexer, "hireDate", hireDateEntry); + pileForIndexer(indexer, "salary", salaryEntry); + pileForIndexer(indexer, "managerId", managerIdEntry); + pileForIndexer(indexer, "departmentId", departmentIdEntry); + + return new QueryIndexViewImpl(indexer); + } + + /** + * Create a {@link QueryIndexView} for post to test. + * <pre> + * | id | title | published | publishTime | owner | + * |-----|--------|-----------|---------------------|-------| + * | 100 | title1 | true | 2024-01-01T00:00:00 | jack | + * | 101 | title2 | true | 2024-01-02T00:00:00 | rose | + * | 102 | title3 | false | null | smith | + * | 103 | title4 | false | null | peter | + * | 104 | title5 | false | null | john | + * | 105 | title6 | true | 2024-01-05 00:00:00 | tom | + * | 106 | title7 | true | 2024-01-05 13:00:00 | jerry | + * | 107 | title8 | true | 2024-01-05 12:00:00 | jerry | + * | 108 | title9 | false | null | jerry | + * </pre> + * + * @return a {@link QueryIndexView} for post to test + */ + public static QueryIndexView createPostIndexViewWithNullCell() { + final var idEntry = List.of( + Map.entry("100", "100"), + Map.entry("101", "101"), + Map.entry("102", "102"), + Map.entry("103", "103"), + Map.entry("104", "104"), + Map.entry("105", "105"), + Map.entry("106", "106"), + Map.entry("107", "107"), + Map.entry("108", "108") + ); + final var titleEntry = List.of( + Map.entry("title1", "100"), + Map.entry("title2", "101"), + Map.entry("title3", "102"), + Map.entry("title4", "103"), + Map.entry("title5", "104"), + Map.entry("title6", "105"), + Map.entry("title7", "106"), + Map.entry("title8", "107"), + Map.entry("title9", "108") + ); + final var publishedEntry = List.of( + Map.entry("true", "100"), + Map.entry("true", "101"), + Map.entry("false", "102"), + Map.entry("false", "103"), + Map.entry("false", "104"), + Map.entry("true", "105"), + Map.entry("true", "106"), + Map.entry("true", "107"), + Map.entry("false", "108") + ); + final var publishTimeEntry = List.of( + Map.entry("2024-01-01T00:00:00", "100"), + Map.entry("2024-01-02T00:00:00", "101"), + Map.entry("2024-01-05 00:00:00", "105"), + Map.entry("2024-01-05 13:00:00", "106"), + Map.entry("2024-01-05 12:00:00", "107") + ); + + final var ownerEntry = List.of( + Map.entry("jack", "100"), + Map.entry("rose", "101"), + Map.entry("smith", "102"), + Map.entry("peter", "103"), + Map.entry("john", "104"), + Map.entry("tom", "105"), + Map.entry("jerry", "106"), + Map.entry("jerry", "107"), + Map.entry("jerry", "108") + ); + + var indexer = mock(Indexer.class); + pileForIndexer(indexer, QueryIndexViewImpl.PRIMARY_INDEX_NAME, idEntry); + pileForIndexer(indexer, "title", titleEntry); + pileForIndexer(indexer, "published", publishedEntry); + pileForIndexer(indexer, "publishTime", publishTimeEntry); + pileForIndexer(indexer, "owner", ownerEntry); + + return new QueryIndexViewImpl(indexer); + } + + /** + * Creates a fake comment index view for below data. + * <pre> + * | Name | Top | Priority | Creation Time | + * | ---- | ----- | -------- | -------------------------------- | + * | 1 | True | 0 | 2024-06-05 02:58:15.633165+00:00 | + * | 2 | True | 1 | 2024-06-05 02:58:16.633165+00:00 | + * | 4 | True | 2 | 2024-06-05 02:58:18.633165+00:00 | + * | 3 | True | 2 | 2024-06-05 02:58:17.633165+00:00 | + * | 5 | True | 3 | 2024-06-05 02:58:18.633165+00:00 | + * | 6 | True | 3 | 2024-06-05 02:58:18.633165+00:00 | + * | 10 | False | 0 | 2024-06-05 02:58:17.633165+00:00 | + * | 9 | False | 0 | 2024-06-05 02:58:17.633165+00:00 | + * | 8 | False | 0 | 2024-06-05 02:58:16.633165+00:00 | + * | 7 | False | 0 | 2024-06-05 02:58:15.633165+00:00 | + * | 11 | False | 0 | 2024-06-05 02:58:14.633165+00:00 | + * | 12 | False | 1 | 2024-06-05 02:58:14.633165+00:00 | + * | 14 | False | 3 | 2024-06-05 02:58:17.633165+00:00 | + * | 13 | False | 3 | 2024-06-05 02:58:14.633165+00:00 | + * </pre> + */ + public static QueryIndexView createCommentIndexView() { + final var idEntry = List.of( + Map.entry("1", "1"), + Map.entry("2", "2"), + Map.entry("3", "3"), + Map.entry("4", "4"), + Map.entry("5", "5"), + Map.entry("6", "6"), + Map.entry("7", "7"), + Map.entry("8", "8"), + Map.entry("9", "9"), + Map.entry("10", "10"), + Map.entry("11", "11"), + Map.entry("12", "12"), + Map.entry("13", "13"), + Map.entry("14", "14") + ); + final var creationTimeEntry = List.of( + Map.entry("2024-06-05 02:58:15.633165", "1"), + Map.entry("2024-06-05 02:58:16.633165", "2"), + Map.entry("2024-06-05 02:58:17.633165", "3"), + Map.entry("2024-06-05 02:58:18.633165", "4"), + Map.entry("2024-06-05 02:58:18.633165", "5"), + Map.entry("2024-06-05 02:58:18.633165", "6"), + Map.entry("2024-06-05 02:58:15.633165", "7"), + Map.entry("2024-06-05 02:58:16.633165", "8"), + Map.entry("2024-06-05 02:58:17.633165", "9"), + Map.entry("2024-06-05 02:58:17.633165", "10"), + Map.entry("2024-06-05 02:58:14.633165", "11"), + Map.entry("2024-06-05 02:58:14.633165", "12"), + Map.entry("2024-06-05 02:58:14.633165", "13"), + Map.entry("2024-06-05 02:58:17.633165", "14") + ); + final var topEntry = List.of( + Map.entry("true", "1"), + Map.entry("true", "2"), + Map.entry("true", "3"), + Map.entry("true", "4"), + Map.entry("true", "5"), + Map.entry("true", "6"), + Map.entry("false", "7"), + Map.entry("false", "8"), + Map.entry("false", "9"), + Map.entry("false", "10"), + Map.entry("false", "11"), + Map.entry("false", "12"), + Map.entry("false", "13"), + Map.entry("false", "14") + ); + final var priorityEntry = List.of( + Map.entry("0", "1"), + Map.entry("1", "2"), + Map.entry("2", "3"), + Map.entry("2", "4"), + Map.entry("3", "5"), + Map.entry("3", "6"), + Map.entry("0", "7"), + Map.entry("0", "8"), + Map.entry("0", "9"), + Map.entry("0", "10"), + Map.entry("0", "11"), + Map.entry("1", "12"), + Map.entry("3", "13"), + Map.entry("3", "14") + ); + + var indexer = mock(Indexer.class); + pileForIndexer(indexer, QueryIndexViewImpl.PRIMARY_INDEX_NAME, idEntry); + pileForIndexer(indexer, "spec.creationTime", creationTimeEntry); + pileForIndexer(indexer, "spec.top", topEntry); + pileForIndexer(indexer, "spec.priority", priorityEntry); + + return new QueryIndexViewImpl(indexer); + } + + public static void pileForIndexer(Indexer indexer, String propertyName, + Collection<Map.Entry<String, String>> entries) { + var indexEntry = mock(IndexEntry.class); + lenient().when(indexer.getIndexEntry(propertyName)).thenReturn(indexEntry); + var sortedEntries = sortEntries(entries); + + lenient().when(indexEntry.entries()).thenReturn(sortedEntries); + lenient().when(indexEntry.getIdPositionMap()).thenReturn(idPositionMap(sortedEntries)); + + var indexedMap = toKeyObjectMap(sortedEntries); + lenient().when(indexEntry.indexedKeys()).thenReturn(new TreeSet<>(indexedMap.keySet())); + lenient().when(indexEntry.getObjectNamesBy(anyString())).thenAnswer(invocation -> { + var key = (String) invocation.getArgument(0); + return indexedMap.get(key); + }); + lenient().when(indexEntry.entries()).thenReturn(entries); + } + + public static List<Map.Entry<String, String>> sortEntries( + Collection<Map.Entry<String, String>> entries) { + return entries.stream() + .sorted((a, b) -> KeyComparator.INSTANCE.compare(a.getKey(), b.getKey())) + .toList(); + } + + public static Map<String, Integer> idPositionMap( + Collection<Map.Entry<String, String>> sortedEntries) { + var asMap = toKeyObjectMap(sortedEntries); + int i = 0; + var idPositionMap = new HashMap<String, Integer>(); + for (var valueIdsEntry : asMap.entrySet()) { + var ids = valueIdsEntry.getValue(); + for (String id : ids) { + idPositionMap.put(id, i); + } + i++; + } + return idPositionMap; + } + + public static LinkedHashMap<String, List<String>> toKeyObjectMap( + Collection<Map.Entry<String, String>> sortedEntries) { + return sortedEntries.stream() + .collect(Collectors.groupingBy(Map.Entry::getKey, + LinkedHashMap::new, + Collectors.mapping(Map.Entry::getValue, Collectors.toList()))); + } +} diff --git a/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java b/application/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java similarity index 75% rename from api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java rename to application/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java index e7b8918d72..8b10ce029c 100644 --- a/api/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java +++ b/application/src/test/java/run/halo/app/extension/index/query/QueryFactoryTest.java @@ -1,6 +1,7 @@ package run.halo.app.extension.index.query; import static org.assertj.core.api.Assertions.assertThat; +import static run.halo.app.extension.index.query.IndexViewDataSet.createEmployeeIndexView; import static run.halo.app.extension.index.query.QueryFactory.all; import static run.halo.app.extension.index.query.QueryFactory.and; import static run.halo.app.extension.index.query.QueryFactory.between; @@ -35,9 +36,11 @@ */ class QueryFactoryTest { + private final String id = QueryIndexViewImpl.PRIMARY_INDEX_NAME; + @Test void allTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = all("firstName").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "102", "103", "104", "105" @@ -64,7 +67,7 @@ void isNotNullTest() { @Test void equalTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = equal("lastName", "Fay").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "104", "105" @@ -73,8 +76,8 @@ void equalTest() { @Test void equalOtherFieldTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = equalOtherField("managerId", "id").matches(indexView); + var indexView = createEmployeeIndexView(); + var resultSet = equalOtherField("managerId", id).matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102", "103" ); @@ -82,7 +85,7 @@ void equalOtherFieldTest() { @Test void notEqualTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = notEqual("lastName", "Fay").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "101", "102", "103" @@ -91,8 +94,8 @@ void notEqualTest() { @Test void notEqualOtherFieldTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = notEqualOtherField("managerId", "id").matches(indexView); + var indexView = createEmployeeIndexView(); + var resultSet = notEqualOtherField("managerId", id).matches(indexView); // 103 102 is equal assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "104", "105" @@ -101,8 +104,8 @@ void notEqualOtherFieldTest() { @Test void lessThanTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = lessThan("id", "103").matches(indexView); + var indexView = createEmployeeIndexView(); + var resultSet = lessThan(id, "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "102" ); @@ -110,8 +113,8 @@ void lessThanTest() { @Test void lessThanOtherFieldTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = lessThanOtherField("id", "managerId").matches(indexView); + var indexView = createEmployeeIndexView(); + var resultSet = lessThanOtherField(id, "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101" ); @@ -119,8 +122,8 @@ void lessThanOtherFieldTest() { @Test void lessThanOrEqualTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = lessThanOrEqual("id", "103").matches(indexView); + var indexView = createEmployeeIndexView(); + var resultSet = lessThanOrEqual(id, "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "102", "103" ); @@ -128,9 +131,9 @@ void lessThanOrEqualTest() { @Test void lessThanOrEqualOtherFieldTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = - lessThanOrEqualOtherField("id", "managerId").matches(indexView); + lessThanOrEqualOtherField(id, "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "101", "102", "103" ); @@ -138,8 +141,8 @@ void lessThanOrEqualOtherFieldTest() { @Test void greaterThanTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = greaterThan("id", "103").matches(indexView); + var indexView = createEmployeeIndexView(); + var resultSet = greaterThan(id, "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "104", "105" ); @@ -147,8 +150,8 @@ void greaterThanTest() { @Test void greaterThanOtherFieldTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = greaterThanOtherField("id", "managerId").matches(indexView); + var indexView = createEmployeeIndexView(); + var resultSet = greaterThanOtherField(id, "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "104", "105" ); @@ -156,8 +159,8 @@ void greaterThanOtherFieldTest() { @Test void greaterThanOrEqualTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = greaterThanOrEqual("id", "103").matches(indexView); + var indexView = createEmployeeIndexView(); + var resultSet = greaterThanOrEqual(id, "103").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "103", "104", "105" ); @@ -165,9 +168,9 @@ void greaterThanOrEqualTest() { @Test void greaterThanOrEqualOtherFieldTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = - greaterThanOrEqualOtherField("id", "managerId").matches(indexView); + greaterThanOrEqualOtherField(id, "managerId").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102", "103", "104", "105" ); @@ -175,8 +178,8 @@ void greaterThanOrEqualOtherFieldTest() { @Test void inTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = in("id", "103", "104").matches(indexView); + var indexView = createEmployeeIndexView(); + var resultSet = in(id, "103", "104").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "103", "104" ); @@ -184,7 +187,7 @@ void inTest() { @Test void inTest2() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = in("lastName", "Fay").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "100", "104", "105" @@ -193,13 +196,13 @@ void inTest2() { @Test void betweenTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); - var resultSet = between("id", "103", "105").matches(indexView); + var indexView = createEmployeeIndexView(); + var resultSet = between(id, "103", "105").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "103", "104", "105" ); - indexView = IndexViewDataSet.createEmployeeIndexView(); + indexView = createEmployeeIndexView(); resultSet = between("salary", "2000", "2400").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "101", "102", "103" @@ -208,7 +211,7 @@ void betweenTest() { @Test void betweenLowerExclusive() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = QueryFactory.betweenLowerExclusive("salary", "2000", "2400").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( @@ -218,7 +221,7 @@ void betweenLowerExclusive() { @Test void betweenUpperExclusive() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = QueryFactory.betweenUpperExclusive("salary", "2000", "2400").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( @@ -228,7 +231,7 @@ void betweenUpperExclusive() { @Test void betweenExclusive() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = QueryFactory.betweenExclusive("salary", "2000", "2400").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102" @@ -237,7 +240,7 @@ void betweenExclusive() { @Test void startsWithTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = startsWith("firstName", "W").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102" @@ -246,7 +249,7 @@ void startsWithTest() { @Test void endsWithTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = endsWith("firstName", "y").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "103" @@ -255,7 +258,7 @@ void endsWithTest() { @Test void containsTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = contains("firstName", "i").matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( "102" @@ -268,7 +271,7 @@ void containsTest() { @Test void notTest() { - var indexView = IndexViewDataSet.createEmployeeIndexView(); + var indexView = createEmployeeIndexView(); var resultSet = QueryFactory.not(QueryFactory.contains("firstName", "i")).matches(indexView); assertThat(resultSet).containsExactlyInAnyOrder( @@ -286,10 +289,10 @@ void getUsedFieldNamesTest() { // and composite query query = and( and(equal("firstName", "W"), equal("lastName", "Fay")), - or(equalOtherField("id", "userId"), lessThan("age", "123")) + or(equalOtherField(id, "userId"), lessThan("age", "123")) ); fieldNames = getFieldNamesUsedInQuery(query); - assertThat(fieldNames).containsExactlyInAnyOrder("firstName", "lastName", "id", "userId", + assertThat(fieldNames).containsExactlyInAnyOrder("firstName", "lastName", id, "userId", "age"); // or composite query diff --git a/application/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java b/application/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java new file mode 100644 index 0000000000..b746295704 --- /dev/null +++ b/application/src/test/java/run/halo/app/extension/index/query/QueryIndexViewImplTest.java @@ -0,0 +1,301 @@ +package run.halo.app.extension.index.query; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static run.halo.app.extension.index.query.IndexViewDataSet.createCommentIndexView; +import static run.halo.app.extension.index.query.IndexViewDataSet.createEmployeeIndexView; +import static run.halo.app.extension.index.query.IndexViewDataSet.createPostIndexViewWithNullCell; +import static run.halo.app.extension.index.query.IndexViewDataSet.pileForIndexer; +import static run.halo.app.extension.index.query.QueryIndexViewImpl.PRIMARY_INDEX_NAME; + +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.domain.Sort; +import run.halo.app.extension.index.IndexEntry; +import run.halo.app.extension.index.Indexer; + +/** + * Tests for {@link QueryIndexViewImpl}. + * + * @author guqing + * @since 2.17.0 + */ +class QueryIndexViewImplTest { + final String id = PRIMARY_INDEX_NAME; + + @Test + void getAllIdsForFieldTest() { + var indexView = createPostIndexViewWithNullCell(); + var resultSet = indexView.getIdsForField("title"); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "102", "103", "104", "105", "106", "107", "108" + ); + + resultSet = indexView.getIdsForField("publishTime"); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "105", "106", "107" + ); + } + + @Test + void findIdsForValueEqualTest() { + var indexView = createEmployeeIndexView(); + var resultSet = indexView.findMatchingIdsWithEqualValues("managerId", id); + assertThat(resultSet).containsExactlyInAnyOrder( + "102", "103" + ); + } + + @Test + void findIdsForFieldValueGreaterThanTest() { + var indexView = createEmployeeIndexView(); + var resultSet = indexView.findMatchingIdsWithGreaterValues(id, "managerId", false); + assertThat(resultSet).containsExactlyInAnyOrder( + "104", "105" + ); + + indexView = createEmployeeIndexView(); + resultSet = indexView.findMatchingIdsWithGreaterValues(id, "managerId", true); + assertThat(resultSet).containsExactlyInAnyOrder( + "103", "102", "104", "105" + ); + } + + @Test + void findIdsForFieldValueGreaterThanTest2() { + var indexView = createEmployeeIndexView(); + var resultSet = indexView.findMatchingIdsWithGreaterValues("managerId", id, false); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101" + ); + + indexView = createEmployeeIndexView(); + resultSet = indexView.findMatchingIdsWithGreaterValues("managerId", id, true); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "102", "103" + ); + } + + @Test + void findIdsForFieldValueLessThanTest() { + var indexView = createEmployeeIndexView(); + var resultSet = indexView.findMatchingIdsWithSmallerValues(id, "managerId", false); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101" + ); + + indexView = createEmployeeIndexView(); + resultSet = indexView.findMatchingIdsWithSmallerValues(id, "managerId", true); + assertThat(resultSet).containsExactlyInAnyOrder( + "100", "101", "102", "103" + ); + } + + @Test + void findIdsForFieldValueLessThanTest2() { + var indexView = createEmployeeIndexView(); + var resultSet = indexView.findMatchingIdsWithSmallerValues("managerId", id, false); + assertThat(resultSet).containsExactlyInAnyOrder( + "104", "105" + ); + + indexView = createEmployeeIndexView(); + resultSet = indexView.findMatchingIdsWithSmallerValues("managerId", id, true); + assertThat(resultSet).containsExactlyInAnyOrder( + "103", "102", "104", "105" + ); + } + + @Nested + @ExtendWith(MockitoExtension.class) + class SortTest { + @Mock + private Indexer indexer; + + @Test + void testSortByUnsorted() { + var idEntry = mock(IndexEntry.class); + when(indexer.getIndexEntry(PRIMARY_INDEX_NAME)) + .thenReturn(idEntry); + var indexView = new QueryIndexViewImpl(indexer); + + var sort = Sort.unsorted(); + + var resultSet = new TreeSet<>(List.of("Item1", "Item2")); + List<String> sortedList = indexView.sortBy(resultSet, sort); + assertThat(sortedList).isEqualTo(List.of("Item1", "Item2")); + } + + @Test + void testSortBySortedAscending() { + pileForIndexer(indexer, "field1", + List.of(Map.entry("key2", "Item2"), Map.entry("key1", "Item1"))); + + pileForIndexer(indexer, PRIMARY_INDEX_NAME, + List.of(Map.entry("Item1", "Item1"), Map.entry("Item2", "Item2"))); + + var indexView = new QueryIndexViewImpl(indexer); + + var sort = Sort.by(Sort.Order.asc("field1")); + + List<String> sortedList = indexView.sortBy(indexView.getAllIds(), sort); + + assertThat(sortedList).containsSequence("Item1", "Item2"); + } + + @Test + void testSortBySortedDescending() { + pileForIndexer(indexer, "field1", + List.of(Map.entry("key1", "Item1"), Map.entry("key2", "Item2"))); + + pileForIndexer(indexer, PRIMARY_INDEX_NAME, + List.of(Map.entry("Item1", "Item1"), Map.entry("Item2", "Item2"))); + + var indexView = new QueryIndexViewImpl(indexer); + + var sort = Sort.by(Sort.Order.desc("field1")); + + var resultSet = new TreeSet<>(List.of("Item1", "Item2")); + List<String> sortedList = indexView.sortBy(resultSet, sort); + + assertThat(sortedList).containsExactly("Item2", "Item1"); + } + + @Test + void testSortByMultipleFields() { + pileForIndexer(indexer, "field1", + List.of(Map.entry("k3", "Item3"), Map.entry("k2", "Item2"))); + + pileForIndexer(indexer, "field2", + List.of(Map.entry("k1", "Item1"), Map.entry("k3", "Item3"))); + + pileForIndexer(indexer, id, + List.of(Map.entry("Item1", "Item1"), Map.entry("Item2", "Item2"), + Map.entry("Item3", "Item3"))); + + var indexView = new QueryIndexViewImpl(indexer); + + var sort = Sort.by(Sort.Order.asc("field1"), Sort.Order.desc("field2")); + + var resultSet = new TreeSet<>(List.of("Item1", "Item2", "Item3")); + List<String> sortedList = indexView.sortBy(resultSet, sort); + + assertThat(sortedList).containsExactly("Item2", "Item3", "Item1"); + } + + @Test + void testSortByMultipleFields2() { + pileForIndexer(indexer, id, List.of()); + + pileForIndexer(indexer, "field1", List.of(Map.entry("John", "John"), + Map.entry("Bob", "Bob"), + Map.entry("Alice", "Alice") + )); + pileForIndexer(indexer, "field2", List.of(Map.entry("David", "David"), + Map.entry("Eva", "Eva"), + Map.entry("Frank", "Frank") + )); + pileForIndexer(indexer, "field3", List.of(Map.entry("George", "George"), + Map.entry("Helen", "Helen"), + Map.entry("Ivy", "Ivy") + )); + + /* + * <pre> + * Row Key | field1 | field2 | field3 + * -------|-------|-------|------- + * John | John | | + * Bob | Bob | | + * Alice | Alice | | + * David | | David | + * Eva | | Eva | + * Frank | | Frank | + * George | | | George + * Helen | | | Helen + * Ivy | | | Ivy + * </pre> + */ + var indexView = new QueryIndexViewImpl(indexer); + var sort = Sort.by(Sort.Order.desc("field1"), Sort.Order.asc("field2"), + Sort.Order.asc("field3")); + + var resultSet = new TreeSet<>( + List.of("Bob", "John", "Eva", "Alice", "Ivy", "David", "Frank", "Helen", "George")); + List<String> sortedList = indexView.sortBy(resultSet, sort); + + assertThat(sortedList).containsSequence("David", "Eva", "Frank", "George", "Helen", + "Ivy", "John", "Bob", "Alice"); + } + + /** + * <p>Result for the following data.</p> + * <pre> + * | id | firstName | lastName | email | hireDate | salary | managerId | departmentId | + * |----|-----------|----------|-------|----------|--------|-----------|--------------| + * | 100| Pat | Fay | p | 17 | 2600 | 101 | 50 | + * | 101| Lee | Day | l | 17 | 2400 | 102 | 40 | + * | 103| Mary | Day | p | 17 | 2000 | 103 | 50 | + * | 104| John | Fay | j | 17 | 1800 | 103 | 50 | + * | 105| Gon | Fay | p | 18 | 1900 | 101 | 40 | + * | 102| William | Jay | w | 19 | 2200 | 102 | 50 | + * </pre> + */ + @Test + void sortByMultipleFieldsWithFirstSame() { + var indexView = createEmployeeIndexView(); + var ids = indexView.getAllIds(); + var result = indexView.sortBy(ids, Sort.by(Sort.Order.asc("hireDate"), + Sort.Order.asc("lastName")) + ); + assertThat(result).containsSequence("101", "103", "100", "104", "105", "102"); + } + + /** + * <p>Result for the following data.</p> + * <pre> + * | id | title | published | publishTime | owner | + * |-----|--------|-----------|---------------------|-------| + * | 100 | title1 | true | 2024-01-01T00:00:00 | jack | + * | 101 | title2 | true | 2024-01-02T00:00:00 | rose | + * | 105 | title6 | true | 2024-01-05 00:00:00 | tom | + * | 107 | title8 | true | 2024-01-05 12:00:00 | jerry | + * | 106 | title7 | true | 2024-01-05 13:00:00 | jerry | + * | 108 | title9 | false | null | jerry | + * | 104 | title5 | false | null | john | + * | 103 | title4 | false | null | peter | + * | 102 | title3 | false | null | smith | + * </pre> + */ + @Test + void sortByMultipleFieldsForPostDataSet() { + var indexView = createPostIndexViewWithNullCell(); + var ids = indexView.getAllIds(); + var result = indexView.sortBy(ids, Sort.by(Sort.Order.asc("publishTime"), + Sort.Order.desc("title")) + ); + assertThat(result).containsSequence("100", "101", "105", "107", "106", "108", "104", + "103", "102"); + } + + @Test + void sortByMultipleFieldsForCommentDataSet() { + var indexView = createCommentIndexView(); + var ids = indexView.getAllIds(); + var sort = Sort.by(Sort.Order.desc("spec.top"), + Sort.Order.asc("spec.priority"), + Sort.Order.desc("spec.creationTime"), + Sort.Order.asc("metadata.name") + ); + var result = indexView.sortBy(ids, sort); + assertThat(result).containsSequence("1", "2", "4", "3", "5", "6", "9", "10", "8", "7", + "11", "12", "14", "13"); + } + } +}