refactor: index mechanism to enhance overall performance

api/src/main/java/run/halo/app/extension/ListOptions.java

@@ -10,4 +10,19 @@
 public class ListOptions {
     private LabelSelector labelSelector;
     private FieldSelector fieldSelector;
-}
+
+    @Override
+    public String toString() {
+        var sb = new StringBuilder();
+        if (fieldSelector != null) {
+            sb.append("fieldSelector: ").append(fieldSelector.query());
+        }
+        if (labelSelector != null) {
+            if (!sb.isEmpty()) {
+                sb.append(", ");
+            }
+            sb.append("labelSelector: ").append(labelSelector);
+        }
+        return sb.toString();
+    }
+}

api/src/main/java/run/halo/app/extension/controller/RequestSynchronizer.java

@@ -2,6 +2,7 @@
 
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.domain.Sort;
 import run.halo.app.extension.Extension;
 import run.halo.app.extension.ExtensionClient;
 import run.halo.app.extension.ExtensionMatcher;
@@ -58,7 +59,7 @@ public void start() {
                 listOptions.setFieldSelector(listMatcher.getFieldSelector());
                 listOptions.setLabelSelector(listMatcher.getLabelSelector());
             }
-            indexedQueryEngine.retrieveAll(type, listOptions)
+            indexedQueryEngine.retrieveAll(type, listOptions, Sort.by("metadata.creationTimestamp"))
                 .forEach(name -> watcher.onAdd(new Request(name)));
         }
         client.watch(this.watcher);

application/src/main/java/run/halo/app/extension/index/IndexDescriptor.java → api/src/main/java/run/halo/app/extension/index/IndexDescriptor.java

@@ -10,7 +10,7 @@ public class IndexDescriptor {
     private final IndexSpec spec;
 
     /**
-     * Record whether the index is ready, managed by {@link IndexBuilder}.
+     * Record whether the index is ready, managed by {@code IndexBuilder}.
      */
     private boolean ready;
 

application/src/main/java/run/halo/app/extension/index/IndexEntry.java → api/src/main/java/run/halo/app/extension/index/IndexEntry.java

@@ -3,7 +3,7 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
+import java.util.NavigableSet;
 import run.halo.app.extension.Metadata;
 
 /**
@@ -34,7 +34,7 @@
 public interface IndexEntry {
 
     /**
-     * Acquires the read lock for reading such as {@link #getByIndexKey(String)},
+     * Acquires the read lock for reading such as {@link #getObjectNamesBy(String)},
      * {@link #entries()}, {@link #indexedKeys()}, because the returned result set of these
      * methods is not immutable.
      */
@@ -87,7 +87,7 @@ public interface IndexEntry {
      *
      * @return distinct indexed keys of this entry.
      */
-    Set<String> indexedKeys();
+    NavigableSet<String> indexedKeys();
 
     /**
      * <p>Returns the entries of this entry in order.</p>
@@ -99,19 +99,34 @@ public interface IndexEntry {
     Collection<Map.Entry<String, String>> entries();
 
     /**
-     * Returns the immutable entries of this entry in order, it is safe to modify the returned
-     * result, but extra cost is made.
-     *
-     * @return immutable entries of this entry.
+     * <p>Returns the position of the object name in the indexed attribute value mapping for
+     * sorting.</p>
+     * For example:
+     * <pre>
+     * metadata.name | field1
+     * ------------- | ------
+     * foo           | 1
+     * bar           | 2
+     * baz           | 2
+     * </pre>
+     * "field1" is the indexed attribute, and the position of the object name in the indexed
+     * attribute
+     * value mapping for sorting is:
+     * <pre>
+     * foo -> 0
+     * bar -> 1
+     * baz -> 1
+     * </pre>
+     * "bar" and "baz" have the same value, so they have the same position.
      */
-    Collection<Map.Entry<String, String>> immutableEntries();
+    Map<String, Integer> getIdPositionMap();
 
     /**
      * Returns the object names of this entry in order.
      *
      * @return object names of this entry.
      */
-    List<String> getByIndexKey(String indexKey);
+    List<String> getObjectNamesBy(String indexKey);
 
     void clear();
 }

api/src/main/java/run/halo/app/extension/index/IndexEntryOperator.java

@@ -0,0 +1,55 @@
+package run.halo.app.extension.index;
+
+import java.util.Collection;
+import java.util.NavigableSet;
+import java.util.Set;
+
+public interface IndexEntryOperator {
+
+    /**
+     * Search all values that key less than the target key.
+     *
+     * @param key target key
+     * @param orEqual whether to include the value of the target key
+     * @return object names that key less than the target key
+     */
+    NavigableSet<String> lessThan(String key, boolean orEqual);
+
+    /**
+     * Search all values that key greater than the target key.
+     *
+     * @param key target key
+     * @param orEqual whether to include the value of the target key
+     * @return object names that key greater than the target key
+     */
+    NavigableSet<String> greaterThan(String key, boolean orEqual);
+
+    /**
+     * Search all values that key in the range of [start, end].
+     *
+     * @param start start key
+     * @param end end key
+     * @param startInclusive whether to include the value of the start key
+     * @param endInclusive whether to include the value of the end key
+     * @return object names that key in the range of [start, end]
+     */
+    NavigableSet<String> range(String start, String end, boolean startInclusive,
+        boolean endInclusive);
+
+    /**
+     * Find all values that key equals to the target key.
+     *
+     * @param key target key
+     * @return object names that key equals to the target key
+     */
+    NavigableSet<String> find(String key);
+
+    NavigableSet<String> findIn(Collection<String> keys);
+
+    /**
+     * Get all values in the index entry.
+     *
+     * @return a set of all object names
+     */
+    Set<String> getValues();
+}

api/src/main/java/run/halo/app/extension/index/IndexEntryOperatorImpl.java

@@ -0,0 +1,112 @@
+package run.halo.app.extension.index;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+import org.springframework.util.Assert;
+
+public class IndexEntryOperatorImpl implements IndexEntryOperator {
+    private final IndexEntry indexEntry;
+
+    public IndexEntryOperatorImpl(IndexEntry indexEntry) {
+        this.indexEntry = indexEntry;
+    }
+
+    private static NavigableSet<String> createNavigableSet() {
+        return new TreeSet<>(KeyComparator.INSTANCE);
+    }
+
+    @Override
+    public NavigableSet<String> lessThan(String key, boolean orEqual) {
+        Assert.notNull(key, "Key must not be null.");
+        indexEntry.acquireReadLock();
+        try {
+            var navigableIndexedKeys = indexEntry.indexedKeys();
+            var headSetKeys = navigableIndexedKeys.headSet(key, orEqual);
+            return findIn(headSetKeys);
+        } finally {
+            indexEntry.releaseReadLock();
+        }
+    }
+
+    @Override
+    public NavigableSet<String> greaterThan(String key, boolean orEqual) {
+        Assert.notNull(key, "Key must not be null.");
+        indexEntry.acquireReadLock();
+        try {
+            var navigableIndexedKeys = indexEntry.indexedKeys();
+            var tailSetKeys = navigableIndexedKeys.tailSet(key, orEqual);
+            return findIn(tailSetKeys);
+        } finally {
+            indexEntry.releaseReadLock();
+        }
+    }
+
+    @Override
+    public NavigableSet<String> range(String start, String end, boolean startInclusive,
+        boolean endInclusive) {
+        Assert.notNull(start, "The start must not be null.");
+        Assert.notNull(end, "The end must not be null.");
+        indexEntry.acquireReadLock();
+        try {
+            var navigableIndexedKeys = indexEntry.indexedKeys();
+            var tailSetKeys = navigableIndexedKeys.subSet(start, startInclusive, end, endInclusive);
+            return findIn(tailSetKeys);
+        } finally {
+            indexEntry.releaseReadLock();
+        }
+    }
+
+    @Override
+    public NavigableSet<String> find(String key) {
+        Assert.notNull(key, "The key must not be null.");
+        indexEntry.acquireReadLock();
+        try {
+            var resultSet = createNavigableSet();
+            var result = indexEntry.getObjectNamesBy(key);
+            if (result != null) {
+                resultSet.addAll(result);
+            }
+            return resultSet;
+        } finally {
+            indexEntry.releaseReadLock();
+        }
+    }
+
+    @Override
+    public NavigableSet<String> findIn(Collection<String> keys) {
+        if (keys == null || keys.isEmpty()) {
+            return createNavigableSet();
+        }
+        indexEntry.acquireReadLock();
+        try {
+            var keysToSearch = new HashSet<>(keys);
+            var resultSet = createNavigableSet();
+            for (var entry : indexEntry.entries()) {
+                if (keysToSearch.contains(entry.getKey())) {
+                    resultSet.add(entry.getValue());
+                }
+            }
+            return resultSet;
+        } finally {
+            indexEntry.releaseReadLock();
+        }
+    }
+
+    @Override
+    public Set<String> getValues() {
+        indexEntry.acquireReadLock();
+        try {
+            Set<String> uniqueValues = new HashSet<>();
+            for (Map.Entry<String, String> entry : indexEntry.entries()) {
+                uniqueValues.add(entry.getValue());
+            }
+            return uniqueValues;
+        } finally {
+            indexEntry.releaseReadLock();
+        }
+    }
+}

api/src/main/java/run/halo/app/extension/index/IndexedQueryEngine.java

@@ -1,6 +1,7 @@
 package run.halo.app.extension.index;
 
 import java.util.List;
+import org.springframework.data.domain.Sort;
 import run.halo.app.extension.GroupVersionKind;
 import run.halo.app.extension.ListOptions;
 import run.halo.app.extension.ListResult;
@@ -36,7 +37,8 @@ public interface IndexedQueryEngine {
      *
      * @param type the type of the object must exist in {@link run.halo.app.extension.SchemeManager}
      * @param options the list options to use for retrieving the object records
+     * @param sort the sort to use for retrieving the object records
      * @return a collection of {@link Metadata#getName()}
      */
-    List<String> retrieveAll(GroupVersionKind type, ListOptions options);
+    List<String> retrieveAll(GroupVersionKind type, ListOptions options, Sort sort);
 }

application/src/main/java/run/halo/app/extension/index/Indexer.java → api/src/main/java/run/halo/app/extension/index/Indexer.java

@@ -2,6 +2,7 @@
 
 import java.util.Iterator;
 import java.util.function.Function;
+import org.springframework.lang.NonNull;
 import run.halo.app.extension.Extension;
 
 /**
@@ -20,7 +21,7 @@ public interface Indexer {
     /**
      * <p>Index the specified {@link Extension} by {@link IndexDescriptor}s.</p>
      * <p>First, the {@link Indexer} will index the {@link Extension} by the
-     * {@link IndexDescriptor}s and record the index entries to {@link IndexerTransaction} and
+     * {@link IndexDescriptor}s and record the index entries to {@code IndexerTransaction} and
      * commit the transaction, if any error occurs, the transaction will be rollback to keep the
      * {@link Indexer} consistent.</p>
      *
@@ -33,7 +34,7 @@ public interface Indexer {
      * <p>Update indexes for the specified {@link Extension} by {@link IndexDescriptor}s.</p>
      * <p>First, the {@link Indexer} will remove the index entries of the {@link Extension} by
      * the old {@link IndexDescriptor}s and reindex the {@link Extension} to generate change logs
-     * to {@link IndexerTransaction} and commit the transaction, if any error occurs, the
+     * to {@code IndexerTransaction} and commit the transaction, if any error occurs, the
      * transaction will be rollback to keep the {@link Indexer} consistent.</p>
      *
      * @param extension the {@link Extension} to be updated
@@ -73,19 +74,33 @@ public interface Indexer {
      */
     void removeIndexRecords(Function<IndexDescriptor, Boolean> matchFn);
 
+    /**
+     * <p>Get the {@link IndexEntry} by index name if found and ready.</p>
+     *
+     * @param name an index name
+     * @return the {@link IndexEntry} if found
+     * @throws IllegalArgumentException if the index name is not found or the index is not ready
+     */
+    @NonNull
+    IndexEntry getIndexEntry(String name);
+
     /**
      * <p>Gets an iterator over all the ready {@link IndexEntry}s, in no particular order.</p>
      *
      * @return an iterator over all the ready {@link IndexEntry}s
-     * @link {@link IndexDescriptor#isReady()}
+     * @see IndexDescriptor#isReady()
      */
     Iterator<IndexEntry> readyIndexesIterator();
 
     /**
      * <p>Gets an iterator over all the {@link IndexEntry}s, in no particular order.</p>
      *
      * @return an iterator over all the {@link IndexEntry}s
-     * @link {@link IndexDescriptor#isReady()}
+     * @see IndexDescriptor#isReady()
      */
     Iterator<IndexEntry> allIndexesIterator();
+
+    void acquireReadLock();
+
+    void releaseReadLock();
 }

api/src/main/java/run/halo/app/extension/index/query/All.java

@@ -10,6 +10,11 @@ public All(String fieldName) {
 
     @Override
     public NavigableSet<String> matches(QueryIndexView indexView) {
-        return indexView.getAllIdsForField(fieldName);
+        return indexView.getIdsForField(fieldName);
+    }
+
+    @Override
+    public String toString() {
+        return fieldName + " != null";
     }
 }

api/src/main/java/run/halo/app/extension/index/query/And.java

@@ -3,6 +3,7 @@
 import com.google.common.collect.Sets;
 import java.util.Collection;
 import java.util.NavigableSet;
+import java.util.stream.Collectors;
 
 public class And extends LogicalQuery {
 
@@ -33,4 +34,10 @@ public NavigableSet<String> matches(QueryIndexView indexView) {
         }
         return resultSet == null ? Sets.newTreeSet() : resultSet;
     }
+
+    @Override
+    public String toString() {
+        return "(" + childQueries.stream().map(Query::toString)
+            .collect(Collectors.joining(" AND ")) + ")";
+    }
 }