Skip to content

Commit

Permalink
Search elements are now returned paged (#149)
Browse files Browse the repository at this point in the history 
* feat: search elements are now returned paged

---------

Signed-off-by: LE SAULNIER Kevin <kevin.lesaulnier@rte-france.com>
Co-authored-by: LE SAULNIER Kevin <kevin.lesaulnier@rte-france.com>
Co-authored-by: jamal-khey <myjamal89@gmail.com>
  • Loading branch information
@klesaulnier @jamal-khey
3 people authored Jul 22, 2024
1 parent a9bd7e4 commit 6607024
Show file tree
Hide file tree
Showing 6 changed files with 130 additions and 35 deletions.
3 changes: 2 additions & 1 deletion src/main/java/org/gridsuite/directory/server/DirectoryController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.gridsuite.directory.server.dto.RootDirectoryAttributes;
import org.gridsuite.directory.server.dto.elasticsearch.DirectoryElementInfos;
import org.gridsuite.directory.server.services.DirectoryRepositoryService;
import org.springframework.data.domain.Page;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
Expand Down Expand Up @@ -253,7 +254,7 @@ public ResponseEntity<Void> elementExists(@PathVariable("directoryUuid") UUID di
@GetMapping(value = "/elements/indexation-infos", produces = MediaType.APPLICATION_JSON_VALUE)
@Operation(summary = "Search elements in elasticsearch")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "List of elements found")})
public ResponseEntity<List<DirectoryElementInfos>> searchElements(
public ResponseEntity<Page<DirectoryElementInfos>> searchElements(
@Parameter(description = "User input") @RequestParam(value = "userInput") String userInput,
@Parameter(description = "Current directory UUID") @RequestParam(value = "directoryUuid", required = false, defaultValue = "") String directoryUuid) {
return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON)
Expand Down
9 changes: 7 additions & 2 deletions src/main/java/org/gridsuite/directory/server/DirectoryService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.messaging.Message;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
Expand Down Expand Up @@ -45,6 +48,7 @@ public class DirectoryService {
public static final String HEADER_STUDY_UUID = "studyUuid";
private static final String CATEGORY_BROKER_INPUT = DirectoryService.class.getName() + ".input-broker-messages";
private static final Logger LOGGER = LoggerFactory.getLogger(DirectoryService.class);
private static final int ES_PAGE_MAX_SIZE = 50;

private final NotificationService notificationService;

Expand Down Expand Up @@ -564,8 +568,9 @@ public String getDuplicateNameCandidate(UUID directoryUuid, String elementName,
return nameCandidate(elementName, i);
}

public List<DirectoryElementInfos> searchElements(@NonNull String userInput, String directoryUuid) {
return directoryElementInfosService.searchElements(userInput, directoryUuid);
public Page<DirectoryElementInfos> searchElements(@NonNull String userInput, String directoryUuid) {
Pageable pageRequest = PageRequest.of(0, ES_PAGE_MAX_SIZE);
return directoryElementInfosService.searchElements(userInput, directoryUuid, pageRequest);
}

public boolean areDirectoryElementsDeletable(List<UUID> elementsUuid, String userId) {
Expand Down
22 changes: 22 additions & 0 deletions src/main/java/org/gridsuite/directory/server/elasticsearch/ESUtils.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package org.gridsuite.directory.server.elasticsearch;

import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;

import java.util.List;

public final class ESUtils {

private ESUtils() {
// This constructor is private to prevent instantiation of the utility class.
throw new UnsupportedOperationException("This is a utility class and cannot be instantiated");
}

public static <T> Page<T> searchHitsToPage(SearchHits<T> searchHits, Pageable pageable) {
List<T> content = searchHits.stream().map(SearchHit::getContent).toList();
return new PageImpl<>(content, pageable, searchHits.getTotalHits());
}
}
35 changes: 20 additions & 15 deletions src/main/java/org/gridsuite/directory/server/services/DirectoryElementInfosService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,39 +6,35 @@
*/
package org.gridsuite.directory.server.services;

import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.MatchQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch._types.query_dsl.TermQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.*;
import lombok.Getter;
import lombok.NonNull;
import org.gridsuite.directory.server.dto.elasticsearch.DirectoryElementInfos;
import org.gridsuite.directory.server.elasticsearch.ESConfig;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.client.elc.NativeQuery;
import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder;
import org.springframework.data.elasticsearch.client.elc.Queries;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.stereotype.Service;

import java.util.List;

import static org.gridsuite.directory.server.DirectoryService.DIRECTORY;
import static org.gridsuite.directory.server.elasticsearch.ESUtils.searchHitsToPage;

/**
* @author Ghazwa Rehili <ghazwa.rehili at rte-france.com>
*/
@Service
public class DirectoryElementInfosService {

private static final int PAGE_MAX_SIZE = 10;

private final ElasticsearchOperations elasticsearchOperations;

private static final String ELEMENT_NAME = "name.fullascii";
private static final String FULL_PATH_UUID = "fullPathUuid.keyword";
private static final String PATH_UUID = "pathUuid.keyword";
private static final String PARENT_ID = "parentId.keyword";
static final String ELEMENT_TYPE = "type.keyword";

Expand All @@ -50,7 +46,7 @@ public DirectoryElementInfosService(ElasticsearchOperations elasticsearchOperati
this.elasticsearchOperations = elasticsearchOperations;
}

public List<DirectoryElementInfos> searchElements(@NonNull String userInput, String currentDirectoryUuid) {
public Page<DirectoryElementInfos> searchElements(@NonNull String userInput, String currentDirectoryUuid, Pageable pageable) {
float defaultBoostValue = 1.0f;

// We don't want to show the directories
Expand All @@ -59,9 +55,16 @@ public List<DirectoryElementInfos> searchElements(@NonNull String userInput, Str
// The documents whose name contains the user input
Query matchNameWilcardQuery = Queries.wildcardQuery(ELEMENT_NAME, "*" + escapeLucene(userInput) + "*")._toQuery();

// Boosting the relevance of starts with input text
Query prefixQuery = PrefixQuery.of(m -> m
.field(ELEMENT_NAME)
.value(userInput)
.boost(defaultBoostValue))
._toQuery();

// The document is in path
Query fullPathQuery = TermQuery.of(m -> m
.field(FULL_PATH_UUID)
.field(PATH_UUID)
.value(currentDirectoryUuid)
.boost(defaultBoostValue)
)._toQuery();
Expand All @@ -74,7 +77,7 @@ public List<DirectoryElementInfos> searchElements(@NonNull String userInput, Str
)._toQuery();

// All queries with default default value
List<Query> queriesWithDefaultBoostValue = List.of(parentIdQuery, fullPathQuery);
List<Query> queriesWithDefaultBoostValue = List.of(parentIdQuery, fullPathQuery, prefixQuery);

// The documents whose name exactly matches the user input
// If parentIdQuery match then fullPathQuery will also match
Expand All @@ -91,13 +94,15 @@ public List<DirectoryElementInfos> searchElements(@NonNull String userInput, Str
.should(queriesWithDefaultBoostValue) // All queries with default default value
.should(exactMatchNameQuery)
.build();

NativeQuery nativeQuery = new NativeQueryBuilder()
.withQuery(query._toQuery())
.withPageable(PageRequest.of(0, PAGE_MAX_SIZE))
.withPageable(pageable)
.build();

return elasticsearchOperations.search(nativeQuery, DirectoryElementInfos.class).stream().map(SearchHit::getContent).toList();
return searchHitsToPage(
elasticsearchOperations.search(nativeQuery, DirectoryElementInfos.class),
pageable
);
}

public static String escapeLucene(String s) {
Expand Down
81 changes: 67 additions & 14 deletions src/test/java/org/gridsuite/directory/server/DirectoryElementInfosServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.test.context.junit.jupiter.SpringExtension;

import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.*;

import static org.gridsuite.directory.server.DirectoryService.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

Expand Down Expand Up @@ -91,15 +92,33 @@ void searchElementInfos() {
List<DirectoryElementInfos> infos = List.of(directoryInfos, element2Infos, element1Infos, element4Infos, element3Infos);
repositoryService.saveElementsInfos(infos);

Set<DirectoryElementInfos> hits = new HashSet<>(directoryElementInfosService.searchElements("a", ""));
Set<DirectoryElementInfos> hits = new HashSet<>(directoryElementInfosService.searchElements("a", "", PageRequest.of(0, 10)).stream().toList());
assertEquals(4, hits.size());
assertTrue(hits.contains(element1Infos));
assertTrue(hits.contains(element4Infos));
assertTrue(hits.contains(element2Infos));
assertTrue(hits.contains(element3Infos));
Page<DirectoryElementInfos> pagedHits = directoryElementInfosService.searchElements("a", "", PageRequest.of(0, 10));
assertEquals(4, pagedHits.getTotalElements());
assertTrue(pagedHits.getContent().contains(element1Infos));
assertTrue(pagedHits.getContent().contains(element4Infos));
assertTrue(pagedHits.getContent().contains(element2Infos));
assertTrue(pagedHits.getContent().contains(element3Infos));

pagedHits = directoryElementInfosService.searchElements("aDirectory", "", PageRequest.of(0, 10));
assertEquals(0, pagedHits.getTotalElements());
}

hits = new HashSet<>(directoryElementInfosService.searchElements("aDirectory", ""));
assertEquals(0, hits.size());
@Test
void searchPagedElementInfos() {
List<DirectoryElementInfos> elements = new ArrayList<>(20);
for (int i = 0; i < 20; i++) {
elements.add(createElements("filter" + i));
}
repositoryService.saveElementsInfos(elements);
Page<DirectoryElementInfos> pagedHits = directoryElementInfosService.searchElements("filter", "", PageRequest.of(0, 10));
assertEquals(20, pagedHits.getTotalElements());
assertEquals(10, pagedHits.getContent().size());
}

@Test
Expand Down Expand Up @@ -129,7 +148,11 @@ void searchSpecialChars() {
}

private void testNameFullAscii(String pat) {
assertEquals(1, directoryElementInfosService.searchElements(pat, "").size());
assertEquals(1, directoryElementInfosService.searchElements(pat, "", PageRequest.of(0, 10)).getTotalElements());
}

private DirectoryElementInfos createElements(String name) {
return DirectoryElementInfos.builder().id(UUID.randomUUID()).name(name).type("TYPE_01").owner("admin").parentId(UUID.randomUUID()).subdirectoriesCount(0L).lastModificationDate(Instant.now().truncatedTo(ChronoUnit.SECONDS)).build();
}

private DirectoryElementInfos makeElementDir(String name) {
Expand Down Expand Up @@ -200,14 +223,14 @@ HashMap<String, DirectoryElementInfos> createFilesElements() {
void testExactMatchFromSubDirectory() {
Map<String, DirectoryElementInfos> allDirs = createFilesElements();
UUID currentDirUuid = allDirs.get("sub_sub_directory1_2").getId();
List<DirectoryElementInfos> hitsCommunFile = directoryElementInfosService.searchElements("common_file", currentDirUuid.toString());
List<DirectoryElementInfos> hitsCommunFile = directoryElementInfosService.searchElements("common_file", currentDirUuid.toString(), PageRequest.of(0, 10)).stream().toList();
assertEquals(6, hitsCommunFile.size());
assertEquals(currentDirUuid, hitsCommunFile.get(0).getParentId()); // we get first the element in the current directory
assertEquals("common_file", hitsCommunFile.get(0).getName());

//now using another current dir , we expect similar results
currentDirUuid = allDirs.get("sub_sub_directory2_2").getId();
hitsCommunFile = directoryElementInfosService.searchElements("common_file", currentDirUuid.toString());
hitsCommunFile = directoryElementInfosService.searchElements("common_file", currentDirUuid.toString(), PageRequest.of(0, 10)).stream().toList();
assertEquals(6, hitsCommunFile.size());
assertEquals(currentDirUuid, hitsCommunFile.get(0).getParentId()); // we get first the element in the current directory
assertEquals("common_file", hitsCommunFile.get(0).getName());
Expand All @@ -217,7 +240,7 @@ void testExactMatchFromSubDirectory() {
void testExactMatchFromOtherDirectory() {
Map<String, DirectoryElementInfos> allDirs = createFilesElements();
UUID currentDirUuid = allDirs.get("sub_sub_directory1_2").getId();
List<DirectoryElementInfos> hits = directoryElementInfosService.searchElements("file3", currentDirUuid.toString());
List<DirectoryElementInfos> hits = directoryElementInfosService.searchElements("file3", currentDirUuid.toString(), PageRequest.of(0, 10)).stream().toList();
assertEquals(1, hits.size());
assertEquals(allDirs.get("sub_directory3").getId(), hits.get(0).getParentId());
assertEquals("file3", hits.get(0).getName());
Expand Down Expand Up @@ -249,7 +272,7 @@ void testExactMatchingParentDirectory() { // when a file is in a sub directory o
//we want to have the files in the current directory if any
// then the files in the path of the current directory (sub directories and parent directories)
// then the files in the other directories
List<DirectoryElementInfos> hitsFile = directoryElementInfosService.searchElements("new-file", currentDirUuid.toString());
List<DirectoryElementInfos> hitsFile = directoryElementInfosService.searchElements("new-file", currentDirUuid.toString(), PageRequest.of(0, 10)).stream().toList();
assertEquals(3, hitsFile.size());
assertEquals(newFile1, hitsFile.get(0));
assertEquals(newFile2, hitsFile.get(1));
Expand All @@ -260,7 +283,7 @@ void testExactMatchingParentDirectory() { // when a file is in a sub directory o
void testPartialMatchFromSubDirectory() {
HashMap<String, DirectoryElementInfos> allDirs = createFilesElements();
UUID currentDirUuid = allDirs.get("sub_sub_directory1_2").getId();
List<DirectoryElementInfos> hitsFile = directoryElementInfosService.searchElements("file", currentDirUuid.toString());
List<DirectoryElementInfos> hitsFile = directoryElementInfosService.searchElements("file", currentDirUuid.toString(), PageRequest.of(0, 10)).stream().toList();
assertEquals(9, hitsFile.size());
assertEquals(currentDirUuid, hitsFile.get(0).getParentId()); // we get first the elements in the current directory
assertEquals("common_file", hitsFile.get(0).getName());
Expand All @@ -269,19 +292,49 @@ void testPartialMatchFromSubDirectory() {

@Test
void testExactMatchInCurrentDir() {
HashMap<String, DirectoryElementInfos> allDirs = createFilesElements();
Map<String, DirectoryElementInfos> allDirs = createFilesElements();
UUID currentDirUuid = allDirs.get("sub_sub_directory1_2").getId();
String fileName = "new-file";
var newFile = makeElementFile(fileName, allDirs.get("sub_sub_directory1_2").getId());
var newFile1 = makeElementFile(fileName + "1", allDirs.get("sub_sub_directory1_2").getId());
var newFile2 = makeElementFile("1" + fileName + "2", allDirs.get("sub_sub_directory1_2").getId());
repositoryService.saveElementsInfos(List.of(newFile1, newFile, newFile2));

List<DirectoryElementInfos> hitsFile = directoryElementInfosService.searchElements(fileName, currentDirUuid.toString());
repositoryService.saveElementsInfos(List.of(newFile, newFile2, newFile1));
List<DirectoryElementInfos> hitsFile = directoryElementInfosService.searchElements(fileName, currentDirUuid.toString(), PageRequest.of(0, 10)).stream().toList();
assertEquals(3, hitsFile.size());
assertEquals(fileName, hitsFile.get(0).getName());
assertEquals(fileName + "1", hitsFile.get(1).getName());
assertEquals("1" + fileName + "2", hitsFile.get(2).getName());
}

/*
root_directory
├── sub_directory1
....
├── sub_directory2
│ ├── bnew-filebbbb
│ ├── anew-file
│ ├── new-file
│ ├── test-new-file
...
*/
@Test
void testTermStartByUserInput() { // when a file start with search term
Map<String, DirectoryElementInfos> allDirs = createFilesElements();
UUID currentDirUuid = allDirs.get("sub_directory2").getId();
var anewFile1 = makeElementFile("anew-file", allDirs.get("sub_directory2").getId());
var newFile2 = makeElementFile("new-file-Ok", allDirs.get("sub_directory2").getId());
var bNewFile = makeElementFile("bnew-filebbbb", allDirs.get("sub_directory2").getId());
var testNewFile = makeElementFile("test-new-file", allDirs.get("sub_directory2").getId());
repositoryService.saveElementsInfos(List.of(bNewFile, newFile2, anewFile1, testNewFile));

//we want to have the files in the current directory if any
// then the files in the path of the current directory (sub directories and parent directories)
// then the files in the other directories
List<DirectoryElementInfos> hitsFile = directoryElementInfosService.searchElements("new-file", currentDirUuid.toString(), PageRequest.of(0, 10)).stream().toList();
assertEquals(4, hitsFile.size());
assertEquals(newFile2, hitsFile.get(0));
assertEquals(bNewFile, hitsFile.get(1));
assertEquals(anewFile1, hitsFile.get(2));
assertEquals(testNewFile, hitsFile.get(3));
}
}
Loading

0 comments on commit 6607024

Please sign in to comment.