Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

imlement scoring in elk for search elements #148

Merged
merged 20 commits into from
Jul 10, 2024
Merged

imlement scoring in elk for search elements #148

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
a80bf21
implement decendent
jamal-khey Jun 11, 2024
499b4dc
re-index update the full path
jamal-khey Jun 11, 2024
e502a84
remove full pathName
jamal-khey Jun 17, 2024
ce5eefa
Merge branch 'refs/heads/main' into jamal-khey/store-path-in-elastic-…
jamal-khey Jun 17, 2024
871a8db
fix build
jamal-khey Jun 17, 2024
5dff01c
implement scoring
jamal-khey Jun 17, 2024
1d4257e
Merge branch 'refs/heads/main' into jamal-khey/store-path-in-elastic-…
jamal-khey Jun 21, 2024
7791be7
Merge branch 'main' into jamal-khey/store-path-in-elastic-search
jamal-khey Jun 24, 2024
f5e0cff
fix reindex test
jamal-khey Jun 24, 2024
78842e8
fix sonar
jamal-khey Jun 24, 2024
c7866f5
adjust sciring
jamal-khey Jun 24, 2024
0899d3c
add base test for scoring
jamal-khey Jul 2, 2024
6cc8436
add unit test for testing search results
jamal-khey Jul 2, 2024
cc28696
refactor code
jamal-khey Jul 2, 2024
abdc408
improve test cases
jamal-khey Jul 2, 2024
7d7e779
filter the Orphan elements from search results
jamal-khey Jul 3, 2024
9a221de
fix sonar issues
jamal-khey Jul 3, 2024
cbf8b95
Merge branch 'main' of https://github.com/gridsuite/directory-server …
Jul 9, 2024
bd27a46
add test , multiple file in the same dir
jamal-khey Jul 9, 2024
b22990d
fix styles
jamal-khey Jul 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/main/java/org/gridsuite/directory/server/DirectoryController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,8 +254,9 @@ public ResponseEntity<Void> elementExists(@PathVariable("directoryUuid") UUID di
@Operation(summary = "Search elements in elasticsearch")
@ApiResponses(value = {@ApiResponse(responseCode = "200", description = "List of elements found")})
public ResponseEntity<List<DirectoryElementInfos>> searchElements(
@Parameter(description = "User input") @RequestParam(value = "userInput") String userInput) {
@Parameter(description = "User input") @RequestParam(value = "userInput") String userInput,
@Parameter(description = "Current directory UUID") @RequestParam(value = "directoryUuid", required = false, defaultValue = "") String directoryUuid) {
return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON)
.body(service.searchElements(userInput));
.body(service.searchElements(userInput, directoryUuid));
}
}
25 changes: 6 additions & 19 deletions src/main/java/org/gridsuite/directory/server/DirectoryService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -589,27 +589,14 @@ public String getDuplicateNameCandidate(UUID directoryUuid, String elementName,
return nameCandidate(elementName, i);
}

public List<DirectoryElementInfos> searchElements(@NonNull String userInput) {
return directoryElementInfosService.searchElements(userInput)
.stream()
.map(this::populatePathInfo)
.filter(Objects::nonNull)
.toList();
public List<DirectoryElementInfos> searchElements(@NonNull String userInput, String directoryUuid) {
return directoryElementInfosService.searchElements(userInput, directoryUuid)
.stream().filter(element -> isElementExists(element.getParentId())) // filter the Orphan elements
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@SlimaneAmar in case of deletion of a directory , we can still have Orphans

//                              root
        //                    /                           \
        // dir1 (deleted but keeping its sub-elements)     dir2
        directoryElementRepository.deleteById(subDirUuid1);`

.toList();
}

private DirectoryElementInfos populatePathInfo(DirectoryElementInfos elementInfos) {
try {
List<ElementAttributes> path = getPath(elementInfos.getParentId());
elementInfos.setPathUuid(path.stream().map(ElementAttributes::getElementUuid).toList());
elementInfos.setPathName(path.stream().map(ElementAttributes::getElementName).toList());
return elementInfos;
} catch (DirectoryException ex) {
if (ex.getType() == DirectoryException.Type.NOT_FOUND) {
LOGGER.error("Error retrieving path for element: '{}' : {}", elementInfos, ex.getMessage());
return null;
}
throw ex;
}
boolean isElementExists(UUID parentDirectoryUuid) {
return !repositoryService.findAllByIdIn(List.of(parentDirectoryUuid)).isEmpty();
}

public boolean areDirectoryElementsDeletable(List<UUID> elementsUuid, String userId) {
Expand Down
5 changes: 0 additions & 5 deletions src/main/java/org/gridsuite/directory/server/dto/elasticsearch/DirectoryElementInfos.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,8 @@
package org.gridsuite.directory.server.dto.elasticsearch;

import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.persistence.Transient;
import lombok.*;
import lombok.experimental.SuperBuilder;

import org.gridsuite.directory.server.elasticsearch.ESConfig;
import org.springframework.data.annotation.Id;
import org.springframework.data.annotation.TypeAlias;
Expand Down Expand Up @@ -58,10 +56,7 @@ public class DirectoryElementInfos {
@Field(type = FieldType.Date, format = DateFormat.date_time)
Instant lastModificationDate;

@Transient
private List<String> pathName;

@Transient
private List<UUID> pathUuid;

}
5 changes: 4 additions & 1 deletion src/main/java/org/gridsuite/directory/server/repository/DirectoryElementEntity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.gridsuite.directory.server.dto.elasticsearch.DirectoryElementInfos;

import java.time.Instant;
import java.util.List;
import java.util.Objects;
import java.util.UUID;

Expand Down Expand Up @@ -101,13 +102,15 @@ public boolean isAttributesUpdatable(@NonNull ElementAttributes newElementAttrib
Objects.isNull(newElementAttributes.getLastModifiedBy());
}

public DirectoryElementInfos toDirectoryElementInfos() {
public DirectoryElementInfos toDirectoryElementInfos(List<DirectoryElementEntity> path) {
return DirectoryElementInfos.builder()
.id(getId())
.name(getName())
.owner(getOwner())
.parentId(getParentId() == null ? getId() : getParentId())
.type(getType())
.pathUuid(path.stream().map(DirectoryElementEntity::getId).toList())
.pathName(path.stream().map(DirectoryElementEntity::getName).toList())
.lastModificationDate(getLastModificationDate())
.build();
}
Expand Down
18 changes: 18 additions & 0 deletions src/main/java/org/gridsuite/directory/server/repository/DirectoryElementRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,4 +88,22 @@ interface SubDirectoryCount {
"WHERE e.id in (SELECT eh.element_id from ElementHierarchy eh) " +
"ORDER BY (SELECT depth FROM ElementHierarchy WHERE element_id = e.id) DESC")
List<DirectoryElementEntity> findElementHierarchy(@Param("elementId") UUID elementId);

@Query(nativeQuery = true, value =
"WITH RECURSIVE DescendantHierarchy (element_id, parent_element_id) AS (" +
" SELECT" +
" id AS element_id, parent_id AS parent_element_id" +
" FROM element where id = :elementId" +
" UNION ALL" +
" select e.id AS element_id, e.parent_id AS parent_element_id" +
" FROM element e" +
" INNER JOIN" +
" DescendantHierarchy dh" +
" ON dh.element_id = e.parent_id" +
" WHERE e.type = 'DIRECTORY' )" +
"SELECT * FROM element e" +
"JOIN" +
" DescendantHierarchy dh" +
" ON e.id = dh.element_id")
List<DirectoryElementEntity> findAllDescendants(@Param("elementId") UUID elementId);
}
50 changes: 45 additions & 5 deletions src/main/java/org/gridsuite/directory/server/services/DirectoryElementInfosService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
*/
package org.gridsuite.directory.server.services;

import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.query_dsl.*;
import lombok.Getter;
import lombok.NonNull;
import org.gridsuite.directory.server.dto.elasticsearch.DirectoryElementInfos;
Expand All @@ -15,7 +16,6 @@
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.client.elc.NativeQuery;
import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder;
import org.springframework.data.elasticsearch.client.elc.Queries;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.stereotype.Service;
Expand All @@ -35,6 +35,8 @@ public class DirectoryElementInfosService {
private final ElasticsearchOperations elasticsearchOperations;

private static final String ELEMENT_NAME = "name.fullascii";
private static final String FULL_PATH_UUID = "fullPathUuid";
private static final String PARENT_ID = "parentId";
static final String ELEMENT_TYPE = "type.keyword";

@Value(ESConfig.DIRECTORY_ELEMENT_INFOS_INDEX_NAME)
Expand All @@ -45,10 +47,48 @@ public DirectoryElementInfosService(ElasticsearchOperations elasticsearchOperati
this.elasticsearchOperations = elasticsearchOperations;
}

public List<DirectoryElementInfos> searchElements(@NonNull String userInput) {
public List<DirectoryElementInfos> searchElements(@NonNull String userInput, String currentDirectoryUuid) {

// We dont want to show the directories
Query directory = TermQuery.of(m -> m
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Queries.termQuery(ELEMENT_TYPE, DIRECTORY)._toQuery() ?

.field(ELEMENT_TYPE)
.value(DIRECTORY)
)._toQuery();

// This query is used to search for elements whose name contains the user input.
Query elementNameContainSearchTerm = WildcardQuery.of(m -> m
.field(ELEMENT_NAME)
.wildcard("*" + escapeLucene(userInput) + "*")
)._toQuery();

// This query is used to search for elements whose name exactly matches the user input.
Query exactMatchName = MatchQuery.of(m -> m
.field(ELEMENT_NAME)
.query(escapeLucene(userInput))
.boost(4.0f)
)._toQuery();

// the element is in path
TermsQueryField termsQueryField = new TermsQueryField.Builder()
.value(List.of(FieldValue.of(currentDirectoryUuid)))
.build();
Query fullPathQuery = TermsQuery.of(t -> t
.field(FULL_PATH_UUID)
.terms(termsQueryField)
.boost(1.0f)
)._toQuery();

// boost the result if the element is in the current search directory
Query parentIdQuery = MatchQuery.of(m -> m
.field(PARENT_ID)
.query(currentDirectoryUuid)
.boost(1.0f)
)._toQuery();

BoolQuery query = new BoolQuery.Builder()
.mustNot(Queries.termQuery(ELEMENT_TYPE, DIRECTORY)._toQuery())
.must(Queries.wildcardQuery(ELEMENT_NAME, "*" + escapeLucene(userInput) + "*")._toQuery())
.mustNot(directory)
.must(elementNameContainSearchTerm) // if a doccument doesn’t match the must clause, it will be filtered out.
.should(fullPathQuery, parentIdQuery, exactMatchName) // boost the query the document match
.build();

NativeQuery nativeQuery = new NativeQueryBuilder()
Expand Down
19 changes: 11 additions & 8 deletions src/main/java/org/gridsuite/directory/server/services/DirectoryRepositoryService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,19 @@

import com.google.common.collect.Lists;
import lombok.NonNull;

import org.gridsuite.directory.server.dto.elasticsearch.DirectoryElementInfos;
import org.gridsuite.directory.server.elasticsearch.DirectoryElementInfosRepository;
import org.gridsuite.directory.server.repository.DirectoryElementEntity;
import org.gridsuite.directory.server.repository.DirectoryElementRepository;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import static org.gridsuite.directory.server.DirectoryService.DIRECTORY;

import java.util.List;
import java.util.Optional;
import java.util.UUID;

import static org.gridsuite.directory.server.DirectoryService.DIRECTORY;

/**
* @author Slimane Amar <slimane.amar at rte-france.com>
*/
Expand Down Expand Up @@ -66,10 +65,13 @@ public void saveElementsInfos(@NonNull List<DirectoryElementInfos> directoryElem
.forEach(directoryElementInfosRepository::saveAll);
}

private DirectoryElementEntity saveElementsInfo(DirectoryElementEntity elementEntity) {
directoryElementInfosRepository.save(elementEntity.toDirectoryElementInfos(findElementHierarchy(elementEntity.getParentId())));
return elementEntity;
}

public DirectoryElementEntity saveElement(DirectoryElementEntity elementEntity) {
DirectoryElementEntity savedElementEntity = directoryElementRepository.save(elementEntity);
directoryElementInfosRepository.save(savedElementEntity.toDirectoryElementInfos());
return savedElementEntity;
return saveElementsInfo(directoryElementRepository.save(elementEntity));
}

public void deleteElement(UUID elementUuid) {
Expand All @@ -88,8 +90,8 @@ public boolean canRead(UUID id, String userId) {

public void reindexElements() {
saveElementsInfos(directoryElementRepository.findAll().stream()
.map(DirectoryElementEntity::toDirectoryElementInfos)
.toList());
.map(directoryElementEntity -> directoryElementEntity.toDirectoryElementInfos(findElementHierarchy(directoryElementEntity.getParentId())))
.toList());
}

public UUID getParentUuid(UUID elementUuid) {
Expand Down Expand Up @@ -130,4 +132,5 @@ public List<String> getNameByTypeAndParentIdAndNameStartWith(String type, UUID p
public List<DirectoryElementEntity> findElementHierarchy(UUID elementId) {
return directoryElementRepository.findElementHierarchy(elementId);
}

}
Loading
Loading