diff --git a/README.md b/README.md index 26bfe68..2af8112 100644 --- a/README.md +++ b/README.md @@ -33,13 +33,18 @@ The Spring Data OpenSearch follows the release model of the Spring Data Elastics ### OpenSearch 2.x / 1.x client libraries -At the moment, Spring Data OpenSearch provides the possibility to use the `RestHighLevelCLient` to connect to OpenSearch clusters. + +At the moment, Spring Data OpenSearch provides the possibility to use either `RestHighLevelCLient` or [OpenSearchClient](https://github.com/opensearch-project/opensearch-java) to connect to OpenSearch clusters. + +#### Using `RestHighLevelCLient` (default) + +By default, the `RestHighLevelCLient` is configured as the means to communicate with OpenSearch clusters. ```xml org.opensearch.client spring-data-opensearch - 1.3.0 + 1.4.0 ``` @@ -49,7 +54,7 @@ To use Spring Boot 3.x auto configuration support: org.opensearch.client spring-data-opensearch-starter - 1.3.0 + 1.4.0 ``` @@ -59,11 +64,80 @@ To use Spring Boot 3.x auto configuration support for testing: org.opensearch.client spring-data-opensearch-test-autoconfigure - 1.3.0 + 1.4.0 test ``` +#### Using `OpenSearchClient` (preferred) + +To switch over to `OpenSearchClient`, the `opensearch-rest-high-level-client` dependency has to be replaced in favor of `opensearch-java`. + +```xml + + org.opensearch.client + spring-data-opensearch + 1.4.0 + + + org.opensearch.client + opensearch-rest-high-level-client + + + + + + org.opensearch.client + opensearch-java + 2.10.1 + +``` + +To use Spring Boot 3.x auto configuration support: + +```xml + + org.opensearch.client + spring-data-opensearch-starter + 1.4.0 + + + org.opensearch.client + opensearch-rest-high-level-client + + + + + + org.opensearch.client + opensearch-java + 2.10.1 + +``` + +To use Spring Boot 3.x auto configuration support for testing: + +```xml + + org.opensearch.client + spring-data-opensearch-test-autoconfigure + 1.4.0 + test + + + org.opensearch.client + opensearch-rest-high-level-client + + + + + + org.opensearch.client + opensearch-java + 2.10.1 + +``` + ## Getting Started Here is a quick teaser of an application using Spring Data Repositories in Java: @@ -224,7 +298,7 @@ Add the Apache Maven dependency: org.opensearch.client spring-data-opensearch - 1.2.1 + 1.4.0 ``` @@ -251,7 +325,7 @@ Add the Gradle dependency: ```groovy dependencies { ... - implementation "org.opensearch.client:spring-data-opensearch:1.2.1" + implementation "org.opensearch.client:spring-data-opensearch:1.4.0" ... } ``` diff --git a/settings.gradle.kts b/settings.gradle.kts index 5649771..a9c4a5f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -31,6 +31,7 @@ dependencyResolutionManagement { create("opensearchLibs") { version("opensearch", "2.14.0") + library("java-client", "org.opensearch.client:opensearch-java:2.10.3") library("client", "org.opensearch.client", "opensearch-rest-client").versionRef("opensearch") library("high-level-client", "org.opensearch.client", "opensearch-rest-high-level-client").versionRef("opensearch") library("sniffer", "org.opensearch.client", "opensearch-rest-client-sniffer").versionRef("opensearch") @@ -43,6 +44,10 @@ dependencyResolutionManagement { library("databind", "com.fasterxml.jackson.core", "jackson-databind").versionRef("jackson") } + create("jakarta") { + library("json-bind", "jakarta.json.bind:jakarta.json.bind-api:2.0.0") + } + create("pluginLibs") { version("spotless", "6.23.1") version("editorconfig", "0.0.3") @@ -78,3 +83,4 @@ include("spring-data-opensearch") include("spring-data-opensearch-starter") include("spring-data-opensearch-test-autoconfigure") include("spring-data-opensearch-examples:spring-boot-gradle") +include("spring-data-opensearch-examples:spring-boot-java-client-gradle") diff --git a/spring-data-opensearch-examples/spring-boot-gradle/README.md b/spring-data-opensearch-examples/spring-boot-gradle/README.md index 8088aac..e1983c3 100644 --- a/spring-data-opensearch-examples/spring-boot-gradle/README.md +++ b/spring-data-opensearch-examples/spring-boot-gradle/README.md @@ -6,7 +6,7 @@ This sample project demonstrates the usage of the [Spring Data OpenSearch](https 1. The easiest way to get [OpenSearch](https://opensearch.org) service up and running is by using [Docker](https://www.docker.com/): ```shell -docker run -p 9200:9200 -e "discovery.type=single-node" opensearchproject/opensearch:2.4.0 +docker run -p 9200:9200 -e "discovery.type=single-node" opensearchproject/opensearch:2.11.1 ``` 2. Build and run the project using [Gradle](https://gradle.org/): diff --git a/spring-data-opensearch-examples/spring-boot-java-client-gradle/README.md b/spring-data-opensearch-examples/spring-boot-java-client-gradle/README.md new file mode 100644 index 0000000..4f5416f --- /dev/null +++ b/spring-data-opensearch-examples/spring-boot-java-client-gradle/README.md @@ -0,0 +1,22 @@ +Spring Data OpenSearch Java Client Spring Boot Example Project +=== + +This sample project demonstrates the usage of the [Spring Data OpenSearch](https://github.com/opensearch-project/spring-data-opensearch/) in the typical Spring Boot web application using [opensearch-java](https://github.com/opensearch-project/opensearch-java) client. The application assumes that there is an [OpenSearch](https://opensearch.org) service up and running on the local machine, available at `https://localhost:9200` (protected by basic authentication with default credentials). + +1. The easiest way to get [OpenSearch](https://opensearch.org) service up and running is by using [Docker](https://www.docker.com/): + +```shell +docker run -p 9200:9200 -e "discovery.type=single-node" opensearchproject/opensearch:2.11.1 +``` + +2. Build and run the project using [Gradle](https://gradle.org/): + +```shell +./gradlew :spring-data-opensearch-examples:spring-boot-java-client-gradle:bootRun +``` + +3. Exercise the REST endpoint available at: `http://localhost:8080/marketplace` + + - Fetch all products: `curl 'http://localhost:8080/marketplace/search'` + - Search products by name: `curl 'http://localhost:8080/marketplace/search?name=pillow'` + - Search products by name and price greater than: `curl 'http://localhost:8080/marketplace/search?name=pillow&price=35.0'` \ No newline at end of file diff --git a/spring-data-opensearch-examples/spring-boot-java-client-gradle/build.gradle.kts b/spring-data-opensearch-examples/spring-boot-java-client-gradle/build.gradle.kts new file mode 100644 index 0000000..07a1db2 --- /dev/null +++ b/spring-data-opensearch-examples/spring-boot-java-client-gradle/build.gradle.kts @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + alias(springLibs.plugins.spring.boot) + alias(pluginLibs.plugins.spotless) + alias(pluginLibs.plugins.editorconfig) + id("java-conventions") +} + +buildscript { + dependencies { + classpath(pluginLibs.editorconfig) + classpath(pluginLibs.spotless) + } +} + +dependencies { + api(project(":spring-data-opensearch")) { + exclude("org.opensearch.client", "opensearch-rest-high-level-client") + } + api(project(":spring-data-opensearch-starter")) { + exclude("org.opensearch.client", "opensearch-rest-high-level-client") + } + implementation(springLibs.boot.web) + implementation(jacksonLibs.core) + implementation(jacksonLibs.databind) + implementation(opensearchLibs.client) + implementation(opensearchLibs.java.client) + testImplementation(springLibs.test) + testImplementation(springLibs.boot.test) + testImplementation(springLibs.boot.test.autoconfigure) + testImplementation(opensearchLibs.testcontainers) + testImplementation(project(":spring-data-opensearch-test-autoconfigure")) { + exclude("org.opensearch.client", "opensearch-rest-high-level-client") + } + + constraints { + implementation("ch.qos.logback:logback-classic") { + version { + require("1.4.12") + } + because("Fixes CVE-2023-6378") + } + } +} + +description = "Spring Data OpenSearch Spring Boot Example Project" + +spotless { + java { + target("src/main/java/**/*.java", "src/test/java/org/opensearch/**/*.java") + + trimTrailingWhitespace() + indentWithSpaces() + endWithNewline() + + removeUnusedImports() + importOrder() + } +} diff --git a/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/MarketplaceApplication.java b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/MarketplaceApplication.java new file mode 100644 index 0000000..6c64e9d --- /dev/null +++ b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/MarketplaceApplication.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.data.example; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration; + +@SpringBootApplication(exclude = ElasticsearchDataAutoConfiguration.class) +public class MarketplaceApplication { + public static void main(String[] args) { + SpringApplication.run(MarketplaceConfiguration.class, args); + } +} diff --git a/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/MarketplaceConfiguration.java b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/MarketplaceConfiguration.java new file mode 100644 index 0000000..1c0e1f8 --- /dev/null +++ b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/MarketplaceConfiguration.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.data.example; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.ssl.SSLContextBuilder; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.data.example.repository.MarketplaceRepository; +import org.opensearch.spring.boot.autoconfigure.RestClientBuilderCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; + +@Configuration +@EnableElasticsearchRepositories(basePackageClasses = MarketplaceRepository.class) +@ComponentScan(basePackageClasses = MarketplaceConfiguration.class) +public class MarketplaceConfiguration { + /** + * Allow to connect to the OpenSearch instance which uses self-signed certificates + */ + @Bean + RestClientBuilderCustomizer customizer() { + return new RestClientBuilderCustomizer() { + @Override + public void customize(HttpAsyncClientBuilder builder) { + try { + builder.setSSLContext(new SSLContextBuilder() + .loadTrustMaterial(null, new TrustSelfSignedStrategy()) + .build()); + } catch (final KeyManagementException | NoSuchAlgorithmException | KeyStoreException ex) { + throw new RuntimeException("Failed to initialize SSL Context instance", ex); + } + } + + @Override + public void customize(RestClientBuilder builder) { + // No additional customizations needed + } + }; + } +} diff --git a/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/model/Product.java b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/model/Product.java new file mode 100644 index 0000000..eea797e --- /dev/null +++ b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/model/Product.java @@ -0,0 +1,92 @@ +/* + * Copyright OpenSearch Contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.data.example.model; + +import java.math.BigDecimal; +import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.FieldType; + +@Document(indexName = "marketplace") +public class Product { + @Id + private String id; + + @Field(type = FieldType.Text, name = "name") + private String name; + + @Field(type = FieldType.Double, name = "price") + private BigDecimal price; + + @Field(type = FieldType.Integer, name = "quantity") + private Integer quantity; + + @Field(type = FieldType.Text, name = "description") + private String description; + + @Field(type = FieldType.Keyword, name = "vendor") + private String vendor; + + public Product() {} + + public Product(String id, String name, BigDecimal price, Integer quantity, String description, String vendor) { + this.id = id; + this.name = name; + this.price = price; + this.quantity = quantity; + this.description = description; + this.vendor = vendor; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getQuantity() { + return quantity; + } + + public void setQuantity(Integer quantity) { + this.quantity = quantity; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getVendor() { + return vendor; + } + + public void setVendor(String vendor) { + this.vendor = vendor; + } + + public BigDecimal getPrice() { + return price; + } + + public void setPrice(BigDecimal price) { + this.price = price; + } +} diff --git a/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/repository/MarketplaceRepository.java b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/repository/MarketplaceRepository.java new file mode 100644 index 0000000..da409e0 --- /dev/null +++ b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/repository/MarketplaceRepository.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.data.example.repository; + +import java.math.BigDecimal; +import java.util.List; +import org.opensearch.data.example.model.Product; +import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; +import org.springframework.stereotype.Repository; + +/** + * See please https://github.com/spring-projects/spring-data-elasticsearch/blob/main/src/main/asciidoc/reference/elasticsearch-repository-queries.adoc + */ +@Repository +public interface MarketplaceRepository extends ElasticsearchRepository { + List findByNameLikeAndPriceGreaterThan(String name, BigDecimal price); +} diff --git a/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/rest/MarketplaceRestController.java b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/rest/MarketplaceRestController.java new file mode 100644 index 0000000..8cee80c --- /dev/null +++ b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/rest/MarketplaceRestController.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.data.example.rest; + +import java.math.BigDecimal; +import java.util.List; +import org.opensearch.data.example.model.Product; +import org.opensearch.data.example.repository.MarketplaceRepository; +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/marketplace") +public class MarketplaceRestController { + private final MarketplaceRepository repository; + + public MarketplaceRestController(MarketplaceRepository repository) { + this.repository = repository; + } + + @GetMapping(value = "/search", produces = MediaType.APPLICATION_JSON_VALUE) + public List search( + @RequestParam(value = "name", required = false, defaultValue = "") String name, + @RequestParam(value = "price", required = false, defaultValue = "0.0") BigDecimal price) { + return repository.findByNameLikeAndPriceGreaterThan(name, price); + } +} diff --git a/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/service/MarketplaceInitializer.java b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/service/MarketplaceInitializer.java new file mode 100644 index 0000000..8122ee3 --- /dev/null +++ b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/java/org/opensearch/data/example/service/MarketplaceInitializer.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.data.example.service; + +import java.math.BigDecimal; +import org.opensearch.data.example.model.Product; +import org.opensearch.data.example.repository.MarketplaceRepository; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.stereotype.Service; + +@Service +public class MarketplaceInitializer implements InitializingBean { + private final MarketplaceRepository repository; + + public MarketplaceInitializer(MarketplaceRepository repository) { + this.repository = repository; + } + + @Override + public void afterPropertiesSet() throws Exception { + repository.save(new Product( + "1", + "Utopia Bedding Bed Pillows", + new BigDecimal(39.99), + 2, + "These professionally finished pillows, with high thread counts, provide great comfort against your skin along with added durability " + + "that easily resists wear and tear to ensure a finished look for your bedroom.", + "Utopia Bedding")); + + repository.save(new Product( + "2", + "Echo Dot Smart speaker", + new BigDecimal(34.99), + 10, + "Our most popular smart speaker with a fabric design. It is our most compact smart speaker that fits perfectly into small spaces.", + "Amazon")); + } +} diff --git a/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/resources/application.yml b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/resources/application.yml new file mode 100644 index 0000000..a6ed7ff --- /dev/null +++ b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/main/resources/application.yml @@ -0,0 +1,14 @@ +# +# Copyright OpenSearch Contributors. +# SPDX-License-Identifier: Apache-2.0 +# + +opensearch: + uris: https://localhost:9200 + username: admin + password: admin + +spring: + jackson: + serialization: + INDENT_OUTPUT: true diff --git a/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/test/java/org/opensearch/data/example/repository/MarketplaceRepositoryIntegrationTests.java b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/test/java/org/opensearch/data/example/repository/MarketplaceRepositoryIntegrationTests.java new file mode 100644 index 0000000..b57a30a --- /dev/null +++ b/spring-data-opensearch-examples/spring-boot-java-client-gradle/src/test/java/org/opensearch/data/example/repository/MarketplaceRepositoryIntegrationTests.java @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.data.example.repository; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Duration; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.opensearch.spring.boot.autoconfigure.test.DataOpenSearchTest; +import org.opensearch.testcontainers.OpensearchContainer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.util.TestPropertyValues; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.test.context.ContextConfiguration; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers(disabledWithoutDocker = true) +@DataOpenSearchTest +@EnableElasticsearchRepositories(basePackageClasses = MarketplaceRepository.class) +@ContextConfiguration(initializers = MarketplaceRepositoryIntegrationTests.Initializer.class) +@Tag("integration-test") +public class MarketplaceRepositoryIntegrationTests { + @Container + static final OpensearchContainer opensearch = new OpensearchContainer<>("opensearchproject/opensearch:2.11.1") + .withStartupAttempts(5) + .withStartupTimeout(Duration.ofMinutes(2)); + + @Test + void testMarketplaceRepository(@Autowired MarketplaceRepository repository) { + assertThat(repository.findAll()).hasSize(0); + } + + static class Initializer implements ApplicationContextInitializer { + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + TestPropertyValues.of("opensearch.uris=" + opensearch.getHttpHostAddress()) + .applyTo(configurableApplicationContext.getEnvironment()); + } + } +} diff --git a/spring-data-opensearch-starter/build.gradle.kts b/spring-data-opensearch-starter/build.gradle.kts index 796411d..53e9392 100644 --- a/spring-data-opensearch-starter/build.gradle.kts +++ b/spring-data-opensearch-starter/build.gradle.kts @@ -26,6 +26,8 @@ dependencies { implementation(opensearchLibs.sniffer) { exclude("commons-logging", "commons-logging") } + compileOnly(opensearchLibs.java.client) + compileOnly(jakarta.json.bind) testImplementation(springLibs.test) { exclude("ch.qos.logback", "logback-classic") } diff --git a/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/OpenSearchClientAutoConfiguration.java b/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/OpenSearchClientAutoConfiguration.java new file mode 100644 index 0000000..9a60259 --- /dev/null +++ b/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/OpenSearchClientAutoConfiguration.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors. + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.spring.boot.autoconfigure; + +import org.opensearch.client.RestClient; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.spring.boot.autoconfigure.OpenSearchClientConfigurations.JsonpMapperConfiguration; +import org.opensearch.spring.boot.autoconfigure.OpenSearchClientConfigurations.OpenSearchClientConfiguration; +import org.opensearch.spring.boot.autoconfigure.OpenSearchClientConfigurations.OpenSearchTransportConfiguration; +import org.springframework.boot.autoconfigure.AutoConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.jsonb.JsonbAutoConfiguration; +import org.springframework.context.annotation.Import; + +/** + * {@link EnableAutoConfiguration Auto-configuration} for OpenSearch's Java client. + */ +@AutoConfiguration(after = { JsonbAutoConfiguration.class, OpenSearchRestClientAutoConfiguration.class }) +@ConditionalOnBean(RestClient.class) +@ConditionalOnClass(OpenSearchClient.class) +@Import({ JsonpMapperConfiguration.class, OpenSearchTransportConfiguration.class, OpenSearchClientConfiguration.class }) +public class OpenSearchClientAutoConfiguration { +} diff --git a/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/OpenSearchClientConfigurations.java b/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/OpenSearchClientConfigurations.java new file mode 100644 index 0000000..5cfcc43 --- /dev/null +++ b/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/OpenSearchClientConfigurations.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors. + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.spring.boot.autoconfigure; + +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.json.bind.Jsonb; +import jakarta.json.spi.JsonProvider; +import org.opensearch.client.RestClient; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.json.jsonb.JsonbJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.rest_client.RestClientOptions; +import org.opensearch.client.transport.rest_client.RestClientTransport; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +class OpenSearchClientConfigurations { + @Import({ JacksonJsonpMapperConfiguration.class, JsonbJsonpMapperConfiguration.class }) + static class JsonpMapperConfiguration { + } + + @ConditionalOnMissingBean(JsonpMapper.class) + @ConditionalOnClass(ObjectMapper.class) + @Configuration(proxyBeanMethods = false) + static class JacksonJsonpMapperConfiguration { + @Bean + JacksonJsonpMapper jacksonJsonpMapper() { + return new JacksonJsonpMapper(); + } + } + + @ConditionalOnMissingBean(JsonpMapper.class) + @ConditionalOnBean(Jsonb.class) + @Configuration(proxyBeanMethods = false) + static class JsonbJsonpMapperConfiguration { + @Bean + JsonbJsonpMapper jsonbJsonpMapper(Jsonb jsonb) { + return new JsonbJsonpMapper(JsonProvider.provider(), jsonb); + } + } + + @Configuration(proxyBeanMethods = false) + @ConditionalOnMissingBean(OpenSearchTransport.class) + static class OpenSearchTransportConfiguration { + @Bean + RestClientTransport restClientTransport(RestClient restClient, JsonpMapper jsonMapper, + ObjectProvider restClientOptions) { + return new RestClientTransport(restClient, jsonMapper, restClientOptions.getIfAvailable()); + } + } + + @Configuration(proxyBeanMethods = false) + @ConditionalOnBean(OpenSearchTransport.class) + static class OpenSearchClientConfiguration { + @Bean + @ConditionalOnMissingBean + OpenSearchClient opensearchClient(OpenSearchTransport transport) { + return new OpenSearchClient(transport); + } + } +} diff --git a/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/data/OpenSearchDataAutoConfiguration.java b/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/data/OpenSearchDataAutoConfiguration.java index bf386a1..3594840 100644 --- a/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/data/OpenSearchDataAutoConfiguration.java +++ b/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/data/OpenSearchDataAutoConfiguration.java @@ -6,6 +6,8 @@ package org.opensearch.spring.boot.autoconfigure.data; import org.opensearch.data.client.orhlc.OpenSearchRestTemplate; +import org.opensearch.data.client.osc.OpenSearchTemplate; +import org.opensearch.spring.boot.autoconfigure.OpenSearchClientAutoConfiguration; import org.opensearch.spring.boot.autoconfigure.OpenSearchRestClientAutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -18,7 +20,7 @@ * Adaptation of the {@link org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration} to * the needs of OpenSearch. */ -@AutoConfiguration(after = {OpenSearchRestClientAutoConfiguration.class}) -@ConditionalOnClass({OpenSearchRestTemplate.class}) -@Import({OpenSearchDataConfiguration.BaseConfiguration.class}) +@AutoConfiguration(after = {OpenSearchClientAutoConfiguration.class, OpenSearchRestClientAutoConfiguration.class}) +@ConditionalOnClass({OpenSearchRestTemplate.class, OpenSearchTemplate.class}) +@Import({OpenSearchDataConfiguration.BaseConfiguration.class, OpenSearchDataConfiguration.JavaClientConfiguration.class}) public class OpenSearchDataAutoConfiguration {} diff --git a/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/data/OpenSearchDataConfiguration.java b/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/data/OpenSearchDataConfiguration.java index a5af7a4..332d730 100644 --- a/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/data/OpenSearchDataConfiguration.java +++ b/spring-data-opensearch-starter/src/main/java/org/opensearch/spring/boot/autoconfigure/data/OpenSearchDataConfiguration.java @@ -6,12 +6,17 @@ package org.opensearch.spring.boot.autoconfigure.data; import java.util.Collections; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.data.client.osc.OpenSearchTemplate; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.domain.EntityScanner; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; import org.springframework.data.elasticsearch.core.convert.ElasticsearchCustomConversions; import org.springframework.data.elasticsearch.core.convert.MappingElasticsearchConverter; @@ -58,4 +63,16 @@ ElasticsearchConverter elasticsearchConverter( return converter; } } + + @Configuration(proxyBeanMethods = false) + @ConditionalOnClass(OpenSearchClient.class) + static class JavaClientConfiguration { + + @Bean + @ConditionalOnMissingBean(value = ElasticsearchOperations.class, name = { "elasticsearchTemplate", "opensearchTemplate" }) + @ConditionalOnBean(OpenSearchClient.class) + OpenSearchTemplate elasticsearchTemplate(OpenSearchClient client, ElasticsearchConverter converter) { + return new OpenSearchTemplate(client, converter); + } + } } diff --git a/spring-data-opensearch-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/spring-data-opensearch-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports index 7fc8b31..156c86c 100644 --- a/spring-data-opensearch-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ b/spring-data-opensearch-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -1,3 +1,4 @@ +org.opensearch.spring.boot.autoconfigure.OpenSearchClientAutoConfiguration org.opensearch.spring.boot.autoconfigure.OpenSearchRestClientAutoConfiguration org.opensearch.spring.boot.autoconfigure.OpenSearchRestHighLevelClientAutoConfiguration org.opensearch.spring.boot.autoconfigure.data.OpenSearchDataAutoConfiguration diff --git a/spring-data-opensearch-test-autoconfigure/src/main/resources/META-INF/spring/org.opensearch.spring.boot.autoconfigure.test.AutoConfigureDataOpenSearch.imports b/spring-data-opensearch-test-autoconfigure/src/main/resources/META-INF/spring/org.opensearch.spring.boot.autoconfigure.test.AutoConfigureDataOpenSearch.imports index 7fc8b31..156c86c 100644 --- a/spring-data-opensearch-test-autoconfigure/src/main/resources/META-INF/spring/org.opensearch.spring.boot.autoconfigure.test.AutoConfigureDataOpenSearch.imports +++ b/spring-data-opensearch-test-autoconfigure/src/main/resources/META-INF/spring/org.opensearch.spring.boot.autoconfigure.test.AutoConfigureDataOpenSearch.imports @@ -1,3 +1,4 @@ +org.opensearch.spring.boot.autoconfigure.OpenSearchClientAutoConfiguration org.opensearch.spring.boot.autoconfigure.OpenSearchRestClientAutoConfiguration org.opensearch.spring.boot.autoconfigure.OpenSearchRestHighLevelClientAutoConfiguration org.opensearch.spring.boot.autoconfigure.data.OpenSearchDataAutoConfiguration diff --git a/spring-data-opensearch/build.gradle.kts b/spring-data-opensearch/build.gradle.kts index d7a1131..c2f47ed 100644 --- a/spring-data-opensearch/build.gradle.kts +++ b/spring-data-opensearch/build.gradle.kts @@ -32,7 +32,9 @@ dependencies { implementation(springLibs.context) implementation(springLibs.tx) compileOnly(springLibs.web) + compileOnly(opensearchLibs.java.client) + testImplementation(opensearchLibs.java.client) testImplementation("jakarta.enterprise:jakarta.enterprise.cdi-api:3.0.0") testImplementation("org.slf4j:log4j-over-slf4j:2.0.13") testImplementation("org.apache.logging.log4j:log4j-core:2.23.1") diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Aggregation.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Aggregation.java new file mode 100644 index 0000000..bc1382a --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Aggregation.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.opensearch.client.opensearch._types.aggregations.Aggregate; + +/** + * Class to combine an OpenSearch {@link org.opensearch.client.opensearch._types.aggregations.Aggregate} with its + * name. Necessary as the OpenSearch Aggregate does not know its name. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class Aggregation { + + private final String name; + private final Aggregate aggregate; + + public Aggregation(String name, Aggregate aggregate) { + this.name = name; + this.aggregate = aggregate; + } + + public String getName() { + return name; + } + + public Aggregate getAggregate() { + return aggregate; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/AutoCloseableOpenSearchClient.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/AutoCloseableOpenSearchClient.java new file mode 100644 index 0000000..5c29335 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/AutoCloseableOpenSearchClient.java @@ -0,0 +1,47 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.opensearch.client.RestClient; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.cluster.OpenSearchClusterClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.springframework.util.Assert; + +/** + * Extension of the {@link OpenSearchClient} class that implements {@link AutoCloseable}. As the underlying + * {@link RestClient} must be closed properly this is handled in the {@link #close()} method. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class AutoCloseableOpenSearchClient extends OpenSearchClient implements AutoCloseable { + + public AutoCloseableOpenSearchClient(OpenSearchTransport transport) { + super(transport); + Assert.notNull(transport, "transport must not be null"); + } + + @Override + public void close() throws Exception { + transport.close(); + } + + @Override + public OpenSearchClusterClient cluster() { + return super.cluster(); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ChildTemplate.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ChildTemplate.java new file mode 100644 index 0000000..58461b4 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ChildTemplate.java @@ -0,0 +1,74 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.io.IOException; +import org.opensearch.client.ApiClient; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch.cluster.OpenSearchClusterClient; +import org.opensearch.client.transport.Transport; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.util.Assert; + +/** + * base class for a template that uses one of the {@link org.opensearch.client.opensearch.OpenSearchClient}'s child + * clients like {@link OpenSearchClusterClient} or + * {@link org.opensearch.client.opensearch.indices.OpenSearchIndicesClient}. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public abstract class ChildTemplate> { + + protected final CLIENT client; + protected final RequestConverter requestConverter; + protected final ResponseConverter responseConverter; + protected final OpenSearchExceptionTranslator exceptionTranslator; + + public ChildTemplate(CLIENT client, ElasticsearchConverter elasticsearchConverter) { + this.client = client; + JsonpMapper jsonpMapper = client._transport().jsonpMapper(); + requestConverter = new RequestConverter(elasticsearchConverter, jsonpMapper); + responseConverter = new ResponseConverter(jsonpMapper); + exceptionTranslator = new OpenSearchExceptionTranslator(jsonpMapper); + } + + /** + * Callback interface to be used with {@link #execute(ClientCallback)} for operating directly on the client. + */ + @FunctionalInterface + public interface ClientCallback { + RESULT doWithClient(CLIENT client) throws IOException; + } + + /** + * Execute a callback with the client and provide exception translation. + * + * @param callback the callback to execute, must not be {@literal null} + * @param the type returned from the callback + * @return the callback result + */ + public RESULT execute(ClientCallback callback) { + + Assert.notNull(callback, "callback must not be null"); + + try { + return callback.doWithClient(client); + } catch (IOException | RuntimeException e) { + throw exceptionTranslator.translateException(e); + } + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ClusterTemplate.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ClusterTemplate.java new file mode 100644 index 0000000..cc94b97 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ClusterTemplate.java @@ -0,0 +1,47 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.opensearch.client.opensearch.cluster.HealthRequest; +import org.opensearch.client.opensearch.cluster.HealthResponse; +import org.opensearch.client.opensearch.cluster.OpenSearchClusterClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.springframework.data.elasticsearch.core.cluster.ClusterHealth; +import org.springframework.data.elasticsearch.core.cluster.ClusterOperations; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; + +/** + * Implementation of the {@link ClusterOperations} interface using en {@link OpenSearchClusterClient}. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class ClusterTemplate extends ChildTemplate + implements ClusterOperations { + + public ClusterTemplate(OpenSearchClusterClient client, ElasticsearchConverter elasticsearchConverter) { + super(client, elasticsearchConverter); + } + + @Override + public ClusterHealth health() { + + HealthRequest healthRequest = requestConverter.clusterHealthRequest(); + HealthResponse healthResponse = execute(client -> client.health(healthRequest)); + return responseConverter.clusterHealth(healthResponse); + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaFilterProcessor.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaFilterProcessor.java new file mode 100644 index 0000000..a14910c --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaFilterProcessor.java @@ -0,0 +1,336 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.GeoDistanceType; +import org.opensearch.client.opensearch._types.GeoShapeRelation; +import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; +import org.opensearch.client.opensearch._types.query_dsl.GeoBoundingBoxQuery; +import org.opensearch.client.opensearch._types.query_dsl.GeoDistanceQuery; +import org.opensearch.client.opensearch._types.query_dsl.GeoShapeQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.QueryBuilders; +import org.opensearch.client.opensearch._types.query_dsl.QueryVariant; +import org.opensearch.client.util.ObjectBuilder; +import org.springframework.data.elasticsearch.core.convert.GeoConverters; +import org.springframework.data.elasticsearch.core.geo.GeoBox; +import org.springframework.data.elasticsearch.core.geo.GeoJson; +import org.springframework.data.elasticsearch.core.geo.GeoPoint; +import org.springframework.data.elasticsearch.core.query.Criteria; +import org.springframework.data.geo.Box; +import org.springframework.data.geo.Distance; +import org.springframework.data.geo.Metrics; +import org.springframework.data.geo.Point; +import org.springframework.util.Assert; + +/** + * Class to convert a {@link org.springframework.data.elasticsearch.core.query.CriteriaQuery} into an OpenSearch + * filter. + * + * @author Peter-Josef Meisch + * @author Junghoon Ban + * @since 4.4 + */ +class CriteriaFilterProcessor { + /** + * Creates a filter query from the given criteria. + * + * @param criteria the criteria to process + * @return the optional query, empty if the criteria did not contain filter relevant elements + */ + public static Optional createQuery(Criteria criteria) { + + Assert.notNull(criteria, "criteria must not be null"); + + List filterQueries = new ArrayList<>(); + + for (Criteria chainedCriteria : criteria.getCriteriaChain()) { + + if (chainedCriteria.isOr()) { + BoolQuery.Builder boolQueryBuilder = QueryBuilders.bool(); + queriesForEntries(chainedCriteria).forEach(boolQueryBuilder::should); + filterQueries.add(new Query(boolQueryBuilder.build())); + } else if (chainedCriteria.isNegating()) { + Collection negatingFilters = buildNegatingFilter(criteria.getField().getName(), + criteria.getFilterCriteriaEntries()); + filterQueries.addAll(negatingFilters); + } else { + filterQueries.addAll(queriesForEntries(chainedCriteria)); + } + } + + if (filterQueries.isEmpty()) { + return Optional.empty(); + } else { + + if (filterQueries.size() == 1) { + return Optional.of(filterQueries.get(0)); + } else { + BoolQuery.Builder boolQueryBuilder = QueryBuilders.bool(); + filterQueries.forEach(boolQueryBuilder::must); + BoolQuery boolQuery = boolQueryBuilder.build(); + return Optional.of(new Query(boolQuery)); + } + } + } + + private static Collection buildNegatingFilter(String fieldName, + Set filterCriteriaEntries) { + + List negationFilters = new ArrayList<>(); + + filterCriteriaEntries.forEach(criteriaEntry -> { + Optional query = queryFor(criteriaEntry.getKey(), criteriaEntry.getValue(), fieldName); + + if (query.isPresent()) { + BoolQuery negatingFilter = QueryBuilders.bool().mustNot(query.get()).build(); + negationFilters.add(new Query(negatingFilter)); + } + }); + + return negationFilters; + } + + private static Collection queriesForEntries(Criteria criteria) { + + Assert.notNull(criteria.getField(), "criteria must have a field"); + String fieldName = criteria.getField().getName(); + Assert.notNull(fieldName, "Unknown field"); + + return criteria.getFilterCriteriaEntries().stream() + .map(entry -> queryFor(entry.getKey(), entry.getValue(), fieldName)) // + .filter(Optional::isPresent) // + .map(Optional::get) // + .collect(Collectors.toList()); + } + + private static Optional queryFor(Criteria.OperationKey key, Object value, String fieldName) { + + ObjectBuilder queryBuilder = null; + + switch (key) { + case WITHIN -> { + Assert.isTrue(value instanceof Object[], "Value of a geo distance filter should be an array of two values."); + queryBuilder = withinQuery(fieldName, (Object[]) value); + } + case BBOX -> { + Assert.isTrue(value instanceof Object[], + "Value of a boundedBy filter should be an array of one or two values."); + queryBuilder = boundingBoxQuery(fieldName, (Object[]) value); + } + case GEO_INTERSECTS -> { + Assert.isTrue(value instanceof GeoJson, "value of a GEO_INTERSECTS filter must be a GeoJson object"); + queryBuilder = geoJsonQuery(fieldName, (GeoJson) value, "intersects"); + } + case GEO_IS_DISJOINT -> { + Assert.isTrue(value instanceof GeoJson, "value of a GEO_IS_DISJOINT filter must be a GeoJson object"); + queryBuilder = geoJsonQuery(fieldName, (GeoJson) value, "disjoint"); + } + case GEO_WITHIN -> { + Assert.isTrue(value instanceof GeoJson, "value of a GEO_WITHIN filter must be a GeoJson object"); + queryBuilder = geoJsonQuery(fieldName, (GeoJson) value, "within"); + } + case GEO_CONTAINS -> { + Assert.isTrue(value instanceof GeoJson, "value of a GEO_CONTAINS filter must be a GeoJson object"); + queryBuilder = geoJsonQuery(fieldName, (GeoJson) value, "contains"); + } + } + + return Optional.ofNullable(queryBuilder != null ? queryBuilder.build()._toQuery() : null); + } + + private static ObjectBuilder withinQuery(String fieldName, Object... values) { + + Assert.noNullElements(values, "Geo distance filter takes 2 not null elements array as parameter."); + Assert.isTrue(values.length == 2, "Geo distance filter takes a 2-elements array as parameter."); + Assert.isTrue(values[0] instanceof GeoPoint || values[0] instanceof String || values[0] instanceof Point, + "First element of a geo distance filter must be a GeoPoint, a Point or a text"); + Assert.isTrue(values[1] instanceof String || values[1] instanceof Distance, + "Second element of a geo distance filter must be a text or a Distance"); + + String dist = (values[1] instanceof Distance distance) ? extractDistanceString(distance) : (String) values[1]; + + return QueryBuilders.geoDistance() // + .field(fieldName) // + .distance(dist) // + .distanceType(GeoDistanceType.Plane) // + .location(location -> { + if (values[0]instanceof GeoPoint loc) { + location.latlon(latlon -> latlon.lat(loc.getLat()).lon(loc.getLon())); + } else if (values[0] instanceof Point point) { + GeoPoint loc = GeoPoint.fromPoint(point); + location.latlon(latlon -> latlon.lat(loc.getLat()).lon(loc.getLon())); + } else { + String loc = (String) values[0]; + if (loc.contains(",")) { + String[] c = loc.split(","); + location.latlon(latlon -> latlon.lat(Double.parseDouble(c[0])).lon(Double.parseDouble(c[1]))); + } else { + location.geohash(geohash -> geohash.geohash(loc)); + } + } + return location; + }); + } + + private static ObjectBuilder boundingBoxQuery(String fieldName, Object... values) { + + Assert.noNullElements(values, "Geo boundedBy filter takes a not null element array as parameter."); + + GeoBoundingBoxQuery.Builder queryBuilder = QueryBuilders.geoBoundingBox() // + .field(fieldName); + + if (values.length == 1) { + // GeoEnvelop + oneParameterBBox(queryBuilder, values[0]); + } else if (values.length == 2) { + // 2x GeoPoint + // 2x text + twoParameterBBox(queryBuilder, values); + } else { + throw new IllegalArgumentException( + "Geo distance filter takes a 1-elements array(GeoBox) or 2-elements array(GeoPoints or Strings(format lat,lon or geohash))."); + } + return queryBuilder; + } + + private static void oneParameterBBox(GeoBoundingBoxQuery.Builder queryBuilder, Object value) { + Assert.isTrue(value instanceof GeoBox || value instanceof Box, + "single-element of boundedBy filter must be type of GeoBox or Box"); + + GeoBox geoBBox; + if (value instanceof Box box) { + geoBBox = GeoBox.fromBox(box); + } else { + geoBBox = (GeoBox) value; + } + + queryBuilder.boundingBox(bb -> bb // + .tlbr(tlbr -> tlbr // + .topLeft(glb -> glb // + .latlon(latlon -> latlon // + .lat(geoBBox.getTopLeft().getLat()) // + .lon(geoBBox.getTopLeft().getLon()))) // + .bottomRight(glb -> glb // + .latlon(latlon -> latlon // + .lat(geoBBox.getBottomRight().getLat())// + .lon(geoBBox.getBottomRight().getLon()// ) + ))))); + } + + private static void twoParameterBBox(GeoBoundingBoxQuery.Builder queryBuilder, Object... values) { + + Assert.isTrue(allElementsAreOfType(values, GeoPoint.class) || allElementsAreOfType(values, String.class), + " both elements of boundedBy filter must be type of GeoPoint or text(format lat,lon or geohash)"); + + if (values[0]instanceof GeoPoint topLeft) { + GeoPoint bottomRight = (GeoPoint) values[1]; + queryBuilder.boundingBox(bb -> bb // + .tlbr(tlbr -> tlbr // + .topLeft(glb -> glb // + .latlon(latlon -> latlon // + .lat(topLeft.getLat()) // + .lon(topLeft.getLon()))) // + .bottomRight(glb -> glb // + .latlon(latlon -> latlon // + .lat(bottomRight.getLat()) // + .lon(bottomRight.getLon()))) // + ) // + ); + } else { + String topLeft = (String) values[0]; + String bottomRight = (String) values[1]; + boolean isGeoHash = !topLeft.contains(","); + queryBuilder.boundingBox(bb -> bb // + .tlbr(tlbr -> tlbr // + .topLeft(glb -> { + if (isGeoHash) { + glb.geohash(gh -> gh.geohash(topLeft)); + } else { + glb.text(topLeft); + } + return glb; + }) // + .bottomRight(glb -> { + if (isGeoHash) { + glb.geohash(gh -> gh.geohash(bottomRight)); + } else { + glb.text(bottomRight); + } + return glb; + }) // + )); + } + } + + private static boolean allElementsAreOfType(Object[] array, Class clazz) { + for (Object o : array) { + if (!clazz.isInstance(o)) { + return false; + } + } + return true; + } + + private static ObjectBuilder geoJsonQuery(String fieldName, GeoJson geoJson, + String relation) { + return buildGeoShapeQuery(fieldName, geoJson, relation); + } + + private static ObjectBuilder buildGeoShapeQuery(String fieldName, GeoJson geoJson, + String relation) { + return QueryBuilders.geoShape().field(fieldName) // + .shape(gsf -> gsf // + .shape(JsonData.of(GeoConverters.GeoJsonToMapConverter.INSTANCE.convert(geoJson))) // + .relation(toRelation(relation))); // + } + + private static GeoShapeRelation toRelation(String relation) { + + for (GeoShapeRelation geoShapeRelation : GeoShapeRelation.values()) { + + if (geoShapeRelation.name().equalsIgnoreCase(relation)) { + return geoShapeRelation; + } + } + throw new IllegalArgumentException("Unknown geo_shape relation: " + relation); + } + + /** + * extract the distance string from a {@link org.springframework.data.geo.Distance} object. + * + * @param distance distance object to extract string from + */ + private static String extractDistanceString(Distance distance) { + + StringBuilder sb = new StringBuilder(); + sb.append((int) distance.getValue()); + switch ((Metrics) distance.getMetric()) { + case KILOMETERS -> sb.append("km"); + case MILES -> sb.append("mi"); + } + + return sb.toString(); + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryException.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryException.java new file mode 100644 index 0000000..982eecf --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.springframework.dao.UncategorizedDataAccessException; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class CriteriaQueryException extends UncategorizedDataAccessException { + public CriteriaQueryException(String msg) { + super(msg, null); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryProcessor.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryProcessor.java new file mode 100644 index 0000000..f5d7a60 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/CriteriaQueryProcessor.java @@ -0,0 +1,392 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.Queries.*; +import static org.springframework.util.StringUtils.*; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.ChildScoreMode; +import org.opensearch.client.opensearch._types.query_dsl.Operator; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.springframework.data.elasticsearch.annotations.FieldType; +import org.springframework.data.elasticsearch.core.query.Criteria; +import org.springframework.data.elasticsearch.core.query.Field; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Class to convert a {@link org.springframework.data.elasticsearch.core.query.CriteriaQuery} into an OpenSearch + * query. + * + * @author Peter-Josef Meisch + * @author Ezequiel Antúnez Camacho + * @since 4.4 + */ +class CriteriaQueryProcessor { + + /** + * creates a query from the criteria + * + * @param criteria the {@link Criteria} + * @return the optional query, null if the criteria did not contain filter relevant elements + */ + @Nullable + public static Query createQuery(Criteria criteria) { + + Assert.notNull(criteria, "criteria must not be null"); + + List shouldQueries = new ArrayList<>(); + List mustNotQueries = new ArrayList<>(); + List mustQueries = new ArrayList<>(); + + Query firstQuery = null; + boolean negateFirstQuery = false; + + for (Criteria chainedCriteria : criteria.getCriteriaChain()) { + Query queryFragment = queryForEntries(chainedCriteria); + + if (queryFragment != null) { + + if (firstQuery == null) { + firstQuery = queryFragment; + negateFirstQuery = chainedCriteria.isNegating(); + continue; + } + + if (chainedCriteria.isOr()) { + shouldQueries.add(queryFragment); + } else if (chainedCriteria.isNegating()) { + mustNotQueries.add(queryFragment); + } else { + mustQueries.add(queryFragment); + } + } + } + + for (Criteria subCriteria : criteria.getSubCriteria()) { + Query subQuery = createQuery(subCriteria); + if (subQuery != null) { + if (criteria.isOr()) { + shouldQueries.add(subQuery); + } else if (criteria.isNegating()) { + mustNotQueries.add(subQuery); + } else { + mustQueries.add(subQuery); + } + } + } + + if (firstQuery != null) { + + if (!shouldQueries.isEmpty() && mustNotQueries.isEmpty() && mustQueries.isEmpty()) { + shouldQueries.add(0, firstQuery); + } else { + + if (negateFirstQuery) { + mustNotQueries.add(0, firstQuery); + } else { + mustQueries.add(0, firstQuery); + } + } + } + + if (shouldQueries.isEmpty() && mustNotQueries.isEmpty() && mustQueries.isEmpty()) { + return null; + } + + Query query = new Query.Builder().bool(boolQueryBuilder -> { + + if (!shouldQueries.isEmpty()) { + boolQueryBuilder.should(shouldQueries); + } + + if (!mustNotQueries.isEmpty()) { + boolQueryBuilder.mustNot(mustNotQueries); + } + + if (!mustQueries.isEmpty()) { + boolQueryBuilder.must(mustQueries); + } + + return boolQueryBuilder; + }).build(); + + return query; + } + + @Nullable + private static Query queryForEntries(Criteria criteria) { + + Field field = criteria.getField(); + + if (field == null || criteria.getQueryCriteriaEntries().isEmpty()) + return null; + + String fieldName = field.getName(); + Assert.notNull(fieldName, "Unknown field " + fieldName); + + Iterator it = criteria.getQueryCriteriaEntries().iterator(); + + Float boost = Float.isNaN(criteria.getBoost()) ? null : criteria.getBoost(); + Query.Builder queryBuilder; + + if (criteria.getQueryCriteriaEntries().size() == 1) { + queryBuilder = queryFor(it.next(), field, boost); + } else { + queryBuilder = new Query.Builder(); + queryBuilder.bool(boolQueryBuilder -> { + while (it.hasNext()) { + Criteria.CriteriaEntry entry = it.next(); + boolQueryBuilder.must(queryFor(entry, field, null).build()); + } + boolQueryBuilder.boost(boost); + return boolQueryBuilder; + }); + + } + + if (hasText(field.getPath())) { + final Query query = queryBuilder.build(); + queryBuilder = new Query.Builder(); + queryBuilder.nested(nqb -> nqb // + .path(field.getPath()) // + .query(query) // + .scoreMode(ChildScoreMode.Avg)); + } + + return queryBuilder.build(); + } + + private static Query.Builder queryFor(Criteria.CriteriaEntry entry, Field field, @Nullable Float boost) { + + String fieldName = field.getName(); + boolean isKeywordField = FieldType.Keyword == field.getFieldType(); + + Criteria.OperationKey key = entry.getKey(); + Object value = key.hasValue() ? entry.getValue() : null; + String searchText = value != null ? escape(value.toString()) : "UNKNOWN_VALUE"; + + Query.Builder queryBuilder = new Query.Builder(); + switch (key) { + case EXISTS: + queryBuilder // + .exists(eb -> eb // + .field(fieldName) // + .boost(boost)); + break; + case EMPTY: + queryBuilder // + .bool(bb -> bb // + .must(mb -> mb // + .exists(eb -> eb // + .field(fieldName) // + )) // + .mustNot(mnb -> mnb // + .wildcard(wb -> wb // + .field(fieldName) // + .wildcard("*"))) // + .boost(boost)); + break; + case NOT_EMPTY: + queryBuilder // + .wildcard(wb -> wb // + .field(fieldName) // + .wildcard("*") // + .boost(boost)); + break; + case EQUALS: + queryBuilder.queryString(queryStringQuery(fieldName, searchText, Operator.And, boost)); + break; + case CONTAINS: + queryBuilder.queryString(queryStringQuery(fieldName, '*' + searchText + '*', true, boost)); + break; + case STARTS_WITH: + queryBuilder.queryString(queryStringQuery(fieldName, searchText + '*', true, boost)); + break; + case ENDS_WITH: + queryBuilder.queryString(queryStringQuery(fieldName, '*' + searchText, true, boost)); + break; + case EXPRESSION: + queryBuilder.queryString(queryStringQuery(fieldName, value.toString(), boost)); + break; + case LESS: + queryBuilder // + .range(rb -> rb // + .field(fieldName) // + .lt(JsonData.of(value)) // + .boost(boost)); // + break; + case LESS_EQUAL: + queryBuilder // + .range(rb -> rb // + .field(fieldName) // + .lte(JsonData.of(value)) // + .boost(boost)); // + break; + case GREATER: + queryBuilder // + .range(rb -> rb // + .field(fieldName) // + .gt(JsonData.of(value)) // + .boost(boost)); // + break; + case GREATER_EQUAL: + queryBuilder // + .range(rb -> rb // + .field(fieldName) // + .gte(JsonData.of(value)) // + .boost(boost)); // + break; + case BETWEEN: + Object[] ranges = (Object[]) value; + queryBuilder // + .range(rb -> { + rb.field(fieldName); + if (ranges[0] != null) { + rb.gte(JsonData.of(ranges[0])); + } + + if (ranges[1] != null) { + rb.lte(JsonData.of(ranges[1])); + } + rb.boost(boost); // + return rb; + }); // + + break; + case FUZZY: + queryBuilder // + .fuzzy(fb -> fb // + .field(fieldName) // + .value(FieldValue.of(searchText)) // + .boost(boost)); // + break; + case MATCHES: + queryBuilder.match(matchQuery(fieldName, value.toString(), Operator.Or, boost)); + break; + case MATCHES_ALL: + queryBuilder.match(matchQuery(fieldName, value.toString(), Operator.And, boost)); + + break; + case IN: + if (value instanceof Iterable iterable) { + if (isKeywordField) { + queryBuilder.bool(bb -> bb // + .must(mb -> mb // + .terms(tb -> tb // + .field(fieldName) // + .terms(tsb -> tsb // + .value(toFieldValueList(iterable))))) // + .boost(boost)); // + } else { + queryBuilder // + .queryString(qsb -> qsb // + .fields(fieldName) // + .query(orQueryString(iterable)) // + .boost(boost)); // + } + } else { + throw new CriteriaQueryException("value for " + fieldName + " is not an Iterable"); + } + break; + case NOT_IN: + if (value instanceof Iterable iterable) { + if (isKeywordField) { + queryBuilder.bool(bb -> bb // + .mustNot(mnb -> mnb // + .terms(tb -> tb // + .field(fieldName) // + .terms(tsb -> tsb // + .value(toFieldValueList(iterable))))) // + .boost(boost)); // + } else { + queryBuilder // + .queryString(qsb -> qsb // + .fields(fieldName) // + .query("NOT(" + orQueryString(iterable) + ')') // + .boost(boost)); // + } + } else { + throw new CriteriaQueryException("value for " + fieldName + " is not an Iterable"); + } + break; + case REGEXP: + queryBuilder // + .regexp(rb -> rb // + .field(fieldName) // + .value(value.toString()) // + .boost(boost)); // + break; + default: + throw new CriteriaQueryException("Could not build query for " + entry); + } + + return queryBuilder; + } + + private static List toFieldValueList(Iterable iterable) { + List list = new ArrayList<>(); + for (Object item : iterable) { + list.add(item != null ? FieldValue.of(item.toString()) : null); + } + return list; + } + + private static String orQueryString(Iterable iterable) { + StringBuilder sb = new StringBuilder(); + + for (Object item : iterable) { + + if (item != null) { + + if (sb.length() > 0) { + sb.append(' '); + } + sb.append('"'); + sb.append(escape(item.toString())); + sb.append('"'); + } + } + + return sb.toString(); + } + + /** + * Returns a String where those characters that TextParser expects to be escaped are escaped by a preceding + * \. Copied from Apachae 2 licensed org.apache.lucene.queryparser.flexible.standard.QueryParserUtil + * class + */ + public static String escape(String s) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + // These characters are part of the query syntax and must be escaped + if (c == '\\' || c == '+' || c == '-' || c == '!' || c == '(' || c == ')' || c == ':' || c == '^' || c == '[' + || c == ']' || c == '\"' || c == '{' || c == '}' || c == '~' || c == '*' || c == '?' || c == '|' || c == '&' + || c == '/') { + sb.append('\\'); + } + sb.append(c); + } + return sb.toString(); + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/DocumentAdapters.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/DocumentAdapters.java new file mode 100644 index 0000000..6a917d3 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/DocumentAdapters.java @@ -0,0 +1,239 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch.core.GetResponse; +import org.opensearch.client.opensearch.core.MgetResponse; +import org.opensearch.client.opensearch.core.explain.ExplanationDetail; +import org.opensearch.client.opensearch.core.get.GetResult; +import org.opensearch.client.opensearch.core.search.CompletionSuggestOption; +import org.opensearch.client.opensearch.core.search.Hit; +import org.opensearch.client.opensearch.core.search.NestedIdentity; +import org.springframework.data.elasticsearch.core.MultiGetItem; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.document.Explanation; +import org.springframework.data.elasticsearch.core.document.NestedMetaData; +import org.springframework.data.elasticsearch.core.document.SearchDocument; +import org.springframework.data.elasticsearch.core.document.SearchDocumentAdapter; +import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Utility class to adapt different Elasticsearch responses to a + * {@link org.springframework.data.elasticsearch.core.document.Document} + * + * @author Peter-Josef Meisch + * @author Haibo Liu + * @since 4.4 + */ +final class DocumentAdapters { + + private static final Log LOGGER = LogFactory.getLog(DocumentAdapters.class); + + private DocumentAdapters() {} + + /** + * Creates a {@link SearchDocument} from a {@link Hit} returned by the Elasticsearch client. + * + * @param hit the hit object + * @param jsonpMapper to map JsonData objects + * @return the created {@link SearchDocument} + */ + @SuppressWarnings("unchecked") + public static SearchDocument from(Hit hit, JsonpMapper jsonpMapper) { + + Assert.notNull(hit, "hit must not be null"); + + Map> highlightFields = hit.highlight(); + + Map innerHits = new LinkedHashMap<>(); + hit.innerHits().forEach((name, innerHitsResult) -> { + // noinspection ReturnOfNull + innerHits.put(name, SearchDocumentResponseBuilder.from(innerHitsResult.hits(), null, null, null, null, + searchDocument -> null, jsonpMapper)); + }); + + NestedMetaData nestedMetaData = from(hit.nested()); + + Explanation explanation = from(hit.explanation()); + + List matchedQueries = hit.matchedQueries(); + + Function, EntityAsMap> fromFields = fields -> { + StringBuilder sb = new StringBuilder("{"); + final boolean[] firstField = { true }; + hit.fields().forEach((key, jsonData) -> { + if (!firstField[0]) { + sb.append(','); + } + sb.append('"').append(key).append("\":") // + .append(jsonData.toJson(jsonpMapper).toString()); + firstField[0] = false; + }); + sb.append('}'); + return new EntityAsMap().fromJson(sb.toString()); + }; + + EntityAsMap hitFieldsAsMap = fromFields.apply(hit.fields()); + + Map> documentFields = new LinkedHashMap<>(); + hitFieldsAsMap.forEach((key, value) -> { + if (value instanceof List) { + // noinspection unchecked + documentFields.put(key, (List) value); + } else { + documentFields.put(key, Collections.singletonList(value)); + } + }); + + Document document; + Object source = hit.source(); + if (source == null) { + document = Document.from(hitFieldsAsMap); + } else { + if (source instanceof EntityAsMap entityAsMap) { + document = Document.from(entityAsMap); + } else if (source instanceof JsonData jsonData) { + document = Document.from(jsonData.to(EntityAsMap.class)); + } else { + + if (LOGGER.isWarnEnabled()) { + LOGGER.warn(String.format("Cannot map from type " + source.getClass().getName())); + } + document = Document.create(); + } + } + document.setIndex(hit.index()); + document.setId(hit.id()); + + if (hit.version() != null) { + document.setVersion(hit.version()); + } + document.setSeqNo(hit.seqNo() != null && hit.seqNo() >= 0 ? hit.seqNo() : -2); // -2 was the default value in the + // old client + document.setPrimaryTerm(hit.primaryTerm() != null && hit.primaryTerm() > 0 ? hit.primaryTerm() : 0); + + float score = hit.score() != null ? hit.score().floatValue() : Float.NaN; + return new SearchDocumentAdapter(document, score, hit.sort().stream().toArray(), + documentFields, highlightFields, innerHits, nestedMetaData, explanation, matchedQueries, hit.routing()); + } + + public static SearchDocument from(CompletionSuggestOption completionSuggestOption) { + + Document document = completionSuggestOption.source() != null ? Document.from(completionSuggestOption.source()) + : Document.create(); + document.setIndex(completionSuggestOption.index()); + + if (completionSuggestOption.id() != null) { + document.setId(completionSuggestOption.id()); + } + + float score = (float) completionSuggestOption.score(); + return new SearchDocumentAdapter(document, score, new Object[] {}, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), null, null, null, completionSuggestOption.routing()); + } + + @Nullable + private static Explanation from(@Nullable org.opensearch.client.opensearch.core.explain.Explanation explanation) { + + if (explanation == null) { + return null; + } + List details = explanation.details().stream().map(DocumentAdapters::from).collect(Collectors.toList()); + return new Explanation(true, (double) explanation.value(), explanation.description(), details); + } + + private static Explanation from(ExplanationDetail explanationDetail) { + + List details = explanationDetail.details().stream().map(DocumentAdapters::from) + .collect(Collectors.toList()); + return new Explanation(null, (double) explanationDetail.value(), explanationDetail.description(), details); + } + + @Nullable + private static NestedMetaData from(@Nullable NestedIdentity nestedIdentity) { + + if (nestedIdentity == null) { + return null; + } + + NestedMetaData child = from(nestedIdentity.nested()); + return NestedMetaData.of(nestedIdentity.field(), nestedIdentity.offset(), child); + } + + /** + * Creates a {@link Document} from a {@link GetResponse} where the found document is contained as {@link EntityAsMap}. + * + * @param getResponse the response instance + * @return the Document + */ + @Nullable + public static Document from(GetResult getResponse) { + + Assert.notNull(getResponse, "getResponse must not be null"); + + if (!getResponse.found()) { + return null; + } + + Document document = getResponse.source() != null ? Document.from(getResponse.source()) : Document.create(); + document.setIndex(getResponse.index()); + document.setId(getResponse.id()); + + if (getResponse.version() != null) { + document.setVersion(getResponse.version()); + } + + if (getResponse.seqNo() != null) { + document.setSeqNo(getResponse.seqNo()); + } + + if (getResponse.primaryTerm() != null) { + document.setPrimaryTerm(getResponse.primaryTerm()); + } + + return document; + } + + /** + * Creates a list of {@link MultiGetItem}s from a {@link MgetResponse} where the data is contained as + * {@link EntityAsMap} instances. + * + * @param mgetResponse the response instance + * @return list of multiget items + */ + public static List> from(MgetResponse mgetResponse) { + + Assert.notNull(mgetResponse, "mgetResponse must not be null"); + + return mgetResponse.docs().stream() // + .map(itemResponse -> MultiGetItem.of( // + itemResponse.isFailure() ? null : from(itemResponse.result()), // + ResponseConverter.getFailure(itemResponse))) + .collect(Collectors.toList()); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/EntityAsMap.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/EntityAsMap.java new file mode 100644 index 0000000..5af5011 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/EntityAsMap.java @@ -0,0 +1,27 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.springframework.data.elasticsearch.support.DefaultStringObjectMap; + +/** + * A Map<String,Object> to represent any entity as it's returned from OpenSearch and before it is converted to a + * {@link org.springframework.data.elasticsearch.core.document.Document}. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class EntityAsMap extends DefaultStringObjectMap {} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/HighlightQueryBuilder.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/HighlightQueryBuilder.java new file mode 100644 index 0000000..db5fe56 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/HighlightQueryBuilder.java @@ -0,0 +1,228 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.TypeUtils.*; + +import java.util.Arrays; +import java.util.stream.Collectors; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty; +import org.springframework.data.elasticsearch.core.query.highlight.Highlight; +import org.springframework.data.elasticsearch.core.query.highlight.HighlightField; +import org.springframework.data.elasticsearch.core.query.highlight.HighlightFieldParameters; +import org.springframework.data.elasticsearch.core.query.highlight.HighlightParameters; +import org.springframework.data.mapping.context.MappingContext; +import org.springframework.lang.Nullable; +import org.springframework.util.StringUtils; + +/** + * Converts the {@link Highlight} annotation from a method to an OpenSearchClient + * {@link org.opensearch.client.opensearch.core.search.Highlight}. + * + * @author Peter-Josef Meisch + * @author Haibo Liu + * @since 4.4 + */ +class HighlightQueryBuilder { + private final MappingContext, ElasticsearchPersistentProperty> mappingContext; + + HighlightQueryBuilder( + MappingContext, ElasticsearchPersistentProperty> mappingContext) { + this.mappingContext = mappingContext; + } + + public org.opensearch.client.opensearch.core.search.Highlight getHighlight(Highlight highlight, + @Nullable Class type) { + + org.opensearch.client.opensearch.core.search.Highlight.Builder highlightBuilder = new org.opensearch.client.opensearch.core.search.Highlight.Builder(); + + // in the old implementation we could use one addParameters method, but in the new Elasticsearch client + // the builder for highlight and highlightfield share no code + addParameters(highlight.getParameters(), highlightBuilder, type); + + for (HighlightField highlightField : highlight.getFields()) { + String mappedName = mapFieldName(highlightField.getName(), type); + highlightBuilder.fields(mappedName, hf -> { + addParameters(highlightField.getParameters(), hf, type); + return hf; + }); + } + + return highlightBuilder.build(); + } + + /* + * the builder for highlight and highlight fields don't share code, so we have these two methods here that basically are almost copies + */ + private void addParameters(HighlightParameters parameters, + org.opensearch.client.opensearch.core.search.Highlight.Builder builder, @Nullable Class type) { + + if (StringUtils.hasLength(parameters.getBoundaryChars())) { + builder.boundaryChars(parameters.getBoundaryChars()); + } + + if (parameters.getBoundaryMaxScan() > -1) { + builder.boundaryMaxScan(parameters.getBoundaryMaxScan()); + } + + if (StringUtils.hasLength(parameters.getBoundaryScanner())) { + builder.boundaryScanner(boundaryScanner(parameters.getBoundaryScanner())); + } + + if (StringUtils.hasLength(parameters.getBoundaryScannerLocale())) { + builder.boundaryScannerLocale(parameters.getBoundaryScannerLocale()); + } + + if (StringUtils.hasLength(parameters.getFragmenter())) { + builder.fragmenter(highlighterFragmenter(parameters.getFragmenter())); + } + + if (parameters.getFragmentSize() > -1) { + builder.fragmentSize(parameters.getFragmentSize()); + } + + if (parameters.getNoMatchSize() > -1) { + builder.noMatchSize(parameters.getNoMatchSize()); + } + + if (parameters.getNumberOfFragments() > -1) { + builder.numberOfFragments(parameters.getNumberOfFragments()); + } + + if (StringUtils.hasLength(parameters.getOrder())) { + builder.order(highlighterOrder(parameters.getOrder())); + } + + if (parameters.getPreTags().length > 0) { + builder.preTags(Arrays.asList(parameters.getPreTags())); + } + + if (parameters.getPostTags().length > 0) { + builder.postTags(Arrays.asList(parameters.getPostTags())); + } + + if (!parameters.getRequireFieldMatch()) { // default is true + builder.requireFieldMatch(false); + } + + if (StringUtils.hasLength(parameters.getType())) { + builder.type(highlighterType(parameters.getType())); + } + + if (StringUtils.hasLength(parameters.getEncoder())) { + builder.encoder(highlighterEncoder(parameters.getEncoder())); + } + + if (StringUtils.hasLength(parameters.getTagsSchema())) { + builder.tagsSchema(highlighterTagsSchema(parameters.getTagsSchema())); + } + } + + /* + * the builder for highlight and highlight fields don't share code, so we have these two methods here that basically are almost copies + */ + private void addParameters(HighlightFieldParameters parameters, + org.opensearch.client.opensearch.core.search.HighlightField.Builder builder, Class type) { + + if (StringUtils.hasLength(parameters.getBoundaryChars())) { + builder.boundaryChars(parameters.getBoundaryChars()); + } + + if (parameters.getBoundaryMaxScan() > -1) { + builder.boundaryMaxScan(parameters.getBoundaryMaxScan()); + } + + if (StringUtils.hasLength(parameters.getBoundaryScanner())) { + builder.boundaryScanner(boundaryScanner(parameters.getBoundaryScanner())); + } + + if (StringUtils.hasLength(parameters.getBoundaryScannerLocale())) { + builder.boundaryScannerLocale(parameters.getBoundaryScannerLocale()); + } + + if (parameters.getForceSource()) { // default is false + builder.forceSource(parameters.getForceSource()); + } + + if (StringUtils.hasLength(parameters.getFragmenter())) { + builder.fragmenter(highlighterFragmenter(parameters.getFragmenter())); + } + + if (parameters.getFragmentSize() > -1) { + builder.fragmentSize(parameters.getFragmentSize()); + } + + if (parameters.getNoMatchSize() > -1) { + builder.noMatchSize(parameters.getNoMatchSize()); + } + + if (parameters.getNumberOfFragments() > -1) { + builder.numberOfFragments(parameters.getNumberOfFragments()); + } + + if (StringUtils.hasLength(parameters.getOrder())) { + builder.order(highlighterOrder(parameters.getOrder())); + } + + if (parameters.getPhraseLimit() > -1) { + builder.phraseLimit(parameters.getPhraseLimit()); + } + + if (parameters.getPreTags().length > 0) { + builder.preTags(Arrays.asList(parameters.getPreTags())); + } + + if (parameters.getPostTags().length > 0) { + builder.postTags(Arrays.asList(parameters.getPostTags())); + } + + if (!parameters.getRequireFieldMatch()) { // default is true + builder.requireFieldMatch(false); + } + + if (StringUtils.hasLength(parameters.getType())) { + builder.type(highlighterType(parameters.getType())); + } + + if ((parameters).getFragmentOffset() > -1) { + builder.fragmentOffset(parameters.getFragmentOffset()); + } + + if (parameters.getMatchedFields().length > 0) { + builder.matchedFields(Arrays.stream(parameters.getMatchedFields()).map(fieldName -> mapFieldName(fieldName, type)) // + .collect(Collectors.toList())); + } + } + + private String mapFieldName(String fieldName, @Nullable Class type) { + + if (type != null) { + ElasticsearchPersistentEntity persistentEntity = mappingContext.getPersistentEntity(type); + + if (persistentEntity != null) { + ElasticsearchPersistentProperty persistentProperty = persistentEntity.getPersistentProperty(fieldName); + + if (persistentProperty != null) { + return persistentProperty.getFieldName(); + } + } + } + + return fieldName; + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/IndicesTemplate.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/IndicesTemplate.java new file mode 100644 index 0000000..b703a17 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/IndicesTemplate.java @@ -0,0 +1,447 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.springframework.util.StringUtils.*; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.opensearch.client.opensearch.indices.*; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.endpoints.BooleanResponse; +import org.springframework.core.annotation.AnnotatedElementUtils; +import org.springframework.dao.InvalidDataAccessApiUsageException; +import org.springframework.data.elasticsearch.UncategorizedElasticsearchException; +import org.springframework.data.elasticsearch.annotations.Mapping; +import org.springframework.data.elasticsearch.core.IndexInformation; +import org.springframework.data.elasticsearch.core.IndexOperations; +import org.springframework.data.elasticsearch.core.ResourceUtil; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.index.*; +import org.springframework.data.elasticsearch.core.index.DeleteIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.DeleteTemplateRequest; +import org.springframework.data.elasticsearch.core.index.ExistsIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.ExistsTemplateRequest; +import org.springframework.data.elasticsearch.core.index.GetIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.GetTemplateRequest; +import org.springframework.data.elasticsearch.core.index.PutIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.PutTemplateRequest; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Implementation of the {@link IndexOperations} interface using en {@link OpenSearchIndicesClient}. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class IndicesTemplate extends ChildTemplate + implements IndexOperations { + + // we need a cluster client as well because ES has put some methods from the indices API into the cluster client + // (component templates) + private final ClusterTemplate clusterTemplate; + protected final ElasticsearchConverter elasticsearchConverter; + @Nullable protected final Class boundClass; + @Nullable protected final IndexCoordinates boundIndex; + + public IndicesTemplate(OpenSearchIndicesClient client, ClusterTemplate clusterTemplate, + ElasticsearchConverter elasticsearchConverter, Class boundClass) { + super(client, elasticsearchConverter); + + Assert.notNull(clusterTemplate, "cluster must not be null"); + Assert.notNull(elasticsearchConverter, "elasticsearchConverter must not be null"); + Assert.notNull(boundClass, "boundClass may not be null"); + + this.clusterTemplate = clusterTemplate; + this.elasticsearchConverter = elasticsearchConverter; + this.boundClass = boundClass; + this.boundIndex = null; + + } + + public IndicesTemplate(OpenSearchIndicesClient client, ClusterTemplate clusterTemplate, + ElasticsearchConverter elasticsearchConverter, IndexCoordinates boundIndex) { + super(client, elasticsearchConverter); + + Assert.notNull(clusterTemplate, "cluster must not be null"); + Assert.notNull(elasticsearchConverter, "elasticsearchConverter must not be null"); + Assert.notNull(boundIndex, "boundIndex must not be null"); + + this.clusterTemplate = clusterTemplate; + this.elasticsearchConverter = elasticsearchConverter; + this.boundClass = null; + this.boundIndex = boundIndex; + + } + + protected Class checkForBoundClass() { + if (boundClass == null) { + throw new InvalidDataAccessApiUsageException("IndexOperations are not bound"); + } + return boundClass; + } + + @Override + public boolean create() { + + Settings settings = boundClass != null ? createSettings(boundClass) : new Settings(); + return doCreate(getIndexCoordinates(), settings, null); + } + + @Override + public boolean create(Map settings) { + + Assert.notNull(settings, "settings must not be null"); + + return doCreate(getIndexCoordinates(), settings, null); + } + + @Override + public boolean create(Map settings, Document mapping) { + + Assert.notNull(settings, "settings must not be null"); + Assert.notNull(mapping, "mapping must not be null"); + + return doCreate(getIndexCoordinates(), settings, mapping); + } + + @Override + public boolean createWithMapping() { + return doCreate(getIndexCoordinates(), createSettings(), createMapping()); + } + + protected boolean doCreate(IndexCoordinates indexCoordinates, Map settings, + @Nullable Document mapping) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + Assert.notNull(settings, "settings must not be null"); + + CreateIndexRequest createIndexRequest = requestConverter.indicesCreateRequest(indexCoordinates, settings, mapping); + CreateIndexResponse createIndexResponse = execute(client -> client.create(createIndexRequest)); + return Boolean.TRUE.equals(createIndexResponse.acknowledged()); + } + + @Override + public boolean delete() { + return doDelete(getIndexCoordinates()); + } + + private boolean doDelete(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + if (doExists(indexCoordinates)) { + DeleteIndexRequest deleteIndexRequest = requestConverter.indicesDeleteRequest(indexCoordinates); + DeleteIndexResponse deleteIndexResponse = execute(client -> client.delete(deleteIndexRequest)); + return deleteIndexResponse.acknowledged(); + } + + return false; + } + + @Override + public boolean exists() { + return doExists(getIndexCoordinates()); + } + + private boolean doExists(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + ExistsRequest existsRequest = requestConverter.indicesExistsRequest(indexCoordinates); + BooleanResponse existsResponse = execute(client -> client.exists(existsRequest)); + return existsResponse.value(); + } + + @Override + public void refresh() { + + RefreshRequest refreshRequest = requestConverter.indicesRefreshRequest(getIndexCoordinates()); + execute(client -> client.refresh(refreshRequest)); + } + + @Override + public Document createMapping() { + return createMapping(checkForBoundClass()); + } + + @Override + public Document createMapping(Class clazz) { + + Assert.notNull(clazz, "clazz must not be null"); + + // load mapping specified in Mapping annotation if present + Mapping mappingAnnotation = AnnotatedElementUtils.findMergedAnnotation(clazz, Mapping.class); + + if (mappingAnnotation != null) { + String mappingPath = mappingAnnotation.mappingPath(); + + if (hasText(mappingPath)) { + String mappings = ResourceUtil.readFileFromClasspath(mappingPath); + + if (hasText(mappings)) { + return Document.parse(mappings); + } + } + } + + // build mapping from field annotations + try { + String mapping = new MappingBuilder(elasticsearchConverter).buildPropertyMapping(clazz); + return Document.parse(mapping); + } catch (Exception e) { + throw new UncategorizedElasticsearchException("Failed to build mapping for " + clazz.getSimpleName(), e); + } + } + + @Override + public boolean putMapping(Document mapping) { + + Assert.notNull(mapping, "mapping must not be null"); + + PutMappingRequest putMappingRequest = requestConverter.indicesPutMappingRequest(getIndexCoordinates(), mapping); + PutMappingResponse putMappingResponse = execute(client -> client.putMapping(putMappingRequest)); + return putMappingResponse.acknowledged(); + } + + @Override + public Map getMapping() { + + IndexCoordinates indexCoordinates = getIndexCoordinates(); + GetMappingRequest getMappingRequest = requestConverter.indicesGetMappingRequest(indexCoordinates); + GetMappingResponse getMappingResponse = execute(client -> client.getMapping(getMappingRequest)); + + Document mappingResponse = responseConverter.indicesGetMapping(getMappingResponse, indexCoordinates); + return mappingResponse; + } + + @Override + public Settings createSettings() { + return createSettings(checkForBoundClass()); + } + + @Override + public Settings createSettings(Class clazz) { + + Assert.notNull(clazz, "clazz must not be null"); + + ElasticsearchPersistentEntity persistentEntity = getRequiredPersistentEntity(clazz); + String settingPath = persistentEntity.settingPath(); + return hasText(settingPath) // + ? Settings.parse(ResourceUtil.readFileFromClasspath(settingPath)) // + : persistentEntity.getDefaultSettings(); + + } + + @Override + public Settings getSettings() { + return getSettings(false); + } + + @Override + public Settings getSettings(boolean includeDefaults) { + + GetIndicesSettingsRequest getIndicesSettingsRequest = requestConverter + .indicesGetSettingsRequest(getIndexCoordinates(), includeDefaults); + GetIndicesSettingsResponse getIndicesSettingsResponse = execute( + client -> client.getSettings(getIndicesSettingsRequest)); + return responseConverter.indicesGetSettings(getIndicesSettingsResponse, getIndexCoordinates().getIndexName()); + } + + @Override + public boolean alias(AliasActions aliasActions) { + + Assert.notNull(aliasActions, "aliasActions must not be null"); + + UpdateAliasesRequest updateAliasesRequest = requestConverter.indicesUpdateAliasesRequest(aliasActions); + UpdateAliasesResponse updateAliasesResponse = execute(client -> client.updateAliases(updateAliasesRequest)); + return updateAliasesResponse.acknowledged(); + } + + @Override + public Map> getAliases(String... aliasNames) { + + Assert.notNull(aliasNames, "aliasNames must not be null"); + + GetAliasRequest getAliasRequest = requestConverter.indicesGetAliasRequest(aliasNames, null); + var getAliasResponse = execute(client -> client.getAlias(getAliasRequest)); + return responseConverter.indicesGetAliasData(getAliasResponse); + } + + @Override + public Map> getAliasesForIndex(String... indexNames) { + + Assert.notNull(indexNames, "indexNames must not be null"); + + GetAliasRequest getAliasRequest = requestConverter.indicesGetAliasRequest(null, indexNames); + var getAliasResponse = execute(client -> client.getAlias(getAliasRequest)); + return responseConverter.indicesGetAliasData(getAliasResponse); + } + + @Override + public boolean putTemplate(PutTemplateRequest putTemplateRequest) { + + Assert.notNull(putTemplateRequest, "putTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.PutTemplateRequest putTemplateRequestES = requestConverter + .indicesPutTemplateRequest(putTemplateRequest); + return execute(client -> client.putTemplate(putTemplateRequestES)).acknowledged(); + } + + @Override + public TemplateData getTemplate(GetTemplateRequest getTemplateRequest) { + + Assert.notNull(getTemplateRequest, "getTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.GetTemplateRequest getTemplateRequestES = requestConverter + .indicesGetTemplateRequest(getTemplateRequest); + GetTemplateResponse getTemplateResponse = execute(client -> client.getTemplate(getTemplateRequestES)); + + return responseConverter.indicesGetTemplateData(getTemplateResponse, getTemplateRequest.getTemplateName()); + } + + @Override + public boolean existsTemplate(ExistsTemplateRequest existsTemplateRequest) { + + Assert.notNull(existsTemplateRequest, "existsTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.ExistsTemplateRequest existsTemplateRequestSO = requestConverter + .indicesExistsTemplateRequest(existsTemplateRequest); + return execute(client -> client.existsTemplate(existsTemplateRequestSO)).value(); + } + + @Override + public boolean deleteTemplate(DeleteTemplateRequest deleteTemplateRequest) { + + Assert.notNull(deleteTemplateRequest, "deleteTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.DeleteTemplateRequest deleteTemplateRequestES = requestConverter + .indicesDeleteTemplateRequest(deleteTemplateRequest); + return execute(client -> client.deleteTemplate(deleteTemplateRequestES)).acknowledged(); + } + + @Override + public boolean putIndexTemplate(PutIndexTemplateRequest putIndexTemplateRequest) { + + org.opensearch.client.opensearch.indices.PutIndexTemplateRequest putIndexTemplateRequestES = requestConverter + .indicesPutIndexTemplateRequest(putIndexTemplateRequest); + + return execute(client -> client.putIndexTemplate(putIndexTemplateRequestES)).acknowledged(); + } + + @Override + public boolean existsIndexTemplate(ExistsIndexTemplateRequest existsIndexTemplateRequest) { + + Assert.notNull(existsIndexTemplateRequest, "existsIndexTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.ExistsIndexTemplateRequest existsTemplateRequestES = requestConverter + .indicesExistsIndexTemplateRequest(existsIndexTemplateRequest); + return execute(client -> client.existsIndexTemplate(existsTemplateRequestES)).value(); + } + + @Override + public List getIndexTemplate(GetIndexTemplateRequest getIndexTemplateRequest) { + + Assert.notNull(getIndexTemplateRequest, "getIndexTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.GetIndexTemplateRequest getIndexTemplateRequestES = requestConverter + .indicesGetIndexTemplateRequest(getIndexTemplateRequest); + var getIndexTemplateResponse = execute(client -> client.getIndexTemplate(getIndexTemplateRequestES)); + return responseConverter.getIndexTemplates(getIndexTemplateResponse); + } + + @Override + public boolean deleteIndexTemplate(DeleteIndexTemplateRequest deleteIndexTemplateRequest) { + + Assert.notNull(deleteIndexTemplateRequest, "deleteIndexTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.DeleteIndexTemplateRequest deleteIndexTemplateRequestES = requestConverter + .indicesDeleteIndexTemplateRequest(deleteIndexTemplateRequest); + return execute(client -> client.deleteIndexTemplate(deleteIndexTemplateRequestES)).acknowledged(); + } + + @Override + public boolean putComponentTemplate(PutComponentTemplateRequest putComponentTemplateRequest) { + + Assert.notNull(putComponentTemplateRequest, "putComponentTemplateRequest must not be null"); + + org.opensearch.client.opensearch.cluster.PutComponentTemplateRequest putComponentTemplateRequestES = requestConverter + .clusterPutComponentTemplateRequest(putComponentTemplateRequest); + // the new Elasticsearch client has this call in the cluster index + return clusterTemplate.execute(client -> client.putComponentTemplate(putComponentTemplateRequestES)).acknowledged(); + } + + @Override + public boolean existsComponentTemplate(ExistsComponentTemplateRequest existsComponentTemplateRequest) { + + Assert.notNull(existsComponentTemplateRequest, "existsComponentTemplateRequest must not be null"); + + org.opensearch.client.opensearch.cluster.ExistsComponentTemplateRequest existsComponentTemplateRequestES = requestConverter + .clusterExistsComponentTemplateRequest(existsComponentTemplateRequest); + return clusterTemplate.execute(client -> client.existsComponentTemplate(existsComponentTemplateRequestES)).value(); + } + + @Override + public List getComponentTemplate(GetComponentTemplateRequest getComponentTemplateRequest) { + + org.opensearch.client.opensearch.cluster.GetComponentTemplateRequest getComponentTemplateRequestES = requestConverter + .clusterGetComponentTemplateRequest(getComponentTemplateRequest); + var response = clusterTemplate.execute(client -> client.getComponentTemplate(getComponentTemplateRequestES)); + return responseConverter.clusterGetComponentTemplates(response); + } + + @Override + public boolean deleteComponentTemplate(DeleteComponentTemplateRequest deleteComponentTemplateRequest) { + + Assert.notNull(deleteComponentTemplateRequest, "deleteComponentTemplateRequest must not be null"); + + org.opensearch.client.opensearch.cluster.DeleteComponentTemplateRequest deleteComponentTemplateRequestES = requestConverter + .clusterDeleteComponentTemplateRequest(deleteComponentTemplateRequest); + return clusterTemplate.execute(client -> client.deleteComponentTemplate(deleteComponentTemplateRequestES)) + .acknowledged(); + } + + @Override + public List getInformation(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + GetIndexRequest getIndexRequest = requestConverter.indicesGetIndexRequest(indexCoordinates); + GetIndexResponse getIndexResponse = execute(client -> client.get(getIndexRequest)); + return responseConverter.indicesGetIndexInformations(getIndexResponse); + } + + // region Helper functions + ElasticsearchPersistentEntity getRequiredPersistentEntity(Class clazz) { + return elasticsearchConverter.getMappingContext().getRequiredPersistentEntity(clazz); + } + + @Override + public IndexCoordinates getIndexCoordinates() { + return (boundClass != null) ? getIndexCoordinatesFor(boundClass) : Objects.requireNonNull(boundIndex); + } + + public IndexCoordinates getIndexCoordinatesFor(Class clazz) { + return getRequiredPersistentEntity(clazz).getIndexCoordinates(); + } + // endregion +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonUtils.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonUtils.java new file mode 100644 index 0000000..02d7f18 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonUtils.java @@ -0,0 +1,68 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import jakarta.json.stream.JsonGenerator; +import java.io.ByteArrayOutputStream; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonpMapper; +import org.springframework.lang.Nullable; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +final class JsonUtils { + + private static final Log LOGGER = LogFactory.getLog(JsonUtils.class); + + private JsonUtils() {} + + public static String toJson(Object object, JsonpMapper mapper) { + + // noinspection SpellCheckingInspection + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonGenerator generator = mapper.jsonProvider().createGenerator(baos); + mapper.serialize(object, generator); + generator.close(); + String json = "{}"; + try { + json = baos.toString("UTF-8"); + } catch (UnsupportedEncodingException e) { + LOGGER.warn("could not read json", e); + } + + return json; + } + + @Nullable + public static String queryToJson(@Nullable org.opensearch.client.opensearch._types.query_dsl.Query query, JsonpMapper mapper) { + + if (query == null) { + return null; + } + + var baos = new ByteArrayOutputStream(); + var generator = mapper.jsonProvider().createGenerator(baos); + query.serialize(generator, mapper); + generator.close(); + return baos.toString(StandardCharsets.UTF_8); + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonpUtils.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonpUtils.java new file mode 100644 index 0000000..907a6a1 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/JsonpUtils.java @@ -0,0 +1,70 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import jakarta.json.JsonException; +import jakarta.json.spi.JsonProvider; +import jakarta.json.stream.JsonGenerator; +import jakarta.json.stream.JsonParser; +import java.io.StringReader; +import org.opensearch.client.json.JsonpDeserializer; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.JsonpMapperBase; +import org.opensearch.client.json.JsonpSerializable; +import org.springframework.data.elasticsearch.core.document.Document; + +final class JsonpUtils { + static final JsonProvider DEFAULT_PROVIDER = provider(); + + static final JsonpMapper DEFAULT_JSONP_MAPPER = new JsonpMapperBase() { + @Override + public JsonProvider jsonProvider() { + return DEFAULT_PROVIDER; + } + + @Override + public void serialize(T value, JsonGenerator generator) { + if (value instanceof JsonpSerializable) { + ((JsonpSerializable) value).serialize(generator, this); + return; + } + + throw new JsonException( + "Cannot find a serializer for type " + value.getClass().getName() + + ". Consider using a full-featured JsonpMapper" + ); + } + + @Override + protected JsonpDeserializer getDefaultDeserializer(Class clazz) { + throw new JsonException( + "Cannot find a default deserializer for type " + clazz.getName() + + ". Consider using a full-featured JsonpMapper"); + } + }; + + private JsonpUtils() {} + + static JsonProvider provider() { + return JsonProvider.provider(); + } + + static T fromJson(Document document, JsonpDeserializer deserializer) { + try (JsonParser parser = DEFAULT_JSONP_MAPPER.jsonProvider().createParser(new StringReader(document.toJson()))) { + return deserializer.deserialize(parser, DEFAULT_JSONP_MAPPER); + } + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQuery.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQuery.java new file mode 100644 index 0000000..4e8bd9d --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQuery.java @@ -0,0 +1,133 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.SortOptions; +import org.opensearch.client.opensearch._types.aggregations.Aggregation; +import org.opensearch.client.opensearch._types.query_dsl.KnnQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch.core.search.FieldCollapse; +import org.opensearch.client.opensearch.core.search.Suggester; +import org.springframework.data.elasticsearch.core.query.BaseQuery; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * A {@link org.springframework.data.elasticsearch.core.query.Query} implementation using query builders from the new + * OpenSearch Client library. + * + * @author Peter-Josef Meisch + * @author Sascha Woo + * @since 4.4 + */ +public class NativeQuery extends BaseQuery { + + @Nullable private final Query query; + @Nullable private org.springframework.data.elasticsearch.core.query.Query springDataQuery; + @Nullable private Query filter; + // note: the new client does not have pipeline aggs, these are just set up as normal aggs + private final Map aggregations = new LinkedHashMap<>(); + @Nullable private Suggester suggester; + @Nullable private FieldCollapse fieldCollapse; + private List sortOptions = Collections.emptyList(); + + private Map searchExtensions = Collections.emptyMap(); + @Nullable private KnnQuery knnQuery; + + public NativeQuery(NativeQueryBuilder builder) { + super(builder); + this.query = builder.getQuery(); + this.filter = builder.getFilter(); + this.aggregations.putAll(builder.getAggregations()); + this.suggester = builder.getSuggester(); + this.fieldCollapse = builder.getFieldCollapse(); + this.sortOptions = builder.getSortOptions(); + this.searchExtensions = builder.getSearchExtensions(); + + if (builder.getSpringDataQuery() != null) { + Assert.isTrue(!NativeQuery.class.isAssignableFrom(builder.getSpringDataQuery().getClass()), + "Cannot add an NativeQuery in a NativeQuery"); + } + this.springDataQuery = builder.getSpringDataQuery(); + this.knnQuery = builder.getKnnQuery(); + } + + public NativeQuery(@Nullable Query query) { + this.query = query; + } + + public static NativeQueryBuilder builder() { + return new NativeQueryBuilder(); + } + + @Nullable + public Query getQuery() { + return query; + } + + @Nullable + public Query getFilter() { + return filter; + } + + public Map getAggregations() { + return aggregations; + } + + @Nullable + public Suggester getSuggester() { + return suggester; + } + + @Nullable + public FieldCollapse getFieldCollapse() { + return fieldCollapse; + } + + public List getSortOptions() { + return sortOptions; + } + + public Map getSearchExtensions() { + return searchExtensions; + } + + /** + * @see NativeQueryBuilder#withQuery(org.springframework.data.elasticsearch.core.query.Query) + * @since 5.1 + */ + public void setSpringDataQuery(@Nullable org.springframework.data.elasticsearch.core.query.Query springDataQuery) { + this.springDataQuery = springDataQuery; + } + + /** + * @since 5.1 + */ + @Nullable + public KnnQuery getKnnQuery() { + return knnQuery; + } + + @Nullable + public org.springframework.data.elasticsearch.core.query.Query getSpringDataQuery() { + return springDataQuery; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQueryBuilder.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQueryBuilder.java new file mode 100644 index 0000000..e9fc032 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/NativeQueryBuilder.java @@ -0,0 +1,231 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.SortOptions; +import org.opensearch.client.opensearch._types.aggregations.Aggregation; +import org.opensearch.client.opensearch._types.query_dsl.KnnQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.QueryVariant; +import org.opensearch.client.opensearch.core.search.FieldCollapse; +import org.opensearch.client.opensearch.core.search.Suggester; +import org.opensearch.client.util.ObjectBuilder; +import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * @author Peter-Josef Meisch + * @author Sascha Woo + * @since 4.4 + */ +public class NativeQueryBuilder extends BaseQueryBuilder { + + @Nullable private Query query; + @Nullable private Query filter; + private final Map aggregations = new LinkedHashMap<>(); + @Nullable private Suggester suggester; + @Nullable private FieldCollapse fieldCollapse; + private List sortOptions = new ArrayList<>(); + private Map searchExtensions = new LinkedHashMap<>(); + + @Nullable private org.springframework.data.elasticsearch.core.query.Query springDataQuery; + @Nullable private KnnQuery knnQuery; + + public NativeQueryBuilder() {} + + @Nullable + public Query getQuery() { + return query; + } + + @Nullable + public Query getFilter() { + return this.filter; + } + + public Map getAggregations() { + return aggregations; + } + + @Nullable + public Suggester getSuggester() { + return suggester; + } + + @Nullable + public FieldCollapse getFieldCollapse() { + return fieldCollapse; + } + + public List getSortOptions() { + return sortOptions; + } + + public Map getSearchExtensions() { + return this.searchExtensions; + } + + @Nullable + public KnnQuery getKnnQuery() { + return knnQuery; + } + + @Nullable + public org.springframework.data.elasticsearch.core.query.Query getSpringDataQuery() { + return springDataQuery; + } + + public NativeQueryBuilder withQuery(QueryVariant variant) { + + Assert.notNull(variant, "query variant must not be null"); + + this.query = variant.toQuery(); + return this; + } + + public NativeQueryBuilder withQuery(ObjectBuilder builder) { + + Assert.notNull(builder, "builder must not be null"); + + this.query = builder.build().toQuery(); + return this; + } + + public NativeQueryBuilder withQuery(Query query) { + + Assert.notNull(query, "query must not be null"); + + this.query = query; + return this; + } + + public NativeQueryBuilder withQuery(Function> fn) { + + Assert.notNull(fn, "fn must not be null"); + + return withQuery(fn.apply(new Query.Builder()).build()); + } + + public NativeQueryBuilder withFilter(@Nullable Query filter) { + this.filter = filter; + return this; + } + + public NativeQueryBuilder withFilter(Function> fn) { + + Assert.notNull(fn, "fn must not be null"); + + return withFilter(fn.apply(new Query.Builder()).build()); + } + + public NativeQueryBuilder withAggregation(String name, Aggregation aggregation) { + + Assert.notNull(name, "name must not be null"); + Assert.notNull(aggregation, "aggregation must not be null"); + + this.aggregations.put(name, aggregation); + return this; + } + + public NativeQueryBuilder withSuggester(@Nullable Suggester suggester) { + this.suggester = suggester; + return this; + } + + public NativeQueryBuilder withFieldCollapse(@Nullable FieldCollapse fieldCollapse) { + this.fieldCollapse = fieldCollapse; + return this; + } + + public NativeQueryBuilder withSort(List values) { + + Assert.notEmpty(values, "values must not be empty"); + + sortOptions.clear(); + sortOptions.addAll(values); + + return this; + } + + public NativeQueryBuilder withSort(SortOptions value, SortOptions... values) { + + Assert.notNull(value, "value must not be null"); + sortOptions.add(value); + if (values.length > 0) { + sortOptions.addAll(Arrays.asList(values)); + } + + return this; + } + + public NativeQueryBuilder withSort(Function> fn) { + + Assert.notNull(fn, "fn must not be null"); + withSort(fn.apply(new SortOptions.Builder()).build()); + + return this; + } + + public NativeQueryBuilder withSearchExtension(String key, JsonData value) { + + Assert.notNull(key, "key must not be null"); + Assert.notNull(value, "value must not be null"); + + searchExtensions.put(key, value); + return this; + } + + public NativeQueryBuilder withSearchExtensions(Map searchExtensions) { + + Assert.notNull(searchExtensions, "searchExtensions must not be null"); + + this.searchExtensions.putAll(searchExtensions); + return this; + } + + /** + * Allows to use a {@link org.springframework.data.elasticsearch.core.query.Query} within a NativeQuery. Cannot be + * used together with {@link #withQuery(Query)} that sets an Elasticsearch query. Passing in a {@link NativeQuery} + * will result in an exception when {@link #build()} is called. + * + * @since 5.1 + */ + public NativeQueryBuilder withQuery(org.springframework.data.elasticsearch.core.query.Query query) { + this.springDataQuery = query; + return this; + } + + /** + * @since 5.1 + */ + public NativeQueryBuilder withKnnQuery(KnnQuery knnQuery) { + this.knnQuery = knnQuery; + return this; + } + + public NativeQuery build() { + Assert.isTrue(query == null || springDataQuery == null, "Cannot have both a native query and a Spring Data query"); + return new NativeQuery(this); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregation.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregation.java new file mode 100644 index 0000000..5d0dd82 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregation.java @@ -0,0 +1,37 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.springframework.data.elasticsearch.core.AggregationContainer; + +/** + * {@link AggregationContainer} for a {@link Aggregation} that holds OpenEearch data. + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class OpenSearchAggregation implements AggregationContainer { + + private final Aggregation aggregation; + + public OpenSearchAggregation(Aggregation aggregation) { + this.aggregation = aggregation; + } + + @Override + public Aggregation aggregation() { + return aggregation; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregations.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregations.java new file mode 100644 index 0000000..42a0d56 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchAggregations.java @@ -0,0 +1,76 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.opensearch.client.opensearch._types.aggregations.Aggregate; +import org.springframework.data.elasticsearch.core.AggregationsContainer; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * AggregationsContainer implementation for the OpenSearch aggregations. + * + * @author Peter-Josef Meisch + * @author Sascha Woo + * @since 4.4 + */ +public class OpenSearchAggregations implements AggregationsContainer> { + + private final List aggregations; + private final Map aggregationsAsMap; + + public OpenSearchAggregations(Map aggregations) { + + Assert.notNull(aggregations, "aggregations must not be null"); + + aggregationsAsMap = new HashMap<>(); + aggregations.forEach((name, aggregate) -> aggregationsAsMap // + .put(name, new OpenSearchAggregation(new Aggregation(name, aggregate)))); + + this.aggregations = new ArrayList<>(aggregationsAsMap.values()); + } + + @Override + public List aggregations() { + return aggregations; + } + + /** + * @return the {@link OpenSearchAggregation}s keyed by aggregation name. + */ + public Map aggregationsAsMap() { + return aggregationsAsMap; + } + + /** + * Returns the aggregation that is associated with the specified name. + * + * @param name the name of the aggregation + * @return the aggregation or {@literal null} if not found + */ + @Nullable + public OpenSearchAggregation get(String name) { + + Assert.notNull(name, "name must not be null"); + + return aggregationsAsMap.get(name); + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientBeanDefinitionParser.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientBeanDefinitionParser.java new file mode 100644 index 0000000..d3ab87c --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientBeanDefinitionParser.java @@ -0,0 +1,47 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.springframework.beans.factory.support.AbstractBeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.xml.AbstractBeanDefinitionParser; +import org.springframework.beans.factory.xml.ParserContext; +import org.w3c.dom.Element; + +/** + * @author Peter-Josef Meisch + * @since 5.0 + */ +public class OpenSearchClientBeanDefinitionParser extends AbstractBeanDefinitionParser { + + @Override + protected AbstractBeanDefinition parseInternal(Element element, ParserContext parserContext) { + BeanDefinitionBuilder builder = BeanDefinitionBuilder.rootBeanDefinition(OpenSearchClientFactoryBean.class); + setConfigurations(element, builder); + return getSourcedBeanDefinition(builder, element, parserContext); + } + + private void setConfigurations(Element element, BeanDefinitionBuilder builder) { + builder.addPropertyValue("hosts", element.getAttribute("hosts")); + } + + private AbstractBeanDefinition getSourcedBeanDefinition(BeanDefinitionBuilder builder, Element source, + ParserContext context) { + AbstractBeanDefinition definition = builder.getBeanDefinition(); + definition.setSource(context.extractSource(source)); + return definition; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientFactoryBean.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientFactoryBean.java new file mode 100644 index 0000000..23f331f --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClientFactoryBean.java @@ -0,0 +1,96 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.FactoryBeanNotInitializedException; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * OpenSearchClientFactoryBean + * + * @author Peter-Josef Meisch + * @since 5.0 + */ +public class OpenSearchClientFactoryBean + implements FactoryBean, InitializingBean, DisposableBean { + + private static final Log LOGGER = LogFactory.getLog(OpenSearchClientFactoryBean.class); + + private @Nullable AutoCloseableOpenSearchClient client; + private String hosts = "http://localhost:9200"; + static final String COMMA = ","; + + @Override + public void destroy() { + try { + LOGGER.info("Closing elasticSearch client"); + if (client != null) { + client.close(); + } + } catch (final Exception e) { + LOGGER.error("Error closing ElasticSearch client: ", e); + } + } + + @Override + public void afterPropertiesSet() throws Exception { + buildClient(); + } + + @Override + public OpenSearchClient getObject() { + + if (client == null) { + throw new FactoryBeanNotInitializedException(); + } + + return client; + } + + @Override + public Class getObjectType() { + return OpenSearchClient.class; + } + + @Override + public boolean isSingleton() { + return false; + } + + protected void buildClient() throws Exception { + + Assert.hasText(hosts, "[Assertion Failed] At least one host must be set."); + + var clientConfiguration = ClientConfiguration.builder().connectedTo(hosts).build(); + client = (AutoCloseableOpenSearchClient) OpenSearchClients.createImperative(clientConfiguration); + } + + public void setHosts(String hosts) { + this.hosts = hosts; + } + + public String getHosts() { + return this.hosts; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClients.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClients.java new file mode 100644 index 0000000..0d11e22 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchClients.java @@ -0,0 +1,411 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.net.InetSocketAddress; +import java.security.KeyManagementException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import javax.net.ssl.SSLContext; +import javax.net.ssl.X509TrustManager; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.message.BasicHeader; +import org.apache.http.protocol.HttpContext; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.TransportOptions; +import org.opensearch.client.transport.rest_client.RestClientOptions; +import org.opensearch.client.transport.rest_client.RestClientTransport; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.support.HttpHeaders; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Utility class to create the different OpenSearch clients + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +@SuppressWarnings("unused") +public final class OpenSearchClients { + public static final String IMPERATIVE_CLIENT = "imperative"; + + /** + * Name of whose value can be used to correlate log messages for this request. + */ + private static final String X_SPRING_DATA_OPENSEARCH_CLIENT = "X-SpringDataOpenSearch-Client"; + private static final JsonpMapper DEFAULT_JSONP_MAPPER = new JacksonJsonpMapper(); + + + // region imperative client + /** + * Creates a new imperative {@link OpenSearchClient} + * + * @param clientConfiguration configuration options, must not be {@literal null}. + * @return the {@link OpenSearchClient} + */ + public static OpenSearchClient createImperative(ClientConfiguration clientConfiguration) { + return createImperative(getRestClient(clientConfiguration), null, DEFAULT_JSONP_MAPPER); + } + + /** + * Creates a new imperative {@link OpenSearchClient} + * + * @param clientConfiguration configuration options, must not be {@literal null}. + * @param transportOptions options to be added to each request. + * @return the {@link OpenSearchClient} + */ + public static OpenSearchClient createImperative(ClientConfiguration clientConfiguration, + TransportOptions transportOptions) { + return createImperative(getRestClient(clientConfiguration), transportOptions, DEFAULT_JSONP_MAPPER); + } + + /** + * Creates a new imperative {@link OpenSearchClient} + * + * @param restClient the RestClient to use + * @return the {@link OpenSearchClient} + */ + public static OpenSearchClient createImperative(RestClient restClient) { + return createImperative(restClient, null, DEFAULT_JSONP_MAPPER); + } + + /** + * Creates a new imperative {@link OpenSearchClient} + * + * @param restClient the RestClient to use + * @param transportOptions options to be added to each request. + * @return the {@link OpenSearchClient} + */ + public static OpenSearchClient createImperative(RestClient restClient, @Nullable TransportOptions transportOptions) { + return createImperative(restClient, transportOptions, DEFAULT_JSONP_MAPPER); + } + + /** + * Creates a new imperative {@link OpenSearchClient} + * + * @param restClient the RestClient to use + * @param transportOptions options to be added to each request. + * @param jsonpMapper the mapper for the transport to use + * @return the {@link OpenSearchClient} + */ + public static OpenSearchClient createImperative(RestClient restClient, @Nullable TransportOptions transportOptions, + JsonpMapper jsonpMapper) { + + Assert.notNull(restClient, "restClient must not be null"); + + OpenSearchTransport transport = getOpenSearchTransport(restClient, IMPERATIVE_CLIENT, transportOptions, + jsonpMapper); + + return createImperative(transport); + } + + /** + * Creates a new {@link OpenSearchClient} that uses the given {@link OpenSearchTransport}. + * + * @param transport the transport to use + * @return the {@link OpenSearchClient} + */ + public static AutoCloseableOpenSearchClient createImperative(OpenSearchTransport transport) { + + Assert.notNull(transport, "transport must not be null"); + + return new AutoCloseableOpenSearchClient(transport); + } + // endregion + + // region low level RestClient + private static RestClientOptions.Builder getRestClientOptionsBuilder(@Nullable TransportOptions transportOptions) { + + if (transportOptions instanceof RestClientOptions restClientOptions) { + return restClientOptions.toBuilder(); + } + + var builder = new RestClientOptions.Builder(RequestOptions.DEFAULT.toBuilder()); + + if (transportOptions != null) { + transportOptions.headers().forEach(header -> builder.addHeader(header.getKey(), header.getValue())); + transportOptions.queryParameters().forEach(builder::setParameter); + builder.onWarnings(transportOptions.onWarnings()); + } + + return builder; + } + + /** + * Creates a low level {@link RestClient} for the given configuration. + * + * @param clientConfiguration must not be {@literal null} + * @return the {@link RestClient} + */ + public static RestClient getRestClient(ClientConfiguration clientConfiguration) { + return getRestClientBuilder(clientConfiguration).build(); + } + + private static RestClientBuilder getRestClientBuilder(ClientConfiguration clientConfiguration) { + HttpHost[] httpHosts = formattedHosts(clientConfiguration.getEndpoints(), clientConfiguration.useSsl()).stream() + .map(HttpHost::create).toArray(HttpHost[]::new); + RestClientBuilder builder = RestClient.builder(httpHosts); + + if (clientConfiguration.getPathPrefix() != null) { + builder.setPathPrefix(clientConfiguration.getPathPrefix()); + } + + HttpHeaders headers = clientConfiguration.getDefaultHeaders(); + + if (!headers.isEmpty()) { + builder.setDefaultHeaders(toHeaderArray(headers)); + } + + builder.setHttpClientConfigCallback(clientBuilder -> { + if (clientConfiguration.getCaFingerprint().isPresent()) { + clientBuilder + .setSSLContext(sslContextFromCaFingerprint(clientConfiguration.getCaFingerprint().get())); + } + clientConfiguration.getSslContext().ifPresent(clientBuilder::setSSLContext); + clientConfiguration.getHostNameVerifier().ifPresent(clientBuilder::setSSLHostnameVerifier); + clientBuilder.addInterceptorLast(new CustomHeaderInjector(clientConfiguration.getHeadersSupplier())); + + RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); + Duration connectTimeout = clientConfiguration.getConnectTimeout(); + + if (!connectTimeout.isNegative()) { + requestConfigBuilder.setConnectTimeout(Math.toIntExact(connectTimeout.toMillis())); + } + + Duration socketTimeout = clientConfiguration.getSocketTimeout(); + + if (!socketTimeout.isNegative()) { + requestConfigBuilder.setSocketTimeout(Math.toIntExact(socketTimeout.toMillis())); + requestConfigBuilder.setConnectionRequestTimeout(Math.toIntExact(socketTimeout.toMillis())); + } + + clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); + + clientConfiguration.getProxy().map(HttpHost::create).ifPresent(clientBuilder::setProxy); + + for (ClientConfiguration.ClientConfigurationCallback clientConfigurer : clientConfiguration + .getClientConfigurers()) { + if (clientConfigurer instanceof OpenSearchHttpClientConfigurationCallback restClientConfigurationCallback) { + clientBuilder = restClientConfigurationCallback.configure(clientBuilder); + } + } + + return clientBuilder; + }); + + for (ClientConfiguration.ClientConfigurationCallback clientConfigurationCallback : clientConfiguration + .getClientConfigurers()) { + if (clientConfigurationCallback instanceof OpenSearchRestClientConfigurationCallback configurationCallback) { + builder = configurationCallback.configure(builder); + } + } + return builder; + } + // endregion + + // region OpenSearch transport + /** + * Creates an {@link OpenSearchTransport} that will use the given client that additionally is customized with a + * header to contain the clientType + * + * @param restClient the client to use + * @param clientType the client type to pass in each request as header + * @param transportOptions options for the transport + * @param jsonpMapper mapper for the transport + * @return OpenSearchTransport + */ + public static OpenSearchTransport getOpenSearchTransport(RestClient restClient, String clientType, + @Nullable TransportOptions transportOptions, JsonpMapper jsonpMapper) { + + Assert.notNull(restClient, "restClient must not be null"); + Assert.notNull(clientType, "clientType must not be null"); + Assert.notNull(jsonpMapper, "jsonpMapper must not be null"); + + TransportOptions.Builder transportOptionsBuilder = transportOptions != null ? transportOptions.toBuilder() + : new RestClientOptions(RequestOptions.DEFAULT).toBuilder(); + + RestClientOptions.Builder restClientOptionsBuilder = getRestClientOptionsBuilder(transportOptions); + + // The "application/vnd.opensearch+json" would be more appropriate here but it is not supported by 1.x line, + // using "application/json" instead. + ContentType jsonContentType = ContentType.APPLICATION_JSON; + + Consumer setHeaderIfNotPresent = header -> { + if (restClientOptionsBuilder.build().headers().stream() // + .noneMatch((h) -> h.getKey().equalsIgnoreCase(header))) { + // need to add the compatibility header, this is only done automatically when not passing in custom options. + // code copied from RestClientTransport as it is not available outside the package + restClientOptionsBuilder.addHeader(header, jsonContentType.toString()); + } + }; + + setHeaderIfNotPresent.accept("Content-Type"); + setHeaderIfNotPresent.accept("Accept"); + + restClientOptionsBuilder.addHeader(X_SPRING_DATA_OPENSEARCH_CLIENT, clientType); + + return new RestClientTransport(restClient, jsonpMapper, restClientOptionsBuilder.build()); + } + // endregion + + private static List formattedHosts(List hosts, boolean useSsl) { + return hosts.stream().map(it -> (useSsl ? "https" : "http") + "://" + it.getHostString() + ':' + it.getPort()) + .collect(Collectors.toList()); + } + + private static org.apache.http.Header[] toHeaderArray(HttpHeaders headers) { + return headers.entrySet().stream() // + .flatMap(entry -> entry.getValue().stream() // + .map(value -> new BasicHeader(entry.getKey(), value))) // + .toArray(org.apache.http.Header[]::new); + } + + /** + * Interceptor to inject custom supplied headers. + * + * @since 4.4 + */ + private record CustomHeaderInjector(Supplier headersSupplier) implements HttpRequestInterceptor { + + @Override + public void process(HttpRequest request, HttpContext context) { + HttpHeaders httpHeaders = headersSupplier.get(); + + if (httpHeaders != null && !httpHeaders.isEmpty()) { + Arrays.stream(toHeaderArray(httpHeaders)).forEach(request::addHeader); + } + } + } + + /** + * {@link org.springframework.data.elasticsearch.client.ClientConfiguration.ClientConfigurationCallback} to configure + * the OpenSearch RestClient's Http client with a {@link HttpAsyncClientBuilder} + * + * @since 4.4 + */ + public interface OpenSearchHttpClientConfigurationCallback + extends ClientConfiguration.ClientConfigurationCallback { + + static OpenSearchHttpClientConfigurationCallback from( + Function httpClientBuilderCallback) { + + Assert.notNull(httpClientBuilderCallback, "httpClientBuilderCallback must not be null"); + + return httpClientBuilderCallback::apply; + } + } + + /** + * {@link org.springframework.data.elasticsearch.client.ClientConfiguration.ClientConfigurationCallback} to configure + * the RestClient client with a {@link RestClientBuilder} + * + * @since 5.0 + */ + public interface OpenSearchRestClientConfigurationCallback + extends ClientConfiguration.ClientConfigurationCallback { + + static OpenSearchRestClientConfigurationCallback from( + Function restClientBuilderCallback) { + + Assert.notNull(restClientBuilderCallback, "restClientBuilderCallback must not be null"); + + return restClientBuilderCallback::apply; + } + } + + /** + * Copy / paste of co.elastic.clients.transport.TransportUtils#sslContextFromCaFingerprint (licensed under ASFv2), since + * OpenSearch Java client does not support such SSL configuration at the moment. + */ + private static SSLContext sslContextFromCaFingerprint(String fingerPrint) { + + fingerPrint = fingerPrint.replace(":", ""); + int len = fingerPrint.length(); + byte[] fpBytes = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + fpBytes[i / 2] = (byte) ( + (Character.digit(fingerPrint.charAt(i), 16) << 4) + + Character.digit(fingerPrint.charAt(i+1), 16) + ); + } + + try { + X509TrustManager tm = new X509TrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + throw new CertificateException("This is a client-side only trust manager"); + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + + // The CA root is the last element of the chain + X509Certificate anchor = chain[chain.length - 1]; + + byte[] bytes; + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + md.update(anchor.getEncoded()); + bytes = md.digest(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + + if (Arrays.equals(fpBytes, bytes)) { + return; + } + + throw new CertificateException("Untrusted certificate: " + anchor.getSubjectX500Principal()); + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + }; + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, new X509TrustManager[] { tm }, null); + return sslContext; + + } catch (NoSuchAlgorithmException | KeyManagementException e) { + // Exceptions that should normally not occur + throw new RuntimeException(e); + } + } + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchConfiguration.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchConfiguration.java new file mode 100644 index 0000000..6113094 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchConfiguration.java @@ -0,0 +1,129 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.opensearch.client.RequestOptions; +import org.opensearch.client.RestClient; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.TransportOptions; +import org.opensearch.client.transport.rest_client.RestClientOptions; +import org.springframework.context.annotation.Bean; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.config.ElasticsearchConfigurationSupport; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.util.Assert; + +/** + * Base class for a @{@link org.springframework.context.annotation.Configuration} class to set up the OpenSearch + * connection using the OpenSearch Client. This class exposes different parts of the setup as Spring beans. Deriving + * classes must provide the {@link ClientConfiguration} to use. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public abstract class OpenSearchConfiguration extends ElasticsearchConfigurationSupport { + + /** + * Must be implemented by deriving classes to provide the {@link ClientConfiguration}. + * + * @return configuration, must not be {@literal null} + */ + @Bean(name = "elasticsearchClientConfiguration") + public abstract ClientConfiguration clientConfiguration(); + + /** + * Provides the underlying low level OpenSearch RestClient. + * + * @param clientConfiguration configuration for the client, must not be {@literal null} + * @return RestClient + */ + @Bean + public RestClient opensearchRestClient(ClientConfiguration clientConfiguration) { + + Assert.notNull(clientConfiguration, "clientConfiguration must not be null"); + + return OpenSearchClients.getRestClient(clientConfiguration); + } + + /** + * Provides the OpenSearch transport to be used. The default implementation uses the {@link RestClient} bean and + * the {@link JsonpMapper} bean provided in this class. + * + * @return the {@link OpenSearchTransport} + * @since 5.2 + */ + @Bean + public OpenSearchTransport opensearchTransport(RestClient restClient, JsonpMapper jsonpMapper) { + + Assert.notNull(restClient, "restClient must not be null"); + Assert.notNull(jsonpMapper, "jsonpMapper must not be null"); + + return OpenSearchClients.getOpenSearchTransport(restClient, OpenSearchClients.IMPERATIVE_CLIENT, + transportOptions(), jsonpMapper); + } + + /** + * Provides the {@link OpenSearchClient} to be used. + * + * @param transport the {@link OpenSearchTransport} to use + * @return OpenSearchClient instance + */ + @Bean + public OpenSearchClient opensearchClient(OpenSearchTransport transport) { + + Assert.notNull(transport, "transport must not be null"); + + return OpenSearchClients.createImperative(transport); + } + + /** + * Creates a {@link ElasticsearchOperations} implementation using an + * {@link org.opensearch.client.opensearch.OpenSearchClient}. + * + * @return never {@literal null}. + */ + @Bean(name = { "elasticsearchOperations", "elasticsearchTemplate", "opensearchOperations", "opensearchTemplate" }) + public ElasticsearchOperations opensearchOperations(ElasticsearchConverter elasticsearchConverter, + OpenSearchClient elasticsearchClient) { + + OpenSearchTemplate template = new OpenSearchTemplate(elasticsearchClient, elasticsearchConverter); + template.setRefreshPolicy(refreshPolicy()); + + return template; + } + + /** + * Provides the JsonpMapper bean that is used in the {@link #opensearchTransport(RestClient, JsonpMapper)} method. + * + * @return the {@link JsonpMapper} to use + * @since 5.2 + */ + @Bean + public JsonpMapper jsonpMapper() { + return new JacksonJsonpMapper(); + } + + /** + * @return the options that should be added to every request. Must not be {@literal null} + */ + public TransportOptions transportOptions() { + return new RestClientOptions(RequestOptions.DEFAULT); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchExceptionTranslator.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchExceptionTranslator.java new file mode 100644 index 0000000..54f0bd5 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchExceptionTranslator.java @@ -0,0 +1,136 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.opensearch.client.ResponseException; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch._types.ErrorResponse; +import org.opensearch.client.opensearch._types.OpenSearchException; +import org.springframework.dao.DataAccessException; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.dao.OptimisticLockingFailureException; +import org.springframework.dao.support.PersistenceExceptionTranslator; +import org.springframework.data.elasticsearch.NoSuchIndexException; +import org.springframework.data.elasticsearch.ResourceNotFoundException; +import org.springframework.data.elasticsearch.UncategorizedElasticsearchException; +import org.springframework.data.elasticsearch.VersionConflictException; + +/** + * Simple {@link PersistenceExceptionTranslator} for OpenSearch. Convert the given runtime exception to an + * appropriate exception from the {@code org.springframework.dao} hierarchy. Return {@literal null} if no translation is + * appropriate: any other exception may have resulted from user code, and should not be translated. + * + * @author Peter-Josef Meisch + * @author Junghoon Ban + * @since 4.4 + */ +public class OpenSearchExceptionTranslator implements PersistenceExceptionTranslator { + + private final JsonpMapper jsonpMapper; + + public OpenSearchExceptionTranslator(JsonpMapper jsonpMapper) { + this.jsonpMapper = jsonpMapper; + } + + /** + * translates an Exception if possible. Exceptions that are no {@link RuntimeException}s are wrapped in a + * RuntimeException + * + * @param throwable the Exception to map + * @return the potentially translated RuntimeException. + */ + public RuntimeException translateException(Throwable throwable) { + + RuntimeException runtimeException = throwable instanceof RuntimeException ex ? ex + : new RuntimeException(throwable.getMessage(), throwable); + RuntimeException potentiallyTranslatedException = translateExceptionIfPossible(runtimeException); + + return potentiallyTranslatedException != null ? potentiallyTranslatedException : runtimeException; + } + + @Override + public DataAccessException translateExceptionIfPossible(RuntimeException ex) { + + checkForConflictException(ex); + + if (ex instanceof OpenSearchException openSearchException) { + + ErrorResponse response = openSearchException.response(); + var errorType = response.error().type(); + var errorReason = response.error().reason() != null ? response.error().reason() : "undefined reason"; + + if (response.status() == 404) { + + if ("index_not_found_exception".equals(errorType)) { + // noinspection RegExpRedundantEscape + Pattern pattern = Pattern.compile(".*no such index \\[(.*)\\]"); + String index = ""; + Matcher matcher = pattern.matcher(errorReason); + if (matcher.matches()) { + index = matcher.group(1); + } + return new NoSuchIndexException(index); + } + + return new ResourceNotFoundException(errorReason); + } + + if (response.status() == 409) { + + } + String body = JsonUtils.toJson(response, jsonpMapper); + + if (errorType != null && errorType.contains("validation_exception")) { + return new DataIntegrityViolationException(errorReason); + } + + return new UncategorizedElasticsearchException(ex.getMessage(), response.status(), body, ex); + } + + Throwable cause = ex.getCause(); + if (cause instanceof IOException) { + return new DataAccessResourceFailureException(ex.getMessage(), ex); + } + + return null; + } + + private void checkForConflictException(Throwable exception) { + Integer status = null; + String message = null; + + if (exception instanceof ResponseException responseException) { + status = responseException.getResponse().getStatusLine().getStatusCode(); + message = responseException.getMessage(); + } else if (exception.getCause() != null) { + checkForConflictException(exception.getCause()); + } + + if (status != null && message != null) { + if (status == 409 && message.contains("type\":\"version_conflict_engine_exception")) + if (message.contains("version conflict, required seqNo")) { + throw new OptimisticLockingFailureException("Cannot index a document due to seq_no+primary_term conflict", + exception); + } else if (message.contains("version conflict, current version [")) { + throw new VersionConflictException("Version conflict", exception); + } + } + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchTemplate.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchTemplate.java new file mode 100644 index 0000000..819cb9a --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/OpenSearchTemplate.java @@ -0,0 +1,723 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.TypeUtils.*; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.Time; +import org.opensearch.client.opensearch.core.*; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; +import org.opensearch.client.opensearch.core.msearch.MultiSearchResponseItem; +import org.opensearch.client.opensearch.core.pit.DeletePitRequest; +import org.opensearch.client.opensearch.core.search.SearchResult; +import org.opensearch.client.transport.Version; +import org.springframework.data.elasticsearch.BulkFailureException; +import org.springframework.data.elasticsearch.client.UnsupportedBackendOperation; +import org.springframework.data.elasticsearch.core.AbstractElasticsearchTemplate; +import org.springframework.data.elasticsearch.core.IndexOperations; +import org.springframework.data.elasticsearch.core.IndexedObjectInformation; +import org.springframework.data.elasticsearch.core.MultiGetItem; +import org.springframework.data.elasticsearch.core.SearchHits; +import org.springframework.data.elasticsearch.core.SearchScrollHits; +import org.springframework.data.elasticsearch.core.cluster.ClusterOperations; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.data.elasticsearch.core.query.BulkOptions; +import org.springframework.data.elasticsearch.core.query.ByQueryResponse; +import org.springframework.data.elasticsearch.core.query.IndexQuery; +import org.springframework.data.elasticsearch.core.query.MoreLikeThisQuery; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.SearchTemplateQuery; +import org.springframework.data.elasticsearch.core.query.UpdateQuery; +import org.springframework.data.elasticsearch.core.query.UpdateResponse; +import org.springframework.data.elasticsearch.core.reindex.ReindexRequest; +import org.springframework.data.elasticsearch.core.reindex.ReindexResponse; +import org.springframework.data.elasticsearch.core.script.Script; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Implementation of {@link org.springframework.data.elasticsearch.core.ElasticsearchOperations} using the new + * OpenSearch client. + * + * @author Peter-Josef Meisch + * @author Hamid Rahimi + * @author Illia Ulianov + * @author Haibo Liu + * @since 4.4 + */ +public class OpenSearchTemplate extends AbstractElasticsearchTemplate { + + private static final Log LOGGER = LogFactory.getLog(OpenSearchTemplate.class); + + private final OpenSearchClient client; + private final RequestConverter requestConverter; + private final ResponseConverter responseConverter; + private final JsonpMapper jsonpMapper; + private final OpenSearchExceptionTranslator exceptionTranslator; + + // region _initialization + public OpenSearchTemplate(OpenSearchClient client) { + + Assert.notNull(client, "client must not be null"); + + this.client = client; + this.jsonpMapper = client._transport().jsonpMapper(); + requestConverter = new RequestConverter(elasticsearchConverter, jsonpMapper); + responseConverter = new ResponseConverter(jsonpMapper); + exceptionTranslator = new OpenSearchExceptionTranslator(jsonpMapper); + } + + public OpenSearchTemplate(OpenSearchClient client, ElasticsearchConverter elasticsearchConverter) { + super(elasticsearchConverter); + + Assert.notNull(client, "client must not be null"); + + this.client = client; + this.jsonpMapper = client._transport().jsonpMapper(); + requestConverter = new RequestConverter(elasticsearchConverter, jsonpMapper); + responseConverter = new ResponseConverter(jsonpMapper); + exceptionTranslator = new OpenSearchExceptionTranslator(jsonpMapper); + } + + @Override + protected AbstractElasticsearchTemplate doCopy() { + return new OpenSearchTemplate(client, elasticsearchConverter); + } + // endregion + + // region child templates + @Override + public IndexOperations indexOps(Class clazz) { + return new IndicesTemplate(client.indices(), getClusterTemplate(), elasticsearchConverter, clazz); + } + + @Override + public IndexOperations indexOps(IndexCoordinates index) { + return new IndicesTemplate(client.indices(), getClusterTemplate(), elasticsearchConverter, index); + } + + @Override + public ClusterOperations cluster() { + return getClusterTemplate(); + } + + private ClusterTemplate getClusterTemplate() { + return new ClusterTemplate(client.cluster(), elasticsearchConverter); + } + // endregion + + // region document operations + @Override + @Nullable + public T get(String id, Class clazz, IndexCoordinates index) { + + GetRequest getRequest = requestConverter.documentGetRequest(elasticsearchConverter.convertId(id), + routingResolver.getRouting(), index); + GetResponse getResponse = execute(client -> client.get(getRequest, EntityAsMap.class)); + + ReadDocumentCallback callback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + return callback.doWith(DocumentAdapters.from(getResponse)); + } + + @Override + public List> multiGet(Query query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + + MgetRequest request = requestConverter.documentMgetRequest(query, clazz, index); + MgetResponse result = execute(client -> client.mget(request, EntityAsMap.class)); + + ReadDocumentCallback callback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + + return DocumentAdapters.from(result).stream() // + .map(multiGetItem -> MultiGetItem.of( // + multiGetItem.isFailed() ? null : callback.doWith(multiGetItem.getItem()), multiGetItem.getFailure())) // + .collect(Collectors.toList()); + } + + @Override + public void bulkUpdate(List queries, BulkOptions bulkOptions, IndexCoordinates index) { + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(bulkOptions, "bulkOptions must not be null"); + Assert.notNull(index, "index must not be null"); + + doBulkOperation(queries, bulkOptions, index); + } + + @Override + public ByQueryResponse delete(Query query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + + DeleteByQueryRequest request = requestConverter.documentDeleteByQueryRequest(query, routingResolver.getRouting(), + clazz, index, getRefreshPolicy()); + + DeleteByQueryResponse response = execute(client -> client.deleteByQuery(request)); + + return responseConverter.byQueryResponse(response); + } + + @Override + public UpdateResponse update(UpdateQuery updateQuery, IndexCoordinates index) { + + UpdateRequest request = requestConverter.documentUpdateRequest(updateQuery, index, getRefreshPolicy(), + routingResolver.getRouting()); + org.opensearch.client.opensearch.core.UpdateResponse response = execute( + client -> client.update(request, Document.class)); + return UpdateResponse.of(result(response.result())); + } + + @Override + public ByQueryResponse updateByQuery(UpdateQuery updateQuery, IndexCoordinates index) { + + Assert.notNull(updateQuery, "updateQuery must not be null"); + Assert.notNull(index, "index must not be null"); + + UpdateByQueryRequest request = requestConverter.documentUpdateByQueryRequest(updateQuery, index, + getRefreshPolicy()); + + UpdateByQueryResponse byQueryResponse = execute(client -> client.updateByQuery(request)); + return responseConverter.byQueryResponse(byQueryResponse); + } + + @Override + public String doIndex(IndexQuery query, IndexCoordinates indexCoordinates) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + IndexRequest indexRequest = requestConverter.documentIndexRequest(query, indexCoordinates, refreshPolicy); + + IndexResponse indexResponse = execute(client -> client.index(indexRequest)); + + Object queryObject = query.getObject(); + + if (queryObject != null) { + query.setObject(entityOperations.updateIndexedObject( + queryObject, + new IndexedObjectInformation( + indexResponse.id(), + indexResponse.index(), + indexResponse.seqNo(), + indexResponse.primaryTerm(), + indexResponse.version()), + elasticsearchConverter, + routingResolver)); + } + + return indexResponse.id(); + } + + @Override + protected boolean doExists(String id, IndexCoordinates index) { + + Assert.notNull(id, "id must not be null"); + Assert.notNull(index, "index must not be null"); + + ExistsRequest request = requestConverter.documentExistsRequest(id, routingResolver.getRouting(), index); + + return execute(client -> client.exists(request)).value(); + } + + @Override + protected String doDelete(String id, @Nullable String routing, IndexCoordinates index) { + + Assert.notNull(id, "id must not be null"); + Assert.notNull(index, "index must not be null"); + + DeleteRequest request = requestConverter.documentDeleteRequest(elasticsearchConverter.convertId(id), routing, index, + getRefreshPolicy()); + return execute(client -> client.delete(request)).id(); + } + + @Override + public ReindexResponse reindex(ReindexRequest reindexRequest) { + + Assert.notNull(reindexRequest, "reindexRequest must not be null"); + + org.opensearch.client.opensearch.core.ReindexRequest reindexRequestES = requestConverter.reindex(reindexRequest, + true); + org.opensearch.client.opensearch.core.ReindexResponse reindexResponse = execute( + client -> client.reindex(reindexRequestES)); + return responseConverter.reindexResponse(reindexResponse); + } + + @Override + public String submitReindex(ReindexRequest reindexRequest) { + + org.opensearch.client.opensearch.core.ReindexRequest reindexRequestES = requestConverter.reindex(reindexRequest, + false); + org.opensearch.client.opensearch.core.ReindexResponse reindexResponse = execute( + client -> client.reindex(reindexRequestES)); + + if (reindexResponse.task() == null) { + throw new UnsupportedBackendOperation("OpenSearchClient did not return a task id on submit request"); + } + + return reindexResponse.task(); + } + + @Override + public List doBulkOperation(List queries, BulkOptions bulkOptions, + IndexCoordinates index) { + + BulkRequest bulkRequest = requestConverter.documentBulkRequest(queries, bulkOptions, index, refreshPolicy); + BulkResponse bulkResponse = execute(client -> client.bulk(bulkRequest)); + List indexedObjectInformationList = checkForBulkOperationFailure(bulkResponse); + updateIndexedObjectsWithQueries(queries, indexedObjectInformationList); + return indexedObjectInformationList; + } + + // endregion + + @Override + public String getClusterVersion() { + return execute(client -> client.info().version().number()); + } + + @Override + public String getVendor() { + return "Elasticsearch"; + } + + @Override + public String getRuntimeLibraryVersion() { + return Version.VERSION != null ? Version.VERSION.toString() : "0.0.0.?"; + } + + // region search operations + @Override + public long count(Query query, @Nullable Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(index, "index must not be null"); + + SearchRequest searchRequest = requestConverter.searchRequest(query, routingResolver.getRouting(), clazz, index, + true); + + SearchResponse searchResponse = execute(client -> client.search(searchRequest, EntityAsMap.class)); + + return searchResponse.hits().total().value(); + } + + @Override + public SearchHits search(Query query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + Assert.notNull(index, "index must not be null"); + + if (query instanceof SearchTemplateQuery searchTemplateQuery) { + return doSearch(searchTemplateQuery, clazz, index); + } else { + return doSearch(query, clazz, index); + } + } + + protected SearchHits doSearch(Query query, Class clazz, IndexCoordinates index) { + SearchRequest searchRequest = requestConverter.searchRequest(query, routingResolver.getRouting(), clazz, index, + false); + SearchResponse searchResponse = execute(client -> client.search(searchRequest, EntityAsMap.class)); + + // noinspection DuplicatedCode + ReadDocumentCallback readDocumentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + SearchDocumentResponse.EntityCreator entityCreator = getEntityCreator(readDocumentCallback); + SearchDocumentResponseCallback> callback = new ReadSearchDocumentResponseCallback<>(clazz, index); + + return callback.doWith(SearchDocumentResponseBuilder.from(searchResponse, entityCreator, jsonpMapper)); + } + + protected SearchHits doSearch(SearchTemplateQuery query, Class clazz, IndexCoordinates index) { + var searchTemplateRequest = requestConverter.searchTemplate(query, routingResolver.getRouting(), index); + var searchTemplateResponse = execute(client -> client.searchTemplate(searchTemplateRequest, EntityAsMap.class)); + + // noinspection DuplicatedCode + ReadDocumentCallback readDocumentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + SearchDocumentResponse.EntityCreator entityCreator = getEntityCreator(readDocumentCallback); + SearchDocumentResponseCallback> callback = new ReadSearchDocumentResponseCallback<>(clazz, index); + + return callback.doWith(SearchDocumentResponseBuilder.from(searchTemplateResponse, entityCreator, jsonpMapper)); + } + + @Override + protected SearchHits doSearch(MoreLikeThisQuery query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + Assert.notNull(index, "index must not be null"); + + return search(NativeQuery.builder() // + .withQuery(q -> q.moreLikeThis(requestConverter.moreLikeThisQuery(query, index)))// + .withPageable(query.getPageable()) // + .build(), clazz, index); + } + + @Override + public SearchScrollHits searchScrollStart(long scrollTimeInMillis, Query query, Class clazz, + IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(query.getPageable(), "pageable of query must not be null."); + + SearchRequest request = requestConverter.searchRequest(query, routingResolver.getRouting(), clazz, index, false, + scrollTimeInMillis); + SearchResponse response = execute(client -> client.search(request, EntityAsMap.class)); + + return getSearchScrollHits(clazz, index, response); + } + + @Override + public SearchScrollHits searchScrollContinue(String scrollId, long scrollTimeInMillis, Class clazz, + IndexCoordinates index) { + + Assert.notNull(scrollId, "scrollId must not be null"); + + ScrollRequest request = ScrollRequest + .of(sr -> sr.scrollId(scrollId).scroll(Time.of(t -> t.time(scrollTimeInMillis + "ms")))); + ScrollResponse response = execute(client -> client.scroll(request, EntityAsMap.class)); + + return getSearchScrollHits(clazz, index, response); + } + + private SearchScrollHits getSearchScrollHits(Class clazz, IndexCoordinates index, + SearchResult response) { + ReadDocumentCallback documentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, index); + SearchDocumentResponseCallback> callback = new ReadSearchScrollDocumentResponseCallback<>(clazz, + index); + + return callback + .doWith(SearchDocumentResponseBuilder.from(response, getEntityCreator(documentCallback), jsonpMapper)); + } + + @Override + public void searchScrollClear(List scrollIds) { + + Assert.notNull(scrollIds, "scrollIds must not be null"); + + if (!scrollIds.isEmpty()) { + ClearScrollRequest request = ClearScrollRequest.of(csr -> csr.scrollId(scrollIds)); + execute(client -> client.clearScroll(request)); + } + } + + @Override + public List> multiSearch(List queries, Class clazz, IndexCoordinates index) { + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + + int size = queries.size(); + // noinspection unchecked + return multiSearch(queries, Collections.nCopies(size, clazz), Collections.nCopies(size, index)) + .stream().map(searchHits -> (SearchHits) searchHits) + .collect(Collectors.toList()); + } + + @Override + public List> multiSearch(List queries, List> classes) { + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(classes, "classes must not be null"); + Assert.isTrue(queries.size() == classes.size(), "queries and classes must have the same size"); + + return multiSearch(queries, classes, classes.stream().map(this::getIndexCoordinatesFor).toList()); + } + + @Override + public List> multiSearch(List queries, List> classes, + IndexCoordinates index) { + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(classes, "classes must not be null"); + Assert.notNull(index, "index must not be null"); + Assert.isTrue(queries.size() == classes.size(), "queries and classes must have the same size"); + + return multiSearch(queries, classes, Collections.nCopies(queries.size(), index)); + } + + @Override + public List> multiSearch(List queries, List> classes, + List indexes) { + + Assert.notNull(queries, "queries must not be null"); + Assert.notNull(classes, "classes must not be null"); + Assert.notNull(indexes, "indexes must not be null"); + Assert.isTrue(queries.size() == classes.size() && queries.size() == indexes.size(), + "queries, classes and indexes must have the same size"); + + List multiSearchQueryParameters = new ArrayList<>(queries.size()); + Iterator> it = classes.iterator(); + Iterator indexesIt = indexes.iterator(); + + Assert.isTrue(!queries.isEmpty(), "queries should have at least 1 query"); + boolean isSearchTemplateQuery = queries.get(0) instanceof SearchTemplateQuery; + + for (Query query : queries) { + Assert.isTrue((query instanceof SearchTemplateQuery) == isSearchTemplateQuery, + "SearchTemplateQuery can't be mixed with other types of query in multiple search"); + + Class clazz = it.next(); + IndexCoordinates index = indexesIt.next(); + multiSearchQueryParameters.add(new MultiSearchQueryParameter(query, clazz, index)); + } + + return multiSearch(multiSearchQueryParameters, isSearchTemplateQuery); + } + + private List> multiSearch(List multiSearchQueryParameters, + boolean isSearchTemplateQuery) { + return isSearchTemplateQuery ? + doMultiTemplateSearch(multiSearchQueryParameters.stream() + .map(p -> new MultiSearchTemplateQueryParameter((SearchTemplateQuery) p.query, p.clazz, p.index)) + .toList()) + : doMultiSearch(multiSearchQueryParameters); + } + + private List> doMultiTemplateSearch(List mSearchTemplateQueryParameters) { + MsearchTemplateRequest request = requestConverter.searchMsearchTemplateRequest(mSearchTemplateQueryParameters, + routingResolver.getRouting()); + + MsearchTemplateResponse response = execute(client -> client.msearchTemplate(request, EntityAsMap.class)); + List> responseItems = response.responses(); + + Assert.isTrue(mSearchTemplateQueryParameters.size() == responseItems.size(), + "number of response items does not match number of requests"); + + int size = mSearchTemplateQueryParameters.size(); + List> classes = mSearchTemplateQueryParameters + .stream().map(MultiSearchTemplateQueryParameter::clazz).collect(Collectors.toList()); + List indices = mSearchTemplateQueryParameters + .stream().map(MultiSearchTemplateQueryParameter::index).collect(Collectors.toList()); + + return getSearchHitsFromMsearchResponse(size, classes, indices, responseItems); + } + + private List> doMultiSearch(List multiSearchQueryParameters) { + + MsearchRequest request = requestConverter.searchMsearchRequest(multiSearchQueryParameters, + routingResolver.getRouting()); + + MsearchResponse msearchResponse = execute(client -> client.msearch(request, EntityAsMap.class)); + List> responseItems = msearchResponse.responses(); + + Assert.isTrue(multiSearchQueryParameters.size() == responseItems.size(), + "number of response items does not match number of requests"); + + int size = multiSearchQueryParameters.size(); + List> classes = multiSearchQueryParameters + .stream().map(MultiSearchQueryParameter::clazz).collect(Collectors.toList()); + List indices = multiSearchQueryParameters + .stream().map(MultiSearchQueryParameter::index).collect(Collectors.toList()); + + return getSearchHitsFromMsearchResponse(size, classes, indices, responseItems); + } + + /** + * {@link MsearchResponse} and {@link MsearchTemplateResponse} share the same {@link MultiSearchResponseItem} + */ + @SuppressWarnings({ "unchecked", "rawtypes" }) + private List> getSearchHitsFromMsearchResponse(int size, List> classes, + List indices, List> responseItems) { + List> searchHitsList = new ArrayList<>(size); + Iterator> clazzIter = classes.iterator(); + Iterator indexIter = indices.iterator(); + Iterator> responseIterator = responseItems.iterator(); + + while (clazzIter.hasNext() && indexIter.hasNext()) { + MultiSearchResponseItem responseItem = responseIterator.next(); + + if (responseItem.isResult()) { + + Class clazz = clazzIter.next(); + IndexCoordinates index = indexIter.next(); + ReadDocumentCallback documentCallback = new ReadDocumentCallback<>(elasticsearchConverter, clazz, + index); + SearchDocumentResponseCallback> callback = new ReadSearchDocumentResponseCallback<>(clazz, + index); + + SearchHits searchHits = callback.doWith( + SearchDocumentResponseBuilder.from(responseItem.result(), getEntityCreator(documentCallback), jsonpMapper)); + + searchHitsList.add(searchHits); + } else { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn(String.format("multisearch response contains failure: %s", + responseItem.failure().error().reason())); + } + } + } + + return searchHitsList; + } + + /** + * value class combining the information needed for a single query in a multisearch request. + */ + record MultiSearchQueryParameter(Query query, Class clazz, IndexCoordinates index) { + } + + /** + * value class combining the information needed for a single query in a template multisearch request. + */ + record MultiSearchTemplateQueryParameter(SearchTemplateQuery query, Class clazz, IndexCoordinates index) { + } + + @Override + public String openPointInTime(IndexCoordinates index, Duration keepAlive, Boolean ignoreUnavailable) { + + Assert.notNull(index, "index must not be null"); + Assert.notNull(keepAlive, "keepAlive must not be null"); + Assert.notNull(ignoreUnavailable, "ignoreUnavailable must not be null"); + + var request = requestConverter.searchOpenPointInTimeRequest(index, keepAlive, ignoreUnavailable); + return execute(client -> client.createPit(request)).pitId(); + } + + @Override + public Boolean closePointInTime(String pit) { + + Assert.notNull(pit, "pit must not be null"); + + DeletePitRequest request = requestConverter.searchClosePointInTime(pit); + var response = execute(client -> client.deletePit(request)); + return !response.pits().isEmpty(); + } + + // endregion + + // region script methods + @Override + public boolean putScript(Script script) { + + Assert.notNull(script, "script must not be null"); + + var request = requestConverter.scriptPut(script); + return execute(client -> client.putScript(request)).acknowledged(); + } + + @Nullable + @Override + public Script getScript(String name) { + + Assert.notNull(name, "name must not be null"); + + var request = requestConverter.scriptGet(name); + return responseConverter.scriptResponse(execute(client -> client.getScript(request))); + } + + public boolean deleteScript(String name) { + + Assert.notNull(name, "name must not be null"); + + DeleteScriptRequest request = requestConverter.scriptDelete(name); + return execute(client -> client.deleteScript(request)).acknowledged(); + } + // endregion + + // region client callback + /** + * Callback interface to be used with {@link #execute(OpenSearchTemplate.ClientCallback)} for operating directly on + * the {@link OpenSearchClient}. + */ + @FunctionalInterface + public interface ClientCallback { + T doWithClient(OpenSearchClient client) throws IOException; + } + + /** + * Execute a callback with the {@link OpenSearchClient} and provide exception translation. + * + * @param callback the callback to execute, must not be {@literal null} + * @param the type returned from the callback + * @return the callback result + */ + public T execute(OpenSearchTemplate.ClientCallback callback) { + + Assert.notNull(callback, "callback must not be null"); + + try { + return callback.doWithClient(client); + } catch (IOException | RuntimeException e) { + throw exceptionTranslator.translateException(e); + } + } + // endregion + + // region helper methods + @Override + public Query matchAllQuery() { + return NativeQuery.builder().withQuery(qb -> qb.matchAll(mab -> mab)).build(); + } + + @Override + public Query idsQuery(List ids) { + return NativeQuery.builder().withQuery(qb -> qb.ids(iq -> iq.values(ids))).build(); + } + + @Override + public BaseQueryBuilder queryBuilderWithIds(List ids) { + return NativeQuery.builder().withIds(ids); + } + + /** + * extract the list of {@link IndexedObjectInformation} from a {@link BulkResponse}. + * + * @param bulkResponse the response to evaluate + * @return the list of the {@link IndexedObjectInformation}s + */ + protected List checkForBulkOperationFailure(BulkResponse bulkResponse) { + + if (bulkResponse.errors()) { + Map failedDocuments = new HashMap<>(); + for (BulkResponseItem item : bulkResponse.items()) { + + if (item.error() != null) { + failedDocuments.put(item.id(), new BulkFailureException.FailureDetails(item.status(), item.error().reason())); + } + } + throw new BulkFailureException( + "Bulk operation has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages [" + + failedDocuments + ']', + failedDocuments); + } + + return bulkResponse.items().stream().map( + item -> new IndexedObjectInformation(item.id(), item.index(), item.seqNo(), item.primaryTerm(), item.version())) + .collect(Collectors.toList()); + + } + // endregion + +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Queries.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Queries.java new file mode 100644 index 0000000..a720d1c --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/Queries.java @@ -0,0 +1,194 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.function.Function; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.LatLonGeoLocation; +import org.opensearch.client.opensearch._types.aggregations.Aggregation; +import org.opensearch.client.opensearch._types.query_dsl.IdsQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchAllQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; +import org.opensearch.client.opensearch._types.query_dsl.Operator; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.QueryStringQuery; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; +import org.opensearch.client.opensearch._types.query_dsl.WildcardQuery; +import org.opensearch.client.opensearch._types.query_dsl.WrapperQuery; +import org.opensearch.client.util.ObjectBuilder; +import org.springframework.data.elasticsearch.core.geo.GeoPoint; +import org.springframework.data.elasticsearch.core.query.BaseQueryBuilder; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Utility class simplifying the creation of some more complex queries and type. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +public final class Queries { + + private Queries() {} + + public static IdsQuery idsQuery(List ids) { + + Assert.notNull(ids, "ids must not be null"); + + return IdsQuery.of(i -> i.values(ids)); + } + + public static Query idsQueryAsQuery(List ids) { + + Assert.notNull(ids, "ids must not be null"); + + Function> builder = b -> b.ids(idsQuery(ids)); + + return builder.apply(new Query.Builder()).build(); + } + + public static MatchQuery matchQuery(String fieldName, String query, @Nullable Operator operator, + @Nullable Float boost) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(query, "query must not be null"); + + return MatchQuery.of(mb -> mb.field(fieldName).query(FieldValue.of(query)).operator(operator).boost(boost)); + } + + public static Query matchQueryAsQuery(String fieldName, String query, @Nullable Operator operator, + @Nullable Float boost) { + + Function> builder = b -> b.match(matchQuery(fieldName, query, operator, boost)); + + return builder.apply(new Query.Builder()).build(); + } + + public static MatchAllQuery matchAllQuery() { + + return MatchAllQuery.of(b -> b); + } + + public static Query matchAllQueryAsQuery() { + + Function> builder = b -> b.matchAll(matchAllQuery()); + + return builder.apply(new Query.Builder()).build(); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Float boost) { + return queryStringQuery(fieldName, query, null, null, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, Operator defaultOperator, + @Nullable Float boost) { + return queryStringQuery(fieldName, query, null, defaultOperator, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Boolean analyzeWildcard, + @Nullable Float boost) { + return queryStringQuery(fieldName, query, analyzeWildcard, null, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Boolean analyzeWildcard, + @Nullable Operator defaultOperator, @Nullable Float boost) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(query, "query must not be null"); + + return QueryStringQuery.of(qs -> qs.fields(fieldName).query(query).analyzeWildcard(analyzeWildcard) + .defaultOperator(defaultOperator).boost(boost)); + } + + public static TermQuery termQuery(String fieldName, String value) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(value, "value must not be null"); + + return TermQuery.of(t -> t.field(fieldName).value(FieldValue.of(value))); + } + + public static Query termQueryAsQuery(String fieldName, String value) { + + Function> builder = q -> q.term(termQuery(fieldName, value)); + return builder.apply(new Query.Builder()).build(); + } + + public static WildcardQuery wildcardQuery(String field, String value) { + + Assert.notNull(field, "field must not be null"); + Assert.notNull(value, "value must not be null"); + + return WildcardQuery.of(w -> w.field(field).wildcard(value)); + } + + public static Query wildcardQueryAsQuery(String field, String value) { + Function> builder = q -> q.wildcard(wildcardQuery(field, value)); + return builder.apply(new Query.Builder()).build(); + } + + public static Query wrapperQueryAsQuery(String query) { + + Function> builder = q -> q.wrapper(wrapperQuery(query)); + + return builder.apply(new Query.Builder()).build(); + } + + public static WrapperQuery wrapperQuery(String query) { + + Assert.notNull(query, "query must not be null"); + + String encodedValue = Base64.getEncoder().encodeToString(query.getBytes(StandardCharsets.UTF_8)); + + return WrapperQuery.of(wq -> wq.query(encodedValue)); + } + + public static LatLonGeoLocation latLon(GeoPoint geoPoint) { + + Assert.notNull(geoPoint, "geoPoint must not be null"); + + return latLon(geoPoint.getLat(), geoPoint.getLon()); + } + + public static LatLonGeoLocation latLon(double lat, double lon) { + return LatLonGeoLocation.of(_0 -> _0.lat(lat).lon(lon)); + } + + public static org.springframework.data.elasticsearch.core.query.Query getTermsAggsQuery(String aggsName, + String aggsField) { + return NativeQuery.builder() // + .withQuery(Queries.matchAllQueryAsQuery()) // + .withAggregation(aggsName, Aggregation.of(a -> a // + .terms(ta -> ta.field(aggsField)))) // + .withMaxResults(0) // + .build(); + } + + public static org.springframework.data.elasticsearch.core.query.Query queryWithIds(String... ids) { + return NativeQuery.builder().withIds(ids).build(); + } + + public static BaseQueryBuilder getBuilderWithMatchAllQuery() { + return NativeQuery.builder().withQuery(matchAllQueryAsQuery()); + } + + public static BaseQueryBuilder getBuilderWithTermQuery(String field, String value) { + return NativeQuery.builder().withQuery(termQueryAsQuery(field, value)); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/QueryBuilders.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/QueryBuilders.java new file mode 100644 index 0000000..d5ca174 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/QueryBuilders.java @@ -0,0 +1,172 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.function.Function; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.LatLonGeoLocation; +import org.opensearch.client.opensearch._types.query_dsl.IdsQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchAllQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; +import org.opensearch.client.opensearch._types.query_dsl.Operator; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.QueryStringQuery; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; +import org.opensearch.client.opensearch._types.query_dsl.WildcardQuery; +import org.opensearch.client.opensearch._types.query_dsl.WrapperQuery; +import org.opensearch.client.util.ObjectBuilder; +import org.springframework.data.elasticsearch.core.geo.GeoPoint; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Utility class simplifying the creation of some more complex queries and type. + * + * @author Peter-Josef Meisch + * @since 4.4 + * @deprecated since 5.1, use {@link Queries} instead. + */ +@Deprecated(forRemoval = true) +public final class QueryBuilders { + + private QueryBuilders() {} + + public static IdsQuery idsQuery(List ids) { + + Assert.notNull(ids, "ids must not be null"); + + return IdsQuery.of(i -> i.values(ids)); + } + + public static Query idsQueryAsQuery(List ids) { + + Assert.notNull(ids, "ids must not be null"); + + Function> builder = b -> b.ids(idsQuery(ids)); + + return builder.apply(new Query.Builder()).build(); + } + + public static MatchQuery matchQuery(String fieldName, String query, @Nullable Operator operator, + @Nullable Float boost) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(query, "query must not be null"); + + return MatchQuery.of(mb -> mb.field(fieldName).query(FieldValue.of(query)).operator(operator).boost(boost)); + } + + public static Query matchQueryAsQuery(String fieldName, String query, @Nullable Operator operator, + @Nullable Float boost) { + + Function> builder = b -> b.match(matchQuery(fieldName, query, operator, boost)); + + return builder.apply(new Query.Builder()).build(); + } + + public static MatchAllQuery matchAllQuery() { + + return MatchAllQuery.of(b -> b); + } + + public static Query matchAllQueryAsQuery() { + + Function> builder = b -> b.matchAll(matchAllQuery()); + + return builder.apply(new Query.Builder()).build(); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Float boost) { + return queryStringQuery(fieldName, query, null, null, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, Operator defaultOperator, + @Nullable Float boost) { + return queryStringQuery(fieldName, query, null, defaultOperator, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Boolean analyzeWildcard, + @Nullable Float boost) { + return queryStringQuery(fieldName, query, analyzeWildcard, null, boost); + } + + public static QueryStringQuery queryStringQuery(String fieldName, String query, @Nullable Boolean analyzeWildcard, + @Nullable Operator defaultOperator, @Nullable Float boost) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(query, "query must not be null"); + + return QueryStringQuery.of(qs -> qs.fields(fieldName).query(query).analyzeWildcard(analyzeWildcard) + .defaultOperator(defaultOperator).boost(boost)); + } + + public static TermQuery termQuery(String fieldName, String value) { + + Assert.notNull(fieldName, "fieldName must not be null"); + Assert.notNull(value, "value must not be null"); + + return TermQuery.of(t -> t.field(fieldName).value(FieldValue.of(value))); + } + + public static Query termQueryAsQuery(String fieldName, String value) { + + Function> builder = q -> q.term(termQuery(fieldName, value)); + return builder.apply(new Query.Builder()).build(); + } + + public static WildcardQuery wildcardQuery(String field, String value) { + + Assert.notNull(field, "field must not be null"); + Assert.notNull(value, "value must not be null"); + + return WildcardQuery.of(w -> w.field(field).wildcard(value)); + } + + public static Query wildcardQueryAsQuery(String field, String value) { + Function> builder = q -> q.wildcard(wildcardQuery(field, value)); + return builder.apply(new Query.Builder()).build(); + } + + public static Query wrapperQueryAsQuery(String query) { + + Function> builder = q -> q.wrapper(wrapperQuery(query)); + + return builder.apply(new Query.Builder()).build(); + } + + public static WrapperQuery wrapperQuery(String query) { + + Assert.notNull(query, "query must not be null"); + + String encodedValue = Base64.getEncoder().encodeToString(query.getBytes(StandardCharsets.UTF_8)); + + return WrapperQuery.of(wq -> wq.query(encodedValue)); + } + + public static LatLonGeoLocation latLon(GeoPoint geoPoint) { + + Assert.notNull(geoPoint, "geoPoint must not be null"); + + return latLon(geoPoint.getLat(), geoPoint.getLon()); + } + + public static LatLonGeoLocation latLon(double lat, double lon) { + return LatLonGeoLocation.of(_0 -> _0.lat(lat).lon(lon)); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/RequestConverter.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/RequestConverter.java new file mode 100644 index 0000000..1a7dd8c --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/RequestConverter.java @@ -0,0 +1,1965 @@ +/* + * Copyright 2021-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.TypeUtils.*; +import static org.springframework.util.CollectionUtils.*; + +import jakarta.json.stream.JsonParser; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.json.JsonpDeserializer; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.ObjectDeserializer; +import org.opensearch.client.opensearch._types.Conflicts; +import org.opensearch.client.opensearch._types.ExpandWildcard; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.InlineScript; +import org.opensearch.client.opensearch._types.NestedSortValue; +import org.opensearch.client.opensearch._types.OpType; +import org.opensearch.client.opensearch._types.SortOptions; +import org.opensearch.client.opensearch._types.SortOrder; +import org.opensearch.client.opensearch._types.VersionType; +import org.opensearch.client.opensearch._types.WaitForActiveShardOptions; +import org.opensearch.client.opensearch._types.mapping.DynamicMapping; +import org.opensearch.client.opensearch._types.mapping.DynamicTemplate; +import org.opensearch.client.opensearch._types.mapping.FieldNamesField; +import org.opensearch.client.opensearch._types.mapping.FieldType; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.RoutingField; +import org.opensearch.client.opensearch._types.mapping.SourceField; +import org.opensearch.client.opensearch._types.query_dsl.FieldAndFormat; +import org.opensearch.client.opensearch._types.query_dsl.Like; +import org.opensearch.client.opensearch.cluster.DeleteComponentTemplateRequest; +import org.opensearch.client.opensearch.cluster.ExistsComponentTemplateRequest; +import org.opensearch.client.opensearch.cluster.GetComponentTemplateRequest; +import org.opensearch.client.opensearch.cluster.HealthRequest; +import org.opensearch.client.opensearch.cluster.PutComponentTemplateRequest; +import org.opensearch.client.opensearch.core.*; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.CreateOperation; +import org.opensearch.client.opensearch.core.bulk.IndexOperation; +import org.opensearch.client.opensearch.core.bulk.UpdateOperation; +import org.opensearch.client.opensearch.core.mget.MultiGetOperation; +import org.opensearch.client.opensearch.core.msearch.MultisearchBody; +import org.opensearch.client.opensearch.core.msearch.MultisearchHeader; +import org.opensearch.client.opensearch.core.pit.CreatePitRequest; +import org.opensearch.client.opensearch.core.pit.DeletePitRequest; +import org.opensearch.client.opensearch.core.search.Highlight; +import org.opensearch.client.opensearch.core.search.Pit; +import org.opensearch.client.opensearch.core.search.Rescore; +import org.opensearch.client.opensearch.core.search.SourceConfig; +import org.opensearch.client.opensearch.indices.*; +import org.opensearch.client.opensearch.indices.ExistsIndexTemplateRequest; +import org.opensearch.client.opensearch.indices.ExistsRequest; +import org.opensearch.client.opensearch.indices.PutMappingRequest.Builder; +import org.opensearch.client.opensearch.indices.update_aliases.Action; +import org.opensearch.client.util.ObjectBuilder; +import org.springframework.dao.InvalidDataAccessApiUsageException; +import org.springframework.data.domain.Sort; +import org.springframework.data.elasticsearch.core.RefreshPolicy; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchConverter; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.index.*; +import org.springframework.data.elasticsearch.core.index.DeleteIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.DeleteTemplateRequest; +import org.springframework.data.elasticsearch.core.index.ExistsTemplateRequest; +import org.springframework.data.elasticsearch.core.index.GetIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.GetTemplateRequest; +import org.springframework.data.elasticsearch.core.index.PutIndexTemplateRequest; +import org.springframework.data.elasticsearch.core.index.PutTemplateRequest; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentEntity; +import org.springframework.data.elasticsearch.core.mapping.ElasticsearchPersistentProperty; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.*; +import org.springframework.data.elasticsearch.core.reindex.ReindexRequest; +import org.springframework.data.elasticsearch.core.reindex.Remote; +import org.springframework.data.elasticsearch.core.script.Script; +import org.springframework.data.elasticsearch.support.DefaultStringObjectMap; +import org.springframework.lang.NonNull; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; + +/** + * Class to create OpenSearch request and request builders. + * + * @author Peter-Josef Meisch + * @author Sascha Woo + * @author cdalxndr + * @author scoobyzhang + * @author Haibo Liu + * @since 4.4 + */ +@SuppressWarnings("ClassCanBeRecord") +class RequestConverter { + + private static final Log LOGGER = LogFactory.getLog(RequestConverter.class); + + // the default max result window size of Elasticsearch + public static final Integer INDEX_MAX_RESULT_WINDOW = 10_000; + + protected final JsonpMapper jsonpMapper; + protected final ElasticsearchConverter elasticsearchConverter; + + public RequestConverter(ElasticsearchConverter elasticsearchConverter, JsonpMapper jsonpMapper) { + this.elasticsearchConverter = elasticsearchConverter; + + Assert.notNull(jsonpMapper, "jsonpMapper must not be null"); + + this.jsonpMapper = jsonpMapper; + } + + // region Cluster client + public org.opensearch.client.opensearch.cluster.HealthRequest clusterHealthRequest() { + return new HealthRequest.Builder().build(); + } + + public org.opensearch.client.opensearch.cluster.PutComponentTemplateRequest clusterPutComponentTemplateRequest( + org.springframework.data.elasticsearch.core.index.PutComponentTemplateRequest putComponentTemplateRequest) { + + Assert.notNull(putComponentTemplateRequest, "putComponentTemplateRequest must not be null"); + + return PutComponentTemplateRequest.of(b -> b // + .name(putComponentTemplateRequest.name()) // + .create(putComponentTemplateRequest.create()) // + .version(putComponentTemplateRequest.version()) // + .masterTimeout(time(putComponentTemplateRequest.masterTimeout())) // + .template(isb -> { + var componentTemplateData = putComponentTemplateRequest.template(); + isb // + .mappings(typeMapping(componentTemplateData.mapping())) // + .settings(indexSettings(componentTemplateData.settings())); + // same code schema, but different Elasticsearch builder types + // noinspection DuplicatedCode + var aliasActions = componentTemplateData.aliasActions(); + if (aliasActions != null) { + aliasActions.getActions().forEach(aliasAction -> { + if (aliasAction instanceof AliasAction.Add add) { + var parameters = add.getParameters(); + // noinspection DuplicatedCode + String[] parametersAliases = parameters.getAliases(); + if (parametersAliases != null) { + for (String aliasName : parametersAliases) { + isb.aliases(aliasName, aliasBuilder -> buildAlias(parameters, aliasBuilder)); + } + } + } + }); + } + return isb; + })); + } + + private Alias.Builder buildAlias(AliasActionParameters parameters, Alias.Builder aliasBuilder) { + + // noinspection DuplicatedCode + if (parameters.getRouting() != null) { + aliasBuilder.routing(parameters.getRouting()); + } + + if (parameters.getIndexRouting() != null) { + aliasBuilder.indexRouting(parameters.getIndexRouting()); + } + + if (parameters.getSearchRouting() != null) { + aliasBuilder.searchRouting(parameters.getSearchRouting()); + } + + if (parameters.getHidden() != null) { + aliasBuilder.isHidden(parameters.getHidden()); + } + + if (parameters.getWriteIndex() != null) { + aliasBuilder.isWriteIndex(parameters.getWriteIndex()); + } + + Query filterQuery = parameters.getFilterQuery(); + + if (filterQuery != null) { + org.opensearch.client.opensearch._types.query_dsl.Query esQuery = getQuery(filterQuery, null); + + if (esQuery != null) { + aliasBuilder.filter(esQuery); + } + } + return aliasBuilder; + } + + public ExistsComponentTemplateRequest clusterExistsComponentTemplateRequest( + org.springframework.data.elasticsearch.core.index.ExistsComponentTemplateRequest existsComponentTemplateRequest) { + + Assert.notNull(existsComponentTemplateRequest, "existsComponentTemplateRequest must not be null"); + + return ExistsComponentTemplateRequest.of(b -> b.name(existsComponentTemplateRequest.templateName())); + } + + public GetComponentTemplateRequest clusterGetComponentTemplateRequest( + org.springframework.data.elasticsearch.core.index.GetComponentTemplateRequest getComponentTemplateRequest) { + + Assert.notNull(getComponentTemplateRequest, "getComponentTemplateRequest must not be null"); + + return GetComponentTemplateRequest.of(b -> b.name(getComponentTemplateRequest.templateName())); + } + + public DeleteComponentTemplateRequest clusterDeleteComponentTemplateRequest( + org.springframework.data.elasticsearch.core.index.DeleteComponentTemplateRequest deleteComponentTemplateRequest) { + return DeleteComponentTemplateRequest.of(b -> b.name(deleteComponentTemplateRequest.templateName())); + } + // endregion + + // region Indices client + public ExistsRequest indicesExistsRequest(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new ExistsRequest.Builder().index(Arrays.asList(indexCoordinates.getIndexNames())).build(); + } + + public CreateIndexRequest indicesCreateRequest(IndexCoordinates indexCoordinates, Map settings, + @Nullable Document mapping) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + Assert.notNull(settings, "settings must not be null"); + + // note: the new client does not support the index.storeType anymore + return new CreateIndexRequest.Builder() // + .index(indexCoordinates.getIndexName()) // + .settings(indexSettings(settings)) // + .mappings(typeMapping(mapping)) // + .build(); + } + + public RefreshRequest indicesRefreshRequest(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new RefreshRequest.Builder().index(Arrays.asList(indexCoordinates.getIndexNames())).build(); + } + + public DeleteIndexRequest indicesDeleteRequest(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new DeleteIndexRequest.Builder().index(Arrays.asList(indexCoordinates.getIndexNames())).build(); + } + + public UpdateAliasesRequest indicesUpdateAliasesRequest(AliasActions aliasActions) { + + Assert.notNull(aliasActions, "aliasActions must not be null"); + + UpdateAliasesRequest.Builder updateAliasRequestBuilder = new UpdateAliasesRequest.Builder(); + + List actions = new ArrayList<>(); + aliasActions.getActions().forEach(aliasAction -> { + + var actionBuilder = getBuilder(aliasAction); + + if (aliasAction instanceof AliasAction.Remove remove) { + AliasActionParameters parameters = remove.getParameters(); + actionBuilder.remove(removeActionBuilder -> { + removeActionBuilder.indices(Arrays.asList(parameters.getIndices())); + + if (parameters.getAliases() != null) { + removeActionBuilder.aliases(Arrays.asList(parameters.getAliases())); + } + + return removeActionBuilder; + }); + } + + if (aliasAction instanceof AliasAction.RemoveIndex removeIndex) { + AliasActionParameters parameters = removeIndex.getParameters(); + actionBuilder.removeIndex( + removeIndexActionBuilder -> removeIndexActionBuilder.indices(Arrays.asList(parameters.getIndices()))); + } + + actions.add(actionBuilder.build()); + }); + + updateAliasRequestBuilder.actions(actions); + + return updateAliasRequestBuilder.build(); + } + + @NonNull + private Action.Builder getBuilder(AliasAction aliasAction) { + Action.Builder actionBuilder = new Action.Builder(); + + if (aliasAction instanceof AliasAction.Add add) { + AliasActionParameters parameters = add.getParameters(); + actionBuilder.add(addActionBuilder -> { + addActionBuilder // + .indices(Arrays.asList(parameters.getIndices())) // + .isHidden(parameters.getHidden()) // + .isWriteIndex(parameters.getWriteIndex()) // + .routing(parameters.getRouting()) // + .indexRouting(parameters.getIndexRouting()) // + .searchRouting(parameters.getSearchRouting()); // + + if (parameters.getAliases() != null) { + addActionBuilder.aliases(Arrays.asList(parameters.getAliases())); + } + + Query filterQuery = parameters.getFilterQuery(); + + if (filterQuery != null) { + elasticsearchConverter.updateQuery(filterQuery, parameters.getFilterQueryClass()); + org.opensearch.client.opensearch._types.query_dsl.Query esQuery = getQuery(filterQuery, null); + if (esQuery != null) { + addActionBuilder.filter(esQuery); + } + } + return addActionBuilder; + }); + } + return actionBuilder; + } + + public PutMappingRequest indicesPutMappingRequest(IndexCoordinates indexCoordinates, Document mapping) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + Assert.notNull(mapping, "mapping must not be null"); + + final ObjectDeserializer deserializer = new ObjectDeserializer<>(PutMappingRequest.Builder::new); + deserializer.add(Builder::fieldNames, FieldNamesField._DESERIALIZER, "_field_names"); + deserializer.add(Builder::meta, JsonpDeserializer.stringMapDeserializer(JsonData._DESERIALIZER), "_meta"); + deserializer.add(Builder::routing, RoutingField._DESERIALIZER, "_routing"); + deserializer.add(Builder::source, SourceField._DESERIALIZER, "_source"); + deserializer.add(Builder::dateDetection, JsonpDeserializer.booleanDeserializer(), "date_detection"); + deserializer.add(Builder::dynamic, DynamicMapping._DESERIALIZER, "dynamic"); + deserializer.add( + Builder::dynamicDateFormats, + JsonpDeserializer.arrayDeserializer(JsonpDeserializer.stringDeserializer()), + "dynamic_date_formats" + ); + deserializer.add( + Builder::dynamicTemplates, + JsonpDeserializer.arrayDeserializer(JsonpDeserializer.stringMapDeserializer(DynamicTemplate._DESERIALIZER)), + "dynamic_templates" + ); + deserializer.add(Builder::numericDetection, JsonpDeserializer.booleanDeserializer(), "numeric_detection"); + deserializer.add(Builder::properties, JsonpDeserializer.stringMapDeserializer(Property._DESERIALIZER), "properties"); + + final PutMappingRequest.Builder builder = JsonpUtils.fromJson(mapping, deserializer); + builder.index(Arrays.asList(indexCoordinates.getIndexNames())); + + return builder.build(); + } + + public GetMappingRequest indicesGetMappingRequest(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new GetMappingRequest.Builder().index(List.of(indexCoordinates.getIndexNames())).build(); + } + + public GetIndicesSettingsRequest indicesGetSettingsRequest(IndexCoordinates indexCoordinates, + boolean includeDefaults) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new GetIndicesSettingsRequest.Builder() // + .index(Arrays.asList(indexCoordinates.getIndexNames())) // + .includeDefaults(includeDefaults) // + .build(); + } + + public GetIndexRequest indicesGetIndexRequest(IndexCoordinates indexCoordinates) { + + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return new GetIndexRequest.Builder() // + .index(Arrays.asList(indexCoordinates.getIndexNames())) // + .includeDefaults(true) // + .build(); // + } + + public GetAliasRequest indicesGetAliasRequest(@Nullable String[] aliasNames, @Nullable String[] indexNames) { + GetAliasRequest.Builder builder = new GetAliasRequest.Builder(); + + if (aliasNames != null) { + builder.name(Arrays.asList(aliasNames)); + } + + if (indexNames != null) { + builder.index(Arrays.asList(indexNames)); + } + + return builder.build(); + } + + public org.opensearch.client.opensearch.indices.PutTemplateRequest indicesPutTemplateRequest( + PutTemplateRequest putTemplateRequest) { + + Assert.notNull(putTemplateRequest, "putTemplateRequest must not be null"); + + org.opensearch.client.opensearch.indices.PutTemplateRequest.Builder builder = new org.opensearch.client.opensearch.indices.PutTemplateRequest.Builder(); + + builder.name(putTemplateRequest.getName()).indexPatterns(Arrays.asList(putTemplateRequest.getIndexPatterns())) + .order(putTemplateRequest.getOrder()); + + if (putTemplateRequest.getSettings() != null) { + Map settings = getTemplateParams(putTemplateRequest.getSettings().entrySet()); + builder.settings(settings); + } + + if (putTemplateRequest.getMappings() != null) { + builder.mappings(typeMapping(putTemplateRequest.getMappings())); + } + + if (putTemplateRequest.getVersion() != null) { + builder.version(Long.valueOf(putTemplateRequest.getVersion())); + } + AliasActions aliasActions = putTemplateRequest.getAliasActions(); + + if (aliasActions != null) { + aliasActions.getActions().forEach(aliasAction -> { + AliasActionParameters parameters = aliasAction.getParameters(); + // noinspection DuplicatedCode + String[] parametersAliases = parameters.getAliases(); + + if (parametersAliases != null) { + for (String aliasName : parametersAliases) { + builder.aliases(aliasName, aliasBuilder -> buildAlias(parameters, aliasBuilder)); + } + } + }); + } + + return builder.build(); + } + + public org.opensearch.client.opensearch.indices.PutIndexTemplateRequest indicesPutIndexTemplateRequest( + PutIndexTemplateRequest putIndexTemplateRequest) { + + Assert.notNull(putIndexTemplateRequest, "putIndexTemplateRequest must not be null"); + + var builder = new org.opensearch.client.opensearch.indices.PutIndexTemplateRequest.Builder() + .name(putIndexTemplateRequest.name()) // + .indexPatterns(Arrays.asList(putIndexTemplateRequest.indexPatterns())) // + .template(t -> { + t // + .settings(indexSettings(putIndexTemplateRequest.settings())) // + .mappings(typeMapping(putIndexTemplateRequest.mapping())); + + // same code schema, but different Elasticsearch builder types + // noinspection DuplicatedCode + var aliasActions = putIndexTemplateRequest.aliasActions(); + if (aliasActions != null) { + aliasActions.getActions().forEach(aliasAction -> { + if (aliasAction instanceof AliasAction.Add add) { + var parameters = add.getParameters(); + // noinspection DuplicatedCode + String[] parametersAliases = parameters.getAliases(); + if (parametersAliases != null) { + for (String aliasName : parametersAliases) { + t.aliases(aliasName, aliasBuilder -> buildAlias(parameters, aliasBuilder)); + } + } + } + }); + } + return t; + }); + + if (!putIndexTemplateRequest.composedOf().isEmpty()) { + builder.composedOf(putIndexTemplateRequest.composedOf()); + } + + return builder.build(); + } + + public ExistsIndexTemplateRequest indicesExistsIndexTemplateRequest( + org.springframework.data.elasticsearch.core.index.ExistsIndexTemplateRequest existsIndexTemplateRequest) { + + Assert.notNull(existsIndexTemplateRequest, "existsIndexTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.ExistsIndexTemplateRequest + .of(b -> b.name(existsIndexTemplateRequest.templateName())); + } + + public org.opensearch.client.opensearch.indices.ExistsTemplateRequest indicesExistsTemplateRequest( + ExistsTemplateRequest existsTemplateRequest) { + + Assert.notNull(existsTemplateRequest, "existsTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.ExistsTemplateRequest + .of(etr -> etr.name(existsTemplateRequest.getTemplateName())); + } + + public org.opensearch.client.opensearch.indices.GetIndexTemplateRequest indicesGetIndexTemplateRequest( + GetIndexTemplateRequest getIndexTemplateRequest) { + + Assert.notNull(getIndexTemplateRequest, "getIndexTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.GetIndexTemplateRequest + .of(gitr -> gitr.name(getIndexTemplateRequest.templateName())); + } + + public org.opensearch.client.opensearch.indices.DeleteIndexTemplateRequest indicesDeleteIndexTemplateRequest( + DeleteIndexTemplateRequest deleteIndexTemplateRequest) { + + Assert.notNull(deleteIndexTemplateRequest, "deleteIndexTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.DeleteIndexTemplateRequest + .of(ditr -> ditr.name(deleteIndexTemplateRequest.templateName())); + } + + public org.opensearch.client.opensearch.indices.DeleteTemplateRequest indicesDeleteTemplateRequest( + DeleteTemplateRequest existsTemplateRequest) { + + Assert.notNull(existsTemplateRequest, "existsTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.DeleteTemplateRequest + .of(dtr -> dtr.name(existsTemplateRequest.getTemplateName())); + } + + public org.opensearch.client.opensearch.indices.GetTemplateRequest indicesGetTemplateRequest( + GetTemplateRequest getTemplateRequest) { + + Assert.notNull(getTemplateRequest, "getTemplateRequest must not be null"); + + return org.opensearch.client.opensearch.indices.GetTemplateRequest + .of(gtr -> gtr.name(getTemplateRequest.getTemplateName()).flatSettings(true)); + } + + // endregion + + // region documents + /* + * the methods documentIndexRequest, bulkIndexOperation and bulkCreateOperation have nearly + * identical code, but the client builders do not have a common accessible base or some reusable parts + * so the code needs to be duplicated. + */ + + public IndexRequest documentIndexRequest(IndexQuery query, IndexCoordinates indexCoordinates, + @Nullable RefreshPolicy refreshPolicy) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + IndexRequest.Builder builder = new IndexRequest.Builder<>(); + + builder.index(query.getIndexName() != null ? query.getIndexName() : indexCoordinates.getIndexName()); + + Object queryObject = query.getObject(); + + if (queryObject != null) { + String id = StringUtils.hasText(query.getId()) ? query.getId() : getPersistentEntityId(queryObject); + builder // + .id(id) // + .document(elasticsearchConverter.mapObject(queryObject)); + } else if (query.getSource() != null) { + builder // + .id(query.getId()) // + .document(new DefaultStringObjectMap<>().fromJson(query.getSource())); + } else { + throw new InvalidDataAccessApiUsageException( + "object or source is null, failed to index the document [id: " + query.getId() + ']'); + } + + if (query.getVersion() != null) { + VersionType versionType = retrieveVersionTypeFromPersistentEntity( + queryObject != null ? queryObject.getClass() : null); + builder.version(query.getVersion()).versionType(versionType); + } + + builder // + .ifSeqNo(query.getSeqNo()) // + .ifPrimaryTerm(query.getPrimaryTerm()) // + .routing(query.getRouting()); // + + if (query.getOpType() != null) { + switch (query.getOpType()) { + case INDEX -> builder.opType(OpType.Index); + case CREATE -> builder.opType(OpType.Create); + } + } + + builder.refresh(refresh(refreshPolicy)); + + return builder.build(); + } + /* + * the methods documentIndexRequest, bulkIndexOperation and bulkCreateOperation have nearly + * identical code, but the client builders do not have a common accessible base or some reusable parts + * so the code needs to be duplicated. + */ + + @SuppressWarnings("DuplicatedCode") + private IndexOperation bulkIndexOperation(IndexQuery query, IndexCoordinates indexCoordinates, + @Nullable RefreshPolicy refreshPolicy) { + + IndexOperation.Builder builder = new IndexOperation.Builder<>(); + + builder.index(query.getIndexName() != null ? query.getIndexName() : indexCoordinates.getIndexName()); + + Object queryObject = query.getObject(); + + if (queryObject != null) { + String id = StringUtils.hasText(query.getId()) ? query.getId() : getPersistentEntityId(queryObject); + builder // + .id(id) // + .document(elasticsearchConverter.mapObject(queryObject)); + } else if (query.getSource() != null) { + builder.document(new DefaultStringObjectMap<>().fromJson(query.getSource())); + } else { + throw new InvalidDataAccessApiUsageException( + "object or source is null, failed to index the document [id: " + query.getId() + ']'); + } + + if (query.getVersion() != null) { + VersionType versionType = retrieveVersionTypeFromPersistentEntity( + queryObject != null ? queryObject.getClass() : null); + builder.version(query.getVersion()).versionType(versionType); + } + + builder // + .ifSeqNo(query.getSeqNo()) // + .ifPrimaryTerm(query.getPrimaryTerm()) // + .routing(query.getRouting()); // + + return builder.build(); + } + /* + * the methods documentIndexRequest, bulkIndexOperation and bulkCreateOperation have nearly + * identical code, but the client builders do not have a common accessible base or some reusable parts + * so the code needs to be duplicated. + */ + + @SuppressWarnings("DuplicatedCode") + private CreateOperation bulkCreateOperation(IndexQuery query, IndexCoordinates indexCoordinates, + @Nullable RefreshPolicy refreshPolicy) { + + CreateOperation.Builder builder = new CreateOperation.Builder<>(); + + builder.index(query.getIndexName() != null ? query.getIndexName() : indexCoordinates.getIndexName()); + + Object queryObject = query.getObject(); + + if (queryObject != null) { + String id = StringUtils.hasText(query.getId()) ? query.getId() : getPersistentEntityId(queryObject); + builder // + .id(id) // + .document(elasticsearchConverter.mapObject(queryObject)); + } else if (query.getSource() != null) { + builder.document(new DefaultStringObjectMap<>().fromJson(query.getSource())); + } else { + throw new InvalidDataAccessApiUsageException( + "object or source is null, failed to index the document [id: " + query.getId() + ']'); + } + + if (query.getVersion() != null) { + VersionType versionType = retrieveVersionTypeFromPersistentEntity( + queryObject != null ? queryObject.getClass() : null); + builder.version(query.getVersion()).versionType(versionType); + } + + builder // + .ifSeqNo(query.getSeqNo()) // + .ifPrimaryTerm(query.getPrimaryTerm()) // + .routing(query.getRouting()); // + + return builder.build(); + } + + private UpdateOperation bulkUpdateOperation(UpdateQuery query, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy) { + + UpdateOperation.Builder uob = new UpdateOperation.Builder<>(); + String indexName = query.getIndexName() != null ? query.getIndexName() : index.getIndexName(); + + uob.index(indexName).id(query.getId()); + uob.script(getScript(query.getScriptData())) // + .document(query.getDocument()) // + .upsert(query.getUpsert()) // + .scriptedUpsert(query.getScriptedUpsert()) // + .docAsUpsert(query.getDocAsUpsert()) // + ; + + if (query.getFetchSource() != null) { + uob.source(sc -> sc.fetch(query.getFetchSource())); + } + + if (query.getFetchSourceIncludes() != null || query.getFetchSourceExcludes() != null) { + List includes = query.getFetchSourceIncludes() != null ? query.getFetchSourceIncludes() + : Collections.emptyList(); + List excludes = query.getFetchSourceExcludes() != null ? query.getFetchSourceExcludes() + : Collections.emptyList(); + uob.source(sc -> sc.filter(sf -> sf.includes(includes).excludes(excludes))); + } + + uob // + .routing(query.getRouting()) // + .ifSeqNo(query.getIfSeqNo() != null ? Long.valueOf(query.getIfSeqNo()) : null) // + .ifPrimaryTerm(query.getIfPrimaryTerm() != null ? Long.valueOf(query.getIfPrimaryTerm()) : null) // + .retryOnConflict(query.getRetryOnConflict()) // + ; + + // no refresh, timeout, waitForActiveShards on UpdateOperation or UpdateAction + + return uob.build(); + } + + @Nullable + private org.opensearch.client.opensearch._types.Script getScript(@Nullable ScriptData scriptData) { + + if (scriptData == null) { + return null; + } + + Map params = new HashMap<>(); + + if (scriptData.params() != null) { + scriptData.params().forEach((key, value) -> params.put(key, JsonData.of(value, jsonpMapper))); + } + return org.opensearch.client.opensearch._types.Script.of(sb -> { + if (scriptData.type() == ScriptType.INLINE) { + sb.inline(is -> is // + .lang(scriptData.language()) // + .source(scriptData.script()) // + .params(params)); // + } else if (scriptData.type() == ScriptType.STORED) { + sb.stored(ss -> ss // + .id(scriptData.script()) // + .params(params) // + ); + } + return sb; + }); + } + + public BulkRequest documentBulkRequest(List queries, BulkOptions bulkOptions, IndexCoordinates indexCoordinates, + @Nullable RefreshPolicy refreshPolicy) { + + BulkRequest.Builder builder = new BulkRequest.Builder(); + + if (bulkOptions.getTimeout() != null) { + builder.timeout(tb -> tb.time(Long.valueOf(bulkOptions.getTimeout().toMillis()).toString() + "ms")); + } + + builder.refresh(refresh(refreshPolicy)); + if (bulkOptions.getRefreshPolicy() != null) { + builder.refresh(refresh(bulkOptions.getRefreshPolicy())); + } + + if (bulkOptions.getWaitForActiveShards() != null) { + builder.waitForActiveShards(wasb -> wasb.count(bulkOptions.getWaitForActiveShards().value())); + } + + if (bulkOptions.getPipeline() != null) { + builder.pipeline(bulkOptions.getPipeline()); + } + + if (bulkOptions.getRoutingId() != null) { + builder.routing(bulkOptions.getRoutingId()); + } + + List operations = queries.stream().map(query -> { + BulkOperation.Builder ob = new BulkOperation.Builder(); + if (query instanceof IndexQuery indexQuery) { + + if (indexQuery.getOpType() == IndexQuery.OpType.CREATE) { + ob.create(bulkCreateOperation(indexQuery, indexCoordinates, refreshPolicy)); + } else { + ob.index(bulkIndexOperation(indexQuery, indexCoordinates, refreshPolicy)); + } + } else if (query instanceof UpdateQuery updateQuery) { + ob.update(bulkUpdateOperation(updateQuery, indexCoordinates, refreshPolicy)); + } + return ob.build(); + }).collect(Collectors.toList()); + + builder.operations(operations); + + return builder.build(); + } + + public GetRequest documentGetRequest(String id, @Nullable String routing, IndexCoordinates indexCoordinates) { + + Assert.notNull(id, "id must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return GetRequest.of(grb -> grb // + .index(indexCoordinates.getIndexName()) // + .id(id) // + .routing(routing)); + } + + public org.opensearch.client.opensearch.core.ExistsRequest documentExistsRequest(String id, @Nullable String routing, + IndexCoordinates indexCoordinates) { + + Assert.notNull(id, "id must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + return org.opensearch.client.opensearch.core.ExistsRequest.of(erb -> erb + .index(indexCoordinates.getIndexName()) + .id(id) + .routing(routing)); + } + + public MgetRequest documentMgetRequest(Query query, Class clazz, IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(clazz, "clazz must not be null"); + Assert.notNull(index, "index must not be null"); + + if (query.getIdsWithRouting().isEmpty()) { + throw new IllegalArgumentException("query does not contain any ids"); + } + + elasticsearchConverter.updateQuery(query, clazz); // to get the SourceConfig right + + SourceConfig sourceConfig = getSourceConfig(query); + + List multiGetOperations = query.getIdsWithRouting().stream() + .map(idWithRouting -> MultiGetOperation.of(mgo -> mgo // + .index(index.getIndexName()) // + .id(idWithRouting.id()) // + .routing(idWithRouting.routing()) // + .source(sourceConfig))) + .collect(Collectors.toList()); + + return MgetRequest.of(mg -> mg// + .docs(multiGetOperations)); + } + + public org.opensearch.client.opensearch.core.ReindexRequest reindex(ReindexRequest reindexRequest, + boolean waitForCompletion) { + + Assert.notNull(reindexRequest, "reindexRequest must not be null"); + + org.opensearch.client.opensearch.core.ReindexRequest.Builder builder = new org.opensearch.client.opensearch.core.ReindexRequest.Builder(); + builder // + .source(s -> { + ReindexRequest.Source source = reindexRequest.getSource(); + s.index(Arrays.asList(source.getIndexes().getIndexNames())) // + .size(source.getSize()); + + ReindexRequest.Slice slice = source.getSlice(); + if (slice != null) { + s.slice(sl -> sl.id(slice.getId()).max(slice.getMax())); + } + + if (source.getQuery() != null) { + s.query(getQuery(source.getQuery(), null)); + } + + if (source.getRemote() != null) { + Remote remote = source.getRemote(); + + s.remote(rs -> { + StringBuilder sb = new StringBuilder(remote.getScheme()); + sb.append("://"); + sb.append(remote.getHost()); + sb.append(':'); + sb.append(remote.getPort()); + + if (remote.getPathPrefix() != null) { + sb.append(remote.getPathPrefix()); + } + + String socketTimeoutSecs = remote.getSocketTimeout() != null + ? remote.getSocketTimeout().getSeconds() + "s" + : "30s"; + String connectTimeoutSecs = remote.getConnectTimeout() != null + ? remote.getConnectTimeout().getSeconds() + "s" + : "30s"; + return rs // + .host(sb.toString()) // + .username(remote.getUsername()) // + .password(remote.getPassword()) // + .socketTimeout(tv -> tv.time(socketTimeoutSecs)) // + .connectTimeout(tv -> tv.time(connectTimeoutSecs)); + }); + } + + SourceFilter sourceFilter = source.getSourceFilter(); + if (sourceFilter != null && sourceFilter.getIncludes() != null) { + s.sourceFields(Arrays.asList(sourceFilter.getIncludes())); + } + return s; + }) // + .dest(d -> { + ReindexRequest.Dest dest = reindexRequest.getDest(); + return d // + .index(dest.getIndex().getIndexName()) // + .versionType(versionType(dest.getVersionType())) // + .opType(opType(dest.getOpType())); + } // + ); + + if (reindexRequest.getConflicts() != null) { + builder.conflicts(conflicts(reindexRequest.getConflicts())); + } + + ReindexRequest.Script script = reindexRequest.getScript(); + if (script != null) { + builder.script(s -> s.inline(InlineScript.of(i -> i.lang(script.getLang()).source(script.getSource())))); + } + + builder.timeout(time(reindexRequest.getTimeout())) // + .scroll(time(reindexRequest.getScroll())); + + if (reindexRequest.getWaitForActiveShards() != null) { + builder.waitForActiveShards(wfas -> wfas // + .count(waitForActiveShardsCount(reindexRequest.getWaitForActiveShards()))); + } + + builder // + .maxDocs(reindexRequest.getMaxDocs()).waitForCompletion(waitForCompletion) // + .refresh(reindexRequest.getRefresh()) // + .requireAlias(reindexRequest.getRequireAlias()) // + .requestsPerSecond(reindexRequest.getRequestsPerSecond()) // + .slices(reindexRequest.getSlices()); + + return builder.build(); + } + + public DeleteRequest documentDeleteRequest(String id, @Nullable String routing, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy) { + + Assert.notNull(id, "id must not be null"); + Assert.notNull(index, "index must not be null"); + + return DeleteRequest.of(r -> { + r.id(id).index(index.getIndexName()); + + if (routing != null) { + r.routing(routing); + } + r.refresh(refresh(refreshPolicy)); + return r; + }); + } + + public DeleteByQueryRequest documentDeleteByQueryRequest(Query query, @Nullable String routing, Class clazz, + IndexCoordinates index, @Nullable RefreshPolicy refreshPolicy) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(index, "index must not be null"); + + return DeleteByQueryRequest.of(b -> { + b.index(Arrays.asList(index.getIndexNames())) // + .query(getQuery(query, clazz))// + .refresh(deleteByQueryRefresh(refreshPolicy)); + + if (query.isLimiting()) { + // noinspection ConstantConditions + b.maxDocs(Long.valueOf(query.getMaxResults())); + } + + b.scroll(time(query.getScrollTime())); + + if (query.getRoute() != null) { + b.routing(query.getRoute()); + } else if (StringUtils.hasText(routing)) { + b.routing(routing); + } + + return b; + }); + } + + public UpdateRequest documentUpdateRequest(UpdateQuery query, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy, @Nullable String routing) { + + String indexName = query.getIndexName() != null ? query.getIndexName() : index.getIndexName(); + return UpdateRequest.of(uqb -> { + uqb.index(indexName).id(query.getId()); + + if (query.getScript() != null) { + Map params = new HashMap<>(); + + if (query.getParams() != null) { + query.getParams().forEach((key, value) -> params.put(key, JsonData.of(value, jsonpMapper))); + } + + uqb.script(sb -> { + if (query.getScriptType() == ScriptType.INLINE) { + sb.inline(is -> is // + .lang(query.getLang()) // + .source(query.getScript()) // + .params(params)); // + } else if (query.getScriptType() == ScriptType.STORED) { + sb.stored(ss -> ss // + .id(query.getScript()) // + .params(params) // + ); + } + return sb; + } + + ); + } + + uqb // + .doc(query.getDocument()) // + .upsert(query.getUpsert()) // + .routing(query.getRouting() != null ? query.getRouting() : routing) // + .scriptedUpsert(query.getScriptedUpsert()) // + .docAsUpsert(query.getDocAsUpsert()) // + .ifSeqNo(query.getIfSeqNo() != null ? Long.valueOf(query.getIfSeqNo()) : null) // + .ifPrimaryTerm(query.getIfPrimaryTerm() != null ? Long.valueOf(query.getIfPrimaryTerm()) : null) // + .refresh(query.getRefreshPolicy() != null ? refresh(query.getRefreshPolicy()) : refresh(refreshPolicy)) // + .retryOnConflict(query.getRetryOnConflict()) // + ; + + if (query.getFetchSource() != null) { + uqb.source(sc -> sc.fetch(query.getFetchSource())); + } + + if (query.getFetchSourceIncludes() != null || query.getFetchSourceExcludes() != null) { + List includes = query.getFetchSourceIncludes() != null ? query.getFetchSourceIncludes() + : Collections.emptyList(); + List excludes = query.getFetchSourceExcludes() != null ? query.getFetchSourceExcludes() + : Collections.emptyList(); + uqb.source(sc -> sc.filter(sf -> sf.includes(includes).excludes(excludes))); + } + + if (query.getTimeout() != null) { + uqb.timeout(tv -> tv.time(query.getTimeout())); + } + + String waitForActiveShards = query.getWaitForActiveShards(); + if (waitForActiveShards != null) { + if ("all".equalsIgnoreCase(waitForActiveShards)) { + uqb.waitForActiveShards(wfa -> wfa.option(WaitForActiveShardOptions.All)); + } else { + int val; + try { + val = Integer.parseInt(waitForActiveShards); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("cannot parse ActiveShardCount[" + waitForActiveShards + ']', e); + } + uqb.waitForActiveShards(wfa -> wfa.count(val)); + } + } + + return uqb; + } // + ); + } + + public UpdateByQueryRequest documentUpdateByQueryRequest(UpdateQuery updateQuery, IndexCoordinates index, + @Nullable RefreshPolicy refreshPolicy) { + + return UpdateByQueryRequest.of(ub -> { + ub // + .index(Arrays.asList(index.getIndexNames())) // + .refresh(refreshPolicy == RefreshPolicy.IMMEDIATE) // + .routing(updateQuery.getRouting()) // + .script(getScript(updateQuery.getScriptData())) // + .maxDocs(updateQuery.getMaxDocs() != null ? Long.valueOf(updateQuery.getMaxDocs()) : null) // + .pipeline(updateQuery.getPipeline()) // + .requestsPerSecond(toLong(updateQuery.getRequestsPerSecond())) // + .slices(updateQuery.getSlices() != null ? Long.valueOf(updateQuery.getSlices()) : null); + + if (updateQuery.getAbortOnVersionConflict() != null) { + ub.conflicts(updateQuery.getAbortOnVersionConflict() ? Conflicts.Abort : Conflicts.Proceed); + } + + if (updateQuery.getQuery() != null) { + Query queryQuery = updateQuery.getQuery(); + + if (updateQuery.getBatchSize() != null) { + ((BaseQuery) queryQuery).setMaxResults(updateQuery.getBatchSize()); + } + ub.query(getQuery(queryQuery, null)); + + // no indicesOptions available like in old client + + ub.scroll(time(queryQuery.getScrollTime())); + + } + + // no maxRetries available like in old client + // no shouldStoreResult + + if (updateQuery.getRefreshPolicy() != null) { + ub.refresh(updateQuery.getRefreshPolicy() == RefreshPolicy.IMMEDIATE); + } + + if (updateQuery.getTimeout() != null) { + ub.timeout(tb -> tb.time(updateQuery.getTimeout())); + } + + if (updateQuery.getWaitForActiveShards() != null) { + ub.waitForActiveShards(w -> w.count(waitForActiveShardsCount(updateQuery.getWaitForActiveShards()))); + } + + return ub; + }); + } + + // endregion + + // region search + + public SearchRequest searchRequest(Query query, @Nullable String routing, @Nullable Class clazz, + IndexCoordinates indexCoordinates, boolean forCount) { + return searchRequest(query, routing, clazz, indexCoordinates, forCount, false, null); + } + + public SearchRequest searchRequest(Query query, @Nullable String routing, @Nullable Class clazz, + IndexCoordinates indexCoordinates, boolean forCount, long scrollTimeInMillis) { + return searchRequest(query, routing, clazz, indexCoordinates, forCount, true, scrollTimeInMillis); + } + + public SearchRequest searchRequest(Query query, @Nullable String routing, @Nullable Class clazz, + IndexCoordinates indexCoordinates, boolean forCount, boolean forBatchedSearch) { + return searchRequest(query, routing, clazz, indexCoordinates, forCount, forBatchedSearch, null); + } + + public SearchRequest searchRequest(Query query, @Nullable String routing, @Nullable Class clazz, + IndexCoordinates indexCoordinates, boolean forCount, boolean forBatchedSearch, + @Nullable Long scrollTimeInMillis) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + elasticsearchConverter.updateQuery(query, clazz); + SearchRequest.Builder builder = new SearchRequest.Builder(); + prepareSearchRequest(query, routing, clazz, indexCoordinates, builder, forCount, forBatchedSearch); + + if (scrollTimeInMillis != null) { + builder.scroll(t -> t.time(scrollTimeInMillis + "ms")); + } + + builder.query(getQuery(query, clazz)); + + if (StringUtils.hasText(query.getRoute())) { + builder.routing(query.getRoute()); + } + if (StringUtils.hasText(routing)) { + builder.routing(routing); + } + + addFilter(query, builder); + + return builder.build(); + } + + public MsearchTemplateRequest searchMsearchTemplateRequest( + List multiSearchTemplateQueryParameters, + @Nullable String routing) { + + // basically the same stuff as in template search + return MsearchTemplateRequest.of(mtrb -> { + multiSearchTemplateQueryParameters.forEach(param -> { + var query = param.query(); + mtrb.searchTemplates(stb -> stb + .header(msearchHeaderBuilder(query, param.index(), routing)) + .body(bb -> { + bb // + .explain(query.getExplain()) // + .id(query.getId()) // + .source(query.getSource()) // + ; + + if (!CollectionUtils.isEmpty(query.getParams())) { + Map params = getTemplateParams(query.getParams().entrySet()); + bb.params(params); + } + + return bb; + }) + ); + }); + return mtrb; + }); + } + + public MsearchRequest searchMsearchRequest( + List multiSearchQueryParameters, @Nullable String routing) { + + // basically the same stuff as in prepareSearchRequest, but the new Elasticsearch has different builders for a + // normal search and msearch + return MsearchRequest.of(mrb -> { + multiSearchQueryParameters.forEach(param -> { + ElasticsearchPersistentEntity persistentEntity = getPersistentEntity(param.clazz()); + + var query = param.query(); + mrb.searches(sb -> sb // + .header(msearchHeaderBuilder(query, param.index(), routing)) // + .body(bb -> { + bb // + .query(getQuery(query, param.clazz()))// + .seqNoPrimaryTerm(persistentEntity != null ? persistentEntity.hasSeqNoPrimaryTermProperty() : null) // + .version(true) // + .trackScores(query.getTrackScores()) // + .source(getSourceConfig(query)) // + .timeout(timeStringMs(query.getTimeout())) // + ; + + if (query.getPageable().isPaged()) { + bb // + .from((int) query.getPageable().getOffset()) // + .size(query.getPageable().getPageSize()); + } + + if (!isEmpty(query.getFields())) { + bb.fields(fb -> { + query.getFields().forEach(fb::field); + return fb; + }); + } + + if (!isEmpty(query.getStoredFields())) { + bb.storedFields(query.getStoredFields()); + } + + if (query.isLimiting()) { + bb.size(query.getMaxResults()); + } + + if (query.getMinScore() > 0) { + bb.minScore((double) query.getMinScore()); + } + + if (query.getSort() != null) { + List sortOptions = getSortOptions(query.getSort(), persistentEntity); + + if (!sortOptions.isEmpty()) { + bb.sort(sortOptions); + } + } + + addHighlight(query, bb); + + if (query.getExplain()) { + bb.explain(true); + } + + if (!isEmpty(query.getSearchAfter())) { + bb.searchAfter(query.getSearchAfter() + .stream() + .map(TypeUtils::toFieldValue) + .map(FieldValue::_toJsonString) + .toList()); + } + + query.getRescorerQueries().forEach(rescorerQuery -> bb.rescore(getRescore(rescorerQuery))); + + if (!isEmpty(query.getIndicesBoost())) { + bb.indicesBoost(query.getIndicesBoost().stream() + .map(indexBoost -> Map.of(indexBoost.getIndexName(), (double) indexBoost.getBoost())) + .collect(Collectors.toList())); + } + + query.getScriptedFields().forEach(scriptedField -> bb.scriptFields(scriptedField.getFieldName(), + sf -> sf.script(getScript(scriptedField.getScriptData())))); + + if (query instanceof NativeQuery nativeQuery) { + prepareNativeSearch(nativeQuery, bb); + } + return bb; + } // + ) // + ); + + }); + + return mrb; + }); + } + + /** + * {@link MsearchRequest} and {@link MsearchTemplateRequest} share the same {@link MultisearchHeader} + */ + private Function> msearchHeaderBuilder(Query query, + IndexCoordinates index, @Nullable String routing) { + return h -> { + var searchType = (query instanceof NativeQuery nativeQuery && nativeQuery.getKnnQuery() != null) ? null + : searchType(query.getSearchType()); + + h // + .index(Arrays.asList(index.getIndexNames())) // + .searchType(searchType) // + .requestCache(query.getRequestCache()) // + ; + + if (StringUtils.hasText(query.getRoute())) { + h.routing(query.getRoute()); + } else if (StringUtils.hasText(routing)) { + h.routing(routing); + } + + if (query.getPreference() != null) { + h.preference(query.getPreference()); + } + + return h; + }; + } + + private void prepareSearchRequest(Query query, @Nullable String routing, @Nullable Class clazz, + IndexCoordinates indexCoordinates, SearchRequest.Builder builder, boolean forCount, boolean forBatchedSearch) { + + String[] indexNames = indexCoordinates.getIndexNames(); + + Assert.notEmpty(indexNames, "indexCoordinates does not contain entries"); + + ElasticsearchPersistentEntity persistentEntity = getPersistentEntity(clazz); + + var searchType = (query instanceof NativeQuery nativeQuery && nativeQuery.getKnnQuery() != null) ? null + : searchType(query.getSearchType()); + + builder // + .version(true) // + .trackScores(query.getTrackScores()) // + .allowNoIndices(query.getAllowNoIndices()) // + .source(getSourceConfig(query)) // + .searchType(searchType) // + .timeout(timeStringMs(query.getTimeout())) // + .requestCache(query.getRequestCache()) // + ; + + var pointInTime = query.getPointInTime(); + if (pointInTime != null) { + builder.pit(new Pit.Builder().id(pointInTime.id()).keepAlive(time(pointInTime.keepAlive()).time()).build()); + } else { + builder // + .index(Arrays.asList(indexNames)) // + ; + + var expandWildcards = query.getExpandWildcards(); + if (expandWildcards != null && !expandWildcards.isEmpty()) { + builder.expandWildcards(expandWildcards(expandWildcards)); + } + + if (query.getRoute() != null) { + builder.routing(query.getRoute()); + } else if (StringUtils.hasText(routing)) { + builder.routing(routing); + } + + if (query.getPreference() != null) { + builder.preference(query.getPreference()); + } + } + + if (persistentEntity != null && persistentEntity.hasSeqNoPrimaryTermProperty()) { + builder.seqNoPrimaryTerm(true); + } + + if (query.getPageable().isPaged()) { + builder // + .from((int) query.getPageable().getOffset()) // + .size(query.getPageable().getPageSize()); + } else { + builder.from(0).size(INDEX_MAX_RESULT_WINDOW); + } + + if (!isEmpty(query.getFields())) { + var fieldAndFormats = query.getFields().stream().map(field -> FieldAndFormat.of(b -> b.field(field))).toList(); + builder.fields(fieldAndFormats); + } + + if (!isEmpty(query.getStoredFields())) { + builder.storedFields(query.getStoredFields()); + } + + if (query.getIndicesOptions() != null) { + addIndicesOptions(builder, query.getIndicesOptions()); + } + + if (query.isLimiting()) { + builder.size(query.getMaxResults()); + } + + if (query.getMinScore() > 0) { + builder.minScore((double) query.getMinScore()); + } + + addHighlight(query, builder); + + query.getScriptedFields().forEach(scriptedField -> builder.scriptFields(scriptedField.getFieldName(), + sf -> sf.script(getScript(scriptedField.getScriptData())))); + + if (query instanceof NativeQuery nativeQuery) { + prepareNativeSearch(nativeQuery, builder); + } + // query.getSort() must be checked after prepareNativeSearch as this already might hav a sort set that must have + // higher priority + if (query.getSort() != null) { + List sortOptions = getSortOptions(query.getSort(), persistentEntity); + + if (!sortOptions.isEmpty()) { + builder.sort(sortOptions); + } + } + + if (query.getTrackTotalHits() != null) { + // logic from the RHLC, choose between -1 and Integer.MAX_VALUE + int value = query.getTrackTotalHits() ? Integer.MAX_VALUE : -1; + builder.trackTotalHits(th -> th.count(value)); + } else if (query.getTrackTotalHitsUpTo() != null) { + builder.trackTotalHits(th -> th.count(query.getTrackTotalHitsUpTo())); + } + + if (query.getExplain()) { + builder.explain(true); + } + + if (!isEmpty(query.getSearchAfter())) { + builder.searchAfter(query.getSearchAfter() + .stream() + .map(TypeUtils::toFieldValue) + .map(FieldValue::_toJsonString) + .toList()); + } + + query.getRescorerQueries().forEach(rescorerQuery -> builder.rescore(getRescore(rescorerQuery))); + + if (forCount) { + builder.size(0) // + .trackTotalHits(th -> th.count(Integer.MAX_VALUE)) // + .source(SourceConfig.of(sc -> sc.fetch(false))); + } else if (forBatchedSearch) { + // request_cache is not allowed on scroll requests. + builder.requestCache(null); + // limit the number of documents in a batch if not already set in a pageable + if (query.getPageable().isUnpaged()) { + builder.size(query.getReactiveBatchSize()); + } + } + + if (!isEmpty(query.getIndicesBoost())) { + builder.indicesBoost(query.getIndicesBoost().stream() + .map(indexBoost -> Map.of(indexBoost.getIndexName(), (double) indexBoost.getBoost())) + .collect(Collectors.toList())); + } + + if (!isEmpty(query.getDocValueFields())) { + builder.docvalueFields(query.getDocValueFields().stream() // + .map(docValueField -> FieldAndFormat.of(b -> b.field(docValueField.field()).format(docValueField.format()))) + .toList()); + } + } + + private void addIndicesOptions(SearchRequest.Builder builder, IndicesOptions indicesOptions) { + + indicesOptions.getOptions().forEach(option -> { + switch (option) { + case ALLOW_NO_INDICES -> builder.allowNoIndices(true); + case IGNORE_UNAVAILABLE -> builder.ignoreUnavailable(true); + case IGNORE_THROTTLED -> builder.ignoreThrottled(true); + // the following ones aren't supported by the builder + case FORBID_ALIASES_TO_MULTIPLE_INDICES, FORBID_CLOSED_INDICES, IGNORE_ALIASES -> { + if (LOGGER.isWarnEnabled()) { + LOGGER + .warn(String.format("indices option %s is not supported by the Elasticsearch client.", option.name())); + } + } + } + }); + + builder.expandWildcards(indicesOptions.getExpandWildcards().stream() + .map(wildcardStates -> switch (wildcardStates) { + case OPEN -> ExpandWildcard.Open; + case CLOSED -> ExpandWildcard.Closed; + case HIDDEN -> ExpandWildcard.Hidden; + case ALL -> ExpandWildcard.All; + case NONE -> ExpandWildcard.None; + }).collect(Collectors.toList())); + } + + private Rescore getRescore(RescorerQuery rescorerQuery) { + + return Rescore.of(r -> r // + .query(rq -> rq // + .query(getQuery(rescorerQuery.getQuery(), null)) // + .scoreMode(scoreMode(rescorerQuery.getScoreMode())) // + .queryWeight(rescorerQuery.getQueryWeight() != null ? Double.valueOf(rescorerQuery.getQueryWeight()) : 1.0) // + .rescoreQueryWeight( + rescorerQuery.getRescoreQueryWeight() != null ? Double.valueOf(rescorerQuery.getRescoreQueryWeight()) + : 1.0) // + + ) // + .windowSize(rescorerQuery.getWindowSize())); + } + + private void addHighlight(Query query, SearchRequest.Builder builder) { + + Highlight highlight = query.getHighlightQuery() + .map(highlightQuery -> new HighlightQueryBuilder(elasticsearchConverter.getMappingContext()) + .getHighlight(highlightQuery.getHighlight(), highlightQuery.getType())) + .orElse(null); + + builder.highlight(highlight); + } + + private void addHighlight(Query query, MultisearchBody.Builder builder) { + + Highlight highlight = query.getHighlightQuery() + .map(highlightQuery -> new HighlightQueryBuilder(elasticsearchConverter.getMappingContext()) + .getHighlight(highlightQuery.getHighlight(), highlightQuery.getType())) + .orElse(null); + + builder.highlight(highlight); + } + + private List getSortOptions(Sort sort, @Nullable ElasticsearchPersistentEntity persistentEntity) { + return sort.stream().map(order -> getSortOptions(order, persistentEntity)).collect(Collectors.toList()); + } + + private SortOptions getSortOptions(Sort.Order order, @Nullable ElasticsearchPersistentEntity persistentEntity) { + SortOrder sortOrder = order.getDirection().isDescending() ? SortOrder.Desc : SortOrder.Asc; + + Order.Mode mode = order.getDirection().isAscending() ? Order.Mode.min : Order.Mode.max; + String unmappedType = null; + String missing = null; + NestedSortValue nestedSortValue = null; + + if (SortOptions.Kind.Score.jsonValue().equals(order.getProperty())) { + return SortOptions.of(so -> so.score(s -> s.order(sortOrder))); + } + + if (order instanceof Order o) { + + if (o.getMode() != null) { + mode = o.getMode(); + } + unmappedType = o.getUnmappedType(); + missing = o.getMissing(); + nestedSortValue = getNestedSort(o.getNested(), persistentEntity); + } + Order.Mode finalMode = mode; + String finalUnmappedType = unmappedType; + var finalNestedSortValue = nestedSortValue; + + ElasticsearchPersistentProperty property = (persistentEntity != null) // + ? persistentEntity.getPersistentProperty(order.getProperty()) // + : null; + String fieldName = property != null ? property.getFieldName() : order.getProperty(); + + if (order instanceof GeoDistanceOrder geoDistanceOrder) { + return getSortOptions(geoDistanceOrder, fieldName, finalMode); + } + + var finalMissing = missing != null ? missing + : (order.getNullHandling() == Sort.NullHandling.NULLS_FIRST) ? "_first" + : ((order.getNullHandling() == Sort.NullHandling.NULLS_LAST) ? "_last" : null); + + return SortOptions.of(so -> so // + .field(f -> { + f.field(fieldName) // + .order(sortOrder) // + .mode(sortMode(finalMode)); + + if (finalUnmappedType != null) { + FieldType fieldType = fieldType(finalUnmappedType); + + if (fieldType != null) { + f.unmappedType(fieldType); + } + } + + if (finalMissing != null) { + f.missing(fv -> fv // + .stringValue(finalMissing)); + } + + if (finalNestedSortValue != null) { + f.nested(finalNestedSortValue); + } + + return f; + })); + } + + @Nullable + private NestedSortValue getNestedSort(@Nullable Order.Nested nested, + @Nullable ElasticsearchPersistentEntity persistentEntity) { + return (nested == null || persistentEntity == null) ? null + : NestedSortValue.of(b -> b // + .path(elasticsearchConverter.updateFieldNames(nested.getPath(), persistentEntity)) // + .maxChildren(nested.getMaxChildren()) // + .nested(getNestedSort(nested.getNested(), persistentEntity)) // + .filter(getQuery(nested.getFilter(), persistentEntity.getType()))); + } + + private static SortOptions getSortOptions(GeoDistanceOrder geoDistanceOrder, String fieldName, Order.Mode finalMode) { + return SortOptions.of(so -> so // + .geoDistance(gd -> gd // + .field(fieldName) // + .location(loc -> loc.latlon(Queries.latLon(geoDistanceOrder.getGeoPoint()))) // + .distanceType(geoDistanceType(geoDistanceOrder.getDistanceType())).mode(sortMode(finalMode)) // + .order(sortOrder(geoDistanceOrder.getDirection())) // + .unit(distanceUnit(geoDistanceOrder.getUnit())) // + .ignoreUnmapped(geoDistanceOrder.getIgnoreUnmapped()))); + } + + @SuppressWarnings("DuplicatedCode") + private void prepareNativeSearch(NativeQuery query, SearchRequest.Builder builder) { + + builder // + .suggest(query.getSuggester()) // + .collapse(query.getFieldCollapse()) // + .sort(query.getSortOptions()) // + ; + + if (query.getKnnQuery() != null) { + builder.query(query.getKnnQuery().toQuery()); + } + + if (!isEmpty(query.getAggregations())) { + builder.aggregations(query.getAggregations()); + } + + if (!isEmpty(query.getSearchExtensions())) { + builder.ext(query.getSearchExtensions()); + } + } + + @SuppressWarnings("DuplicatedCode") + private void prepareNativeSearch(NativeQuery query, MultisearchBody.Builder builder) { + + builder // + .suggest(query.getSuggester()) // + .collapse(query.getFieldCollapse()) // + .sort(query.getSortOptions()); + + if (query.getKnnQuery() != null) { + builder.query(query.getKnnQuery().toQuery()); + } + + if (!isEmpty(query.getAggregations())) { + builder.aggregations(query.getAggregations()); + } + + if (!isEmpty(query.getSearchExtensions())) { + builder.ext(query.getSearchExtensions()); + } + } + + @Nullable + org.opensearch.client.opensearch._types.query_dsl.Query getQuery(@Nullable Query query, + @Nullable Class clazz) { + + if (query == null) { + return null; + } + + elasticsearchConverter.updateQuery(query, clazz); + + org.opensearch.client.opensearch._types.query_dsl.Query esQuery = null; + + if (query instanceof CriteriaQuery) { + esQuery = CriteriaQueryProcessor.createQuery(((CriteriaQuery) query).getCriteria()); + } else if (query instanceof StringQuery) { + esQuery = Queries.wrapperQueryAsQuery(((StringQuery) query).getSource()); + } else if (query instanceof NativeQuery nativeQuery) { + + if (nativeQuery.getQuery() != null) { + esQuery = nativeQuery.getQuery(); + } else if (nativeQuery.getSpringDataQuery() != null) { + esQuery = getQuery(nativeQuery.getSpringDataQuery(), clazz); + } + } else { + throw new IllegalArgumentException("unhandled Query implementation " + query.getClass().getName()); + } + + return esQuery; + } + + private void addFilter(Query query, SearchRequest.Builder builder) { + + if (query instanceof CriteriaQuery) { + CriteriaFilterProcessor.createQuery(((CriteriaQuery) query).getCriteria()).ifPresent(builder::postFilter); + } else // noinspection StatementWithEmptyBody + if (query instanceof StringQuery) { + // no filter for StringQuery + } else if (query instanceof NativeQuery nativeQuery) { + if (nativeQuery.getFilter() != null) { + builder.postFilter(nativeQuery.getFilter()); + } else if (nativeQuery.getSpringDataQuery() != null) { + addFilter(nativeQuery.getSpringDataQuery(), builder); + } + } else { + throw new IllegalArgumentException("unhandled Query implementation " + query.getClass().getName()); + } + } + + public org.opensearch.client.opensearch._types.query_dsl.MoreLikeThisQuery moreLikeThisQuery(MoreLikeThisQuery query, + IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + Assert.notNull(index, "index must not be null"); + + return org.opensearch.client.opensearch._types.query_dsl.MoreLikeThisQuery + .of(q -> { + q.like(Like.of(l -> l.document(ld -> ld.index(index.getIndexName()).id(query.getId())))) + .fields(query.getFields()); + + if (query.getMinTermFreq() != null) { + q.minTermFreq(query.getMinTermFreq()); + } + + if (query.getMaxQueryTerms() != null) { + q.maxQueryTerms(query.getMaxQueryTerms()); + } + + if (!isEmpty(query.getStopWords())) { + q.stopWords(query.getStopWords()); + } + + if (query.getMinDocFreq() != null) { + q.minDocFreq(query.getMinDocFreq()); + } + + if (query.getMaxDocFreq() != null) { + q.maxDocFreq(query.getMaxDocFreq()); + } + + if (query.getMinWordLen() != null) { + q.minWordLength(query.getMinWordLen()); + } + + if (query.getMaxWordLen() != null) { + q.maxWordLength(query.getMaxWordLen()); + } + + if (query.getBoostTerms() != null) { + q.boostTerms(Double.valueOf(query.getBoostTerms())); + } + + return q; + }); + } + + public CreatePitRequest searchOpenPointInTimeRequest(IndexCoordinates index, Duration keepAlive, + Boolean allowPartialPitCreation) { + + Assert.notNull(index, "index must not be null"); + Assert.notNull(keepAlive, "keepAlive must not be null"); + Assert.notNull(allowPartialPitCreation, "allowPartialPitCreation must not be null"); + + return CreatePitRequest.of(opit -> opit // + .targetIndexes(Arrays.asList(index.getIndexNames())) // + .allowPartialPitCreation(allowPartialPitCreation) // + .keepAlive(time(keepAlive)) // + ); + } + + public DeletePitRequest searchClosePointInTime(String pit) { + + Assert.notNull(pit, "pit must not be null"); + + return DeletePitRequest.of(cpit -> cpit.pitId(Collections.singletonList(pit))); + } + + public SearchTemplateRequest searchTemplate(SearchTemplateQuery query, @Nullable String routing, + IndexCoordinates index) { + + Assert.notNull(query, "query must not be null"); + + return SearchTemplateRequest.of(builder -> { + builder // + .allowNoIndices(query.getAllowNoIndices()) // + .explain(query.getExplain()) // + .id(query.getId()) // + .index(Arrays.asList(index.getIndexNames())) // + .preference(query.getPreference()) // + .searchType(searchType(query.getSearchType())) // + .source(query.getSource()) // + ; + + if (query.getRoute() != null) { + builder.routing(query.getRoute()); + } else if (StringUtils.hasText(routing)) { + builder.routing(routing); + } + + var expandWildcards = query.getExpandWildcards(); + if (expandWildcards != null && !expandWildcards.isEmpty()) { + builder.expandWildcards(expandWildcards(expandWildcards)); + } + + if (query.hasScrollTime()) { + builder.scroll(time(query.getScrollTime())); + } + + if (!CollectionUtils.isEmpty(query.getParams())) { + Map params = getTemplateParams(query.getParams().entrySet()); + builder.params(params); + } + + return builder; + }); + } + + @NonNull + private Map getTemplateParams(Set> query) { + Function, String> keyMapper = Map.Entry::getKey; + Function, JsonData> valueMapper = entry -> JsonData.of(entry.getValue(), jsonpMapper); + return query.stream() + .collect(Collectors.toMap(keyMapper, valueMapper)); + } + + // endregion + + public PutScriptRequest scriptPut(Script script) { + + Assert.notNull(script, "script must not be null"); + + return PutScriptRequest.of(b -> b // + .id(script.id()) // + .script(sb -> sb // + .lang(script.language()) // + .source(script.source()))); + } + + public GetScriptRequest scriptGet(String name) { + + Assert.notNull(name, "name must not be null"); + + return GetScriptRequest.of(b -> b.id(name)); + } + + public DeleteScriptRequest scriptDelete(String name) { + + Assert.notNull(name, "name must not be null"); + + return DeleteScriptRequest.of(b -> b.id(name)); + } + // region helper functions + + public T fromJson(String json, JsonpDeserializer deserializer) { + + Assert.notNull(json, "json must not be null"); + Assert.notNull(deserializer, "deserializer must not be null"); + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)); + return fromJson(byteArrayInputStream, deserializer); + } + + public T fromJson(ByteArrayInputStream byteArrayInputStream, JsonpDeserializer deserializer) { + + Assert.notNull(byteArrayInputStream, "byteArrayInputStream must not be null"); + Assert.notNull(deserializer, "deserializer must not be null"); + + JsonParser parser = jsonpMapper.jsonProvider().createParser(byteArrayInputStream); + return deserializer.deserialize(parser, jsonpMapper); + } + + @Nullable + private ElasticsearchPersistentEntity getPersistentEntity(Object entity) { + return elasticsearchConverter.getMappingContext().getPersistentEntity(entity.getClass()); + } + + @Nullable + private ElasticsearchPersistentEntity getPersistentEntity(@Nullable Class clazz) { + return clazz != null ? elasticsearchConverter.getMappingContext().getPersistentEntity(clazz) : null; + } + + @Nullable + private String getPersistentEntityId(Object entity) { + + ElasticsearchPersistentEntity persistentEntity = getPersistentEntity(entity); + + if (persistentEntity != null) { + Object identifier = persistentEntity // + .getIdentifierAccessor(entity).getIdentifier(); + + if (identifier != null) { + return identifier.toString(); + } + } + + return null; + } + + private VersionType retrieveVersionTypeFromPersistentEntity(@Nullable Class clazz) { + + ElasticsearchPersistentEntity persistentEntity = getPersistentEntity(clazz); + + VersionType versionType = null; + + if (persistentEntity != null) { + org.springframework.data.elasticsearch.annotations.Document.VersionType entityVersionType = persistentEntity + .getVersionType(); + + if (entityVersionType != null) { + versionType = switch (entityVersionType) { + case INTERNAL -> VersionType.Internal; + case EXTERNAL -> VersionType.External; + case EXTERNAL_GTE -> VersionType.ExternalGte; + case FORCE -> VersionType.Force; + }; + } + } + + return versionType != null ? versionType : VersionType.External; + } + + @Nullable + private SourceConfig getSourceConfig(Query query) { + + if (query.getSourceFilter() != null) { + return SourceConfig.of(s -> s // + .filter(sfb -> { + SourceFilter sourceFilter = query.getSourceFilter(); + String[] includes = sourceFilter.getIncludes(); + String[] excludes = sourceFilter.getExcludes(); + + if (includes != null) { + sfb.includes(Arrays.asList(includes)); + } + + if (excludes != null) { + sfb.excludes(Arrays.asList(excludes)); + } + + return sfb; + })); + } else { + return null; + } + } + + @Nullable + static Boolean deleteByQueryRefresh(@Nullable RefreshPolicy refreshPolicy) { + + if (refreshPolicy == null) { + return null; + } + + return switch (refreshPolicy) { + case IMMEDIATE -> true; + case WAIT_UNTIL -> null; + case NONE -> false; + }; + } + + // endregion +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ResponseConverter.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ResponseConverter.java new file mode 100644 index 0000000..c747fff --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/ResponseConverter.java @@ -0,0 +1,567 @@ +/* + * Copyright 2021-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.JsonUtils.toJson; +import static org.opensearch.data.client.osc.TypeUtils.removePrefixFromJson; +import static org.opensearch.data.client.osc.TypeUtils.typeMapping; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch._types.BulkIndexByScrollFailure; +import org.opensearch.client.opensearch._types.ErrorCause; +import org.opensearch.client.opensearch._types.Time; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch.cluster.ComponentTemplateSummary; +import org.opensearch.client.opensearch.cluster.GetComponentTemplateResponse; +import org.opensearch.client.opensearch.cluster.HealthResponse; +import org.opensearch.client.opensearch.core.DeleteByQueryResponse; +import org.opensearch.client.opensearch.core.GetScriptResponse; +import org.opensearch.client.opensearch.core.UpdateByQueryResponse; +import org.opensearch.client.opensearch.core.mget.MultiGetError; +import org.opensearch.client.opensearch.core.mget.MultiGetResponseItem; +import org.opensearch.client.opensearch.indices.*; +import org.opensearch.client.opensearch.indices.get_index_template.IndexTemplateItem; +import org.opensearch.client.opensearch.indices.get_index_template.IndexTemplateSummary; +import org.opensearch.client.opensearch.indices.get_mapping.IndexMappingRecord; +import org.springframework.data.elasticsearch.ElasticsearchErrorCause; +import org.springframework.data.elasticsearch.core.IndexInformation; +import org.springframework.data.elasticsearch.core.MultiGetItem; +import org.springframework.data.elasticsearch.core.cluster.ClusterHealth; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.index.*; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.ByQueryResponse; +import org.springframework.data.elasticsearch.core.query.StringQuery; +import org.springframework.data.elasticsearch.core.reindex.ReindexResponse; +import org.springframework.data.elasticsearch.core.script.Script; +import org.springframework.data.elasticsearch.support.DefaultStringObjectMap; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Class to convert Elasticsearch responses into Spring Data Elasticsearch classes. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +class ResponseConverter { + + private static final Log LOGGER = LogFactory.getLog(ResponseConverter.class); + + private final JsonpMapper jsonpMapper; + + public ResponseConverter(JsonpMapper jsonpMapper) { + this.jsonpMapper = jsonpMapper; + } + + // region cluster client + public ClusterHealth clusterHealth(HealthResponse healthResponse) { + + Assert.notNull(healthResponse, "healthResponse must not be null"); + + return ClusterHealth.builder() // + .withActivePrimaryShards(healthResponse.activePrimaryShards()) // + .withActiveShards(healthResponse.activeShards()) // + .withActiveShardsPercent(Double.parseDouble(healthResponse.activeShardsPercentAsNumber()))// + .withClusterName(healthResponse.clusterName()) // + .withDelayedUnassignedShards(healthResponse.delayedUnassignedShards()) // + .withInitializingShards(healthResponse.initializingShards()) // + .withNumberOfDataNodes(healthResponse.numberOfDataNodes()) // + .withNumberOfInFlightFetch(healthResponse.numberOfInFlightFetch()) // + .withNumberOfNodes(healthResponse.numberOfNodes()) // + .withNumberOfPendingTasks(healthResponse.numberOfPendingTasks()) // + .withRelocatingShards(healthResponse.relocatingShards()) // + .withStatus(healthResponse.status().toString()) // + .withTaskMaxWaitingTimeMillis(Long.parseLong(healthResponse.taskMaxWaitingInQueueMillis())) // + .withTimedOut(healthResponse.timedOut()) // + .withUnassignedShards(healthResponse.unassignedShards()) // + .build(); // + } + + public List clusterGetComponentTemplates( + GetComponentTemplateResponse getComponentTemplateResponse) { + + Assert.notNull(getComponentTemplateResponse, "getComponentTemplateResponse must not be null"); + + var componentTemplates = new ArrayList(); + getComponentTemplateResponse.componentTemplates().forEach(componentTemplate -> { + componentTemplates.add(clusterGetComponentTemplate(componentTemplate)); + }); + + return componentTemplates; + } + + private TemplateResponse clusterGetComponentTemplate( + org.opensearch.client.opensearch.cluster.ComponentTemplate componentTemplate) { + var componentTemplateNode = componentTemplate.componentTemplate(); + var componentTemplateSummary = componentTemplateNode.template(); + return TemplateResponse.builder() // + .withName(componentTemplate.name()) // + .withVersion(componentTemplateNode.version()) // + .withTemplateData(clusterGetComponentTemplateData(componentTemplateSummary)) // + .build(); + } + + private TemplateResponseData clusterGetComponentTemplateData(ComponentTemplateSummary componentTemplateSummary) { + + var mapping = typeMapping(componentTemplateSummary.mappings()); + var settings = new Settings(); + componentTemplateSummary.settings().forEach((key, indexSettings) -> { + settings.put(key, Settings.parse(removePrefixFromJson(indexSettings.toString()))); + }); + + Function, String> keyMapper = Map.Entry::getKey; + Function, AliasData> valueMapper = entry -> indicesGetAliasData( + entry.getKey(), entry.getValue()); + + Map aliases = componentTemplateSummary.aliases().entrySet().stream() + .collect(Collectors.toMap(keyMapper, valueMapper)); + + return TemplateResponseData.builder() // + .withMapping(mapping) // + .withSettings(settings) // + .withAliases(aliases) // + .build(); + } + + // endregion + + // region indices client + public Settings indicesGetSettings(GetIndicesSettingsResponse getIndicesSettingsResponse, String indexName) { + + Assert.notNull(getIndicesSettingsResponse, "getIndicesSettingsResponse must not be null"); + Assert.notNull(indexName, "indexName must not be null"); + + Settings settings = new Settings(); + IndexState indexState = getIndicesSettingsResponse.get(indexName); + + if (indexState != null) { + + Function indexSettingsToSettings = indexSettings -> { + Settings parsedSettings = Settings.parse(toJson(indexSettings, jsonpMapper)); + return (indexSettings.index() != null) ? parsedSettings : new Settings().append("index", parsedSettings); + }; + + if (indexState.defaults() != null) { + Settings defaultSettings = indexSettingsToSettings.apply(indexState.defaults()); + settings.merge(defaultSettings); + } + + if (indexState.settings() != null) { + Settings nonDefaultSettings = indexSettingsToSettings.apply(indexState.settings()); + settings.merge(nonDefaultSettings); + } + } + + return settings; + } + + public Document indicesGetMapping(GetMappingResponse getMappingResponse, IndexCoordinates indexCoordinates) { + + Assert.notNull(getMappingResponse, "getMappingResponse must not be null"); + Assert.notNull(indexCoordinates, "indexCoordinates must not be null"); + + Map mappings = getMappingResponse.result(); + + if (mappings == null || mappings.size() == 0) { + return Document.create(); + } + + IndexMappingRecord indexMappingRecord = mappings.get(indexCoordinates.getIndexName()); + + // this can happen when the mapping was requested with an alias + if (indexMappingRecord == null) { + + if (mappings.size() != 1) { + LOGGER.warn(String.format("no mapping returned for index %s", indexCoordinates.getIndexName())); + return Document.create(); + } + String index = mappings.keySet().iterator().next(); + indexMappingRecord = mappings.get(index); + } + + return Document.parse(toJson(indexMappingRecord.mappings(), jsonpMapper)); + } + + public List indicesGetIndexInformations(GetIndexResponse getIndexResponse) { + + Assert.notNull(getIndexResponse, "getIndexResponse must not be null"); + + List indexInformationList = new ArrayList<>(); + + getIndexResponse.result().forEach((indexName, indexState) -> { + Settings settings = indexState.settings() != null ? Settings.parse(toJson(indexState.settings(), jsonpMapper)) + : new Settings(); + Document mappings = indexState.mappings() != null ? Document.parse(toJson(indexState.mappings(), jsonpMapper)) + : Document.create(); + + List aliasDataList = new ArrayList<>(); + indexState.aliases().forEach((aliasName, alias) -> aliasDataList.add(indicesGetAliasData(aliasName, alias))); + + indexInformationList.add(IndexInformation.of(indexName, settings, mappings, aliasDataList)); + + }); + return indexInformationList; + } + + public Map> indicesGetAliasData(GetAliasResponse getAliasResponse) { + + Assert.notNull(getAliasResponse, "getAliasResponse must not be null"); + + Map> aliasDataMap = new HashMap<>(); + getAliasResponse.result().forEach((indexName, alias) -> { + Set aliasDataSet = new HashSet<>(); + alias.aliases() + .forEach((aliasName, aliasDefinition) -> aliasDataSet.add(indicesGetAliasData(aliasName, aliasDefinition))); + aliasDataMap.put(indexName, aliasDataSet); + }); + return aliasDataMap; + } + + private AliasData indicesGetAliasData(String aliasName, Alias alias) { + + Query filter = alias.filter(); + String filterJson = filter != null ? toJson(filter, jsonpMapper) : null; + var filterQuery = filterJson != null ? StringQuery.builder(filterJson).build() : null; + return AliasData.of(aliasName, filterQuery, alias.indexRouting(), alias.searchRouting(), alias.isWriteIndex(), + alias.isHidden()); + } + + private AliasData indicesGetAliasData(String aliasName, AliasDefinition alias) { + Query filter = alias.filter(); + String filterJson = filter != null ? toJson(filter, jsonpMapper) : null; + var filterQuery = filterJson != null ? StringQuery.builder(filterJson).build() : null; + return AliasData.of(aliasName, filterQuery, alias.indexRouting(), alias.searchRouting(), alias.isWriteIndex(), + null); + } + + @Nullable + public TemplateData indicesGetTemplateData(GetTemplateResponse getTemplateResponse, String templateName) { + + Assert.notNull(getTemplateResponse, "getTemplateResponse must not be null"); + Assert.notNull(templateName, "templateName must not be null"); + + TemplateMapping templateMapping = getTemplateResponse.get(templateName); + if (templateMapping != null) { + + Settings settings = new Settings(); + templateMapping.settings().forEach((key, jsonData) -> { + + if (key.contains(".")) { + // returned string contains " quotes + settings.put(key, jsonData.toJson().toString().replaceAll("^\"|\"$", "")); + } else { + settings.put(key, new DefaultStringObjectMap<>().fromJson(jsonData.toJson().toString())); + } + }); + + Function, String> keyMapper = Map.Entry::getKey; + Function, AliasData> valueMapper = entry -> indicesGetAliasData(entry.getKey(), + entry.getValue()); + + Map aliases = templateMapping.aliases().entrySet().stream() + .collect(Collectors.toMap(keyMapper, valueMapper)); + + Document mapping = Document.parse(toJson(templateMapping.mappings(), jsonpMapper)); + + TemplateData.TemplateDataBuilder builder = TemplateData.builder() // + .withIndexPatterns(templateMapping.indexPatterns().toArray(new String[0])) // + .withOrder(templateMapping.order()) // + .withSettings(settings) // + .withMapping(mapping) // + .withAliases(aliases) // + ; + + if (templateMapping.version() != null) { + builder.withVersion(templateMapping.version().intValue()); + } + + return builder.build(); + } + + return null; + } + + public List getIndexTemplates(GetIndexTemplateResponse getIndexTemplateResponse) { + + Assert.notNull(getIndexTemplateResponse, "getIndexTemplateResponse must not be null"); + + var componentTemplates = new ArrayList(); + getIndexTemplateResponse.indexTemplates().forEach(indexTemplateItem -> { + componentTemplates.add(indexGetComponentTemplate(indexTemplateItem)); + }); + + return componentTemplates; + } + + private TemplateResponse indexGetComponentTemplate(IndexTemplateItem indexTemplateItem) { + var indexTemplate = indexTemplateItem.indexTemplate(); + var composedOf = indexTemplate.composedOf(); + var indexTemplateSummary = indexTemplate.template(); + return TemplateResponse.builder() // + .withName(indexTemplateItem.name()) // + .withVersion(indexTemplate.version()) // + .withTemplateData(indexGetComponentTemplateData(indexTemplateSummary, composedOf)) // + .build(); + } + + private TemplateResponseData indexGetComponentTemplateData(IndexTemplateSummary indexTemplateSummary, + List composedOf) { + var mapping = typeMapping(indexTemplateSummary.mappings()); + + Function, Settings> indexSettingsToSettings = indexSettings -> { + + if (indexSettings == null) { + return null; + } + + Settings parsedSettings = Settings.parse(toJson(indexSettings, jsonpMapper)); + return (indexSettings.get("index") != null) ? parsedSettings : new Settings().append("index", parsedSettings); + }; + var settings = indexSettingsToSettings.apply(indexTemplateSummary.settings()); + + Function, String> keyMapper = Map.Entry::getKey; + Function, AliasData> valueMapper = entry -> indicesGetAliasData(entry.getKey(), + entry.getValue()); + + Map aliases1 = indexTemplateSummary.aliases(); + Map aliases = aliases1.entrySet().stream().collect(Collectors.toMap(keyMapper, valueMapper)); + + return TemplateResponseData.builder() // + .withMapping(mapping) // + .withSettings(settings) // + .withAliases(aliases) // + .withComposedOf(composedOf) // + .build(); + } + + // endregion + + // region document operations + public ReindexResponse reindexResponse(org.opensearch.client.opensearch.core.ReindexResponse reindexResponse) { + + Assert.notNull(reindexResponse, "reindexResponse must not be null"); + + List failures = reindexResponse.failures() // + .stream() // + .map(this::reindexResponseFailureOf) // + .collect(Collectors.toList()); + + // noinspection ConstantConditions + return ReindexResponse.builder() // + .withTook(timeToLong(reindexResponse.took())) // + .withTimedOut(reindexResponse.timedOut()) // + .withTotal(reindexResponse.total()) // + .withCreated(reindexResponse.created()) // + .withUpdated(reindexResponse.updated()) // + .withDeleted(reindexResponse.deleted()) // + .withBatches(reindexResponse.batches()) // + .withVersionConflicts(reindexResponse.versionConflicts()) // + .withNoops(reindexResponse.noops()) // + .withBulkRetries(reindexResponse.retries().bulk()) // + .withSearchRetries(reindexResponse.retries().search()) // + .withThrottledMillis(timeToLong(reindexResponse.throttledMillis())) // + .withRequestsPerSecond(reindexResponse.requestsPerSecond()) // + .withThrottledUntilMillis(timeToLong(reindexResponse.throttledUntilMillis())) // + .withFailures(failures) // + .build(); + } + + private ReindexResponse.Failure reindexResponseFailureOf(BulkIndexByScrollFailure failure) { + return ReindexResponse.Failure.builder() // + .withIndex(failure.index()) // + .withType(failure.type()) // + .withId(failure.id()) // + .withStatus(failure.status())// + .withErrorCause(toErrorCause(failure.cause())) // + // seqno, term, aborted are not available in the new client + .build(); + } + + private ByQueryResponse.Failure byQueryResponseFailureOf(BulkIndexByScrollFailure failure) { + return ByQueryResponse.Failure.builder() // + .withIndex(failure.index()) // + .withType(failure.type()) // + .withId(failure.id()) // + .withStatus(failure.status())// + .withErrorCause(toErrorCause(failure.cause())).build(); + } + + @Nullable + public static MultiGetItem.Failure getFailure(MultiGetResponseItem itemResponse) { + + MultiGetError responseFailure = itemResponse.isFailure() ? itemResponse.failure() : null; + + return responseFailure != null + ? MultiGetItem.Failure.of(responseFailure.index(), null, responseFailure.id(), null, + toErrorCause(responseFailure.error())) + : null; + } + + public ByQueryResponse byQueryResponse(DeleteByQueryResponse response) { + // the code for the methods taking a DeleteByQueryResponse or a UpdateByQueryResponse is duplicated because the + // Elasticsearch responses do not share a common class + // noinspection DuplicatedCode + List failures = response.failures().stream().map(this::byQueryResponseFailureOf) + .collect(Collectors.toList()); + + ByQueryResponse.ByQueryResponseBuilder builder = ByQueryResponse.builder(); + + if (response.took() != null) { + builder.withTook(response.took()); + } + + if (response.timedOut() != null) { + builder.withTimedOut(response.timedOut()); + } + + if (response.total() != null) { + builder.withTotal(response.total()); + } + + if (response.deleted() != null) { + builder.withDeleted(response.deleted()); + } + + if (response.batches() != null) { + builder.withBatches(Math.toIntExact(response.batches())); + } + + if (response.versionConflicts() != null) { + builder.withVersionConflicts(response.versionConflicts()); + } + + if (response.noops() != null) { + builder.withNoops(response.noops()); + } + + if (response.retries() != null) { + builder.withBulkRetries(response.retries().bulk()); + builder.withSearchRetries(response.retries().search()); + } + + builder.withFailures(failures); + + return builder.build(); + } + + public ByQueryResponse byQueryResponse(UpdateByQueryResponse response) { + // the code for the methods taking a DeleteByQueryResponse or a UpdateByQueryResponse is duplicated because the + // Elasticsearch responses do not share a common class + // noinspection DuplicatedCode + List failures = response.failures().stream().map(this::byQueryResponseFailureOf) + .collect(Collectors.toList()); + + ByQueryResponse.ByQueryResponseBuilder builder = ByQueryResponse.builder(); + + if (response.took() != null) { + builder.withTook(response.took()); + } + + if (response.timedOut() != null) { + builder.withTimedOut(response.timedOut()); + } + + if (response.total() != null) { + builder.withTotal(response.total()); + } + + if (response.deleted() != null) { + builder.withDeleted(response.deleted()); + } + + if (response.batches() != null) { + builder.withBatches(Math.toIntExact(response.batches())); + } + + if (response.versionConflicts() != null) { + builder.withVersionConflicts(response.versionConflicts()); + } + + if (response.noops() != null) { + builder.withNoops(response.noops()); + } + + if (response.retries() != null) { + builder.withBulkRetries(response.retries().bulk()); + builder.withSearchRetries(response.retries().search()); + } + + builder.withFailures(failures); + + return builder.build(); + } + + // endregion + + // region script API + @Nullable + public Script scriptResponse(GetScriptResponse response) { + + Assert.notNull(response, "response must not be null"); + + return response.found() // + ? Script.builder() // + .withId(response.id()) // + .withLanguage(response.script().lang()) // + .withSource(response.script().source()).build() // + : null; + } + // endregion + + // region helper functions + + private long timeToLong(String time) { + + if (time == null) { + return 0L; + } else { + return Long.parseLong(time); + } + } + + private long timeToLong(Time time) { + + if (time.isTime()) { + return Long.parseLong(time.time()); + } else { + return time.offset(); + } + } + + @Nullable + static ElasticsearchErrorCause toErrorCause(@Nullable ErrorCause errorCause) { + + if (errorCause != null) { + return new ElasticsearchErrorCause( // + errorCause.type(), // + errorCause.reason(), // + errorCause.stackTrace(), // + toErrorCause(errorCause.causedBy()), // + errorCause.rootCause().stream().map(ResponseConverter::toErrorCause).collect(Collectors.toList()), // + errorCause.suppressed().stream().map(ResponseConverter::toErrorCause).collect(Collectors.toList())); + } else { + return null; + } + } + // endregion +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/SearchDocumentResponseBuilder.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/SearchDocumentResponseBuilder.java new file mode 100644 index 0000000..96d3590 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/SearchDocumentResponseBuilder.java @@ -0,0 +1,264 @@ +/* + * Copyright 2021-2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.opensearch._types.aggregations.Aggregate; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.SearchTemplateResponse; +import org.opensearch.client.opensearch.core.search.CompletionSuggest; +import org.opensearch.client.opensearch.core.search.CompletionSuggestOption; +import org.opensearch.client.opensearch.core.search.Hit; +import org.opensearch.client.opensearch.core.search.HitsMetadata; +import org.opensearch.client.opensearch.core.search.SearchResult; +import org.opensearch.client.opensearch.core.search.TotalHits; +import org.springframework.data.elasticsearch.core.TotalHitsRelation; +import org.springframework.data.elasticsearch.core.document.SearchDocument; +import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; +import org.springframework.data.elasticsearch.core.suggest.response.CompletionSuggestion; +import org.springframework.data.elasticsearch.core.suggest.response.PhraseSuggestion; +import org.springframework.data.elasticsearch.core.suggest.response.Suggest; +import org.springframework.data.elasticsearch.core.suggest.response.TermSuggestion; +import org.springframework.data.elasticsearch.support.ScoreDoc; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +/** + * Factory class to create {@link SearchDocumentResponse} instances. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +class SearchDocumentResponseBuilder { + + private static final Log LOGGER = LogFactory.getLog(SearchDocumentResponseBuilder.class); + + /** + * creates a SearchDocumentResponse from the {@link SearchResponse} + * + * @param responseBody the Elasticsearch response body + * @param entityCreator function to create an entity from a {@link SearchDocument} + * @param jsonpMapper to map JsonData objects + * @return the SearchDocumentResponse + */ + public static SearchDocumentResponse from(SearchResult responseBody, + SearchDocumentResponse.EntityCreator entityCreator, JsonpMapper jsonpMapper) { + + Assert.notNull(responseBody, "responseBody must not be null"); + Assert.notNull(entityCreator, "entityCreator must not be null"); + Assert.notNull(jsonpMapper, "jsonpMapper must not be null"); + + HitsMetadata hitsMetadata = responseBody.hits(); + String scrollId = responseBody.scrollId(); + Map aggregations = responseBody.aggregations(); + Map>> suggest = responseBody.suggest(); + var pointInTimeId = responseBody.pitId(); + + return from(hitsMetadata, scrollId, pointInTimeId, aggregations, suggest, entityCreator, jsonpMapper); + } + + /** + * creates a SearchDocumentResponse from the {@link SearchTemplateResponse} + * + * @param response the Elasticsearch response body + * @param entityCreator function to create an entity from a {@link SearchDocument} + * @param jsonpMapper to map JsonData objects + * @return the SearchDocumentResponse + * @since 5.1 + */ + public static SearchDocumentResponse from(SearchTemplateResponse response, + SearchDocumentResponse.EntityCreator entityCreator, JsonpMapper jsonpMapper) { + + Assert.notNull(response, "response must not be null"); + Assert.notNull(entityCreator, "entityCreator must not be null"); + Assert.notNull(jsonpMapper, "jsonpMapper must not be null"); + + var hitsMetadata = response.hits(); + var scrollId = response.scrollId(); + var aggregations = response.aggregations(); + var suggest = response.suggest(); + var pointInTimeId = response.pitId(); + + return from(hitsMetadata, scrollId, pointInTimeId, aggregations, suggest, entityCreator, jsonpMapper); + } + + /** + * creates a {@link SearchDocumentResponseBuilder} from {@link HitsMetadata} with the given scrollId aggregations and + * suggestES + * + * @param entity type + * @param hitsMetadata the {@link HitsMetadata} to process + * @param scrollId scrollId + * @param aggregations aggregations + * @param suggestES the suggestion response from Elasticsearch + * @param entityCreator function to create an entity from a {@link SearchDocument}, needed in mapping the suggest data + * @param jsonpMapper to map JsonData objects + * @return the {@link SearchDocumentResponse} + */ + public static SearchDocumentResponse from(HitsMetadata hitsMetadata, + @Nullable String scrollId, @Nullable String pointInTimeId, @Nullable Map aggregations, + Map>> suggestES, SearchDocumentResponse.EntityCreator entityCreator, + JsonpMapper jsonpMapper) { + + Assert.notNull(hitsMetadata, "hitsMetadata must not be null"); + + long totalHits; + String totalHitsRelation; + + TotalHits responseTotalHits = hitsMetadata.total(); + if (responseTotalHits != null) { + totalHits = responseTotalHits.value(); + totalHitsRelation = switch (responseTotalHits.relation().jsonValue()) { + case "eq" -> TotalHitsRelation.EQUAL_TO.name(); + case "gte" -> TotalHitsRelation.GREATER_THAN_OR_EQUAL_TO.name(); + default -> TotalHitsRelation.OFF.name(); + }; + } else { + totalHits = hitsMetadata.hits().size(); + totalHitsRelation = "OFF"; + } + + float maxScore = hitsMetadata.maxScore() != null ? hitsMetadata.maxScore().floatValue() : Float.NaN; + + List searchDocuments = new ArrayList<>(); + for (Hit hit : hitsMetadata.hits()) { + searchDocuments.add(DocumentAdapters.from(hit, jsonpMapper)); + } + + OpenSearchAggregations aggregationsContainer = aggregations != null ? new OpenSearchAggregations(aggregations) + : null; + + Suggest suggest = suggestFrom(suggestES, entityCreator); + + return new SearchDocumentResponse(totalHits, totalHitsRelation, maxScore, scrollId, pointInTimeId, searchDocuments, + aggregationsContainer, suggest); + } + + @Nullable + private static Suggest suggestFrom(Map>> suggestES, + SearchDocumentResponse.EntityCreator entityCreator) { + + if (CollectionUtils.isEmpty(suggestES)) { + return null; + } + + List>> suggestions = new ArrayList<>(); + + suggestES.forEach((name, suggestionsES) -> { + + if (!suggestionsES.isEmpty()) { + // take the type from the first entry + switch (suggestionsES.get(0)._kind()) { + case Term -> { + suggestions.add(getTermSuggestion(name, suggestionsES)); + break; + } + case Phrase -> { + suggestions.add(getPhraseSuggestion(name, suggestionsES)); + break; + } + case Completion -> { + suggestions.add(getCompletionSuggestion(name, suggestionsES, entityCreator)); + break; + } + default -> {} + } + } + }); + + // todo: hasScoreDocs checks if any one + boolean hasScoreDocs = false; + + return new Suggest(suggestions, hasScoreDocs); + } + + private static TermSuggestion getTermSuggestion(String name, List> suggestionsES) { + + List entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + var termSuggest = suggestionES.term(); + var termSuggestOptions = termSuggest.options(); + List options = new ArrayList<>(); + termSuggestOptions.forEach(optionES -> options.add(new TermSuggestion.Entry.Option(optionES.text(), null, + optionES.score(), null, Math.toIntExact(optionES.freq())))); + entries.add(new TermSuggestion.Entry(termSuggest.text(), termSuggest.offset(), termSuggest.length(), options)); + }); + return new TermSuggestion(name, suggestionsES.size(), entries, null); + } + + private static PhraseSuggestion getPhraseSuggestion(String name, List> suggestionsES) { + + List entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + var phraseSuggest = suggestionES.phrase(); + var phraseSuggestOptions = phraseSuggest.options(); + List options = new ArrayList<>(); + phraseSuggestOptions.forEach(optionES -> options.add(new PhraseSuggestion.Entry.Option(optionES.text(), + optionES.highlighted(), optionES.score(), optionES.collateMatch()))); + entries.add(new PhraseSuggestion.Entry(phraseSuggest.text(), phraseSuggest.offset(), phraseSuggest.length(), + options, null)); + }); + return new PhraseSuggestion(name, suggestionsES.size(), entries); + } + + private static CompletionSuggestion getCompletionSuggestion(String name, + List> suggestionsES, SearchDocumentResponse.EntityCreator entityCreator) { + List> entries = new ArrayList<>(); + suggestionsES.forEach(suggestionES -> { + CompletionSuggest completionSuggest = suggestionES.completion(); + List> options = new ArrayList<>(); + List> optionsES = completionSuggest.options(); + optionsES.forEach(optionES -> { + SearchDocument searchDocument = (optionES.source() != null) ? DocumentAdapters.from(optionES) : null; + T hitEntity = null; + + if (searchDocument != null) { + try { + hitEntity = entityCreator.apply(searchDocument).get(); + } catch (Exception e) { + if (LOGGER.isWarnEnabled()) { + LOGGER.warn("Error creating entity from SearchDocument: " + e.getMessage()); + } + } + } + + Map> contexts = new HashMap<>(); + optionES.contexts().forEach((key, contextList) -> contexts.put(key, + contextList.stream().map(context -> context._get().toString()).collect(Collectors.toSet()))); + + // response from the new client does not have a doc and shardindex as the ScoreDoc from the old client responses + + options.add(new CompletionSuggestion.Entry.Option<>(optionES.text(), null, optionES.score(), + optionES.collateMatch() != null ? optionES.collateMatch() : false, contexts, + new ScoreDoc(optionES.score(), null, null), searchDocument, + hitEntity)); + }); + + entries.add(new CompletionSuggestion.Entry<>(completionSuggest.text(), completionSuggest.offset(), + completionSuggest.length(), options)); + }); + return new CompletionSuggestion<>(name, suggestionsES.size(), entries); + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/TypeUtils.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/TypeUtils.java new file mode 100644 index 0000000..41c61f4 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/TypeUtils.java @@ -0,0 +1,480 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.time.Duration; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.opensearch.client.opensearch._types.*; +import org.opensearch.client.opensearch._types.mapping.FieldType; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch.core.search.BoundaryScanner; +import org.opensearch.client.opensearch.core.search.BuiltinHighlighterType; +import org.opensearch.client.opensearch.core.search.HighlighterEncoder; +import org.opensearch.client.opensearch.core.search.HighlighterFragmenter; +import org.opensearch.client.opensearch.core.search.HighlighterOrder; +import org.opensearch.client.opensearch.core.search.HighlighterTagsSchema; +import org.opensearch.client.opensearch.core.search.HighlighterType; +import org.opensearch.client.opensearch.core.search.ScoreMode; +import org.opensearch.client.opensearch.indices.IndexSettings; +import org.springframework.data.domain.Sort; +import org.springframework.data.elasticsearch.core.RefreshPolicy; +import org.springframework.data.elasticsearch.core.document.Document; +import org.springframework.data.elasticsearch.core.query.GeoDistanceOrder; +import org.springframework.data.elasticsearch.core.query.IndexQuery; +import org.springframework.data.elasticsearch.core.query.IndicesOptions; +import org.springframework.data.elasticsearch.core.query.Order; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.RescorerQuery; +import org.springframework.data.elasticsearch.core.query.UpdateResponse; +import org.springframework.data.elasticsearch.core.reindex.ReindexRequest; +import org.springframework.lang.Nullable; + +/** + * Utility to handle new OpenSearch client type values. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +final class TypeUtils { + + @Nullable + static BoundaryScanner boundaryScanner(@Nullable String value) { + + if (value != null) { + return switch (value.toLowerCase()) { + case "chars" -> BoundaryScanner.Chars; + case "sentence" -> BoundaryScanner.Sentence; + case "word" -> BoundaryScanner.Word; + default -> null; + }; + } + return null; + } + + static Conflicts conflicts(ReindexRequest.Conflicts conflicts) { + return switch (conflicts) { + case ABORT -> Conflicts.Abort; + case PROCEED -> Conflicts.Proceed; + }; + } + + @Nullable + static DistanceUnit distanceUnit(String unit) { + + return switch (unit.toLowerCase()) { + case "in", "inch" -> DistanceUnit.Inches; + case "yd", "yards" -> DistanceUnit.Yards; + case "ft", "feet" -> DistanceUnit.Feet; + case "km", "kilometers" -> DistanceUnit.Kilometers; + case "nm", "nmi" -> DistanceUnit.NauticMiles; + case "mm", "millimeters" -> DistanceUnit.Millimeters; + case "cm", "centimeters" -> DistanceUnit.Centimeters; + case "mi", "miles" -> DistanceUnit.Miles; + case "m", "meters" -> DistanceUnit.Meters; + default -> null; + }; + } + + @Nullable + static FieldType fieldType(String type) { + + for (FieldType fieldType : FieldType.values()) { + + if (fieldType.jsonValue().equals(type)) { + return fieldType; + } + } + return null; + } + + @Nullable + static String toString(@Nullable FieldValue fieldValue) { + + if (fieldValue == null) { + return null; + } + + switch (fieldValue._kind()) { + case Double -> { + return String.valueOf(fieldValue.doubleValue()); + } + case Long -> { + return String.valueOf(fieldValue.longValue()); + } + case Boolean -> { + return String.valueOf(fieldValue.booleanValue()); + } + case String -> { + return fieldValue.stringValue(); + } + case Null -> { + return null; + } + + default -> throw new IllegalStateException("Unexpected value: " + fieldValue._kind()); + } + } + + @Nullable + static Object toObject(@Nullable FieldValue fieldValue) { + + if (fieldValue == null) { + return null; + } + + switch (fieldValue._kind()) { + case Double -> { + return Double.valueOf(fieldValue.doubleValue()); + } + case Long -> { + return Long.valueOf(fieldValue.longValue()); + } + case Boolean -> { + return Boolean.valueOf(fieldValue.booleanValue()); + } + case String -> { + return fieldValue.stringValue(); + } + case Null -> { + return null; + } + + default -> throw new IllegalStateException("Unexpected value: " + fieldValue._kind()); + } + } + + @Nullable + static FieldValue toFieldValue(@Nullable Object fieldValue) { + + if (fieldValue == null) { + return FieldValue.NULL; + } + + if (fieldValue instanceof Boolean b) { + return b ? FieldValue.TRUE : FieldValue.FALSE; + } + + if (fieldValue instanceof String s) { + return FieldValue.of(s); + } + + if (fieldValue instanceof Long l) { + return FieldValue.of(l); + } + + if (fieldValue instanceof Integer i) { + return FieldValue.of((long) i); + } + + if (fieldValue instanceof Double d) { + return FieldValue.of(d); + } + + if (fieldValue instanceof Float f) { + return FieldValue.of((double) f); + } + + throw new IllegalStateException("Unexpected value: " + fieldValue); + } + + @Nullable + static GeoDistanceType geoDistanceType(GeoDistanceOrder.DistanceType distanceType) { + + return switch (distanceType) { + case arc -> GeoDistanceType.Arc; + case plane -> GeoDistanceType.Plane; + }; + + } + + @Nullable + static SortOrder sortOrder(@Nullable Sort.Direction direction) { + + if (direction == null) { + return null; + } + + return switch (direction) { + case ASC -> SortOrder.Asc; + case DESC -> SortOrder.Desc; + }; + + } + + @Nullable + static HighlighterFragmenter highlighterFragmenter(@Nullable String value) { + + if (value != null) { + return switch (value.toLowerCase()) { + case "simple" -> HighlighterFragmenter.Simple; + case "span" -> HighlighterFragmenter.Span; + default -> null; + }; + } + + return null; + } + + @Nullable + static HighlighterOrder highlighterOrder(@Nullable String value) { + + if (value != null) { + if ("score".equals(value.toLowerCase())) { + return HighlighterOrder.Score; + } + } + + return null; + } + + @Nullable + static HighlighterType highlighterType(@Nullable String value) { + + if (value != null) { + return switch (value.toLowerCase()) { + case "unified" -> HighlighterType.of(fn -> fn.builtin(BuiltinHighlighterType.Unified)); + case "plain" -> HighlighterType.of(fn -> fn.builtin(BuiltinHighlighterType.Plain)); + case "fvh" -> HighlighterType.of(fn -> fn.builtin(BuiltinHighlighterType.FastVector)); + default -> null; + }; + } + + return null; + } + + @Nullable + static HighlighterEncoder highlighterEncoder(@Nullable String value) { + + if (value != null) { + return switch (value.toLowerCase()) { + case "default" -> HighlighterEncoder.Default; + case "html" -> HighlighterEncoder.Html; + default -> null; + }; + } + + return null; + } + + @Nullable + static HighlighterTagsSchema highlighterTagsSchema(@Nullable String value) { + + if (value != null) { + if ("styled".equals(value.toLowerCase())) { + return HighlighterTagsSchema.Styled; + } + } + + return null; + } + + @Nullable + static OpType opType(@Nullable IndexQuery.OpType opType) { + + if (opType != null) { + return switch (opType) { + case INDEX -> OpType.Index; + case CREATE -> OpType.Create; + }; + } + return null; + } + + static Refresh refresh(@Nullable RefreshPolicy refreshPolicy) { + + if (refreshPolicy == null) { + return Refresh.False; + } + + return switch (refreshPolicy) { + case IMMEDIATE -> Refresh.True; + case WAIT_UNTIL -> Refresh.WaitFor; + case NONE -> Refresh.False; + }; + } + + @Nullable + static UpdateResponse.Result result(@Nullable Result result) { + + if (result == null) { + return null; + } + + return switch (result) { + case Created -> UpdateResponse.Result.CREATED; + case Updated -> UpdateResponse.Result.UPDATED; + case Deleted -> UpdateResponse.Result.DELETED; + case NotFound -> UpdateResponse.Result.NOT_FOUND; + case NoOp -> UpdateResponse.Result.NOOP; + }; + + } + + @Nullable + static ScoreMode scoreMode(@Nullable RescorerQuery.ScoreMode scoreMode) { + + if (scoreMode == null) { + return null; + } + + return switch (scoreMode) { + case Default -> null; + case Avg -> ScoreMode.Avg; + case Max -> ScoreMode.Max; + case Min -> ScoreMode.Min; + case Total -> ScoreMode.Total; + case Multiply -> ScoreMode.Multiply; + }; + + } + + @Nullable + static SearchType searchType(@Nullable Query.SearchType searchType) { + + if (searchType == null) { + return null; + } + + return switch (searchType) { + case QUERY_THEN_FETCH -> SearchType.QueryThenFetch; + case DFS_QUERY_THEN_FETCH -> SearchType.DfsQueryThenFetch; + }; + + } + + @Nullable + static SortMode sortMode(Order.Mode mode) { + + return switch (mode) { + case min -> SortMode.Min; + case max -> SortMode.Max; + case median -> SortMode.Median; + case avg -> SortMode.Avg; + }; + + } + + @Nullable + static Time time(@Nullable Duration duration) { + + if (duration == null) { + return null; + } + + return Time.of(t -> t.time(duration.toMillis() + "ms")); + } + + @Nullable + static String timeStringMs(@Nullable Duration duration) { + + if (duration == null) { + return null; + } + + return duration.toMillis() + "ms"; + } + + @Nullable + static VersionType versionType( + @Nullable org.springframework.data.elasticsearch.annotations.Document.VersionType versionType) { + + if (versionType != null) { + return switch (versionType) { + case INTERNAL -> VersionType.Internal; + case EXTERNAL -> VersionType.External; + case EXTERNAL_GTE -> VersionType.ExternalGte; + case FORCE -> VersionType.Force; + }; + } + + return null; + } + + static Integer waitForActiveShardsCount(@Nullable String value) { + // values taken from the RHLC implementation + if (value == null) { + return -2; + } else if ("all".equals(value.toUpperCase())) { + return -1; + } else { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Illegale value for waitForActiveShards" + value); + } + } + } + + /** + * Converts a Long to a Float, returning null if the input is null. + * + * @param value the long value + * @return a FLoat with the given value + * @since 5.0 + */ + @Nullable + static Float toFloat(@Nullable Long value) { + return value != null ? Float.valueOf(value) : null; + } + + /** + * Converts a Float to a Long, returning null if the input is null. + * + * @param value the float value + * @return a LOng with the given value + * @since 5.0 + */ + @Nullable + static Long toLong(@Nullable Float value) { + return value != null ? value.longValue() : null; + } + + /** + * @sice 5.1 + */ + @Nullable + public static List expandWildcards(@Nullable EnumSet wildcardStates) { + return (wildcardStates != null && !wildcardStates.isEmpty()) ? wildcardStates.stream() + .map(wildcardState -> ExpandWildcard.valueOf(wildcardState.name().toLowerCase())).collect(Collectors.toList()) + : null; + } + + @Nullable + static TypeMapping typeMapping(@Nullable Document mapping) { + if (mapping != null) { + return JsonpUtils.fromJson(mapping, TypeMapping._DESERIALIZER); + } + return null; + } + + @Nullable + static Document typeMapping(@Nullable TypeMapping typeMapping) { + return (typeMapping != null) ? Document.parse(removePrefixFromJson(typeMapping.toString())) : null; + } + + public static String removePrefixFromJson(String jsonWithPrefix) { + return jsonWithPrefix.substring(jsonWithPrefix.indexOf("{")); + } + + @Nullable + static IndexSettings indexSettings(@Nullable Map settings) { + return settings != null ? JsonpUtils.fromJson(Document.from(settings), IndexSettings._DESERIALIZER) + : null; + } +} diff --git a/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/package-info.java b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/package-info.java new file mode 100644 index 0000000..0f392d1 --- /dev/null +++ b/spring-data-opensearch/src/main/java/org/opensearch/data/client/osc/package-info.java @@ -0,0 +1,23 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains classes that use the new OpenSearch client library (org.opensearch.client:opensearch-java) + * to access OpenSearch. + */ +@org.springframework.lang.NonNullApi +@org.springframework.lang.NonNullFields +package org.opensearch.data.client.osc; diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/NestedObjectORHLCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/NestedObjectORHLCIntegrationTests.java index a1bdf52..0aee6b6 100644 --- a/spring-data-opensearch/src/test/java/org/opensearch/data/client/NestedObjectORHLCIntegrationTests.java +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/NestedObjectORHLCIntegrationTests.java @@ -12,6 +12,7 @@ import static org.opensearch.index.query.QueryBuilders.*; import org.apache.lucene.search.join.ScoreMode; +import org.jetbrains.annotations.NotNull; import org.opensearch.data.client.junit.jupiter.OpenSearchRestTemplateConfiguration; import org.opensearch.data.client.orhlc.NativeSearchQueryBuilder; import org.springframework.context.annotation.Bean; @@ -20,7 +21,6 @@ import org.springframework.data.elasticsearch.NestedObjectIntegrationTests; import org.springframework.data.elasticsearch.core.query.Query; import org.springframework.data.elasticsearch.utils.IndexNameProvider; -import org.springframework.lang.NonNull; import org.springframework.test.context.ContextConfiguration; @ContextConfiguration(classes = {NestedObjectORHLCIntegrationTests.Config.class}) @@ -34,7 +34,7 @@ IndexNameProvider indexNameProvider() { } } - @NonNull + @NotNull protected Query getNestedQuery1() { return new NativeSearchQueryBuilder() .withQuery( // @@ -47,7 +47,7 @@ protected Query getNestedQuery1() { .build(); } - @NonNull + @NotNull protected Query getNestedQuery2() { return new NativeSearchQueryBuilder() .withQuery( // @@ -63,7 +63,7 @@ protected Query getNestedQuery2() { .build(); } - @NonNull + @NotNull protected Query getNestedQuery3() { return new NativeSearchQueryBuilder() .withQuery( // @@ -75,7 +75,7 @@ protected Query getNestedQuery3() { .build(); } - @NonNull + @NotNull protected Query getNestedQuery4() { return new NativeSearchQueryBuilder() .withQuery( // diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/NestedObjectOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/NestedObjectOSCIntegrationTests.java new file mode 100644 index 0000000..2ccabc0 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/NestedObjectOSCIntegrationTests.java @@ -0,0 +1,95 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client; + +import static org.opensearch.client.opensearch._types.query_dsl.QueryBuilders.bool; +import static org.opensearch.client.opensearch._types.query_dsl.QueryBuilders.nested; +import static org.opensearch.data.client.osc.Queries.termQueryAsQuery; + +import org.jetbrains.annotations.NotNull; +import org.opensearch.client.opensearch._types.query_dsl.ChildScoreMode; +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.opensearch.data.client.osc.NativeQuery; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.NestedObjectIntegrationTests; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {NestedObjectOSCIntegrationTests.Config.class}) +public class NestedObjectOSCIntegrationTests extends NestedObjectIntegrationTests { + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("nestedobject-os"); + } + } + + @Override + protected @NotNull Query getNestedQuery1() { + return NativeQuery.builder().withQuery( // + nested() // + .path("car") // + .query(q -> q.bool(b -> b // + .must(termQueryAsQuery("car.name", "saturn")) // + .must(termQueryAsQuery("car.model", "imprezza")) // + )) // + .scoreMode(ChildScoreMode.None) // + )// + .build(); + } + + @Override + protected @NotNull Query getNestedQuery2() { + return NativeQuery.builder().withQuery( // + bool() // + .must(q -> q.nested(n -> n // + .path("girlFriends") // + .query(termQueryAsQuery("girlFriends.type", "temp")) // + .scoreMode(ChildScoreMode.None) // + ) // + ) // + .must(q -> q.nested(n -> n // + .path("girlFriends.cars") // + .query(termQueryAsQuery("girlFriends.cars.name", "Ford".toLowerCase())) // + .scoreMode(ChildScoreMode.None) // + ) // + ) // + ) // + .build(); + } + + @Override + protected @NotNull Query getNestedQuery3() { + return NativeQuery.builder().withQuery( // + nested() // + .path("books") // + .query(bool().must(termQueryAsQuery("books.name", "java")).build().toQuery() // + ) // + .scoreMode(ChildScoreMode.None) // + )// + .build(); + } + + @Override + protected @NotNull Query getNestedQuery4() { + return NativeQuery.builder().withQuery( // + nested() // + .path("buckets") // + .query(termQueryAsQuery("buckets.1", "test3")) // + .scoreMode(ChildScoreMode.None) // + )// + .build(); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/AuditingOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/AuditingOSCIntegrationTests.java new file mode 100644 index 0000000..e2ec4e9 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/AuditingOSCIntegrationTests.java @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.config; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.config.AuditingIntegrationTests; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {AuditingOSCIntegrationTests.Config.class}) +public class AuditingOSCIntegrationTests extends AuditingIntegrationTests { + + @Import({OpenSearchTemplateConfiguration.class, AuditingIntegrationTests.Config.class}) + static class Config {} +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/configuration/OpenSearchConfigurationOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/configuration/OpenSearchConfigurationOSCIntegrationTests.java new file mode 100644 index 0000000..b62becb --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/configuration/OpenSearchConfigurationOSCIntegrationTests.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.config.configuration; + +import static org.assertj.core.api.Assertions.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.opensearch.client.RestClient; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.data.client.osc.OpenSearchConfiguration; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.lang.NonNull; +import org.springframework.lang.Nullable; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +/** + * Tests for {@link OpenSearchConfiguration}. + */ +@ExtendWith(SpringExtension.class) +@ContextConfiguration +public class OpenSearchConfigurationOSCIntegrationTests { + + /* + * using a repository with an entity that is set to createIndex = false as we have no elastic running for this test + * and just check that all the necessary beans are created. + */ + @Autowired private RestClient restClient; + @Autowired private OpenSearchClient opensearchClient; + @Autowired private ElasticsearchOperations elasticsearchOperations; + + @Autowired + private CreateIndexFalseRepository repository; + + @Configuration + @EnableElasticsearchRepositories( + basePackages = {"org.opensearch.data.client.config.configuration"}, + considerNestedRepositories = true) + static class Config extends OpenSearchConfiguration { + + @NonNull + @Override + public ClientConfiguration clientConfiguration() { + return ClientConfiguration.builder() // + .connectedTo("localhost:9200") // + .build(); + } + } + + @Test + public void providesRequiredBeans() { + assertThat(restClient).isNotNull(); + assertThat(opensearchClient).isNotNull(); + assertThat(elasticsearchOperations).isNotNull(); + assertThat(repository).isNotNull(); + } + + @Document(indexName = "test-index-config-configuration", createIndex = false) + static class CreateIndexFalseEntity { + + @Nullable + @Id + private String id; + } + + interface CreateIndexFalseRepository extends ElasticsearchRepository {} +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/nested/EnableNestedRepositoriesOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/nested/EnableNestedRepositoriesOSCIntegrationTests.java new file mode 100644 index 0000000..59aa690 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/nested/EnableNestedRepositoriesOSCIntegrationTests.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.config.nested; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.config.nested.EnableNestedRepositoriesIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {EnableNestedRepositoriesOSCIntegrationTests.Config.class}) +public class EnableNestedRepositoriesOSCIntegrationTests extends EnableNestedRepositoriesIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = { + "org.opensearch.data.client.config.nested", + "org.springframework.data.elasticsearch.config.nested" + }, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("nested-repositories-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/notnested/EnableRepositoriesOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/notnested/EnableRepositoriesOSCIntegrationTests.java new file mode 100644 index 0000000..8929495 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/config/notnested/EnableRepositoriesOSCIntegrationTests.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.config.notnested; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.config.notnested.EnableRepositoriesIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; + +public class EnableRepositoriesOSCIntegrationTests extends EnableRepositoriesIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories(basePackages = {"org.springframework.data.elasticsearch.config.notnested"}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("repositories-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/InnerHitsOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/InnerHitsOSCIntegrationTests.java new file mode 100644 index 0000000..f3a629d --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/InnerHitsOSCIntegrationTests.java @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core; + +import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; +import org.opensearch.client.opensearch._types.query_dsl.NestedQuery; +import org.opensearch.client.opensearch.core.search.InnerHits; +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.opensearch.data.client.osc.NativeQuery; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.InnerHitsIntegrationTests; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {InnerHitsOSCIntegrationTests.Config.class}) +public class InnerHitsOSCIntegrationTests extends InnerHitsIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("innerhits-os"); + } + } + + @Override + protected Query buildQueryForInnerHits( + String innerHitName, String nestedQueryPath, String matchField, String matchValue) { + return NativeQuery.builder() // + .withQuery(q -> q.nested( // + NestedQuery.of(n -> n // + .path(nestedQueryPath) // + .query(q2 -> q2.match( // + MatchQuery.of(m -> m // + .field(matchField) // + .query(fv -> fv.stringValue(matchValue)) // + ))) // + .innerHits(InnerHits.of(ih -> ih.name(innerHitName))) // + ))) // + .build(); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/LogEntityOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/LogEntityOSCIntegrationTests.java new file mode 100644 index 0000000..1fbb10b --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/LogEntityOSCIntegrationTests.java @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core; + +import org.opensearch.client.json.JsonData; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.opensearch.data.client.osc.NativeQuery; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.LogEntityIntegrationTests; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {LogEntityOSCIntegrationTests.Config.class}) +public class LogEntityOSCIntegrationTests extends LogEntityIntegrationTests { + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("logentity-os"); + } + } + + @Override + public Query termQueryForIp(String ip) { + return NativeQuery.builder() // + .withQuery(qb -> qb // + .term(tq -> tq // + .field("ip") // + .value(FieldValue.of(ip)))) + .build(); + } + + @Override + public Query rangeQueryForIp(String from, String to) { + return NativeQuery.builder() // + .withQuery(qb -> qb // + .range(rqb -> rqb // + .field("ip") // + .gte(JsonData.of(from))// + .lte(JsonData.of(to))// + )).build(); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/ReindexOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/ReindexOSCIntegrationTests.java new file mode 100644 index 0000000..26250fa --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/ReindexOSCIntegrationTests.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core; + + +import static org.opensearch.data.client.osc.Queries.termQueryAsQuery; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.opensearch.data.client.osc.NativeQuery; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.ReindexIntegrationTests; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {ReindexOSCIntegrationTests.Config.class}) +public class ReindexOSCIntegrationTests extends ReindexIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("reindex-os"); + } + } + + @Override + protected Query queryForId(String id) { + return NativeQuery.builder().withQuery(termQueryAsQuery("_id", id)).build(); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/SearchAsYouTypeOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/SearchAsYouTypeOSCIntegrationTests.java new file mode 100644 index 0000000..f3c0e3e --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/SearchAsYouTypeOSCIntegrationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core; + +import org.opensearch.client.opensearch._types.query_dsl.TextQueryType; +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.opensearch.data.client.osc.NativeQuery; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.SearchAsYouTypeIntegrationTests; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {SearchAsYouTypeOSCIntegrationTests.Config.class}) +public class SearchAsYouTypeOSCIntegrationTests extends SearchAsYouTypeIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("search-as-you-type-os"); + } + } + + @Override + protected Query buildMultiMatchQuery(String text) { + return NativeQuery.builder() // + .withQuery(q -> q // + .multiMatch(mm -> mm // + .query(text) // + .fields("suggest", "suggest._2gram", "suggest._3gram", "suggest._4gram") // + .type(TextQueryType.BoolPrefix))) + .build(); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/SourceFilterOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/SourceFilterOSCIntegrationTests.java new file mode 100644 index 0000000..0eda885 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/SourceFilterOSCIntegrationTests.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.SourceFilterIntegrationTests; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {SourceFilterOSCIntegrationTests.Config.class}) +public class SourceFilterOSCIntegrationTests extends SourceFilterIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("source-filter-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/aggregation/AggregationOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/aggregation/AggregationOSCIntegrationTests.java new file mode 100644 index 0000000..b865bcc --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/aggregation/AggregationOSCIntegrationTests.java @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.aggregation; + +import static org.assertj.core.api.Assertions.*; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.opensearch.client.opensearch._types.aggregations.Aggregate; +import org.opensearch.client.opensearch._types.aggregations.Aggregation; +import org.opensearch.client.opensearch._types.aggregations.StatsBucketAggregate; +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.opensearch.data.client.osc.NativeQuery; +import org.opensearch.data.client.osc.OpenSearchAggregation; +import org.opensearch.data.client.osc.OpenSearchAggregations; +import org.opensearch.data.client.osc.Queries; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.AggregationsContainer; +import org.springframework.data.elasticsearch.core.aggregation.AggregationIntegrationTests; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {AggregationOSCIntegrationTests.Config.class}) +public class AggregationOSCIntegrationTests extends AggregationIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories(considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("aggs-es7"); + } + } + + protected Query getTermsAggsQuery(String aggsName, String aggsField) { + return Queries.getTermsAggsQuery(aggsName, aggsField); + } + + @Override + protected void assertThatAggsHasResult(AggregationsContainer aggregationsContainer, String aggsName) { + List aggregations = ((OpenSearchAggregations) aggregationsContainer).aggregations(); + List aggNames = aggregations.stream() // + .map(OpenSearchAggregation::aggregation) // + .map(org.opensearch.data.client.osc.Aggregation::getName) // + .collect(Collectors.toList()); + assertThat(aggNames).contains(aggsName); + + } + + protected Query getPipelineAggsQuery( + String aggsName, String aggsField, String aggsNamePipeline, String bucketsPath) { + return NativeQuery.builder() // + .withQuery(Queries.matchAllQueryAsQuery()) // + .withAggregation(aggsName, Aggregation.of(a -> a // + .terms(ta -> ta.field(aggsField)))) // + .withAggregation(aggsNamePipeline, Aggregation.of(a -> a // + .statsBucket(sb -> sb.bucketsPath(bp -> bp.single(bucketsPath))))) // + .withMaxResults(0) // + .build(); + + } + + protected void assertThatPipelineAggsAreCorrect( + AggregationsContainer aggregationsContainer, String aggsName, String pipelineAggsName) { + Map aggregates = ((OpenSearchAggregations) aggregationsContainer).aggregations().stream() // + .map(OpenSearchAggregation::aggregation) // + .collect(Collectors.toMap(org.opensearch.data.client.osc.Aggregation::getName, + org.opensearch.data.client.osc.Aggregation::getAggregate)); + + assertThat(aggregates).containsKey(aggsName); + Aggregate aggregate = aggregates.get(pipelineAggsName); + assertThat(aggregate.isStatsBucket()).isTrue(); + StatsBucketAggregate statsBucketAggregate = aggregate.statsBucket(); + assertThat(statsBucketAggregate.min()).isEqualTo(1.0); + assertThat(statsBucketAggregate.max()).isEqualTo(3.0); + assertThat(statsBucketAggregate.avg()).isEqualTo(2.0); + assertThat(statsBucketAggregate.sum()).isEqualTo(6.0); + assertThat(statsBucketAggregate.count()).isEqualTo(3L); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/cluster/ClusterOperationsOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/cluster/ClusterOperationsOSCIntegrationTests.java new file mode 100644 index 0000000..ab5212a --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/cluster/ClusterOperationsOSCIntegrationTests.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.cluster; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.data.elasticsearch.core.cluster.ClusterOperationsIntegrationTests; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {OpenSearchTemplateConfiguration.class}) +public class ClusterOperationsOSCIntegrationTests extends ClusterOperationsIntegrationTests {} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/event/CallbackOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/event/CallbackOSCIntegrationTests.java new file mode 100644 index 0000000..ae00973 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/event/CallbackOSCIntegrationTests.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.event; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.event.CallbackIntegrationTests; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {CallbackOSCIntegrationTests.Config.class}) +class CallbackOSCIntegrationTests extends CallbackIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class, CallbackIntegrationTests.Config.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("callback-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/geo/GeoJsonOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/geo/GeoJsonOSCIntegrationTests.java new file mode 100644 index 0000000..0e386c1 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/geo/GeoJsonOSCIntegrationTests.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.geo; + +import org.junit.jupiter.api.DisplayName; +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.geo.GeoJsonIntegrationTests; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {GeoJsonOSCIntegrationTests.Config.class}) +@DisplayName("GeoJson integration test with OpenSearchClient") +public class GeoJsonOSCIntegrationTests extends GeoJsonIntegrationTests { + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("geojson-integration-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/geo/GeoOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/geo/GeoOSCIntegrationTests.java new file mode 100644 index 0000000..9ec8821 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/geo/GeoOSCIntegrationTests.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.geo; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.opensearch.data.client.osc.NativeQuery; +import org.opensearch.data.client.osc.Queries; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.geo.GeoIntegrationTests; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.data.elasticsearch.utils.geohash.Geohash; +import org.springframework.data.elasticsearch.utils.geohash.Rectangle; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {GeoOSCIntegrationTests.Config.class}) +public class GeoOSCIntegrationTests extends GeoIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories(considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("geo-integration-os"); + } + } + + @Override + protected Query nativeQueryForBoundingBox(String fieldName, double top, double left, double bottom, double right) { + return NativeQuery.builder() // + .withQuery(q -> q // + .geoBoundingBox(bb -> bb // + .field(fieldName) // + .boundingBox(gb -> gb // + .tlbr(tlbr -> tlbr // + .topLeft(tl -> tl // + .latlon(Queries.latLon(top, left))) + .bottomRight(br -> br // + .latlon(Queries.latLon(bottom, right))))))) + .build(); + + } + + @Override + protected Query nativeQueryForBoundingBox(String fieldName, String geoHash) { + Rectangle rect = Geohash.toBoundingBox(geoHash); + return nativeQueryForBoundingBox(fieldName, rect.getMaxY(), rect.getMinX(), rect.getMinY(), rect.getMaxX()); + + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/index/IndexTemplateOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/index/IndexTemplateOSCIntegrationTests.java new file mode 100644 index 0000000..093d29b --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/index/IndexTemplateOSCIntegrationTests.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.index; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.data.elasticsearch.core.index.IndexTemplateIntegrationTests; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {OpenSearchTemplateConfiguration.class}) +public class IndexTemplateOSCIntegrationTests extends IndexTemplateIntegrationTests {} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/index/MappingBuilderOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/index/MappingBuilderOSCIntegrationTests.java new file mode 100644 index 0000000..9ab34c6 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/index/MappingBuilderOSCIntegrationTests.java @@ -0,0 +1,104 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.index; + +import java.util.List; +import java.util.Map; +import org.junit.Ignore; +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.annotations.Dynamic; +import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.FieldType; +import org.springframework.data.elasticsearch.core.IndexOperations; +import org.springframework.data.elasticsearch.core.index.MappingBuilderIntegrationTests; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.lang.Nullable; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {MappingBuilderOSCIntegrationTests.Config.class}) +public class MappingBuilderOSCIntegrationTests extends MappingBuilderIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("mappingbuilder-os"); + } + } + + @Ignore + @Override + public void shouldWriteDenseVectorFieldMapping() { + // see please https://github.com/opensearch-project/OpenSearch/pull/3659 + } + + @Ignore + @Override + public void shouldWriteRuntimeFields() { + // Not supported by OpenSearch + } + + @Ignore + @Override + public void shouldWriteWildcardFieldMapping() { + // Not supported by OpenSearch + } + + @Override + public void shouldWriteDynamicMapping() { + IndexOperations indexOps = operations.indexOps(DynamicMappingEntity.class); + indexOps.createWithMapping(); + } + + @Document(indexName = "#{@indexNameProvider.indexName()}", dynamic = Dynamic.FALSE) + static class DynamicMappingEntity { + + @Nullable + @Field(type = FieldType.Object) // + private Map objectInherit; + + @Nullable + @Field(type = FieldType.Object, dynamic = Dynamic.FALSE) // + private Map objectFalse; + + @Nullable + @Field(type = FieldType.Object, dynamic = Dynamic.STRICT) // + private Map objectStrict; + + @Nullable + @Field(type = FieldType.Object, dynamic = Dynamic.RUNTIME) // + private Map objectRuntime; + + @Nullable + @Field(type = FieldType.Nested) // + private List> nestedObjectInherit; + + @Nullable + @Field(type = FieldType.Nested, dynamic = Dynamic.FALSE) // + private List> nestedObjectFalse; + + @Nullable + @Field(type = FieldType.Nested, dynamic = Dynamic.TRUE) // + private List> nestedObjectTrue; + + @Nullable + @Field(type = FieldType.Nested, dynamic = Dynamic.STRICT) // + private List> nestedObjectStrict; + + @Nullable + @Field(type = FieldType.Nested, dynamic = Dynamic.RUNTIME) // + private List> nestedObjectRuntime; + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/indices/IndexOperationsOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/indices/IndexOperationsOSCIntegrationTests.java new file mode 100644 index 0000000..943b32e --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/indices/IndexOperationsOSCIntegrationTests.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.indices; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.indices.IndexOperationsIntegrationTests; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {IndexOperationsOSCIntegrationTests.Config.class}) +public class IndexOperationsOSCIntegrationTests extends IndexOperationsIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("indexoperations-es7"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/mapping/EntityCustomConversionOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/mapping/EntityCustomConversionOSCIntegrationTests.java new file mode 100644 index 0000000..e3e571f --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/mapping/EntityCustomConversionOSCIntegrationTests.java @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.mapping; + +import java.util.Arrays; +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.convert.ElasticsearchCustomConversions; +import org.springframework.data.elasticsearch.core.mapping.EntityCustomConversionIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {EntityCustomConversionOSCIntegrationTests.Config.class}) +public class EntityCustomConversionOSCIntegrationTests extends EntityCustomConversionIntegrationTests { + + @Configuration + @Import({EntityCustomConversionIntegrationTests.Config.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.core.mapping"}, + considerNestedRepositories = true) + static class Config extends OpenSearchTemplateConfiguration { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("entity-customconversions-operations-os"); + } + + @Override + public ElasticsearchCustomConversions elasticsearchCustomConversions() { + return new ElasticsearchCustomConversions( + Arrays.asList(new EntityToMapConverter(), new MapToEntityConverter())); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/mapping/FieldNamingStrategyOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/mapping/FieldNamingStrategyOSCIntegrationTests.java new file mode 100644 index 0000000..bb69217 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/mapping/FieldNamingStrategyOSCIntegrationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.mapping; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.opensearch.data.client.osc.NativeQuery; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.elasticsearch.core.mapping.FieldNamingStrategyIntegrationTests; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.data.mapping.model.FieldNamingStrategy; +import org.springframework.data.mapping.model.SnakeCaseFieldNamingStrategy; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {FieldNamingStrategyOSCIntegrationTests.Config.class}) +public class FieldNamingStrategyOSCIntegrationTests extends FieldNamingStrategyIntegrationTests { + + @Configuration + static class Config extends OpenSearchTemplateConfiguration { + + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("fieldnaming-strategy-os"); + } + + @Override + protected FieldNamingStrategy fieldNamingStrategy() { + return new SnakeCaseFieldNamingStrategy(); + } + } + + @Override + protected Query nativeMatchQuery(String fieldName, String value) { + return NativeQuery.builder() // + .withQuery(q -> q.match(mq -> mq.field(fieldName).query(fv -> fv.stringValue(value)))).build(); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/paginating/SearchAfterOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/paginating/SearchAfterOSCIntegrationTests.java new file mode 100644 index 0000000..b4b09b9 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/paginating/SearchAfterOSCIntegrationTests.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.paginating; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.paginating.SearchAfterIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {SearchAfterOSCIntegrationTests.Config.class}) +public class SearchAfterOSCIntegrationTests extends SearchAfterIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories(considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("search-after-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/query/CriteriaQueryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/query/CriteriaQueryOSCIntegrationTests.java new file mode 100644 index 0000000..553ea8c --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/query/CriteriaQueryOSCIntegrationTests.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.query; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.query.CriteriaQueryIntegrationTests; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {CriteriaQueryOSCIntegrationTests.Config.class}) +public class CriteriaQueryOSCIntegrationTests extends CriteriaQueryIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("criteria-query-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/query/NativeQueryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/query/NativeQueryOSCIntegrationTests.java new file mode 100644 index 0000000..12f3cac --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/query/NativeQueryOSCIntegrationTests.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.core.query; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.query.NativeQueryIntegrationTests; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +/** + * @author Peter-Josef Meisch + */ +@ContextConfiguration(classes = { NativeQueryOSCIntegrationTests.Config.class }) +public class NativeQueryOSCIntegrationTests extends NativeQueryIntegrationTests { + @Configuration + @Import({ OpenSearchTemplateConfiguration.class }) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("criteria-os"); + } + } + +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/routing/RoutingOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/routing/RoutingOSCIntegrationTests.java new file mode 100644 index 0000000..d0b58de --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/routing/RoutingOSCIntegrationTests.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.routing; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.routing.RoutingIntegrationTests; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {RoutingOSCIntegrationTests.Config.class}) +public class RoutingOSCIntegrationTests extends RoutingIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("routing-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/suggest/CompletionOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/suggest/CompletionOSCIntegrationTests.java new file mode 100644 index 0000000..46e7797 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/core/suggest/CompletionOSCIntegrationTests.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.core.suggest; + +import org.opensearch.client.opensearch.core.search.FieldSuggester; +import org.opensearch.client.opensearch.core.search.SuggestFuzziness; +import org.opensearch.client.opensearch.core.search.Suggester; +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.opensearch.data.client.osc.NativeQuery; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.suggest.CompletionIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {CompletionOSCIntegrationTests.Config.class}) +public class CompletionOSCIntegrationTests extends CompletionIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories(considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("completion-es7"); + } + } + + @Override + protected Query getSuggestQuery(String suggestionName, String fieldName, String prefix) { + return NativeQuery.builder() // + .withSuggester(Suggester.of(s -> s // + .suggesters(suggestionName, FieldSuggester.of(fs -> fs // + .prefix(prefix)// + .completion(cs -> cs // + .field(fieldName) // + .fuzzy(SuggestFuzziness.of(f -> f // + .fuzziness("AUTO") // + .minLength(3) // + .prefixLength(1) // + .transpositions(true) // + .unicodeAware(false))))// + ))) // + ).build(); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/immutable/ImmutableRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/immutable/ImmutableRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..a8e323f --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/immutable/ImmutableRepositoryOSCIntegrationTests.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.immutable; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.immutable.ImmutableRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = ImmutableRepositoryOSCIntegrationTests.Config.class) +public class ImmutableRepositoryOSCIntegrationTests extends ImmutableRepositoryIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.immutable"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("immutable-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/junit/jupiter/OpenSearchTemplateConfiguration.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/junit/jupiter/OpenSearchTemplateConfiguration.java new file mode 100644 index 0000000..5b7a769 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/junit/jupiter/OpenSearchTemplateConfiguration.java @@ -0,0 +1,79 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.junit.jupiter; + +import static org.springframework.util.StringUtils.*; + +import java.time.Duration; +import org.opensearch.data.client.osc.OpenSearchConfiguration; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate; +import org.springframework.data.elasticsearch.core.RefreshPolicy; +import org.springframework.data.elasticsearch.junit.jupiter.ClusterConnectionInfo; + +/** + * Configuration for Spring Data Elasticsearch tests using an {@link ElasticsearchTemplate}. + * + * @author Peter-Josef Meisch + * @since 4.4 + */ +@Configuration +public class OpenSearchTemplateConfiguration extends OpenSearchConfiguration { + + @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") + @Autowired private ClusterConnectionInfo clusterConnectionInfo; + + @Override + public ClientConfiguration clientConfiguration() { + String elasticsearchHostPort = clusterConnectionInfo.getHost() + ':' + clusterConnectionInfo.getHttpPort(); + + ClientConfiguration.TerminalClientConfigurationBuilder configurationBuilder = ClientConfiguration.builder() + .connectedTo(elasticsearchHostPort); + + String proxy = System.getenv("DATAES_ELASTICSEARCH_PROXY"); + + if (proxy != null) { + configurationBuilder = configurationBuilder.withProxy(proxy); + } + + if (clusterConnectionInfo.isUseSsl()) { + configurationBuilder = ((ClientConfiguration.MaybeSecureClientConfigurationBuilder) configurationBuilder) + .usingSsl(); + } + + String user = System.getenv("DATAES_ELASTICSEARCH_USER"); + String password = System.getenv("DATAES_ELASTICSEARCH_PASSWORD"); + + if (hasText(user) && hasText(password)) { + configurationBuilder.withBasicAuth(user, password); + } + + // noinspection UnnecessaryLocalVariable + ClientConfiguration clientConfiguration = configurationBuilder // + .withConnectTimeout(Duration.ofSeconds(20)) // + .withSocketTimeout(Duration.ofSeconds(20)) // + .build(); + + return clientConfiguration; + } + + @Override + protected RefreshPolicy refreshPolicy() { + return RefreshPolicy.IMMEDIATE; + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/AutoCloseableOpenSearchClientTest.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/AutoCloseableOpenSearchClientTest.java new file mode 100644 index 0000000..70cab21 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/AutoCloseableOpenSearchClientTest.java @@ -0,0 +1,50 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.mockito.Mockito.*; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.client.RestClient; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.rest_client.RestClientTransport; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +@ExtendWith(MockitoExtension.class) +class AutoCloseableOpenSearchClientTest { + + @Mock private RestClient restClient; + @Mock private JsonpMapper jsonMapper; + + @Test // #1973 + @DisplayName("should close the RestClient") + void shouldCloseTheRestClient() throws Exception { + + OpenSearchTransport transport = new RestClientTransport(restClient, jsonMapper); + // noinspection EmptyTryBlock + try (AutoCloseableOpenSearchClient ignored = new AutoCloseableOpenSearchClient(transport)) {} + + verify(restClient).close(); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/CriteriaQueryMappingUnitTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/CriteriaQueryMappingUnitTests.java new file mode 100644 index 0000000..03096b4 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/CriteriaQueryMappingUnitTests.java @@ -0,0 +1,488 @@ +/* + * Copyright 2019-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.JsonUtils.*; +import static org.skyscreamer.jsonassert.JSONAssert.*; + +import java.time.LocalDate; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import org.assertj.core.api.SoftAssertions; +import org.json.JSONException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.springframework.core.convert.support.GenericConversionService; +import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.annotations.DateFormat; +import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.FieldType; +import org.springframework.data.elasticsearch.annotations.InnerField; +import org.springframework.data.elasticsearch.annotations.MultiField; +import org.springframework.data.elasticsearch.core.convert.MappingElasticsearchConverter; +import org.springframework.data.elasticsearch.core.geo.GeoJson; +import org.springframework.data.elasticsearch.core.geo.GeoJsonPoint; +import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext; +import org.springframework.data.elasticsearch.core.query.Criteria; +import org.springframework.data.elasticsearch.core.query.CriteriaQuery; +import org.springframework.data.elasticsearch.core.query.FetchSourceFilterBuilder; +import org.springframework.data.elasticsearch.core.query.Query; +import org.springframework.data.elasticsearch.core.query.SourceFilter; +import org.springframework.lang.Nullable; + +/** + * Tests for the mapping of {@link CriteriaQuery} by a {@link MappingElasticsearchConverter}. In the same package as + * {@link org.opensearch.data.client.osc.CriteriaQueryProcessor} as this is needed to get the String + * representation to assert. + * + * @author Peter-Josef Meisch + * @author Sascha Woo + * @author vdisk + */ +public class CriteriaQueryMappingUnitTests { + + private JsonpMapper mapper = new JacksonJsonpMapper(); + + MappingElasticsearchConverter mappingElasticsearchConverter; + + // region setup + @BeforeEach + void setUp() { + SimpleElasticsearchMappingContext mappingContext = new SimpleElasticsearchMappingContext(); + mappingContext.setInitialEntitySet(Collections.singleton(Person.class)); + mappingContext.afterPropertiesSet(); + + mappingElasticsearchConverter = new MappingElasticsearchConverter(mappingContext, new GenericConversionService()); + mappingElasticsearchConverter.afterPropertiesSet(); + + } + // endregion + + // region tests + @Test // DATAES-716 + void shouldMapNamesAndConvertValuesInCriteriaQuery() throws JSONException { + + // use POJO properties and types in the query building + CriteriaQuery criteriaQuery = new CriteriaQuery( // + new Criteria("birthDate") // + .between(LocalDate.of(1989, 11, 9), LocalDate.of(1990, 11, 9)) // + .or("birthDate").is(LocalDate.of(2019, 12, 28)) // + ); + + // mapped field name and converted parameter + var expected = """ + { + "bool": { + "should": [ + { + "range": { + "birth-date": { + "gte": "09.11.1989", + "lte": "09.11.1990" + } + } + }, + { + "query_string": { + "default_operator": "and", + "fields": [ + "birth-date" + ], + "query": "28.12.2019" + } + } + ] + } + } + """; + + mappingElasticsearchConverter.updateQuery(criteriaQuery, Person.class); + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteriaQuery.getCriteria()), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // #1668 + void shouldMapNamesAndConvertValuesInCriteriaQueryForSubCriteria() throws JSONException { + + // use POJO properties and types in the query building + CriteriaQuery criteriaQuery = new CriteriaQuery( // + Criteria.or().subCriteria(Criteria.where("birthDate") // + .between(LocalDate.of(1989, 11, 9), LocalDate.of(1990, 11, 9))) // + .subCriteria(Criteria.where("birthDate").is(LocalDate.of(2019, 12, 28))) // + ); + + // mapped field name and converted parameter + String expected = """ + { + "bool": { + "should": [ + { + "bool": { + "must": [ + { + "range": { + "birth-date": { + "gte": "09.11.1989", + "lte": "09.11.1990" + } + } + } + ] + } + }, + { + "bool": { + "must": [ + { + "query_string": { + "default_operator": "and", + "fields": [ + "birth-date" + ], + "query": "28.12.2019" + } + } + ] + } + } + ] + } + } + """; + + mappingElasticsearchConverter.updateQuery(criteriaQuery, Person.class); + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteriaQuery.getCriteria()), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // #1668 + void shouldMapNamesAndConvertValuesInCriteriaQueryForSubCriteriaWithDate() throws JSONException { + // use POJO properties and types in the query building + CriteriaQuery criteriaQuery = new CriteriaQuery( // + Criteria.or().subCriteria(Criteria.where("birthDate") // + .between(LocalDate.of(1989, 11, 9), LocalDate.of(1990, 11, 9))) // + .subCriteria(Criteria.where("createdDate").is(new Date(383745721653L))) // + ); + + // mapped field name and converted parameter + String expected = """ + { + "bool": { + "should": [ + { + "bool": { + "must": [ + { + "range": { + "birth-date": { + "gte": "09.11.1989", + "lte": "09.11.1990" + } + } + } + ] + } + }, + { + "bool": { + "must": [ + { + "query_string": { + "default_operator": "and", + "fields": [ + "created-date" + ], + "query": "383745721653" + } + } + ] + } + } + ] + } + }"""; + + mappingElasticsearchConverter.updateQuery(criteriaQuery, Person.class); + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteriaQuery.getCriteria()), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // DATAES-706 + void shouldMapNamesAndValuesInSubCriteriaQuery() throws JSONException { + + CriteriaQuery criteriaQuery = new CriteriaQuery( // + new Criteria("firstName").matches("John") // + .subCriteria(new Criteria("birthDate") // + .between(LocalDate.of(1989, 11, 9), LocalDate.of(1990, 11, 9)) // + .or("birthDate").is(LocalDate.of(2019, 12, 28)))); + + String expected = """ + { + "bool": { + "must": [ + { + "match": { + "first-name": { + "query": "John" + } + } + }, + { + "bool": { + "should": [ + { + "range": { + "birth-date": { + "gte": "09.11.1989", + "lte": "09.11.1990" + } + } + }, + { + "query_string": { + "default_operator": "and", + "fields": [ + "birth-date" + ], + "query": "28.12.2019" + } + } + ] + } + } + ] + } + }"""; + + mappingElasticsearchConverter.updateQuery(criteriaQuery, Person.class); + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteriaQuery.getCriteria()), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // DATAES-931 + @DisplayName("should map names in GeoJson query") + void shouldMapNamesInGeoJsonQuery() throws JSONException { + + GeoJsonPoint geoJsonPoint = GeoJsonPoint.of(1.2, 3.4); + CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria("geoShapeField").intersects(geoJsonPoint)); + + String expected = """ + { + "geo_shape": { + "geo-shape-field": { + "shape": { + "type": "Point", + "coordinates": [ + 1.2, + 3.4 + ] + }, + "relation": "intersects" + } + } + } + """; + + mappingElasticsearchConverter.updateQuery(criteriaQuery, GeoShapeEntity.class); + var queryString = queryToJson(CriteriaFilterProcessor.createQuery(criteriaQuery.getCriteria()).get(), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // #1753 + @DisplayName("should map names and value in nested entities") + void shouldMapNamesAndValueInNestedEntities() throws JSONException { + + String expected = """ + { + "bool": { + "must": [ + { + "nested": { + "path": "per-sons", + "query": { + "query_string": { + "default_operator": "and", + "fields": [ + "per-sons.birth-date" + ], + "query": "03.10.1999" + } + }, + "score_mode": "avg" + } + } + ] + } + } + """; + + CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria("persons.birthDate").is(LocalDate.of(1999, 10, 3))); + mappingElasticsearchConverter.updateQuery(criteriaQuery, House.class); + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteriaQuery.getCriteria()), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // #1753 + @DisplayName("should map names and value in nested entities with sub-fields") + void shouldMapNamesAndValueInNestedEntitiesWithSubfields() throws JSONException { + + String expected = """ + { + "bool": { + "must": [ + { + "nested": { + "path": "per-sons", + "query": { + "query_string": { + "default_operator": "and", + "fields": [ + "per-sons.nick-name.keyword" + ], + "query": "Foobar" + } + }, + "score_mode": "avg" + } + } + ] + } + }"""; + + CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria("persons.nickName.keyword").is("Foobar")); + mappingElasticsearchConverter.updateQuery(criteriaQuery, House.class); + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteriaQuery.getCriteria()), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // #1761 + @DisplayName("should map names and value in object entities") + void shouldMapNamesAndValueInObjectEntities() throws JSONException { + + String expected = """ + { + "bool": { + "must": [ + { + "query_string": { + "default_operator": "and", + "fields": [ + "per-sons.birth-date" + ], + "query": "03.10.1999" + } + } + ] + } + }"""; + + CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria("persons.birthDate").is(LocalDate.of(1999, 10, 3))); + mappingElasticsearchConverter.updateQuery(criteriaQuery, ObjectWithPerson.class); + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteriaQuery.getCriteria()), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // #1778 + @DisplayName("should map names in source fields and SourceFilters") + void shouldMapNamesInSourceFieldsAndSourceFilters() { + + Query query = Query.findAll(); + // Note: we don't care if these filters make sense here, this test is only about name mapping + query.addFields("firstName", "lastName"); + query.addSourceFilter(new FetchSourceFilterBuilder().withIncludes("firstName").withExcludes("lastName").build()); + + mappingElasticsearchConverter.updateQuery(query, Person.class); + + SoftAssertions softly = new SoftAssertions(); + softly.assertThat(query.getFields()).containsExactly("first-name", "last-name"); + SourceFilter sourceFilter = query.getSourceFilter(); + softly.assertThat(sourceFilter).isNotNull(); + softly.assertThat(sourceFilter.getIncludes()).containsExactly("first-name"); + softly.assertThat(sourceFilter.getExcludes()).containsExactly("last-name"); + softly.assertAll(); + } + + @Test + @DisplayName("should map names in source stored fields") + void shouldMapNamesInSourceStoredFields() { + + Query query = Query.findAll(); + query.addStoredFields("firstName", "lastName"); + + mappingElasticsearchConverter.updateQuery(query, Person.class); + + SoftAssertions softly = new SoftAssertions(); + List storedFields = query.getStoredFields(); + softly.assertThat(storedFields).isNotNull(); + softly.assertThat(storedFields).containsExactly("first-name", "last-name"); + softly.assertAll(); + } + + // endregion + // region helper functions + + // endregion + + // region test entities + static class Person { + + @Nullable + @Id String id; + @Nullable + @Field(name = "first-name") String firstName; + @Nullable + @Field(name = "last-name") String lastName; + @Nullable + @MultiField(mainField = @Field(name = "nick-name"), + otherFields = { @InnerField(suffix = "keyword", type = FieldType.Keyword) }) String nickName; + @Nullable + @Field(name = "created-date", type = FieldType.Date, format = DateFormat.epoch_millis) Date createdDate; + @Nullable + @Field(name = "birth-date", type = FieldType.Date, format = {}, pattern = "dd.MM.uuuu") LocalDate birthDate; + } + + static class House { + @Nullable + @Id String id; + @Nullable + @Field(name = "per-sons", type = FieldType.Nested) List persons; + } + + static class ObjectWithPerson { + @Nullable + @Id String id; + @Nullable + @Field(name = "per-sons", type = FieldType.Object) List persons; + } + + static class GeoShapeEntity { + @Nullable + @Field(name = "geo-shape-field") GeoJson geoShapeField; + } + // endregion +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/CriteriaQueryProcessorUnitTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/CriteriaQueryProcessorUnitTests.java new file mode 100644 index 0000000..c8846d2 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/CriteriaQueryProcessorUnitTests.java @@ -0,0 +1,483 @@ +/* + * Copyright 2020-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.opensearch.data.client.osc.JsonUtils.*; +import static org.skyscreamer.jsonassert.JSONAssert.*; + +import org.json.JSONException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.springframework.data.elasticsearch.core.query.Criteria; + +/** + * @author Peter-Josef Meisch + * @author Ezequiel Antúnez Camacho + */ +@SuppressWarnings("ConstantConditions") +class CriteriaQueryProcessorUnitTests { + + private JsonpMapper mapper = new JacksonJsonpMapper(); + + @Test // DATAES-706 + void shouldProcessTwoCriteriaWithAnd() throws JSONException { + + String expected = """ + { + "bool": { + "must": [ + { + "query_string": { + "fields": [ + "field1" + ], + "query": "value1" + } + }, + { + "query_string": { + "fields": [ + "field2" + ], + "query": "value2" + } + } + ] + } + } + + """; // + + Criteria criteria = new Criteria("field1").is("value1").and("field2").is("value2"); + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // DATAES-706 + void shouldProcessTwoCriteriaWithOr() throws JSONException { + + String expected = """ + { + "bool": { + "should": [ + { + "query_string": { + "fields": [ + "field1" + ], + "query": "value1" + } + }, + { + "query_string": { + "fields": [ + "field2" + ], + "query": "value2" + } + } + ] + } + } + """; + + Criteria criteria = new Criteria("field1").is("value1").or("field2").is("value2"); + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // DATAES-706 + void shouldProcessMixedCriteriaWithOrAnd() throws JSONException { + + String expected = """ + { + "bool": { + "must": [ + { + "query_string": { + "fields": [ + "field1" + ], + "query": "value1" + } + }, + { + "query_string": { + "fields": [ + "field3" + ], + "query": "value3" + } + } + ], + "should": [ + { + "query_string": { + "fields": [ + "field2" + ], + "query": "value2" + } + }, + { + "query_string": { + "fields": [ + "field4" + ], + "query": "value4" + } + } + ] + } + } + """; + + Criteria criteria = new Criteria("field1").is("value1") // + .or("field2").is("value2") // + .and("field3").is("value3") // + .or("field4").is("value4"); // + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // DATAES-706 + void shouldAddSubQuery() throws JSONException { + + String expected = """ + { + "bool": { + "must": [ + { + "query_string": { + "fields": [ + "lastName" + ], + "query": "Miller" + } + }, + { + "bool": { + "should": [ + { + "query_string": { + "fields": [ + "firstName" + ], + "query": "John" + } + }, + { + "query_string": { + "fields": [ + "firstName" + ], + "query": "Jack" + } + } + ] + } + } + ] + } + }"""; + + Criteria criteria = new Criteria("lastName").is("Miller") + .subCriteria(new Criteria().or("firstName").is("John").or("firstName").is("Jack")); + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // DATAES-706 + void shouldProcessNestedSubCriteria() throws JSONException { + + String expected = """ + { + "bool": { + "should": [ + { + "bool": { + "must": [ + { + "query_string": { + "fields": [ + "lastName" + ], + "query": "Miller" + } + }, + { + "bool": { + "should": [ + { + "query_string": { + "fields": [ + "firstName" + ], + "query": "John" + } + }, + { + "query_string": { + "fields": [ + "firstName" + ], + "query": "Jack" + } + } + ] + } + } + ] + } + }, + { + "bool": { + "must": [ + { + "query_string": { + "fields": [ + "lastName" + ], + "query": "Smith" + } + }, + { + "bool": { + "should": [ + { + "query_string": { + "fields": [ + "firstName" + ], + "query": "Emma" + } + }, + { + "query_string": { + "fields": [ + "firstName" + ], + "query": "Lucy" + } + } + ] + } + } + ] + } + } + ] + } + } + """; + + Criteria criteria = Criteria.or() + .subCriteria(new Criteria("lastName").is("Miller") + .subCriteria(new Criteria().or("firstName").is("John").or("firstName").is("Jack"))) + .subCriteria(new Criteria("lastName").is("Smith") + .subCriteria(new Criteria().or("firstName").is("Emma").or("firstName").is("Lucy"))); + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // DATAES-706 + void shouldBuildMatchQuery() throws JSONException { + + String expected = """ + { + "bool": { + "must": [ + { + "match": { + "field1": { + "operator": "or", + "query": "value1 value2" + } + } + } + ] + } + } + """; + + Criteria criteria = new Criteria("field1").matches("value1 value2"); + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // DATAES-706 + void shouldBuildMatchAllQuery() throws JSONException { + + String expected = """ + { + "bool": { + "must": [ + { + "match": { + "field1": { + "operator": "and", + "query": "value1 value2" + } + } + } + ] + } + }"""; + + Criteria criteria = new Criteria("field1").matchesAll("value1 value2"); + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // #1753 + @DisplayName("should build nested query") + void shouldBuildNestedQuery() throws JSONException { + + String expected = """ + { + "bool": { + "must": [ + { + "nested": { + "path": "houses.inhabitants", + "query": { + "query_string": { + "fields": [ + "houses.inhabitants.lastName" + ], + "query": "murphy" + } + } + } + } + ] + } + }"""; + + Criteria criteria = new Criteria("houses.inhabitants.lastName").is("murphy"); + criteria.getField().setPath("houses.inhabitants"); + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // #1909 + @DisplayName("should build query for empty property") + void shouldBuildQueryForEmptyProperty() throws JSONException { + + String expected = """ + { + "bool" : { + "must" : [ + { + "bool" : { + "must" : [ + { + "exists" : { + "field" : "lastName" } + } + ], + "must_not" : [ + { + "wildcard" : { + "lastName" : { + "wildcard" : "*" } + } + } + ] + } + } + ] + } + }"""; // + + Criteria criteria = new Criteria("lastName").empty(); + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // #1909 + @DisplayName("should build query for non-empty property") + void shouldBuildQueryForNonEmptyProperty() throws JSONException { + + String expected = """ + { + "bool" : { + "must" : [ + { + "wildcard" : { + "lastName" : { + "wildcard" : "*" + } + } + } + ] + } + } + """; // + + Criteria criteria = new Criteria("lastName").notEmpty(); + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + + @Test // #2418 + void shouldBuildRegexpQuery() throws JSONException { + String expected = """ + { + "bool": { + "must": [ + { + "regexp": { + "field1": { + "value": "[^abc]" + } + } + } + ] + } + } + """; + + Criteria criteria = new Criteria("field1").regexp("[^abc]"); + + var queryString = queryToJson(CriteriaQueryProcessor.createQuery(criteria), mapper); + + assertEquals(expected, queryString, false); + } + +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/DevTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/DevTests.java new file mode 100644 index 0000000..64c6963 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/DevTests.java @@ -0,0 +1,335 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.assertj.core.api.Assertions.*; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.function.Function; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch.cluster.HealthRequest; +import org.opensearch.client.opensearch.cluster.HealthResponse; +import org.opensearch.client.opensearch.core.IndexRequest; +import org.opensearch.client.opensearch.core.IndexResponse; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.indices.GetIndicesSettingsRequest; +import org.opensearch.client.opensearch.indices.GetIndicesSettingsResponse; +import org.opensearch.client.opensearch.indices.IndexSettings; +import org.opensearch.client.opensearch.indices.OpenSearchIndicesClient; +import org.opensearch.client.transport.TransportOptions; +import org.opensearch.client.transport.rest_client.RestClientOptions; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.core.convert.MappingElasticsearchConverter; +import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext; +import org.springframework.data.elasticsearch.support.HttpHeaders; +import org.springframework.lang.Nullable; + +/** + * Not really tests, but a class to check the first implementations of the new OpenSearch client. Needs OpenSearch + * on port 9200 and an intercepting proxy on port 8080. + * + * @author Peter-Josef Meisch + */ +@Disabled +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class DevTests { + private static final String INDEX = "appdata-index-os"; + + private static final SimpleElasticsearchMappingContext mappingContext = new SimpleElasticsearchMappingContext(); + private static final MappingElasticsearchConverter converter = new MappingElasticsearchConverter(mappingContext); + + private final TransportOptions transportOptions = new RestClientOptions(RequestOptions.DEFAULT).toBuilder() + .addHeader("X-SpringDataElasticsearch-AlwaysThere", "true").setParameter("pretty", "true").build(); + + private final OpenSearchClient imperativeOpensearchClient = OpenSearchClients + .createImperative(OpenSearchClients.getRestClient(clientConfiguration()), transportOptions); + + @Test + void someTest() throws IOException { + + OpenSearchClient client = imperativeOpensearchClient; + OpenSearchIndicesClient indicesClient = client.indices(); + + indicesClient.create(b -> b.index("testindex")); + + GetIndicesSettingsResponse getIndicesSettingsResponse = indicesClient + .getSettings(GetIndicesSettingsRequest.of(b -> b.index("testindex").includeDefaults(true))); + } + + static class Product { + @Nullable String id; + @Nullable Double price; + + public Product() {} + + public Product(@Nullable String id, @Nullable Double price) { + this.id = id; + this.price = price; + } + + @Nullable + public String getId() { + return id; + } + + public void setId(@Nullable String id) { + this.id = id; + } + + @Nullable + public Double getPrice() { + return price; + } + + public void setPrice(@Nullable Double price) { + this.price = price; + } + } + + static class Person { + @Nullable String id; + @Nullable Name name; + + public Person() {} + + public Person(String id, Name name) { + this.id = id; + this.name = name; + } + + @Nullable + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + @Nullable + public Name getName() { + return name; + } + + public void setName(Name name) { + this.name = name; + } + + @Override + public String toString() { + return "Person{" + "id='" + id + '\'' + ", name=" + name + '}'; + } + } + + static class Name { + @Nullable String first; + @Nullable String last; + + public Name() {} + + public Name(String first, String last) { + this.first = first; + this.last = last; + } + + @Nullable + public String getFirst() { + return first; + } + + public void setFirst(String first) { + this.first = first; + } + + @Nullable + public String getLast() { + return last; + } + + public void setLast(String last) { + this.last = last; + } + + @Override + public String toString() { + return "Name{" + "first='" + first + '\'' + ", last='" + last + '\'' + '}'; + } + } + + // region cluster health + @Test + @Order(10) + void clusterHealth() { + + HealthRequest healthRequest = new HealthRequest.Builder().build(); + + try { + HealthResponse healthResponse = clusterHealthImperative(healthRequest); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private HealthResponse clusterHealthImperative(HealthRequest healthRequest) throws IOException { + return imperativeOpensearchClient.cluster().health(healthRequest); + } + // endregion + + @Test + @Order(15) + void indexCreation() throws IOException { + + RequestConverter requestConverter = new RequestConverter(converter, + imperativeOpensearchClient._transport().jsonpMapper()); + + String index = "pjtestindex"; + OpenSearchIndicesClient indicesClient = imperativeOpensearchClient.indices(); + + if (indicesClient.exists(erb -> erb.index(index)).value()) { + indicesClient.delete(drb -> drb.index(index)); + } + + String jsonSettings = """ + { + "index": { + "number_of_shards": "1", + "number_of_replicas": "0", + "analysis": { + "analyzer": { + "emailAnalyzer": { + "type": "custom", + "tokenizer": "uax_url_email" + } + } + } + } + } + """; + + String jsonMapping = """ + { + "properties": { + "email": { + "type": "text", + "analyzer": "emailAnalyzer" + } + } + } + """; + + indicesClient.create(crb -> crb // + .index(index) // + .settings(requestConverter.fromJson(jsonSettings, IndexSettings._DESERIALIZER)) // + .mappings(requestConverter.fromJson(jsonMapping, TypeMapping._DESERIALIZER))); + } + + // region save + @Test + @Order(20) + void save() { + + Function> indexRequestBuilder = (Integer id) -> { + AppData appData = new AppData(); + appData.setId("id" + id); + appData.setContent("content" + id); + + return new IndexRequest.Builder().id(appData.getId()).document(appData).index(INDEX).build(); + }; + + try { + indexImperative(indexRequestBuilder.apply(1)); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + private IndexResponse indexImperative(IndexRequest indexRequest) throws IOException { + return imperativeOpensearchClient.index(indexRequest); + } + + // endregion + // region search + @Test + @Order(30) + void search() { + + SearchRequest searchRequest = new SearchRequest.Builder().index(INDEX) + .query(query -> query.match(matchQuery -> matchQuery.field("content").query(FieldValue.of("content1")))) + .build(); + + SearchResponse searchResponse = null; + try { + searchResponse = searchImperative(searchRequest); + assertThat(searchResponse).isNotNull(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private SearchResponse searchImperative(SearchRequest searchRequest) throws IOException { + return imperativeOpensearchClient.search(searchRequest, EntityAsMap.class); + } + + // endregion + + private ClientConfiguration clientConfiguration() { + return ClientConfiguration.builder() // + .connectedTo("localhost:9200")// + .withBasicAuth("elastic", "hcraescitsale").withProxy("localhost:8080") // + .withHeaders(() -> { + HttpHeaders headers = new HttpHeaders(); + headers.add("X-SpringDataElasticsearch-timestamp", + LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); + return headers; + }) // + .build(); + } + + private static class AppData { + @Nullable private String id; + @Nullable private String content; + + @Nullable + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + @Nullable + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/DocumentAdaptersUnitTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/DocumentAdaptersUnitTests.java new file mode 100644 index 0000000..05fb229 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/DocumentAdaptersUnitTests.java @@ -0,0 +1,158 @@ +/* + * Copyright 2021-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opensearch.client.json.JsonData; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.core.search.Hit; +import org.springframework.data.elasticsearch.core.document.Explanation; +import org.springframework.data.elasticsearch.core.document.SearchDocument; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +class DocumentAdaptersUnitTests { + + private final JsonpMapper jsonpMapper = new JacksonJsonpMapper(); + + @Test // #1973 + @DisplayName("should adapt search Hit from fields") + void shouldAdaptSearchHitFromFields() { + + Hit searchHit = new Hit.Builder() // + .index("index") // + .id("my-id") // + .score(42d) // + .fields("field1", JsonData.of(Collections.singletonList("listValue"))) // + .fields("field2", JsonData.of("stringValue")) // + .seqNo(1l) // + .primaryTerm(2l) // + .build(); // + + SearchDocument document = DocumentAdapters.from(searchHit, jsonpMapper); + + SoftAssertions softly = new SoftAssertions(); + + softly.assertThat(document.getIndex()).isEqualTo("index"); + softly.assertThat(document.hasId()).isTrue(); + softly.assertThat(document.getId()).isEqualTo("my-id"); + softly.assertThat(document.hasVersion()).isFalse(); + softly.assertThat(document.getScore()).isBetween(42f, 42f); + Object field1 = document.get("field1"); + softly.assertThat(field1).isInstanceOf(List.class); + // noinspection unchecked + List fieldList = (List) field1; + softly.assertThat(fieldList).containsExactly("listValue"); + softly.assertThat(document.get("field2")).isEqualTo("stringValue"); + softly.assertThat(document.hasSeqNo()).isTrue(); + softly.assertThat(document.getSeqNo()).isEqualTo(1); + softly.assertThat(document.hasPrimaryTerm()).isTrue(); + softly.assertThat(document.getPrimaryTerm()).isEqualTo(2); + + softly.assertAll(); + } + + @Test // #1973 + @DisplayName("should adapt search Hit from source") + void shouldAdaptSearchHitFromSource() { + + EntityAsMap eam = new EntityAsMap(); + eam.put("field", "value"); + Hit searchHit = new Hit.Builder() // + .index("index") // + .id("my-id") // + .score(42d) // + .seqNo(1l) // + .primaryTerm(2l) // + .source(eam) // + .build(); // + + SearchDocument document = DocumentAdapters.from(searchHit, jsonpMapper); + + SoftAssertions softly = new SoftAssertions(); + + softly.assertThat(document.getIndex()).isEqualTo("index"); + softly.assertThat(document.hasId()).isTrue(); + softly.assertThat(document.getId()).isEqualTo("my-id"); + softly.assertThat(document.hasVersion()).isFalse(); + softly.assertThat(document.getScore()).isBetween(42f, 42f); + softly.assertThat(document.get("field")).isEqualTo("value"); + softly.assertThat(document.hasSeqNo()).isTrue(); + softly.assertThat(document.getSeqNo()).isEqualTo(1); + softly.assertThat(document.hasPrimaryTerm()).isTrue(); + softly.assertThat(document.getPrimaryTerm()).isEqualTo(2); + + softly.assertAll(); + } + + @Test // #725 #1973 + @DisplayName("should adapt returned explanations") + void shouldAdaptReturnedExplanations() { + + Hit searchHit = new Hit.Builder() // + .index("index") // + .id("42") // + .explanation(eb -> eb // + .value(3.14f) // + .description("explanation 3.14") // + .details(edb -> edb.description("explanation noMatch").value(0f))) + .build(); + + SearchDocument searchDocument = DocumentAdapters.from(searchHit, jsonpMapper); + + SoftAssertions softly = new SoftAssertions(); + + Explanation explanation = searchDocument.getExplanation(); + softly.assertThat(explanation).isNotNull(); + softly.assertThat(explanation.isMatch()).isTrue(); + softly.assertThat(explanation.getValue()).isCloseTo(3.14, Offset.offset(0.001)); + softly.assertThat(explanation.getDescription()).isEqualTo("explanation 3.14"); + List details = explanation.getDetails(); + softly.assertThat(details) + .containsExactly(new Explanation(null, 0.0, "explanation noMatch", Collections.emptyList())); + softly.assertAll(); + } + + @Test // DATAES-979 #1973 + @DisplayName("should adapt returned matched queries") + void shouldAdaptReturnedMatchedQueries() { + + Hit searchHit = new Hit.Builder() // + .index("index") // + .id("42") // + .matchedQueries("query1", "query2") // + .build(); + + SearchDocument searchDocument = DocumentAdapters.from(searchHit, jsonpMapper); + + SoftAssertions softly = new SoftAssertions(); + + List matchedQueries = searchDocument.getMatchedQueries(); + softly.assertThat(matchedQueries).isNotNull(); + softly.assertThat(matchedQueries).hasSize(2); + softly.assertThat(matchedQueries).isEqualTo(Arrays.asList("query1", "query2")); + softly.assertAll(); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/ElasticsearchPartQueryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/ElasticsearchPartQueryOSCIntegrationTests.java new file mode 100644 index 0000000..8ce0b3c --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/ElasticsearchPartQueryOSCIntegrationTests.java @@ -0,0 +1,47 @@ +/* + * Copyright 2022-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.query.ElasticsearchPartQueryIntegrationTests; +import org.springframework.data.elasticsearch.core.query.Query; + +/** + * @author Peter-Josef Meisch + * @since 4.4 + */ +public class ElasticsearchPartQueryOSCIntegrationTests extends ElasticsearchPartQueryIntegrationTests { + + @Configuration + @Import({ OpenSearchTemplateConfiguration.class }) + static class Config {} + + @Override + protected String buildQueryString(Query query, Class clazz) { + + JacksonJsonpMapper jsonpMapper = new JacksonJsonpMapper(); + RequestConverter requestConverter = new RequestConverter(operations.getElasticsearchConverter(), jsonpMapper); + SearchRequest request = requestConverter.searchRequest(query, null, clazz, IndexCoordinates.of("dummy"), false); + + return JsonUtils.toJson(request, jsonpMapper); + // return "{\"query\":" + JsonUtils.toJson(request.query(), jsonpMapper) + "}"; + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/RequestConverterTest.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/RequestConverterTest.java new file mode 100644 index 0000000..92f438e --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/RequestConverterTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import static org.assertj.core.api.Assertions.*; + +import java.util.List; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.FieldType; +import org.springframework.data.elasticsearch.core.convert.MappingElasticsearchConverter; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.core.mapping.SimpleElasticsearchMappingContext; +import org.springframework.data.elasticsearch.core.query.DocValueField; +import org.springframework.data.elasticsearch.core.query.StringQuery; +import org.springframework.lang.Nullable; + +/** + * @author Peter-Josef Meisch + */ +class RequestConverterTest { + + private static final SimpleElasticsearchMappingContext mappingContext = new SimpleElasticsearchMappingContext(); + private static final MappingElasticsearchConverter converter = new MappingElasticsearchConverter(mappingContext); + private JacksonJsonpMapper jsonpMapper = new JacksonJsonpMapper(); + private RequestConverter requestConverter = new RequestConverter(converter, jsonpMapper); + + @Test // #2316 + @DisplayName("should add docvalue_fields") + void shouldAddDocvalueFields() { + + var docValueFields = List.of( // + new DocValueField("field1"), // + new DocValueField("field2", "format2") // + ); + // doesn't matter what type of query is used, the relevant part for docvalue_fields is in the base builder. + var query = StringQuery.builder(""" + { + "match_all":{} + } + """) // + .withDocValueFields(docValueFields) // + .build(); + + var searchRequest = requestConverter.searchRequest(query,null, SampleEntity.class, IndexCoordinates.of("foo"), true); + + var fieldAndFormats = searchRequest.docvalueFields(); + assertThat(fieldAndFormats).hasSize(2); + assertThat(fieldAndFormats.get(0).field()).isEqualTo("field1"); + assertThat(fieldAndFormats.get(0).format()).isNull(); + assertThat(fieldAndFormats.get(1).field()).isEqualTo("field2"); + assertThat(fieldAndFormats.get(1).format()).isEqualTo("format2"); + } + + @Document(indexName = "does-not-matter") + static class SampleEntity { + @Nullable + @Id private String id; + @Nullable + @Field(type = FieldType.Text) private String text; + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/SearchDocumentResponseBuilderUnitTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/SearchDocumentResponseBuilderUnitTests.java new file mode 100644 index 0000000..56ee27b --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/osc/SearchDocumentResponseBuilderUnitTests.java @@ -0,0 +1,108 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.opensearch.data.client.osc; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.List; +import org.assertj.core.api.SoftAssertions; +import org.json.JSONException; +import org.junit.jupiter.api.Test; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.core.search.HitsMetadata; +import org.opensearch.client.opensearch.core.search.Suggest; +import org.opensearch.client.opensearch.core.search.TotalHitsRelation; +import org.springframework.data.elasticsearch.core.document.SearchDocumentResponse; + +/** + * Tests for the factory class to create {@link SearchDocumentResponse} instances. + * + * @author Sébastien Comeau + * @since 5.2 + */ +class SearchDocumentResponseBuilderUnitTests { + + private JacksonJsonpMapper jsonpMapper = new JacksonJsonpMapper(); + + @Test // #2681 + void shouldGetPhraseSuggestion() throws JSONException { + // arrange + final var hitsMetadata = new HitsMetadata.Builder() + .total(total -> total + .value(0) + .relation(TotalHitsRelation.Eq)) + .hits(new ArrayList<>()) + .build(); + + final var suggestionTest = new Suggest.Builder() + .phrase(phrase -> phrase + .text("National") + .offset(0) + .length(8) + .options(option -> option + .text("nations") + .highlighted("highlighted-nations") + .score(0.11480146) + .collateMatch(false)) + .options(option -> option + .text("national") + .highlighted("highlighted-national") + .score(0.08063514) + .collateMatch(false))) + .build(); + + final var sortProperties = ImmutableMap.>> builder() + .put("suggestionTest", ImmutableList.of(suggestionTest)) + .build(); + + // act + final var actual = SearchDocumentResponseBuilder.from(hitsMetadata, null, null, null, sortProperties, null, + jsonpMapper); + + // assert + SoftAssertions softly = new SoftAssertions(); + + softly.assertThat(actual).isNotNull(); + softly.assertThat(actual.getSuggest()).isNotNull(); + softly.assertThat(actual.getSuggest().getSuggestions()).isNotNull().hasSize(1); + + final var actualSuggestion = actual.getSuggest().getSuggestions().get(0); + softly.assertThat(actualSuggestion.getName()).isEqualTo("suggestionTest"); + softly.assertThat(actualSuggestion.getEntries()).isNotNull().hasSize(1); + + final var actualEntry = actualSuggestion.getEntries().get(0); + softly.assertThat(actualEntry).isNotNull(); + softly.assertThat(actualEntry.getText()).isEqualTo("National"); + softly.assertThat(actualEntry.getOffset()).isEqualTo(0); + softly.assertThat(actualEntry.getLength()).isEqualTo(8); + softly.assertThat(actualEntry.getOptions()).isNotNull().hasSize(2); + + final var actualOption1 = actualEntry.getOptions().get(0); + softly.assertThat(actualOption1.getText()).isEqualTo("nations"); + softly.assertThat(actualOption1.getHighlighted()).isEqualTo("highlighted-nations"); + softly.assertThat(actualOption1.getScore()).isEqualTo(0.11480146); + softly.assertThat(actualOption1.getCollateMatch()).isEqualTo(false); + + final var actualOption2 = actualEntry.getOptions().get(1); + softly.assertThat(actualOption2.getText()).isEqualTo("national"); + softly.assertThat(actualOption2.getHighlighted()).isEqualTo("highlighted-national"); + softly.assertThat(actualOption2.getScore()).isEqualTo(0.08063514); + softly.assertThat(actualOption2.getCollateMatch()).isEqualTo(false); + + softly.assertAll(); + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/complex/custommethod/autowiring/ComplexCustomMethodRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/complex/custommethod/autowiring/ComplexCustomMethodRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..41ce54e --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/complex/custommethod/autowiring/ComplexCustomMethodRepositoryOSCIntegrationTests.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.complex.custommethod.autowiring; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.complex.custommethod.autowiring.ComplexCustomMethodRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {ComplexCustomMethodRepositoryOSCIntegrationTests.Config.class}) +public class ComplexCustomMethodRepositoryOSCIntegrationTests extends ComplexCustomMethodRepositoryIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.complex.custommethod.autowiring"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("complex-custom-method-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/complex/custommethod/manualwiring/ComplexCustomMethodRepositoryManualWiringOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/complex/custommethod/manualwiring/ComplexCustomMethodRepositoryManualWiringOSCIntegrationTests.java new file mode 100644 index 0000000..4100626 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/complex/custommethod/manualwiring/ComplexCustomMethodRepositoryManualWiringOSCIntegrationTests.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.complex.custommethod.manualwiring; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.complex.custommethod.manualwiring.ComplexCustomMethodRepositoryManualWiringIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {ComplexCustomMethodRepositoryManualWiringOSCIntegrationTests.Config.class}) +public class ComplexCustomMethodRepositoryManualWiringOSCIntegrationTests + extends ComplexCustomMethodRepositoryManualWiringIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.complex.custommethod.manualwiring"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("complex-custom-method-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/custommethod/CustomMethodRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/custommethod/CustomMethodRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..6e8d03c --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/custommethod/CustomMethodRepositoryOSCIntegrationTests.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.custommethod; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.custommethod.CustomMethodRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {CustomMethodRepositoryOSCIntegrationTests.Config.class}) +public class CustomMethodRepositoryOSCIntegrationTests extends CustomMethodRepositoryIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.custommethod"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("custom-method-repository-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/doubleid/DoubleIDRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/doubleid/DoubleIDRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..924e8a0 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/doubleid/DoubleIDRepositoryOSCIntegrationTests.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.doubleid; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.doubleid.DoubleIDRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {DoubleIDRepositoryOSCIntegrationTests.Config.class}) +public class DoubleIDRepositoryOSCIntegrationTests extends DoubleIDRepositoryIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.doubleid"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("doubleid-repository-es7"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/dynamicindex/DynamicIndexEntityOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/dynamicindex/DynamicIndexEntityOSCIntegrationTests.java new file mode 100644 index 0000000..4289fbc --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/dynamicindex/DynamicIndexEntityOSCIntegrationTests.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.dynamicindex; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.dynamicindex.DynamicIndexEntityIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {DynamicIndexEntityOSCIntegrationTests.Config.class}) +public class DynamicIndexEntityOSCIntegrationTests extends DynamicIndexEntityIntegrationTests { + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.dynamicindex"}, + considerNestedRepositories = true) + static class Config { + @Bean + public IndexNameProvider indexNameProvider() { + return new IndexNameProvider(); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/geo/GeoRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/geo/GeoRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..afc6add --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/geo/GeoRepositoryOSCIntegrationTests.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.geo; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.geo.GeoRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {GeoRepositoryOSCIntegrationTests.Config.class}) +public class GeoRepositoryOSCIntegrationTests extends GeoRepositoryIntegrationTests { + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.geo"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("geo-repository-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/integer/IntegerIDRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/integer/IntegerIDRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..a2aad6f --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/integer/IntegerIDRepositoryOSCIntegrationTests.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.integer; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.integer.IntegerIDRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {IntegerIDRepositoryOSCIntegrationTests.Config.class}) +public class IntegerIDRepositoryOSCIntegrationTests extends IntegerIDRepositoryIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.integer"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("integerid-repository-es7"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/nestedobject/InnerObjectOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/nestedobject/InnerObjectOSCIntegrationTests.java new file mode 100644 index 0000000..4ce8d76 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/nestedobject/InnerObjectOSCIntegrationTests.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.nestedobject; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.nestedobject.InnerObjectIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {InnerObjectOSCIntegrationTests.Config.class}) +public class InnerObjectOSCIntegrationTests extends InnerObjectIntegrationTests { + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.nestedobject"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("inner-object-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/setting/dynamic/DynamicSettingAndMappingEntityRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/setting/dynamic/DynamicSettingAndMappingEntityRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..e485664 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/setting/dynamic/DynamicSettingAndMappingEntityRepositoryOSCIntegrationTests.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.setting.dynamic; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.setting.dynamic.DynamicSettingAndMappingEntityRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {DynamicSettingAndMappingEntityRepositoryOSCIntegrationTests.Config.class}) +public class DynamicSettingAndMappingEntityRepositoryOSCIntegrationTests + extends DynamicSettingAndMappingEntityRepositoryIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.setting.dynamic"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("dynamic-setting-and-mapping-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/setting/fielddynamic/FieldDynamicMappingEntityRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/setting/fielddynamic/FieldDynamicMappingEntityRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..2304bef --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/setting/fielddynamic/FieldDynamicMappingEntityRepositoryOSCIntegrationTests.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.setting.fielddynamic; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.setting.fielddynamic.FieldDynamicMappingEntityRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {FieldDynamicMappingEntityRepositoryOSCIntegrationTests.Config.class}) +public class FieldDynamicMappingEntityRepositoryOSCIntegrationTests + extends FieldDynamicMappingEntityRepositoryIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.setting.fielddynamic"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("field-dynamic-mapping-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/spel/SpELEntityOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/spel/SpELEntityOSCIntegrationTests.java new file mode 100644 index 0000000..106817c --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/spel/SpELEntityOSCIntegrationTests.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.spel; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.spel.SpELEntityIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {SpELEntityOSCIntegrationTests.Config.class}) +public class SpELEntityOSCIntegrationTests extends SpELEntityIntegrationTests { + @Configuration + @Import(OpenSearchTemplateConfiguration.class) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.spel"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("spel-entity-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/synonym/SynonymRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/synonym/SynonymRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..4948a47 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/synonym/SynonymRepositoryOSCIntegrationTests.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.synonym; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.synonym.SynonymRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {SynonymRepositoryOSCIntegrationTests.Config.class}) +public class SynonymRepositoryOSCIntegrationTests extends SynonymRepositoryIntegrationTests { + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.synonym"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("sysnonym-entity-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/uuidkeyed/UUIDElasticsearchRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/uuidkeyed/UUIDElasticsearchRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..cfec956 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repositories/uuidkeyed/UUIDElasticsearchRepositoryOSCIntegrationTests.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repositories.uuidkeyed; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repositories.uuidkeyed.UUIDElasticsearchRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {UUIDElasticsearchRepositoryOSCIntegrationTests.Config.class}) +public class UUIDElasticsearchRepositoryOSCIntegrationTests extends UUIDElasticsearchRepositoryIntegrationTests { + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repositories.uuidkeyed"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("uuid-keyed-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repository/query/keywords/QueryKeywordsOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repository/query/keywords/QueryKeywordsOSCIntegrationTests.java new file mode 100644 index 0000000..3a8f46a --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repository/query/keywords/QueryKeywordsOSCIntegrationTests.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repository.query.keywords; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.repository.query.keywords.QueryKeywordsIntegrationTests; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +/** + * {@link QueryKeywordsIntegrationTests} using a Repository backed by an + * {@link org.opensearch.data.client.orhlc.OpenSearchRestTemplate}. + */ +@ContextConfiguration(classes = {QueryKeywordsOSCIntegrationTests.Config.class}) +public class QueryKeywordsOSCIntegrationTests extends QueryKeywordsIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repository.query.keywords"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("query-keywords-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/opensearch/data/client/repository/support/ElasticsearchRepositoryOSCIntegrationTests.java b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repository/support/ElasticsearchRepositoryOSCIntegrationTests.java new file mode 100644 index 0000000..34a8aa7 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/opensearch/data/client/repository/support/ElasticsearchRepositoryOSCIntegrationTests.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.data.client.repository.support; + +import org.opensearch.data.client.junit.jupiter.OpenSearchTemplateConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; +import org.springframework.data.elasticsearch.repository.support.ElasticsearchRepositoryIntegrationTests; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(classes = {ElasticsearchRepositoryOSCIntegrationTests.Config.class}) +public class ElasticsearchRepositoryOSCIntegrationTests extends ElasticsearchRepositoryIntegrationTests { + + @Configuration + @Import({OpenSearchTemplateConfiguration.class}) + @EnableElasticsearchRepositories( + basePackages = {"org.springframework.data.elasticsearch.repository.support"}, + considerNestedRepositories = true) + static class Config { + @Bean + IndexNameProvider indexNameProvider() { + return new IndexNameProvider("repository-os"); + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/NestedObjectIntegrationTests.java b/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/NestedObjectIntegrationTests.java index 869fa3f..2333e4f 100644 --- a/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/NestedObjectIntegrationTests.java +++ b/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/NestedObjectIntegrationTests.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; @@ -44,7 +45,6 @@ import org.springframework.data.elasticsearch.core.query.Query; import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; import org.springframework.data.elasticsearch.utils.IndexNameProvider; -import org.springframework.lang.NonNull; import org.springframework.lang.Nullable; /** @@ -130,7 +130,7 @@ public void shouldIndexInitialLevelNestedObject() { assertThat(persons).hasSize(1); } - @NonNull + @NotNull abstract protected Query getNestedQuery1(); @Test @@ -185,7 +185,7 @@ public void shouldSearchUsingNestedQueryOnMultipleLevelNestedObject() { assertThat(personIndexed.getSearchHit(0).getContent().getId()).isEqualTo("1"); } - @NonNull + @NotNull abstract protected Query getNestedQuery2(); private List createPerson() { @@ -320,7 +320,7 @@ public void shouldSearchBooksForPersonInitialLevelNestedType() { assertThat(persons).hasSize(1); } - @NonNull + @NotNull abstract protected Query getNestedQuery3(); @Test // DATAES-73 @@ -368,7 +368,7 @@ public void shouldIndexAndSearchMapAsNestedType() { assertThat(books.getSearchHit(0).getContent().getId()).isEqualTo(book2.getId()); } - @NonNull + @NotNull abstract protected Query getNestedQuery4(); @Document(indexName = "#{@indexNameProvider.indexName()}-book") diff --git a/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/core/query/NativeQueryIntegrationTests.java b/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/core/query/NativeQueryIntegrationTests.java new file mode 100644 index 0000000..ea49776 --- /dev/null +++ b/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/core/query/NativeQueryIntegrationTests.java @@ -0,0 +1,173 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.elasticsearch.core.query; + +import static org.assertj.core.api.Assertions.*; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.opensearch.data.client.osc.NativeQuery; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.annotation.Id; +import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.FieldType; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.geo.GeoPoint; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.data.elasticsearch.junit.jupiter.SpringIntegrationTest; +import org.springframework.data.elasticsearch.utils.IndexNameProvider; +import org.springframework.lang.Nullable; + +/** + * @author Peter-Josef Meisch + */ +@SpringIntegrationTest +public abstract class NativeQueryIntegrationTests { + @Autowired private ElasticsearchOperations operations; + @Autowired private IndexNameProvider indexNameProvider; + + @BeforeEach + public void before() { + indexNameProvider.increment(); + operations.indexOps(SampleEntity.class).createWithMapping(); + } + + @Test + @Order(java.lang.Integer.MAX_VALUE) + void cleanup() { + operations.indexOps(IndexCoordinates.of(indexNameProvider.getPrefix() + '*')).delete(); + } + + @Test // #2391 + @DisplayName("should be able to use CriteriaQuery in a NativeQuery") + void shouldBeAbleToUseCriteriaQueryInANativeQuery() { + + var entity = new SampleEntity(); + entity.setId("7"); + entity.setText("seven"); + operations.save(entity); + entity = new SampleEntity(); + entity.setId("42"); + entity.setText("criteria"); + operations.save(entity); + + var criteriaQuery = CriteriaQuery.builder(Criteria.where("text").is("criteria")).build(); + var nativeQuery = NativeQuery.builder().withQuery(criteriaQuery).build(); + + var searchHits = operations.search(nativeQuery, SampleEntity.class); + + assertThat(searchHits.getTotalHits()).isEqualTo(1); + assertThat(searchHits.getSearchHit(0).getId()).isEqualTo(entity.getId()); + } + + @Test // #2840 + @DisplayName("should be able to use CriteriaQuery with filter arguments in a NativeQuery") + void shouldBeAbleToUseCriteriaQueryWithFilterArgumentsInANativeQuery() { + var entity1 = new SampleEntity(); + entity1.setId("60"); + var location1 = new GeoPoint(60.0, 60.0); + entity1.setLocation(location1); + var entity2 = new SampleEntity(); + entity2.setId("70"); + var location70 = new GeoPoint(70.0, 70.0); + entity2.setLocation(location70); + operations.save(entity1, entity2); + + var criteriaQuery = new CriteriaQuery(Criteria.where("location").within(location1, "10km")); + var nativeQuery = NativeQuery.builder().withQuery(criteriaQuery).build(); + + var searchHits = operations.search(nativeQuery, SampleEntity.class); + + assertThat(searchHits.getTotalHits()).isEqualTo(1); + assertThat(searchHits.getSearchHit(0).getId()).isEqualTo(entity1.getId()); + } + + @Test // #2391 + @DisplayName("should be able to use StringQuery in a NativeQuery") + void shouldBeAbleToUseStringQueryInANativeQuery() { + + var entity = new SampleEntity(); + entity.setId("7"); + entity.setText("seven"); + operations.save(entity); + entity = new SampleEntity(); + entity.setId("42"); + entity.setText("string"); + operations.save(entity); + + var stringQuery = StringQuery.builder(""" + { + "bool": { + "must": [ + { + "match": { + "text": "string" + } + } + ] + } + } + """).build(); + var nativeQuery = NativeQuery.builder().withQuery(stringQuery).build(); + + var searchHits = operations.search(nativeQuery, SampleEntity.class); + + assertThat(searchHits.getTotalHits()).isEqualTo(1); + assertThat(searchHits.getSearchHit(0).getId()).isEqualTo(entity.getId()); + } + + @Document(indexName = "#{@indexNameProvider.indexName()}") + static class SampleEntity { + + @Nullable + @Id private String id; + + @Nullable + @Field(type = FieldType.Text) private String text; + + @Nullable private GeoPoint location; + + @Nullable + public String getId() { + return id; + } + + public void setId(@Nullable String id) { + this.id = id; + } + + @Nullable + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + @Nullable + public GeoPoint getLocation() { + return location; + } + + public void setLocation(GeoPoint location) { + this.location = location; + } + } +} diff --git a/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/repositories/setting/dynamic/DynamicSettingAndMappingEntityRepositoryIntegrationTests.java b/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/repositories/setting/dynamic/DynamicSettingAndMappingEntityRepositoryIntegrationTests.java index e15c99c..e1e9bfe 100644 --- a/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/repositories/setting/dynamic/DynamicSettingAndMappingEntityRepositoryIntegrationTests.java +++ b/spring-data-opensearch/src/test/java/org/springframework/data/elasticsearch/repositories/setting/dynamic/DynamicSettingAndMappingEntityRepositoryIntegrationTests.java @@ -71,7 +71,7 @@ void cleanup() { public void shouldCreateGivenDynamicSettingsForGivenIndex() { assertThat(indexOperations.exists()).isTrue(); - Map map = indexOperations.getSettings(); + Map map = indexOperations.getSettings().flatten(); assertThat(map.containsKey("index.number_of_replicas")).isTrue(); assertThat(map.containsKey("index.number_of_shards")).isTrue(); assertThat(map.containsKey("index.analysis.analyzer.emailAnalyzer.tokenizer")).isTrue(); diff --git a/version.properties b/version.properties index 6d41bff..7dc9496 100644 --- a/version.properties +++ b/version.properties @@ -1 +1 @@ -version=1.3.1 +version=1.4.0