diff --git a/docs/changelog/119995.yaml b/docs/changelog/119995.yaml new file mode 100644 index 0000000000000..e9ee8bc5b8458 --- /dev/null +++ b/docs/changelog/119995.yaml @@ -0,0 +1,5 @@ +pr: 119995 +summary: "apm-data: Use representative count as event.success_count if available" +area: Ingest Node +type: bug +issues: [] diff --git a/docs/changelog/122134.yaml b/docs/changelog/122134.yaml new file mode 100644 index 0000000000000..25ca556789525 --- /dev/null +++ b/docs/changelog/122134.yaml @@ -0,0 +1,5 @@ +pr: 122134 +summary: Adding integration for VoyageAI embeddings and rerank models +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/122586.yaml b/docs/changelog/122586.yaml new file mode 100644 index 0000000000000..1555148b57917 --- /dev/null +++ b/docs/changelog/122586.yaml @@ -0,0 +1,6 @@ +pr: 122586 +summary: "ESQL: Fix inconsistent results in using scaled_float field" +area: ES|QL +type: bug +issues: + - 122547 diff --git a/docs/changelog/122601.yaml b/docs/changelog/122601.yaml new file mode 100644 index 0000000000000..11f44a806917d --- /dev/null +++ b/docs/changelog/122601.yaml @@ -0,0 +1,6 @@ +pr: 122601 +summary: Implicit numeric casting for CASE/GREATEST/LEAST +area: ES|QL +type: bug +issues: + - 121890 diff --git a/docs/changelog/122938.yaml b/docs/changelog/122938.yaml new file mode 100644 index 0000000000000..cfb6e319c6cd2 --- /dev/null +++ b/docs/changelog/122938.yaml @@ -0,0 +1,5 @@ +pr: 122938 +summary: Fix geoip databases index access after system feature migration (again) +area: Ingest Node +type: bug +issues: [] diff --git a/docs/changelog/123010.yaml b/docs/changelog/123010.yaml new file mode 100644 index 0000000000000..3d95d775b8a54 --- /dev/null +++ b/docs/changelog/123010.yaml @@ -0,0 +1,6 @@ +pr: 123010 +summary: Hold store reference in `InternalEngine#performActionWithDirectoryReader(...)` +area: Engine +type: bug +issues: + - 122974 diff --git a/docs/changelog/123085.yaml b/docs/changelog/123085.yaml new file mode 100644 index 0000000000000..316b1f6f26705 --- /dev/null +++ b/docs/changelog/123085.yaml @@ -0,0 +1,5 @@ +pr: 123085 +summary: Remove duplicated nested commands +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/123155.yaml b/docs/changelog/123155.yaml new file mode 100644 index 0000000000000..73027c87510ba --- /dev/null +++ b/docs/changelog/123155.yaml @@ -0,0 +1,5 @@ +pr: 123155 +summary: Add `ElasticInferenceServiceCompletionServiceSettings` +area: Machine Learning +type: bug +issues: [] diff --git a/libs/entitlement/bridge/src/main/java/module-info.java b/libs/entitlement/bridge/src/main/java/module-info.java index 518a0a1ef29ec..6a85013c1f1f5 100644 --- a/libs/entitlement/bridge/src/main/java/module-info.java +++ b/libs/entitlement/bridge/src/main/java/module-info.java @@ -12,6 +12,7 @@ module org.elasticsearch.entitlement.bridge { requires java.net.http; requires jdk.net; + requires java.logging; exports org.elasticsearch.entitlement.bridge; } diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index f0bbcd9b7d09e..ce162abed60c6 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -88,6 +88,7 @@ import java.util.concurrent.ForkJoinPool; import java.util.function.BiPredicate; import java.util.function.Consumer; +import java.util.logging.FileHandler; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; @@ -882,9 +883,34 @@ public interface EntitlementChecker { void check$java_nio_file_Files$$lines(Class callerClass, Path path); - // file system providers void check$java_nio_file_spi_FileSystemProvider$(Class callerClass); + void check$java_util_logging_FileHandler$(Class callerClass); + + void check$java_util_logging_FileHandler$(Class callerClass, String pattern); + + void check$java_util_logging_FileHandler$(Class callerClass, String pattern, boolean append); + + void check$java_util_logging_FileHandler$(Class callerClass, String pattern, int limit, int count); + + void check$java_util_logging_FileHandler$(Class callerClass, String pattern, int limit, int count, boolean append); + + void check$java_util_logging_FileHandler$(Class callerClass, String pattern, long limit, int count, boolean append); + + void check$java_util_logging_FileHandler$close(Class callerClass, FileHandler that); + + void check$java_net_http_HttpRequest$BodyPublishers$$ofFile(Class callerClass, Path path); + + void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class callerClass, Path path); + + void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class callerClass, Path path, OpenOption... options); + + void check$java_net_http_HttpResponse$BodyHandlers$$ofFileDownload(Class callerClass, Path directory, OpenOption... openOptions); + + void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class callerClass, Path directory); + + void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class callerClass, Path directory, OpenOption... openOptions); + void checkNewFileSystem(Class callerClass, FileSystemProvider that, URI uri, Map env); void checkNewFileSystem(Class callerClass, FileSystemProvider that, Path path, Map env); diff --git a/libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java b/libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java index eafac9006daec..74559a12a4da4 100644 --- a/libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java +++ b/libs/entitlement/qa/entitled-plugin/src/main/java/module-info.java @@ -12,6 +12,7 @@ requires org.elasticsearch.entitlement; requires org.elasticsearch.base; // SuppressForbidden requires org.elasticsearch.logging; + requires java.logging; exports org.elasticsearch.entitlement.qa.entitled; // Must be unqualified so non-modular IT tests can call us } diff --git a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java index d62fbf458be0f..b22643c90064e 100644 --- a/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java +++ b/libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java @@ -22,6 +22,8 @@ import java.io.FileWriter; import java.io.IOException; import java.io.RandomAccessFile; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; @@ -29,10 +31,13 @@ import java.security.KeyStore; import java.util.Scanner; import java.util.jar.JarFile; +import java.util.logging.FileHandler; import java.util.zip.ZipException; import java.util.zip.ZipFile; import static java.nio.charset.Charset.defaultCharset; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.WRITE; import static java.util.zip.ZipFile.OPEN_DELETE; import static java.util.zip.ZipFile.OPEN_READ; import static org.elasticsearch.entitlement.qa.entitled.EntitledActions.createTempFileForWrite; @@ -477,5 +482,86 @@ static void createScannerFileWithCharsetName() throws FileNotFoundException { new Scanner(readFile().toFile(), "UTF-8"); } + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler() throws IOException { + new FileHandler(); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler_String() throws IOException { + new FileHandler(readFile().toString()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler_StringBoolean() throws IOException { + new FileHandler(readFile().toString(), false); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler_StringIntInt() throws IOException { + new FileHandler(readFile().toString(), 1, 2); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler_StringIntIntBoolean() throws IOException { + new FileHandler(readFile().toString(), 1, 2, false); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void fileHandler_StringLongIntBoolean() throws IOException { + new FileHandler(readFile().toString(), 1L, 2, false); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void httpRequestBodyPublishersOfFile() throws IOException { + HttpRequest.BodyPublishers.ofFile(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void httpResponseBodyHandlersOfFile() { + HttpResponse.BodyHandlers.ofFile(readWriteFile()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void httpResponseBodyHandlersOfFile_readOnly() { + HttpResponse.BodyHandlers.ofFile(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void httpResponseBodyHandlersOfFileDownload() { + HttpResponse.BodyHandlers.ofFileDownload(readWriteDir()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void httpResponseBodyHandlersOfFileDownload_readOnly() { + HttpResponse.BodyHandlers.ofFileDownload(readDir()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void httpResponseBodySubscribersOfFile_File() { + HttpResponse.BodySubscribers.ofFile(readWriteFile()); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void httpResponseBodySubscribersOfFile_File_readOnly() { + HttpResponse.BodySubscribers.ofFile(readFile()); + } + + @EntitlementTest(expectedAccess = PLUGINS) + static void httpResponseBodySubscribersOfFile_FileOpenOptions() { + // Note that, unlike other methods like BodyHandlers.ofFile, this is indeed + // an overload distinct from ofFile with no OpenOptions, and so it needs its + // own instrumentation and its own test. + HttpResponse.BodySubscribers.ofFile(readWriteFile(), CREATE, WRITE); + } + + @EntitlementTest(expectedAccess = ALWAYS_DENIED) + static void httpResponseBodySubscribersOfFile_FileOpenOptions_readOnly() { + // Note that, unlike other methods like BodyHandlers.ofFile, this is indeed + // an overload distinct from ofFile with no OpenOptions, and so it needs its + // own instrumentation and its own test. + HttpResponse.BodySubscribers.ofFile(readFile(), CREATE, WRITE); + } + private FileCheckActions() {} } diff --git a/libs/entitlement/src/main/java/module-info.java b/libs/entitlement/src/main/java/module-info.java index 697d26747b806..d6737a14a0b88 100644 --- a/libs/entitlement/src/main/java/module-info.java +++ b/libs/entitlement/src/main/java/module-info.java @@ -8,12 +8,13 @@ */ module org.elasticsearch.entitlement { + requires org.elasticsearch.base; requires org.elasticsearch.xcontent; requires org.elasticsearch.logging; requires java.instrument; - requires org.elasticsearch.base; - requires jdk.attach; + requires java.logging; requires java.net.http; + requires jdk.attach; requires jdk.net; requires static org.elasticsearch.entitlement.bridge; // At runtime, this will be in java.base diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index f6a45cce3c56c..028d03056724d 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -97,6 +97,7 @@ import java.util.concurrent.ForkJoinPool; import java.util.function.BiPredicate; import java.util.function.Consumer; +import java.util.logging.FileHandler; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; @@ -1845,6 +1846,78 @@ public void checkSelectorProviderInheritedChannel(Class callerClass, Selector policyManager.checkChangeJVMGlobalState(callerClass); } + @Override + public void check$java_util_logging_FileHandler$(Class callerClass) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass, String pattern) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass, String pattern, boolean append) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass, String pattern, int limit, int count) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass, String pattern, int limit, int count, boolean append) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$(Class callerClass, String pattern, long limit, int count, boolean append) { + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_util_logging_FileHandler$close(Class callerClass, FileHandler that) { + // Note that there's no IT test for this one, because there's no way to create + // a FileHandler. However, we have this check just in case someone does manage + // to get their hands on a FileHandler and uses close() to cause its lock file to be deleted. + policyManager.checkLoggingFileHandler(callerClass); + } + + @Override + public void check$java_net_http_HttpRequest$BodyPublishers$$ofFile(Class callerClass, Path path) { + policyManager.checkFileRead(callerClass, path); + } + + @Override + public void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class callerClass, Path path) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_net_http_HttpResponse$BodyHandlers$$ofFile(Class callerClass, Path path, OpenOption... options) { + policyManager.checkFileWrite(callerClass, path); + } + + @Override + public void check$java_net_http_HttpResponse$BodyHandlers$$ofFileDownload( + Class callerClass, + Path directory, + OpenOption... openOptions + ) { + policyManager.checkFileWrite(callerClass, directory); + } + + @Override + public void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class callerClass, Path directory) { + policyManager.checkFileWrite(callerClass, directory); + } + + @Override + public void check$java_net_http_HttpResponse$BodySubscribers$$ofFile(Class callerClass, Path directory, OpenOption... openOptions) { + policyManager.checkFileWrite(callerClass, directory); + } + @Override public void checkNewFileSystem(Class callerClass, FileSystemProvider that, URI uri, Map env) { policyManager.checkChangeJVMGlobalState(callerClass); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 2999377cac28b..c85198b26a2aa 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -295,6 +295,10 @@ public void checkChangeJVMGlobalState(Class callerClass) { neverEntitled(callerClass, () -> walkStackForCheckMethodName().orElse("change JVM global state")); } + public void checkLoggingFileHandler(Class callerClass) { + neverEntitled(callerClass, () -> walkStackForCheckMethodName().orElse("create logging file handler")); + } + private Optional walkStackForCheckMethodName() { // Look up the check$ method to compose an informative error message. // This way, we don't need to painstakingly describe every individual global-state change. diff --git a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java index 0ba3b4ebb69f5..15e85c23bf51e 100644 --- a/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java +++ b/modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java @@ -123,6 +123,9 @@ public void testGeoIpSystemFeaturesMigration() throws Exception { // as should a normal get * assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndex)); + + // and getting data streams + assertBusy(() -> testGetDatastreams()); } else { // after the upgrade, but before the migration, Kibana should work assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), maybeSecurityIndex)); @@ -130,6 +133,9 @@ public void testGeoIpSystemFeaturesMigration() throws Exception { // as should a normal get * assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndex)); + // and getting data streams + assertBusy(() -> testGetDatastreams()); + // migrate the system features and give the cluster a moment to settle Request migrateSystemFeatures = new Request("POST", "/_migration/system_features"); assertOK(client().performRequest(migrateSystemFeatures)); @@ -144,6 +150,9 @@ public void testGeoIpSystemFeaturesMigration() throws Exception { // as should a normal get * assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndexReindexed)); + // and getting data streams + assertBusy(() -> testGetDatastreams()); + Request disableDownloader = new Request("PUT", "/_cluster/settings"); disableDownloader.setJsonEntity(""" {"persistent": {"ingest.geoip.downloader.enabled": false}} @@ -257,4 +266,15 @@ private void testGetStarAsKibana(List indexNames, @Nullable List Map map = responseAsMap(response); assertThat(map.keySet(), is(new HashSet<>(indexNames))); } + + private void testGetDatastreams() throws IOException { + Request getStar = new Request("GET", "_data_stream"); + getStar.setOptions( + RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors + ); + Response response = client().performRequest(getStar); + assertOK(response); + + // note: we don't actually care about the response, just that there was one and that it didn't error out on us + } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java index 0ef4686dc033e..37b406c1403c7 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java @@ -50,6 +50,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -341,7 +342,7 @@ private String mockSearches(String databaseName, int firstChunk, int lastChunk) } SearchHits hits = SearchHits.unpooled(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f); - SearchResponse searchResponse = new SearchResponse(hits, null, null, false, null, null, 0, null, 1, 1, 0, 1L, null, null); + SearchResponse searchResponse = SearchResponseUtils.successfulResponse(hits); toRelease.add(searchResponse::decRef); @SuppressWarnings("unchecked") ActionFuture actionFuture = mock(ActionFuture.class); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 8f269ad4066c0..17729d7c57dde 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -321,8 +321,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { return BlockLoader.CONSTANT_NULLS; } if (hasDocValues()) { - double scalingFactorInverse = 1d / scalingFactor; - return new BlockDocValuesReader.DoublesBlockLoader(name(), l -> l * scalingFactorInverse); + return new BlockDocValuesReader.DoublesBlockLoader(name(), l -> l / scalingFactor); } if (isSyntheticSource) { return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldBlockLoaderTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldBlockLoaderTests.java index 5541b5677f00e..c44efd2e52da2 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldBlockLoaderTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldBlockLoaderTests.java @@ -26,16 +26,6 @@ public ScaledFloatFieldBlockLoaderTests() { protected Double convert(Number value, Map fieldMapping) { var scalingFactor = ((Number) fieldMapping.get("scaling_factor")).doubleValue(); - var docValues = (boolean) fieldMapping.getOrDefault("doc_values", false); - - // There is a slight inconsistency between values that are read from doc_values and from source. - // Due to how precision reduction is applied to source values so that they are consistent with doc_values. - // See #122547. - if (docValues) { - var reverseScalingFactor = 1d / scalingFactor; - return Math.round(value.doubleValue() * scalingFactor) * reverseScalingFactor; - } - // Adjust values coming from source to the way they are stored in doc_values. // See mapper implementation. return Math.round(value.doubleValue() * scalingFactor) / scalingFactor; diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java index 28f2eafc20a6e..db642bbdc5105 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java @@ -68,6 +68,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; @@ -574,22 +575,7 @@ protected RequestWrapper buildRequest(Hit doc) { new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0 ); - SearchResponse searchResponse = new SearchResponse( - hits, - null, - null, - false, - false, - null, - 1, - scrollId(), - 5, - 4, - 0, - randomLong(), - null, - SearchResponse.Clusters.EMPTY - ); + SearchResponse searchResponse = SearchResponseUtils.response(hits).scrollId(scrollId()).shards(5, 4, 0).build(); try { client.lastSearch.get().listener.onResponse(searchResponse); diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java index 5f4e2b3a55156..26922c62d3931 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.reindex.ScrollableHitSource; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; @@ -166,22 +167,7 @@ private SearchResponse createSearchResponse() { new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0 ); - return new SearchResponse( - hits, - null, - null, - false, - false, - null, - 1, - randomSimpleString(random(), 1, 10), - 5, - 4, - 0, - randomLong(), - null, - SearchResponse.Clusters.EMPTY - ); + return SearchResponseUtils.response(hits).scrollId(randomSimpleString(random(), 1, 10)).shards(5, 4, 0).build(); } private void assertSameHits(List actual, SearchHit[] expected) { diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java index 3ed492881afa9..22c600445c609 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java @@ -41,7 +41,7 @@ class GoogleCloudStorageBlobContainer extends AbstractBlobContainer { @Override public boolean blobExists(OperationPurpose purpose, String blobName) { try { - return blobStore.blobExists(buildKey(blobName)); + return blobStore.blobExists(purpose, buildKey(blobName)); } catch (Exception e) { throw new BlobStoreException("Failed to check if blob [" + blobName + "] exists", e); } @@ -49,39 +49,39 @@ public boolean blobExists(OperationPurpose purpose, String blobName) { @Override public Map listBlobs(OperationPurpose purpose) throws IOException { - return blobStore.listBlobs(path); + return blobStore.listBlobs(purpose, path); } @Override public Map children(OperationPurpose purpose) throws IOException { - return blobStore.listChildren(path()); + return blobStore.listChildren(purpose, path()); } @Override public Map listBlobsByPrefix(OperationPurpose purpose, String prefix) throws IOException { - return blobStore.listBlobsByPrefix(path, prefix); + return blobStore.listBlobsByPrefix(purpose, path, prefix); } @Override public InputStream readBlob(OperationPurpose purpose, String blobName) throws IOException { - return blobStore.readBlob(buildKey(blobName)); + return blobStore.readBlob(purpose, buildKey(blobName)); } @Override public InputStream readBlob(OperationPurpose purpose, final String blobName, final long position, final long length) throws IOException { - return blobStore.readBlob(buildKey(blobName), position, length); + return blobStore.readBlob(purpose, buildKey(blobName), position, length); } @Override public void writeBlob(OperationPurpose purpose, String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { - blobStore.writeBlob(buildKey(blobName), inputStream, blobSize, failIfAlreadyExists); + blobStore.writeBlob(purpose, buildKey(blobName), inputStream, blobSize, failIfAlreadyExists); } @Override public void writeBlob(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { - blobStore.writeBlob(buildKey(blobName), bytes, failIfAlreadyExists); + blobStore.writeBlob(purpose, buildKey(blobName), bytes, failIfAlreadyExists); } @Override @@ -92,7 +92,7 @@ public void writeMetadataBlob( boolean atomic, CheckedConsumer writer ) throws IOException { - blobStore.writeBlob(buildKey(blobName), failIfAlreadyExists, writer); + blobStore.writeBlob(purpose, buildKey(blobName), failIfAlreadyExists, writer); } @Override @@ -114,12 +114,12 @@ public void writeBlobAtomic(OperationPurpose purpose, String blobName, BytesRefe @Override public DeleteResult delete(OperationPurpose purpose) throws IOException { - return blobStore.deleteDirectory(path().buildAsString()); + return blobStore.deleteDirectory(purpose, path().buildAsString()); } @Override public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator blobNames) throws IOException { - blobStore.deleteBlobs(new Iterator<>() { + blobStore.deleteBlobs(purpose, new Iterator<>() { @Override public boolean hasNext() { return blobNames.hasNext(); @@ -145,11 +145,14 @@ public void compareAndExchangeRegister( BytesReference updated, ActionListener listener ) { - ActionListener.completeWith(listener, () -> blobStore.compareAndExchangeRegister(buildKey(key), path, key, expected, updated)); + ActionListener.completeWith( + listener, + () -> blobStore.compareAndExchangeRegister(purpose, buildKey(key), path, key, expected, updated) + ); } @Override public void getRegister(OperationPurpose purpose, String key, ActionListener listener) { - ActionListener.completeWith(listener, () -> blobStore.getRegister(buildKey(key), path, key)); + ActionListener.completeWith(listener, () -> blobStore.getRegister(purpose, buildKey(key), path, key)); } } diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java index 48192e9173ffa..e09c601050a8e 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStore.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.BlobStoreActionStats; import org.elasticsearch.common.blobstore.DeleteResult; +import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.blobstore.OptionalBytesReference; import org.elasticsearch.common.blobstore.support.BlobContainerUtils; import org.elasticsearch.common.blobstore.support.BlobMetadata; @@ -128,8 +129,8 @@ class GoogleCloudStorageBlobStore implements BlobStore { this.casBackoffPolicy = casBackoffPolicy; } - private Storage client() throws IOException { - return storageService.client(clientName, repositoryName, stats); + private Storage client(OperationPurpose purpose) throws IOException { + return storageService.client(clientName, repositoryName, purpose, stats); } @Override @@ -139,33 +140,35 @@ public BlobContainer blobContainer(BlobPath path) { @Override public void close() { - storageService.closeRepositoryClient(repositoryName); + storageService.closeRepositoryClients(repositoryName); } /** * List blobs in the specific bucket under the specified path. The path root is removed. * + * @param purpose the operation purpose * @param path base path of the blobs to list * @return a map of blob names and their metadata */ - Map listBlobs(String path) throws IOException { - return listBlobsByPrefix(path, ""); + Map listBlobs(OperationPurpose purpose, String path) throws IOException { + return listBlobsByPrefix(purpose, path, ""); } /** * List all blobs in the specific bucket with names prefixed * + * @param purpose the operation purpose * @param path * base path of the blobs to list. This path is removed from the * names of the blobs returned. * @param prefix prefix of the blobs to list. * @return a map of blob names and their metadata. */ - Map listBlobsByPrefix(String path, String prefix) throws IOException { + Map listBlobsByPrefix(OperationPurpose purpose, String path, String prefix) throws IOException { final String pathPrefix = buildKey(path, prefix); final Map mapBuilder = new HashMap<>(); SocketAccess.doPrivilegedVoidIOException( - () -> client().list(bucketName, BlobListOption.currentDirectory(), BlobListOption.prefix(pathPrefix)) + () -> client(purpose).list(bucketName, BlobListOption.currentDirectory(), BlobListOption.prefix(pathPrefix)) .iterateAll() .forEach(blob -> { assert blob.getName().startsWith(path); @@ -178,11 +181,11 @@ Map listBlobsByPrefix(String path, String prefix) throws I return Map.copyOf(mapBuilder); } - Map listChildren(BlobPath path) throws IOException { + Map listChildren(OperationPurpose purpose, BlobPath path) throws IOException { final String pathStr = path.buildAsString(); final Map mapBuilder = new HashMap<>(); SocketAccess.doPrivilegedVoidIOException( - () -> client().list(bucketName, BlobListOption.currentDirectory(), BlobListOption.prefix(pathStr)) + () -> client(purpose).list(bucketName, BlobListOption.currentDirectory(), BlobListOption.prefix(pathStr)) .iterateAll() .forEach(blob -> { if (blob.isDirectory()) { @@ -202,34 +205,37 @@ Map listChildren(BlobPath path) throws IOException { /** * Returns true if the blob exists in the specific bucket * + * @param purpose the operation purpose * @param blobName name of the blob * @return true iff the blob exists */ - boolean blobExists(String blobName) throws IOException { + boolean blobExists(OperationPurpose purpose, String blobName) throws IOException { final BlobId blobId = BlobId.of(bucketName, blobName); - final Blob blob = SocketAccess.doPrivilegedIOException(() -> client().get(blobId)); + final Blob blob = SocketAccess.doPrivilegedIOException(() -> client(purpose).get(blobId)); return blob != null; } /** * Returns an {@link java.io.InputStream} for the given blob name * + * @param purpose the operation purpose * @param blobName name of the blob * @return the InputStream used to read the blob's content */ - InputStream readBlob(String blobName) throws IOException { - return new GoogleCloudStorageRetryingInputStream(client(), BlobId.of(bucketName, blobName)); + InputStream readBlob(OperationPurpose purpose, String blobName) throws IOException { + return new GoogleCloudStorageRetryingInputStream(client(purpose), BlobId.of(bucketName, blobName)); } /** * Returns an {@link java.io.InputStream} for the given blob's position and length * + * @param purpose the operation purpose * @param blobName name of the blob * @param position starting position to read from * @param length length of bytes to read * @return the InputStream used to read the blob's content */ - InputStream readBlob(String blobName, long position, long length) throws IOException { + InputStream readBlob(OperationPurpose purpose, String blobName, long position, long length) throws IOException { if (position < 0L) { throw new IllegalArgumentException("position must be non-negative"); } @@ -240,7 +246,7 @@ InputStream readBlob(String blobName, long position, long length) throws IOExcep return new ByteArrayInputStream(new byte[0]); } else { return new GoogleCloudStorageRetryingInputStream( - client(), + client(purpose), BlobId.of(bucketName, blobName), position, Math.addExact(position, length - 1) @@ -250,16 +256,18 @@ InputStream readBlob(String blobName, long position, long length) throws IOExcep /** * Writes a blob in the specific bucket + * @param purpose the operation purpose * @param bytes content of the blob to be written * @param failIfAlreadyExists whether to throw a FileAlreadyExistsException if the given blob already exists */ - void writeBlob(String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + void writeBlob(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { if (bytes.length() > getLargeBlobThresholdInBytes()) { // Compute md5 here so #writeBlobResumable forces the integrity check on the resumable upload. // This is needed since we rely on atomic write behavior when writing BytesReferences in BlobStoreRepository which is not // guaranteed for resumable uploads. final String md5 = Base64.getEncoder().encodeToString(MessageDigests.digest(bytes, MessageDigests.md5())); writeBlobResumable( + purpose, BlobInfo.newBuilder(bucketName, blobName).setMd5(md5).build(), bytes.streamInput(), bytes.length(), @@ -268,30 +276,33 @@ void writeBlob(String blobName, BytesReference bytes, boolean failIfAlreadyExist } else { final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); if (bytes.hasArray()) { - writeBlobMultipart(blobInfo, bytes.array(), bytes.arrayOffset(), bytes.length(), failIfAlreadyExists); + writeBlobMultipart(purpose, blobInfo, bytes.array(), bytes.arrayOffset(), bytes.length(), failIfAlreadyExists); } else { - writeBlob(bytes.streamInput(), bytes.length(), failIfAlreadyExists, blobInfo); + writeBlob(purpose, bytes.streamInput(), bytes.length(), failIfAlreadyExists, blobInfo); } } } /** * Writes a blob in the specific bucket + * @param purpose the operation purpose * @param inputStream content of the blob to be written * @param blobSize expected size of the blob to be written * @param failIfAlreadyExists whether to throw a FileAlreadyExistsException if the given blob already exists */ - void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { - writeBlob(inputStream, blobSize, failIfAlreadyExists, BlobInfo.newBuilder(bucketName, blobName).build()); + void writeBlob(OperationPurpose purpose, String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) + throws IOException { + writeBlob(purpose, inputStream, blobSize, failIfAlreadyExists, BlobInfo.newBuilder(bucketName, blobName).build()); } - private void writeBlob(InputStream inputStream, long blobSize, boolean failIfAlreadyExists, BlobInfo blobInfo) throws IOException { + private void writeBlob(OperationPurpose purpose, InputStream inputStream, long blobSize, boolean failIfAlreadyExists, BlobInfo blobInfo) + throws IOException { if (blobSize > getLargeBlobThresholdInBytes()) { - writeBlobResumable(blobInfo, inputStream, blobSize, failIfAlreadyExists); + writeBlobResumable(purpose, blobInfo, inputStream, blobSize, failIfAlreadyExists); } else { final byte[] buffer = new byte[Math.toIntExact(blobSize)]; Streams.readFully(inputStream, buffer); - writeBlobMultipart(blobInfo, buffer, 0, Math.toIntExact(blobSize), failIfAlreadyExists); + writeBlobMultipart(purpose, blobInfo, buffer, 0, Math.toIntExact(blobSize), failIfAlreadyExists); } } @@ -308,7 +319,12 @@ long getLargeBlobThresholdInBytes() { Storage.BlobWriteOption.md5Match() }; private static final Storage.BlobWriteOption[] OVERWRITE_CHECK_MD5 = { Storage.BlobWriteOption.md5Match() }; - void writeBlob(String blobName, boolean failIfAlreadyExists, CheckedConsumer writer) throws IOException { + void writeBlob( + OperationPurpose purpose, + String blobName, + boolean failIfAlreadyExists, + CheckedConsumer writer + ) throws IOException { final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build(); final Storage.BlobWriteOption[] writeOptions = failIfAlreadyExists ? NO_OVERWRITE_NO_MD5 : OVERWRITE_NO_MD5; @@ -354,7 +370,7 @@ public void write(byte[] b, int off, int len) throws IOException { private void initResumableStream() throws IOException { final WriteChannel writeChannel = SocketAccess.doPrivilegedIOException( - () -> client().writer(blobInfo, writeOptions) + () -> client(purpose).writer(blobInfo, writeOptions) ); channelRef.set(writeChannel); resumableStream = new FilterOutputStream(Channels.newOutputStream(new WritableBlobChannel(writeChannel))) { @@ -379,7 +395,7 @@ public void write(byte[] b, int off, int len) throws IOException { SocketAccess.doPrivilegedVoidIOException(writeChannel::close); stats.trackPutOperation(); } else { - writeBlob(blobName, buffer.bytes(), failIfAlreadyExists); + writeBlob(purpose, blobName, buffer.bytes(), failIfAlreadyExists); } return; } catch (final StorageException se) { @@ -405,12 +421,19 @@ public void write(byte[] b, int off, int len) throws IOException { * Uploads a blob using the "resumable upload" method (multiple requests, which * can be independently retried in case of failure, see * https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload + * @param purpose the operation purpose * @param blobInfo the info for the blob to be uploaded * @param inputStream the stream containing the blob data * @param size expected size of the blob to be written * @param failIfAlreadyExists whether to throw a FileAlreadyExistsException if the given blob already exists */ - private void writeBlobResumable(BlobInfo blobInfo, InputStream inputStream, long size, boolean failIfAlreadyExists) throws IOException { + private void writeBlobResumable( + OperationPurpose purpose, + BlobInfo blobInfo, + InputStream inputStream, + long size, + boolean failIfAlreadyExists + ) throws IOException { // We retry 410 GONE errors to cover the unlikely but possible scenario where a resumable upload session becomes broken and // needs to be restarted from scratch. Given how unlikely a 410 error should be according to SLAs we retry only twice. assert inputStream.markSupported(); @@ -427,7 +450,9 @@ private void writeBlobResumable(BlobInfo blobInfo, InputStream inputStream, long } for (int retry = 0; retry < 3; ++retry) { try { - final WriteChannel writeChannel = SocketAccess.doPrivilegedIOException(() -> client().writer(blobInfo, writeOptions)); + final WriteChannel writeChannel = SocketAccess.doPrivilegedIOException( + () -> client(purpose).writer(blobInfo, writeOptions) + ); /* * It is not enough to wrap the call to Streams#copy, we have to wrap the privileged calls too; this is because Streams#copy * is in the stacktrace and is not granted the permissions needed to close and write the channel. @@ -465,20 +490,27 @@ private void writeBlobResumable(BlobInfo blobInfo, InputStream inputStream, long * 'multipart/related' request containing both data and metadata. The request is * gziped), see: * https://cloud.google.com/storage/docs/json_api/v1/how-tos/multipart-upload + * @param purpose the operation purpose * @param blobInfo the info for the blob to be uploaded * @param buffer the byte array containing the data * @param offset offset at which the blob contents start in the buffer * @param blobSize the size * @param failIfAlreadyExists whether to throw a FileAlreadyExistsException if the given blob already exists */ - private void writeBlobMultipart(BlobInfo blobInfo, byte[] buffer, int offset, int blobSize, boolean failIfAlreadyExists) - throws IOException { + private void writeBlobMultipart( + OperationPurpose purpose, + BlobInfo blobInfo, + byte[] buffer, + int offset, + int blobSize, + boolean failIfAlreadyExists + ) throws IOException { assert blobSize <= getLargeBlobThresholdInBytes() : "large blob uploads should use the resumable upload method"; try { final Storage.BlobTargetOption[] targetOptions = failIfAlreadyExists ? new Storage.BlobTargetOption[] { Storage.BlobTargetOption.doesNotExist() } : new Storage.BlobTargetOption[0]; - SocketAccess.doPrivilegedVoidIOException(() -> client().create(blobInfo, buffer, offset, blobSize, targetOptions)); + SocketAccess.doPrivilegedVoidIOException(() -> client(purpose).create(blobInfo, buffer, offset, blobSize, targetOptions)); // We don't track this operation on the http layer as // we do with the GET/LIST operations since this operations // can trigger multiple underlying http requests but only one @@ -495,17 +527,18 @@ private void writeBlobMultipart(BlobInfo blobInfo, byte[] buffer, int offset, in /** * Deletes the given path and all its children. * + * @param purpose the operation purpose * @param pathStr Name of path to delete */ - DeleteResult deleteDirectory(String pathStr) throws IOException { + DeleteResult deleteDirectory(OperationPurpose purpose, String pathStr) throws IOException { return SocketAccess.doPrivilegedIOException(() -> { DeleteResult deleteResult = DeleteResult.ZERO; - Page page = client().list(bucketName, BlobListOption.prefix(pathStr)); + Page page = client(purpose).list(bucketName, BlobListOption.prefix(pathStr)); do { final AtomicLong blobsDeleted = new AtomicLong(0L); final AtomicLong bytesDeleted = new AtomicLong(0L); final Iterator blobs = page.getValues().iterator(); - deleteBlobs(new Iterator<>() { + deleteBlobs(purpose, new Iterator<>() { @Override public boolean hasNext() { return blobs.hasNext(); @@ -529,9 +562,10 @@ public String next() { /** * Deletes multiple blobs from the specific bucket using a batch request * + * @param purpose the operation purpose * @param blobNames names of the blobs to delete */ - void deleteBlobs(Iterator blobNames) throws IOException { + void deleteBlobs(OperationPurpose purpose, Iterator blobNames) throws IOException { if (blobNames.hasNext() == false) { return; } @@ -550,7 +584,7 @@ public BlobId next() { try { SocketAccess.doPrivilegedVoidIOException(() -> { final AtomicReference ioe = new AtomicReference<>(); - StorageBatch batch = client().batch(); + StorageBatch batch = client(purpose).batch(); int pendingDeletesInBatch = 0; while (blobIdsToDelete.hasNext()) { BlobId blob = blobIdsToDelete.next(); @@ -574,7 +608,7 @@ public void error(StorageException exception) { pendingDeletesInBatch++; if (pendingDeletesInBatch % MAX_DELETES_PER_BATCH == 0) { batch.submit(); - batch = client().batch(); + batch = client(purpose).batch(); pendingDeletesInBatch = 0; } } @@ -628,10 +662,10 @@ public void close() { } } - OptionalBytesReference getRegister(String blobName, String container, String key) throws IOException { + OptionalBytesReference getRegister(OperationPurpose purpose, String blobName, String container, String key) throws IOException { final var blobId = BlobId.of(bucketName, blobName); try ( - var readChannel = SocketAccess.doPrivilegedIOException(() -> client().reader(blobId)); + var readChannel = SocketAccess.doPrivilegedIOException(() -> client(purpose).reader(blobId)); var stream = new PrivilegedReadChannelStream(readChannel) ) { return OptionalBytesReference.of(BlobContainerUtils.getRegisterUsingConsistentRead(stream, container, key)); @@ -648,6 +682,7 @@ OptionalBytesReference getRegister(String blobName, String container, String key } OptionalBytesReference compareAndExchangeRegister( + OperationPurpose purpose, String blobName, String container, String key, @@ -657,7 +692,7 @@ OptionalBytesReference compareAndExchangeRegister( BlobContainerUtils.ensureValidRegisterContent(updated); final var blobId = BlobId.of(bucketName, blobName); - final var blob = SocketAccess.doPrivilegedIOException(() -> client().get(blobId)); + final var blob = SocketAccess.doPrivilegedIOException(() -> client(purpose).get(blobId)); final long generation; if (blob == null || blob.getGeneration() == null) { @@ -670,7 +705,7 @@ OptionalBytesReference compareAndExchangeRegister( try ( var stream = new PrivilegedReadChannelStream( SocketAccess.doPrivilegedIOException( - () -> client().reader(blobId, Storage.BlobSourceOption.generationMatch(generation)) + () -> client(purpose).reader(blobId, Storage.BlobSourceOption.generationMatch(generation)) ) ) ) { @@ -702,7 +737,7 @@ OptionalBytesReference compareAndExchangeRegister( while (true) { try { SocketAccess.doPrivilegedVoidIOException( - () -> client().create( + () -> client(purpose).create( blobInfo, bytesRef.bytes, bytesRef.offset, @@ -710,6 +745,7 @@ OptionalBytesReference compareAndExchangeRegister( Storage.BlobTargetOption.generationMatch() ) ); + stats.trackPostOperation(); return OptionalBytesReference.of(expected); } catch (Exception e) { final var serviceException = unwrapServiceException(e); diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java index e7e21360de545..9cd657f34c9fe 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageHttpStatsCollector.java @@ -14,6 +14,8 @@ import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpResponseInterceptor; +import org.elasticsearch.common.blobstore.OperationPurpose; + import java.util.List; import java.util.Locale; import java.util.function.Consumer; @@ -43,10 +45,12 @@ final class GoogleCloudStorageHttpStatsCollector implements HttpResponseIntercep ); private final GoogleCloudStorageOperationsStats gcsOperationStats; + private final OperationPurpose operationPurpose; private final List trackers; - GoogleCloudStorageHttpStatsCollector(final GoogleCloudStorageOperationsStats gcsOperationStats) { + GoogleCloudStorageHttpStatsCollector(final GoogleCloudStorageOperationsStats gcsOperationStats, OperationPurpose operationPurpose) { this.gcsOperationStats = gcsOperationStats; + this.operationPurpose = operationPurpose; this.trackers = trackerFactories.stream() .map(trackerFactory -> trackerFactory.apply(gcsOperationStats.getTrackedBucket())) .toList(); diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java index 6a4eeeeabbb6f..befd67b5aa565 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java @@ -25,6 +25,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; @@ -40,6 +41,7 @@ import java.net.URL; import java.security.KeyStore; import java.util.Map; +import java.util.stream.Collectors; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.emptyMap; @@ -51,12 +53,15 @@ public class GoogleCloudStorageService { private volatile Map clientSettings = emptyMap(); + private record ClientKey(OperationPurpose purpose, String repositoryName) {} + /** * Dictionary of client instances. Client instances are built lazily from the - * latest settings. Each repository has its own client instance identified by - * the repository name. + * latest settings. Clients are cached by a composite OperationPurpose/repositoryName + * key. + * @see ClientKey */ - private volatile Map clientCache = emptyMap(); + private volatile Map clientCache = emptyMap(); /** * Refreshes the client settings and clears the client cache. Subsequent calls to @@ -79,20 +84,26 @@ public synchronized void refreshAndClearCache(Map format("creating GCS client with client_name [%s], endpoint [%s]", clientName, settings.getHost())); - final Storage storage = createClient(settings, stats); - clientCache = Maps.copyMapWithAddedEntry(clientCache, repositoryName, storage); + final Storage storage = createClient(settings, stats, operationPurpose); + clientCache = Maps.copyMapWithAddedEntry(clientCache, clientKey, storage); return storage; } } - synchronized void closeRepositoryClient(String repositoryName) { - clientCache = Maps.copyMapWithRemovedEntry(clientCache, repositoryName); + synchronized void closeRepositoryClients(String repositoryName) { + clientCache = clientCache.entrySet() + .stream() + .filter(entry -> entry.getKey().repositoryName().equals(repositoryName) == false) + .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); } /** @@ -125,11 +139,15 @@ synchronized void closeRepositoryClient(String repositoryName) { * * @param gcsClientSettings client settings to use, including secure settings * @param stats the stats collector to use by the underlying SDK + * @param operationPurpose the purpose this client will be used for * @return a new client storage instance that can be used to manage objects * (blobs) */ - private Storage createClient(GoogleCloudStorageClientSettings gcsClientSettings, GoogleCloudStorageOperationsStats stats) - throws IOException { + private Storage createClient( + GoogleCloudStorageClientSettings gcsClientSettings, + GoogleCloudStorageOperationsStats stats, + OperationPurpose operationPurpose + ) throws IOException { final HttpTransport httpTransport = SocketAccess.doPrivilegedIOException(() -> { final NetHttpTransport.Builder builder = new NetHttpTransport.Builder(); // requires java.lang.RuntimePermission "setFactory" @@ -149,7 +167,7 @@ private Storage createClient(GoogleCloudStorageClientSettings gcsClientSettings, return builder.build(); }); - final GoogleCloudStorageHttpStatsCollector httpStatsCollector = new GoogleCloudStorageHttpStatsCollector(stats); + final GoogleCloudStorageHttpStatsCollector httpStatsCollector = new GoogleCloudStorageHttpStatsCollector(stats, operationPurpose); final HttpTransportOptions httpTransportOptions = new HttpTransportOptions( HttpTransportOptions.newBuilder() diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerStatsTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerStatsTests.java new file mode 100644 index 0000000000000..c028c89ff8a65 --- /dev/null +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerStatsTests.java @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.gcs; + +import fixture.gcs.FakeOAuth2HttpHandler; +import fixture.gcs.GoogleCloudStorageHttpHandler; + +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.sun.net.httpserver.HttpServer; + +import org.elasticsearch.common.BackoffPolicy; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStoreActionStats; +import org.elasticsearch.common.blobstore.support.BlobContainerUtils; +import org.elasticsearch.common.blobstore.support.BlobMetadata; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.mocksocket.MockHttpServer; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.Closeable; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.URI; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CONNECT_TIMEOUT_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROJECT_ID_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.READ_TIMEOUT_SETTING; + +@SuppressForbidden(reason = "Uses a HttpServer to emulate a Google Cloud Storage endpoint") +public class GoogleCloudStorageBlobContainerStatsTests extends ESTestCase { + private static final String BUCKET = "bucket"; + private static final ByteSizeValue BUFFER_SIZE = ByteSizeValue.ofKb(128); + + private HttpServer httpServer; + private ThreadPool threadPool; + private GoogleCloudStorageService googleCloudStorageService; + private GoogleCloudStorageHttpHandler googleCloudStorageHttpHandler; + private ContainerAndBlobStore containerAndStore; + + @Before + public void createStorageService() throws Exception { + threadPool = new TestThreadPool(getTestClass().getName()); + httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + httpServer.start(); + googleCloudStorageService = new GoogleCloudStorageService(); + googleCloudStorageHttpHandler = new GoogleCloudStorageHttpHandler(BUCKET); + httpServer.createContext("/", googleCloudStorageHttpHandler); + httpServer.createContext("/token", new FakeOAuth2HttpHandler()); + containerAndStore = createBlobContainer(randomIdentifier()); + } + + @After + public void stopHttpServer() { + IOUtils.closeWhileHandlingException(containerAndStore); + httpServer.stop(0); + ThreadPool.terminate(threadPool, 10L, TimeUnit.SECONDS); + } + + @Test + public void testSingleMultipartWrite() throws Exception { + final GoogleCloudStorageBlobContainer container = containerAndStore.blobContainer(); + final GoogleCloudStorageBlobStore store = containerAndStore.blobStore(); + + final String blobName = randomIdentifier(); + final int blobLength = randomIntBetween(1, (int) store.getLargeBlobThresholdInBytes() - 1); + final BytesArray blobContents = new BytesArray(randomByteArrayOfLength(blobLength)); + container.writeBlob(randomPurpose(), blobName, blobContents, true); + assertEquals(createStats(1, 0, 0), store.stats()); + + try (InputStream is = container.readBlob(randomPurpose(), blobName)) { + assertEquals(blobContents, Streams.readFully(is)); + } + assertEquals(createStats(1, 0, 1), store.stats()); + } + + @Test + public void testResumableWrite() throws Exception { + final GoogleCloudStorageBlobContainer container = containerAndStore.blobContainer(); + final GoogleCloudStorageBlobStore store = containerAndStore.blobStore(); + + final String blobName = randomIdentifier(); + final int size = randomIntBetween((int) store.getLargeBlobThresholdInBytes(), (int) store.getLargeBlobThresholdInBytes() * 2); + final BytesArray blobContents = new BytesArray(randomByteArrayOfLength(size)); + container.writeBlob(randomPurpose(), blobName, blobContents, true); + assertEquals(createStats(1, 0, 0), store.stats()); + + try (InputStream is = container.readBlob(randomPurpose(), blobName)) { + assertEquals(blobContents, Streams.readFully(is)); + } + assertEquals(createStats(1, 0, 1), store.stats()); + } + + @Test + public void testDeleteDirectory() throws Exception { + final GoogleCloudStorageBlobContainer container = containerAndStore.blobContainer(); + final GoogleCloudStorageBlobStore store = containerAndStore.blobStore(); + + final String directoryName = randomIdentifier(); + final BytesArray contents = new BytesArray(randomByteArrayOfLength(50)); + final int numberOfFiles = randomIntBetween(1, 20); + for (int i = 0; i < numberOfFiles; i++) { + container.writeBlob(randomPurpose(), String.format("%s/file_%d", directoryName, i), contents, true); + } + assertEquals(createStats(numberOfFiles, 0, 0), store.stats()); + + container.delete(randomPurpose()); + // We only count the list because we can't track the bulk delete + assertEquals(createStats(numberOfFiles, 1, 0), store.stats()); + } + + @Test + public void testListBlobsAccountsForPaging() throws Exception { + final GoogleCloudStorageBlobContainer container = containerAndStore.blobContainer(); + final GoogleCloudStorageBlobStore store = containerAndStore.blobStore(); + + final int pageSize = randomIntBetween(3, 20); + googleCloudStorageHttpHandler.setDefaultPageLimit(pageSize); + final int numberOfPages = randomIntBetween(1, 10); + final int numberOfObjects = randomIntBetween((numberOfPages - 1) * pageSize, numberOfPages * pageSize - 1); + final BytesArray contents = new BytesArray(randomByteArrayOfLength(50)); + for (int i = 0; i < numberOfObjects; i++) { + container.writeBlob(randomPurpose(), String.format("file_%d", i), contents, true); + } + assertEquals(createStats(numberOfObjects, 0, 0), store.stats()); + + final Map stringBlobMetadataMap = container.listBlobs(randomPurpose()); + assertEquals(numberOfObjects, stringBlobMetadataMap.size()); + // There should be {numberOfPages} pages of blobs + assertEquals(createStats(numberOfObjects, numberOfPages, 0), store.stats()); + } + + public void testCompareAndSetRegister() { + final GoogleCloudStorageBlobContainer container = containerAndStore.blobContainer(); + final GoogleCloudStorageBlobStore store = containerAndStore.blobStore(); + + // update from empty (adds a single insert) + final BytesArray contents = new BytesArray(randomByteArrayOfLength(BlobContainerUtils.MAX_REGISTER_CONTENT_LENGTH)); + final String registerName = randomIdentifier(); + assertTrue(safeAwait(l -> container.compareAndSetRegister(randomPurpose(), registerName, BytesArray.EMPTY, contents, l))); + assertEquals(createStats(1, 0, 0), store.stats()); + + // successful update from non-null (adds two gets, one insert) + final BytesArray nextContents = new BytesArray(randomByteArrayOfLength(BlobContainerUtils.MAX_REGISTER_CONTENT_LENGTH)); + assertTrue(safeAwait(l -> container.compareAndSetRegister(randomPurpose(), registerName, contents, nextContents, l))); + assertEquals(createStats(2, 0, 2), store.stats()); + + // failed update (adds two gets, zero inserts) + final BytesArray wrongContents = randomValueOtherThan( + nextContents, + () -> new BytesArray(randomByteArrayOfLength(BlobContainerUtils.MAX_REGISTER_CONTENT_LENGTH)) + ); + assertFalse(safeAwait(l -> container.compareAndSetRegister(randomPurpose(), registerName, wrongContents, contents, l))); + assertEquals(createStats(2, 0, 4), store.stats()); + } + + private Map createStats(int insertCount, int listCount, int getCount) { + return Map.of( + "GetObject", + new BlobStoreActionStats(getCount, getCount), + "ListObjects", + new BlobStoreActionStats(listCount, listCount), + "InsertObject", + new BlobStoreActionStats(insertCount, insertCount) + ); + } + + private record ContainerAndBlobStore(GoogleCloudStorageBlobContainer blobContainer, GoogleCloudStorageBlobStore blobStore) + implements + Closeable { + + @Override + public void close() { + blobStore.close(); + } + } + + private ContainerAndBlobStore createBlobContainer(final String repositoryName) throws Exception { + final String clientName = randomIdentifier(); + + final Tuple serviceAccountCredentialsTuple = GoogleCloudStorageTestUtilities.randomCredential( + clientName + ); + final GoogleCloudStorageClientSettings clientSettings = new GoogleCloudStorageClientSettings( + serviceAccountCredentialsTuple.v1(), + getEndpointForServer(httpServer), + PROJECT_ID_SETTING.getDefault(Settings.EMPTY), + CONNECT_TIMEOUT_SETTING.getDefault(Settings.EMPTY), + READ_TIMEOUT_SETTING.getDefault(Settings.EMPTY), + APPLICATION_NAME_SETTING.getDefault(Settings.EMPTY), + new URI(getEndpointForServer(httpServer) + "/token"), + null + ); + googleCloudStorageService.refreshAndClearCache(Map.of(clientName, clientSettings)); + final GoogleCloudStorageBlobStore blobStore = new GoogleCloudStorageBlobStore( + BUCKET, + clientName, + repositoryName, + googleCloudStorageService, + BigArrays.NON_RECYCLING_INSTANCE, + Math.toIntExact(BUFFER_SIZE.getBytes()), + BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(10), 10) + ); + final GoogleCloudStorageBlobContainer googleCloudStorageBlobContainer = new GoogleCloudStorageBlobContainer( + BlobPath.EMPTY, + blobStore + ); + return new ContainerAndBlobStore(googleCloudStorageBlobContainer, blobStore); + } + + protected String getEndpointForServer(final HttpServer server) { + final InetSocketAddress address = server.getAddress(); + return "http://" + address.getHostString() + ":" + address.getPort(); + } +} diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java index 81509c7f2183b..c3693d70c8469 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.test.ESTestCase; @@ -79,7 +80,14 @@ public void testDeleteBlobsIgnoringIfNotExistsThrowsIOException() throws Excepti when(storage.batch()).thenReturn(batch); final GoogleCloudStorageService storageService = mock(GoogleCloudStorageService.class); - when(storageService.client(any(String.class), any(String.class), any(GoogleCloudStorageOperationsStats.class))).thenReturn(storage); + when( + storageService.client( + any(String.class), + any(String.class), + any(OperationPurpose.class), + any(GoogleCloudStorageOperationsStats.class) + ) + ).thenReturn(storage); try ( BlobStore store = new GoogleCloudStorageBlobStore( diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java index 3f42e00dce8a3..dc3d3e018e09f 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java @@ -8,7 +8,6 @@ */ package org.elasticsearch.repositories.gcs; -import com.google.api.services.storage.StorageScopes; import com.google.auth.oauth2.ServiceAccountCredentials; import org.apache.http.HttpRequest; @@ -29,11 +28,9 @@ import java.net.Proxy; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.security.KeyPair; import java.security.KeyPairGenerator; import java.util.ArrayList; import java.util.Base64; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -47,6 +44,7 @@ import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.READ_TIMEOUT_SETTING; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.getClientSettings; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.loadCredential; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageTestUtilities.randomCredential; import static org.hamcrest.Matchers.equalTo; public class GoogleCloudStorageClientSettingsTests extends ESTestCase { @@ -292,32 +290,6 @@ private static GoogleCloudStorageClientSettings randomClient( ); } - /** Generates a random GoogleCredential along with its corresponding Service Account file provided as a byte array **/ - private static Tuple randomCredential(final String clientName) throws Exception { - final KeyPair keyPair = KeyPairGenerator.getInstance("RSA").generateKeyPair(); - final ServiceAccountCredentials.Builder credentialBuilder = ServiceAccountCredentials.newBuilder(); - credentialBuilder.setClientId("id_" + clientName); - credentialBuilder.setClientEmail(clientName); - credentialBuilder.setProjectId("project_id_" + clientName); - credentialBuilder.setPrivateKey(keyPair.getPrivate()); - credentialBuilder.setPrivateKeyId("private_key_id_" + clientName); - credentialBuilder.setScopes(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)); - URI tokenServerUri = URI.create("http://localhost/oauth2/token"); - credentialBuilder.setTokenServerUri(tokenServerUri); - final String encodedPrivateKey = Base64.getEncoder().encodeToString(keyPair.getPrivate().getEncoded()); - final String serviceAccount = Strings.format(""" - { - "type": "service_account", - "project_id": "project_id_%s", - "private_key_id": "private_key_id_%s", - "private_key": "-----BEGIN PRIVATE KEY-----\\n%s\\n-----END PRIVATE KEY-----\\n", - "client_email": "%s", - "client_id": "id_%s", - "token_uri": "%s" - }""", clientName, clientName, encodedPrivateKey, clientName, clientName, tokenServerUri); - return Tuple.tuple(credentialBuilder.build(), serviceAccount.getBytes(StandardCharsets.UTF_8)); - } - private static TimeValue randomTimeout() { return randomFrom(TimeValue.MINUS_ONE, TimeValue.ZERO, randomPositiveTimeValue()); } diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java index c6f6b0315f595..8c7e01188f826 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java @@ -18,11 +18,13 @@ import org.apache.http.entity.StringEntity; import org.apache.http.protocol.HttpContext; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.hamcrest.Matchers; @@ -36,6 +38,7 @@ import java.util.Locale; import java.util.UUID; +import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -82,13 +85,13 @@ void notifyProxyIsSet(Proxy p) { GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket"); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> service.client("another_client", "repo", statsCollector) + () -> service.client("another_client", "repo", randomPurpose(), statsCollector) ); assertThat(e.getMessage(), Matchers.startsWith("Unknown client name")); assertSettingDeprecationsAndWarnings( new Setting[] { GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING.getConcreteSettingForNamespace(clientName) } ); - final Storage storage = service.client(clientName, "repo", statsCollector); + final Storage storage = service.client(clientName, "repo", randomPurpose(), statsCollector); assertThat(storage.getOptions().getApplicationName(), Matchers.containsString(applicationName)); assertThat(storage.getOptions().getHost(), Matchers.is(endpoint)); assertThat(storage.getOptions().getProjectId(), Matchers.is(projectIdName)); @@ -116,14 +119,14 @@ public void testReinitClientSettings() throws Exception { try (GoogleCloudStoragePlugin plugin = new GoogleCloudStoragePlugin(settings1)) { final GoogleCloudStorageService storageService = plugin.storageService; GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket"); - final Storage client11 = storageService.client("gcs1", "repo1", statsCollector); + final Storage client11 = storageService.client("gcs1", "repo1", randomPurpose(), statsCollector); assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11")); - final Storage client12 = storageService.client("gcs2", "repo2", statsCollector); + final Storage client12 = storageService.client("gcs2", "repo2", randomPurpose(), statsCollector); assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12")); // client 3 is missing final IllegalArgumentException e1 = expectThrows( IllegalArgumentException.class, - () -> storageService.client("gcs3", "repo3", statsCollector) + () -> storageService.client("gcs3", "repo3", randomPurpose(), statsCollector) ); assertThat(e1.getMessage(), containsString("Unknown client name [gcs3].")); // update client settings @@ -131,18 +134,18 @@ public void testReinitClientSettings() throws Exception { // old client 1 not changed assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11")); // new client 1 is changed - final Storage client21 = storageService.client("gcs1", "repo1", statsCollector); + final Storage client21 = storageService.client("gcs1", "repo1", randomPurpose(), statsCollector); assertThat(client21.getOptions().getProjectId(), equalTo("project_gcs21")); // old client 2 not changed assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12")); // new client2 is gone final IllegalArgumentException e2 = expectThrows( IllegalArgumentException.class, - () -> storageService.client("gcs2", "repo2", statsCollector) + () -> storageService.client("gcs2", "repo2", randomPurpose(), statsCollector) ); assertThat(e2.getMessage(), containsString("Unknown client name [gcs2].")); // client 3 emerged - final Storage client23 = storageService.client("gcs3", "repo3", statsCollector); + final Storage client23 = storageService.client("gcs3", "repo3", randomPurpose(), statsCollector); assertThat(client23.getOptions().getProjectId(), equalTo("project_gcs23")); } } @@ -154,15 +157,35 @@ public void testClientsAreNotSharedAcrossRepositories() throws Exception { try (GoogleCloudStoragePlugin plugin = new GoogleCloudStoragePlugin(settings)) { final GoogleCloudStorageService storageService = plugin.storageService; - final Storage repo1Client = storageService.client("gcs1", "repo1", new GoogleCloudStorageOperationsStats("bucket")); - final Storage repo2Client = storageService.client("gcs1", "repo2", new GoogleCloudStorageOperationsStats("bucket")); + final OperationPurpose operationPurpose = randomPurpose(); + final OperationPurpose differentOperationPurpose = randomValueOtherThan(operationPurpose, BlobStoreTestUtil::randomPurpose); + final Storage repo1Client = storageService.client( + "gcs1", + "repo1", + operationPurpose, + new GoogleCloudStorageOperationsStats("bucket") + ); + final Storage repo1ClientOtherPurpose = storageService.client( + "gcs1", + "repo1", + differentOperationPurpose, + new GoogleCloudStorageOperationsStats("bucket") + ); + final Storage repo2Client = storageService.client( + "gcs1", + "repo2", + operationPurpose, + new GoogleCloudStorageOperationsStats("bucket") + ); final Storage repo1ClientSecondInstance = storageService.client( "gcs1", "repo1", + operationPurpose, new GoogleCloudStorageOperationsStats("bucket") ); assertNotSame(repo1Client, repo2Client); + assertNotSame(repo1Client, repo1ClientOtherPurpose); assertSame(repo1Client, repo1ClientSecondInstance); } } diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestUtilities.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestUtilities.java new file mode 100644 index 0000000000000..27ab000dea92f --- /dev/null +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestUtilities.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.gcs; + +import com.google.api.services.storage.StorageScopes; +import com.google.auth.oauth2.ServiceAccountCredentials; + +import org.elasticsearch.core.Strings; +import org.elasticsearch.core.Tuple; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.util.Base64; +import java.util.Collections; + +public class GoogleCloudStorageTestUtilities { + + /** Generates a random GoogleCredential along with its corresponding Service Account file provided as a byte array **/ + public static Tuple randomCredential(final String clientName) throws Exception { + final KeyPair keyPair = KeyPairGenerator.getInstance("RSA").generateKeyPair(); + final ServiceAccountCredentials.Builder credentialBuilder = ServiceAccountCredentials.newBuilder(); + credentialBuilder.setClientId("id_" + clientName); + credentialBuilder.setClientEmail(clientName); + credentialBuilder.setProjectId("project_id_" + clientName); + credentialBuilder.setPrivateKey(keyPair.getPrivate()); + credentialBuilder.setPrivateKeyId("private_key_id_" + clientName); + credentialBuilder.setScopes(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)); + URI tokenServerUri = URI.create("http://localhost/oauth2/token"); + credentialBuilder.setTokenServerUri(tokenServerUri); + final String encodedPrivateKey = Base64.getEncoder().encodeToString(keyPair.getPrivate().getEncoded()); + final String serviceAccount = Strings.format(""" + { + "type": "service_account", + "project_id": "project_id_%s", + "private_key_id": "private_key_id_%s", + "private_key": "-----BEGIN PRIVATE KEY-----\\n%s\\n-----END PRIVATE KEY-----\\n", + "client_email": "%s", + "client_id": "id_%s", + "token_uri": "%s" + }""", clientName, clientName, encodedPrivateKey, clientName, clientName, tokenServerUri); + return Tuple.tuple(credentialBuilder.build(), serviceAccount.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/muted-tests.yml b/muted-tests.yml index 90ad056d4f881..3344b6e4335b8 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -274,9 +274,6 @@ tests: - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/cat/health/cat-health-example} issue: https://github.com/elastic/elasticsearch/issues/122335 -- class: org.elasticsearch.xpack.esql.action.CrossClusterCancellationIT - method: testCloseSkipUnavailable - issue: https://github.com/elastic/elasticsearch/issues/122336 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/alias/line_260} issue: https://github.com/elastic/elasticsearch/issues/122343 @@ -317,15 +314,36 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/122913 - class: org.elasticsearch.xpack.search.AsyncSearchSecurityIT issue: https://github.com/elastic/elasticsearch/issues/122940 -- class: org.elasticsearch.action.admin.indices.create.ShrinkIndexIT - method: testShrinkIndexPrimaryTerm - issue: https://github.com/elastic/elasticsearch/issues/122974 - class: org.elasticsearch.test.apmintegration.TracesApmIT method: testApmIntegration issue: https://github.com/elastic/elasticsearch/issues/122129 - class: org.elasticsearch.test.apmintegration.MetricsApmIT method: testApmIntegration issue: https://github.com/elastic/elasticsearch/issues/123022 +- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageServiceTests + method: testClientsAreNotSharedAcrossRepositories + issue: https://github.com/elastic/elasticsearch/issues/123090 +- class: org.elasticsearch.xpack.esql.action.EnrichIT + method: testAvgDurationByArtist + issue: https://github.com/elastic/elasticsearch/issues/123093 +- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT + method: test {yaml=reference/troubleshooting/common-issues/disk-usage-exceeded/line_65} + issue: https://github.com/elastic/elasticsearch/issues/123094 +- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithPartialResultsIT + method: testPartialResults + issue: https://github.com/elastic/elasticsearch/issues/123101 +- class: org.elasticsearch.index.mapper.extras.ScaledFloatFieldMapperTests + method: testBlockLoaderFromRowStrideReader + issue: https://github.com/elastic/elasticsearch/issues/123126 +- class: org.elasticsearch.index.mapper.extras.ScaledFloatFieldMapperTests + method: testBlockLoaderFromRowStrideReaderWithSyntheticSource + issue: https://github.com/elastic/elasticsearch/issues/123145 +- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT + method: testStopQueryLocal + issue: https://github.com/elastic/elasticsearch/issues/121672 +- class: org.elasticsearch.index.mapper.extras.ScaledFloatFieldMapperTests + method: testBlockLoaderFromColumnReaderWithSyntheticSource + issue: https://github.com/elastic/elasticsearch/issues/123149 # Examples: # diff --git a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java index d912ccbe07454..c80bcc79f8f64 100644 --- a/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java +++ b/qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java @@ -12,16 +12,13 @@ import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; -import org.apache.lucene.search.TotalHits; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchShardsRequest; import org.elasticsearch.action.search.SearchShardsResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchShardsAction; import org.elasticsearch.client.Request; @@ -33,11 +30,11 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.node.VersionInformation; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.ObjectPath; @@ -102,21 +99,8 @@ private static MockTransportService startTransport( EsExecutors.DIRECT_EXECUTOR_SERVICE, SearchRequest::new, (request, channel, task) -> { - var searchResponse = new SearchResponse( - SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY + var searchResponse = SearchResponseUtils.successfulResponse( + SearchHits.empty(Lucene.TOTAL_HITS_EQUAL_TO_ZERO, Float.NaN) ); try { channel.sendResponse(searchResponse); diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java index 1eb7cbd3f70c2..63672156c6a2f 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java @@ -40,7 +40,7 @@ public class LogsIndexModeRollingUpgradeIT extends AbstractRollingUpgradeTestCas .module("x-pack-aggregate-metric") .module("x-pack-stack") .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.license.self_generated.type", initTestSeed().nextBoolean() ? "trial" : "basic") // We upgrade from standard to logsdb, so we need to start with logsdb disabled, // then later cluster.logsdb.enabled gets set to true and next rollover data stream is in logsdb mode. .setting("cluster.logsdb.enabled", "false") diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json index 7c929dca1370f..d31188fe7f7fc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json @@ -18,17 +18,6 @@ ] } ] - }, - "params":{ - "help":{ - "type":"boolean", - "description":"Return help information", - "default":false - }, - "s":{ - "type":"list", - "description":"Comma-separated list of column names or column aliases to sort by" - } } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json index 7fe66ea3ba887..eee7e5a5c4b29 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json @@ -36,6 +36,14 @@ "type":"string", "description":"a short version of the Accept header, e.g. json, yaml" }, + "local":{ + "type":"boolean", + "description":"Return local information, do not retrieve the state from master node (default: false)" + }, + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + }, "bytes":{ "type":"enum", "description":"The unit in which to display byte values", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json index 6969a1c1e595a..e7329cf8dbbf0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json @@ -70,6 +70,16 @@ "type":"boolean", "description":"Verbose mode. Display column headers", "default":false + }, + "timeout":{ + "type":"time", + "default":"30s", + "description":"Period to wait for a response. If no response is received before the timeout expires, the request fails and returns an error." + }, + "wait_for_completion":{ + "type":"boolean", + "default":false, + "description":"If `true`, the request blocks until the task has completed." } } } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 03c55ecaac40b..1022dfc3b48fe 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -180,12 +180,14 @@ static TransportVersion def(int id) { public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_19 = def(8_841_0_02); public static final TransportVersion ESQL_RETRY_ON_SHARD_LEVEL_FAILURE_BACKPORT_8_19 = def(8_841_0_03); public static final TransportVersion ESQL_SUPPORT_PARTIAL_RESULTS_BACKPORT_8_19 = def(8_841_0_04); + public static final TransportVersion VOYAGE_AI_INTEGRATION_ADDED_BACKPORT_8_X = def(8_841_0_05); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0 = def(9_000_0_00); public static final TransportVersion REMOVE_SNAPSHOT_FAILURES_90 = def(9_000_0_01); public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED_90 = def(9_000_0_02); public static final TransportVersion REMOVE_DESIRED_NODE_VERSION_90 = def(9_000_0_03); public static final TransportVersion ESQL_DRIVER_TASK_DESCRIPTION_90 = def(9_000_0_04); public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_9_0 = def(9_000_0_05); + public static final TransportVersion BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_90 = def(9_000_0_06); public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00); public static final TransportVersion REMOVE_SNAPSHOT_FAILURES = def(9_002_0_00); public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED = def(9_003_0_00); @@ -199,6 +201,8 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_SUPPORT_PARTIAL_RESULTS = def(9_011_0_00); public static final TransportVersion REMOVE_REPOSITORY_CONFLICT_MESSAGE = def(9_012_0_00); public static final TransportVersion RERANKER_FAILURES_ALLOWED = def(9_013_0_00); + public static final TransportVersion VOYAGE_AI_INTEGRATION_ADDED = def(9_014_0_00); + public static final TransportVersion BYTE_SIZE_VALUE_ALWAYS_USES_BYTES = def(9_015_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java index fe7199f8332d2..562d2363905d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; @@ -158,7 +159,26 @@ public static boolean isIndexVisible( if (indexAbstraction.isSystem()) { // check if it is net new if (resolver.getNetNewSystemIndexPredicate().test(indexAbstraction.getName())) { - return isSystemIndexVisible(resolver, indexAbstraction); + // don't give this code any particular credit for being *correct*. it's just trying to resolve a combination of + // issues in a way that happens to *work*. there's probably a better way of writing things such that this won't + // be necessary, but for the moment, it happens to be expedient to write things this way. + + // unwrap the alias and re-run the function on the write index of the alias -- that is, the alias is visible if + // the concrete index that it refers to is visible + Index writeIndex = indexAbstraction.getWriteIndex(); + if (writeIndex == null) { + return false; + } else { + return isIndexVisible( + expression, + selectorString, + writeIndex.getName(), + indicesOptions, + metadata, + resolver, + includeDataStreams + ); + } } } diff --git a/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java b/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java index 23d76abdec2f2..093b8ca33c6b3 100644 --- a/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java +++ b/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java @@ -25,7 +25,9 @@ import java.util.Locale; import java.util.Objects; +import static org.elasticsearch.TransportVersions.BYTE_SIZE_VALUE_ALWAYS_USES_BYTES; import static org.elasticsearch.TransportVersions.BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1; +import static org.elasticsearch.TransportVersions.BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_90; import static org.elasticsearch.TransportVersions.REVERT_BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1; import static org.elasticsearch.common.unit.ByteSizeUnit.BYTES; import static org.elasticsearch.common.unit.ByteSizeUnit.GB; @@ -113,8 +115,7 @@ static ByteSizeValue newByteSizeValue(long sizeInBytes, ByteSizeUnit desiredUnit public static ByteSizeValue readFrom(StreamInput in) throws IOException { long size = in.readZLong(); ByteSizeUnit unit = ByteSizeUnit.readFrom(in); - TransportVersion tv = in.getTransportVersion(); - if (tv.onOrAfter(BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1) && tv.before(REVERT_BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1)) { + if (alwaysUseBytes(in.getTransportVersion())) { return newByteSizeValue(size, unit); } else { return of(size, unit); @@ -123,8 +124,7 @@ public static ByteSizeValue readFrom(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { - TransportVersion tv = out.getTransportVersion(); - if (tv.onOrAfter(BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1) && tv.before(REVERT_BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1)) { + if (alwaysUseBytes(out.getTransportVersion())) { out.writeZLong(sizeInBytes); } else { out.writeZLong(Math.divideExact(sizeInBytes, desiredUnit.toBytes(1))); @@ -132,6 +132,12 @@ public void writeTo(StreamOutput out) throws IOException { desiredUnit.writeTo(out); } + private static boolean alwaysUseBytes(TransportVersion tv) { + return tv.onOrAfter(BYTE_SIZE_VALUE_ALWAYS_USES_BYTES) + || tv.isPatchFrom(BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_90) + || tv.between(BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1, REVERT_BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1); + } + ByteSizeValue(long sizeInBytes, ByteSizeUnit desiredUnit) { this.sizeInBytes = sizeInBytes; this.desiredUnit = desiredUnit; diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 94e8989bc5a1c..fc7f6eab0856c 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -3470,7 +3470,9 @@ protected long getPreCommitSegmentGeneration() { T performActionWithDirectoryReader(SearcherScope scope, CheckedFunction action) throws EngineException { assert scope == SearcherScope.INTERNAL : "performActionWithDirectoryReader(...) isn't prepared for external usage"; - assert store.hasReferences(); + if (store.tryIncRef() == false) { + throw new AlreadyClosedException(shardId + " store is closed", failedEngine.get()); + } try { ReferenceManager referenceManager = getReferenceManager(scope); ElasticsearchDirectoryReader acquire = referenceManager.acquire(); @@ -3486,6 +3488,8 @@ T performActionWithDirectoryReader(SearcherScope scope, CheckedFunction listener) { assertTrue(executedMultiSearch.compareAndSet(false, true)); - SearchResponse searchResponse = new SearchResponse( - collapsedHits, - null, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); + SearchResponse searchResponse = SearchResponseUtils.successfulResponse(collapsedHits); ActionListener.respondAndRelease( listener, new MultiSearchResponse( diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java index a3ac361f5b055..24eb98952d74a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java @@ -29,6 +29,7 @@ import java.util.function.Supplier; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; +import static org.elasticsearch.indices.SystemIndices.EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; import static org.elasticsearch.indices.SystemIndices.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.contains; @@ -218,18 +219,6 @@ public void testIsIndexVisible() { assertThat(isIndexVisible("data-stream1", "failures"), is(true)); } - private boolean isIndexVisible(String index, String selector) { - return IndexAbstractionResolver.isIndexVisible( - "*", - selector, - index, - IndicesOptions.strictExpandHidden(), - metadata, - indexNameExpressionResolver, - true - ); - } - public void testIsNetNewSystemIndexVisible() { final Settings settings = Settings.builder() .put("index.number_of_replicas", 0) @@ -269,16 +258,71 @@ public void testIsNetNewSystemIndexVisible() { List.of(new SystemIndices.Feature("name", "description", List.of(fooDescriptor, barDescriptor))) ); - final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "false"); - indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices); - indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver); - metadata = Metadata.builder().put(foo, true).put(barReindexed, true).put(other, true).build(); - assertThat(isIndexVisible("other", "*"), is(true)); - assertThat(isIndexVisible(".foo", "*"), is(false)); - assertThat(isIndexVisible(".bar", "*"), is(false)); + // these indices options are for the GET _data_streams case + final IndicesOptions noHiddenNoAliases = IndicesOptions.builder() + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(true) + .includeHidden(false) + .resolveAliases(false) + .build() + ) + .build(); + + { + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "true"); + indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices); + indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver); + + // this covers the GET * case -- with system access, you can see everything + assertThat(isIndexVisible("other", "*"), is(true)); + assertThat(isIndexVisible(".foo", "*"), is(true)); + assertThat(isIndexVisible(".bar", "*"), is(true)); + + // but if you don't ask for hidden and aliases, you won't see hidden indices or aliases, naturally + assertThat(isIndexVisible("other", "*", noHiddenNoAliases), is(true)); + assertThat(isIndexVisible(".foo", "*", noHiddenNoAliases), is(false)); + assertThat(isIndexVisible(".bar", "*", noHiddenNoAliases), is(false)); + } + + { + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "false"); + indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices); + indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver); + + // this covers the GET * case -- without system access, you can't see everything + assertThat(isIndexVisible("other", "*"), is(true)); + assertThat(isIndexVisible(".foo", "*"), is(false)); + assertThat(isIndexVisible(".bar", "*"), is(false)); + + // no difference here in the datastream case, you can't see these then, either + assertThat(isIndexVisible("other", "*", noHiddenNoAliases), is(true)); + assertThat(isIndexVisible(".foo", "*", noHiddenNoAliases), is(false)); + assertThat(isIndexVisible(".bar", "*", noHiddenNoAliases), is(false)); + } + + { + final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "true"); + threadContext.putHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "some-elastic-product"); + indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices); + indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver); + + // this covers the GET * case -- with product (only) access, you can't see everything + assertThat(isIndexVisible("other", "*"), is(true)); + assertThat(isIndexVisible(".foo", "*"), is(false)); + assertThat(isIndexVisible(".bar", "*"), is(false)); + + // no difference here in the datastream case, you can't see these then, either + assertThat(isIndexVisible("other", "*", noHiddenNoAliases), is(true)); + assertThat(isIndexVisible(".foo", "*", noHiddenNoAliases), is(false)); + assertThat(isIndexVisible(".bar", "*", noHiddenNoAliases), is(false)); + } } private static XContentBuilder mappings() { @@ -306,4 +350,12 @@ private List resolveAbstractionsSelectorAllowed(List expressions private List resolveAbstractions(List expressions, IndicesOptions indicesOptions, Supplier> mask) { return indexAbstractionResolver.resolveIndexAbstractions(expressions, indicesOptions, metadata, mask, (idx) -> true, true); } + + private boolean isIndexVisible(String index, String selector) { + return isIndexVisible(index, selector, IndicesOptions.strictExpandHidden()); + } + + private boolean isIndexVisible(String index, String selector, IndicesOptions indicesOptions) { + return IndexAbstractionResolver.isIndexVisible("*", selector, index, indicesOptions, metadata, indexNameExpressionResolver, true); + } } diff --git a/server/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java b/server/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java index 79b06926cdef6..6992cc5808007 100644 --- a/server/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java +++ b/server/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -22,6 +21,10 @@ import java.util.List; import java.util.function.Function; +import static org.elasticsearch.TransportVersions.BYTE_SIZE_VALUE_ALWAYS_USES_BYTES; +import static org.elasticsearch.TransportVersions.BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_90; +import static org.elasticsearch.TransportVersions.INITIAL_ELASTICSEARCH_9_0; +import static org.elasticsearch.TransportVersions.V_8_16_0; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -520,44 +523,44 @@ protected void assertEqualInstances(ByteSizeValue expectedInstance, ByteSizeValu public void testBWCTransportFormat() throws IOException { var tenMegs = ByteSizeValue.ofMb(10); - try (BytesStreamOutput expected = new BytesStreamOutput(); BytesStreamOutput actual = new BytesStreamOutput()) { - expected.writeZLong(10); - ByteSizeUnit.MB.writeTo(expected); - actual.setTransportVersion(TransportVersions.V_8_16_0); - tenMegs.writeTo(actual); - assertArrayEquals( - "Size denominated in the desired unit for backward compatibility", - expected.bytes().array(), - actual.bytes().array() - ); + for (var tv : List.of(V_8_16_0, INITIAL_ELASTICSEARCH_9_0)) { + try (BytesStreamOutput expected = new BytesStreamOutput(); BytesStreamOutput actual = new BytesStreamOutput()) { + expected.writeZLong(10); + ByteSizeUnit.MB.writeTo(expected); + actual.setTransportVersion(tv); + tenMegs.writeTo(actual); + assertArrayEquals( + "Size denominated in the desired unit for backward compatibility", + expected.bytes().array(), + actual.bytes().array() + ); + } } } - /** - * @see TransportVersions#REVERT_BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1 - */ - @AwaitsFix(bugUrl = "https://elasticco.atlassian.net/browse/ES-10585") - public void testTwoDigitTransportRoundTrips() throws IOException { - TransportVersion tv = TransportVersion.current(); - for (var desiredUnit : ByteSizeUnit.values()) { - if (desiredUnit == ByteSizeUnit.BYTES) { - continue; - } - checkTransportRoundTrip(ByteSizeValue.parseBytesSizeValue("23" + desiredUnit.getSuffix(), "test"), tv); - for (int tenths = 1; tenths <= 9; tenths++) { - checkTransportRoundTrip(ByteSizeValue.parseBytesSizeValue("23." + tenths + desiredUnit.getSuffix(), "test"), tv); - for (int hundredths = 1; hundredths <= 9; hundredths++) { - checkTransportRoundTrip( - ByteSizeValue.parseBytesSizeValue("23." + tenths + hundredths + desiredUnit.getSuffix(), "test"), - tv - ); + public void testTransportRoundTripsWithTwoDigitFractions() throws IOException { + for (var tv : List.of(TransportVersion.current(), BYTE_SIZE_VALUE_ALWAYS_USES_BYTES, BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_90)) { + for (var desiredUnit : ByteSizeUnit.values()) { + if (desiredUnit == ByteSizeUnit.BYTES) { + // Can't have a fraction of a byte! + continue; + } + checkTransportRoundTrip(ByteSizeValue.parseBytesSizeValue("23" + desiredUnit.getSuffix(), "test"), tv); + for (int tenths = 1; tenths <= 9; tenths++) { + checkTransportRoundTrip(ByteSizeValue.parseBytesSizeValue("23." + tenths + desiredUnit.getSuffix(), "test"), tv); + for (int hundredths = 1; hundredths <= 9; hundredths++) { + checkTransportRoundTrip( + ByteSizeValue.parseBytesSizeValue("23." + tenths + hundredths + desiredUnit.getSuffix(), "test"), + tv + ); + } } } } } public void testIntegerTransportRoundTrips() throws IOException { - for (var tv : List.of(TransportVersion.current(), TransportVersions.V_8_16_0)) { + for (var tv : List.of(TransportVersion.current(), V_8_16_0)) { checkTransportRoundTrip(ByteSizeValue.ONE, tv); checkTransportRoundTrip(ByteSizeValue.ZERO, tv); checkTransportRoundTrip(ByteSizeValue.MINUS_ONE, tv); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 21346bb93ef8e..a4ee85310632f 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -17,10 +17,8 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchShardsRequest; import org.elasticsearch.action.search.SearchShardsResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchShardsAction; import org.elasticsearch.action.support.PlainActionFuture; @@ -46,7 +44,7 @@ import org.elasticsearch.mocksocket.MockServerSocket; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; @@ -157,26 +155,7 @@ public static MockTransportService startTransport( } else { searchHits = SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN); } - try ( - var searchResponseRef = ReleasableRef.of( - new SearchResponse( - searchHits, - InternalAggregations.EMPTY, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) - ) - ) { + try (var searchResponseRef = ReleasableRef.of(SearchResponseUtils.successfulResponse(searchHits))) { channel.sendResponse(searchResponseRef.get()); } } diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java index b43c516d2b246..0c6aae605c8a7 100644 --- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java @@ -17,29 +17,30 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestUtils; import org.elasticsearch.test.fixture.HttpHeaderParser; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.URLDecoder; -import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import static fixture.gcs.MockGcsBlobStore.failAndThrow; @@ -56,6 +57,7 @@ public class GoogleCloudStorageHttpHandler implements HttpHandler { private static final Logger logger = LogManager.getLogger(GoogleCloudStorageHttpHandler.class); private static final String IF_GENERATION_MATCH = "ifGenerationMatch"; + private final AtomicInteger defaultPageLimit = new AtomicInteger(1_000); private final MockGcsBlobStore mockGcsBlobStore; private final String bucket; @@ -64,6 +66,15 @@ public GoogleCloudStorageHttpHandler(final String bucket) { this.mockGcsBlobStore = new MockGcsBlobStore(); } + /** + * Set the default page limit + * + * @param limit The new limit + */ + public void setDefaultPageLimit(final int limit) { + this.defaultPageLimit.set(limit); + } + @Override public void handle(final HttpExchange exchange) throws IOException { final String request = exchange.getRequestMethod() + " " + exchange.getRequestURI().toString(); @@ -82,40 +93,31 @@ public void handle(final HttpExchange exchange) throws IOException { final String key = exchange.getRequestURI().getPath().replace("/storage/v1/b/" + bucket + "/o/", ""); final Long ifGenerationMatch = parseOptionalLongParameter(exchange, IF_GENERATION_MATCH); final MockGcsBlobStore.BlobVersion blob = mockGcsBlobStore.getBlob(key, ifGenerationMatch); - final byte[] response = buildBlobInfoJson(blob).getBytes(UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); + writeBlobVersionAsJson(exchange, blob); } else if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "/o*", request)) { // List Objects https://cloud.google.com/storage/docs/json_api/v1/objects/list final Map params = new HashMap<>(); RestUtils.decodeQueryString(exchange.getRequestURI(), params); final String prefix = params.getOrDefault("prefix", ""); - final String delimiter = params.get("delimiter"); - - final Set prefixes = new HashSet<>(); - final List listOfBlobs = new ArrayList<>(); - - for (final Map.Entry blob : mockGcsBlobStore.listBlobs().entrySet()) { - final String blobName = blob.getKey(); - if (prefix.isEmpty() || blobName.startsWith(prefix)) { - int delimiterPos = (delimiter != null) ? blobName.substring(prefix.length()).indexOf(delimiter) : -1; - if (delimiterPos > -1) { - prefixes.add("\"" + blobName.substring(0, prefix.length() + delimiterPos + 1) + "\""); - } else { - listOfBlobs.add(buildBlobInfoJson(blob.getValue())); - } - } - } + final int maxResults = Integer.parseInt(params.getOrDefault("maxResults", String.valueOf(defaultPageLimit.get()))); + final String delimiter = params.getOrDefault("delimiter", ""); + final String pageToken = params.get("pageToken"); - byte[] response = (String.format(Locale.ROOT, """ - {"kind":"storage#objects","items":[%s],"prefixes":[%s]}\ - """, String.join(",", listOfBlobs), String.join(",", prefixes))).getBytes(UTF_8); - - exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); + final MockGcsBlobStore.PageOfBlobs pageOfBlobs; + if (pageToken != null) { + pageOfBlobs = mockGcsBlobStore.listBlobs(pageToken); + } else { + pageOfBlobs = mockGcsBlobStore.listBlobs(maxResults, delimiter, prefix); + } + ListBlobsResponse response = new ListBlobsResponse(bucket, pageOfBlobs); + try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + BytesReference responseBytes = BytesReference.bytes(builder); + exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), responseBytes.length()); + responseBytes.writeTo(exchange.getResponseBody()); + } } else if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "*", request)) { // GET Bucket https://cloud.google.com/storage/docs/json_api/v1/buckets/get throw new AssertionError("Should not call get bucket API"); @@ -193,10 +195,7 @@ public void handle(final HttpExchange exchange) throws IOException { ifGenerationMatch, content.get().v2() ); - byte[] response = buildBlobInfoJson(newBlobVersion).getBytes(UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); + writeBlobVersionAsJson(exchange, newBlobVersion); } else { throw new AssertionError( "Could not read multi-part request to [" @@ -266,6 +265,48 @@ public void handle(final HttpExchange exchange) throws IOException { } } + private void writeBlobVersionAsJson(HttpExchange exchange, MockGcsBlobStore.BlobVersion newBlobVersion) throws IOException { + try (XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON)) { + writeBlobAsXContent(newBlobVersion, builder, bucket); + BytesReference responseBytes = BytesReference.bytes(builder); + exchange.getResponseHeaders().add("Content-Type", "application/json; charset=utf-8"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), responseBytes.length()); + responseBytes.writeTo(exchange.getResponseBody()); + } + } + + record ListBlobsResponse(String bucket, MockGcsBlobStore.PageOfBlobs pageOfBlobs) implements ToXContent { + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("kind", "storage#objects"); + if (pageOfBlobs.nextPageToken() != null) { + builder.field("nextPageToken", pageOfBlobs.nextPageToken()); + } + builder.startArray("items"); + for (MockGcsBlobStore.BlobVersion blobVersion : pageOfBlobs().blobs()) { + writeBlobAsXContent(blobVersion, builder, bucket); + } + builder.endArray(); + builder.field("prefixes", pageOfBlobs.prefixes()); + builder.endObject(); + return builder; + } + } + + private static void writeBlobAsXContent(MockGcsBlobStore.BlobVersion blobVersion, XContentBuilder builder, String bucket) + throws IOException { + builder.startObject(); + builder.field("kind", "storage#object"); + builder.field("bucket", bucket); + builder.field("name", blobVersion.path()); + builder.field("id", blobVersion.path()); + builder.field("size", String.valueOf(blobVersion.contents().length())); + builder.field("generation", String.valueOf(blobVersion.generation())); + builder.endObject(); + } + private void sendError(HttpExchange exchange, MockGcsBlobStore.GcsRestException e) throws IOException { final String responseBody = Strings.format(""" { @@ -280,21 +321,10 @@ private void sendError(HttpExchange exchange, MockGcsBlobStore.GcsRestException exchange.getResponseBody().write(responseBody.getBytes(UTF_8)); } - private String buildBlobInfoJson(MockGcsBlobStore.BlobVersion blobReference) { - return String.format( - Locale.ROOT, - """ - {"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"%s","generation":"%d"}""", - bucket, - blobReference.path(), - blobReference.path(), - blobReference.contents().length(), - blobReference.generation() - ); - } - public Map blobs() { - return Maps.transformValues(mockGcsBlobStore.listBlobs(), MockGcsBlobStore.BlobVersion::contents); + return mockGcsBlobStore.listBlobs() + .stream() + .collect(Collectors.toMap(MockGcsBlobStore.BlobVersion::path, MockGcsBlobStore.BlobVersion::contents)); } private static String httpServerUrl(final HttpExchange exchange) { diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java index a5b055566f78d..3b92680fc804f 100644 --- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/MockGcsBlobStore.java @@ -10,22 +10,30 @@ package fixture.gcs; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.CompositeBytesReference; +import org.elasticsearch.core.Nullable; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.fixture.HttpHeaderParser; -import java.util.Map; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicReference; public class MockGcsBlobStore { private static final int RESUME_INCOMPLETE = 308; - private final ConcurrentMap blobs = new ConcurrentHashMap<>(); + // we use skip-list map so we can do paging right + private final ConcurrentMap blobs = new ConcurrentSkipListMap<>(); private final ConcurrentMap resumableUploads = new ConcurrentHashMap<>(); record BlobVersion(String path, long generation, BytesReference contents) {} @@ -164,8 +172,114 @@ void deleteBlob(String path) { blobs.remove(path); } - Map listBlobs() { - return Map.copyOf(blobs); + private String stripPrefixIfPresent(@Nullable String prefix, String toStrip) { + if (prefix != null && toStrip.startsWith(prefix)) { + return toStrip.substring(prefix.length()); + } + return toStrip; + } + + PageOfBlobs listBlobs(String pageToken) { + final PageToken parsedToken = PageToken.fromString(pageToken); + return calculatePageOfBlobs(parsedToken); + } + + /** + * Calculate the requested page of blobs taking into account the request parameters + * + * @see Description of prefix/delimiter + * @see List objects parameters + * @param pageToken The token containing the prefix/delimiter/maxResults/pageNumber parameters + * @return The filtered list + */ + private PageOfBlobs calculatePageOfBlobs(PageToken pageToken) { + final String previousBlob = pageToken.previousBlob(); + final int maxResults = pageToken.maxResults(); + final String prefix = pageToken.prefix(); + final String delimiter = pageToken.delimiter(); + final SortedSet prefixes = new TreeSet<>(); + final List matchingBlobs = new ArrayList<>(); + String lastBlobPath = null; + for (BlobVersion blob : blobs.values()) { + if (Strings.hasLength(previousBlob) && previousBlob.compareTo(blob.path()) >= 0) { + continue; + } + if (blob.path().startsWith(prefix)) { + final String pathWithoutPrefix = stripPrefixIfPresent(prefix, blob.path()); + if (Strings.hasLength(delimiter) && pathWithoutPrefix.contains(delimiter)) { + // This seems counter to what is described in the example at the top of + // https://cloud.google.com/storage/docs/json_api/v1/objects/list, + // but it's required to make the third party tests pass + prefixes.add(prefix + pathWithoutPrefix.substring(0, pathWithoutPrefix.indexOf(delimiter) + 1)); + } else { + matchingBlobs.add(blob); + } + } + lastBlobPath = blob.path(); + if (prefixes.size() + matchingBlobs.size() == maxResults) { + return new PageOfBlobs( + new PageToken(prefix, delimiter, maxResults, previousBlob), + new ArrayList<>(prefixes), + matchingBlobs, + lastBlobPath, + false + ); + } + } + return new PageOfBlobs( + new PageToken(prefix, delimiter, maxResults, previousBlob), + new ArrayList<>(prefixes), + matchingBlobs, + lastBlobPath, + true + ); + } + + PageOfBlobs listBlobs(int maxResults, String delimiter, String prefix) { + final PageToken pageToken = new PageToken(prefix, delimiter, maxResults, ""); + return calculatePageOfBlobs(pageToken); + } + + List listBlobs() { + return new ArrayList<>(blobs.values()); + } + + /** + * We serialise this as a tuple with base64 encoded components so we don't need to escape the delimiter + */ + record PageToken(String prefix, String delimiter, int maxResults, String previousBlob) { + public static PageToken fromString(String pageToken) { + final String[] parts = pageToken.split("\\."); + assert parts.length == 4; + return new PageToken(decode(parts[0]), decode(parts[1]), Integer.parseInt(decode(parts[2])), decode(parts[3])); + } + + public String toString() { + return encode(prefix) + "." + encode(delimiter) + "." + encode(String.valueOf(maxResults)) + "." + encode(previousBlob); + } + + public PageToken nextPageToken(String previousBlob) { + return new PageToken(prefix, delimiter, maxResults, previousBlob); + } + + private static String encode(String value) { + return Base64.getEncoder().encodeToString(value.getBytes()); + } + + private static String decode(String value) { + return new String(Base64.getDecoder().decode(value)); + } + } + + record PageOfBlobs(PageToken pageToken, List prefixes, List blobs, String lastBlobIncluded, boolean lastPage) { + + public boolean isLastPage() { + return lastPage; + } + + public String nextPageToken() { + return isLastPage() ? null : pageToken.nextPageToken(lastBlobIncluded).toString(); + } } static class BlobNotFoundException extends GcsRestException { diff --git a/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java b/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java index f83da9e1679a9..61d20295b1fd9 100644 --- a/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java +++ b/test/fixtures/gcs-fixture/src/test/java/fixture/gcs/GoogleCloudStorageHttpHandlerTests.java @@ -22,8 +22,10 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.test.fixture.HttpHeaderParser; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; @@ -33,13 +35,19 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.HashSet; import java.util.List; -import java.util.Objects; +import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPOutputStream; +import static java.util.Objects.requireNonNull; + public class GoogleCloudStorageHttpHandlerTests extends ESTestCase { private static final String HOST = "http://127.0.0.1:12345"; @@ -72,7 +80,7 @@ public void testSimpleObjectOperations() { assertEquals( new TestHttpResponse(RestStatus.OK, "{\"kind\":\"storage#objects\",\"items\":[],\"prefixes\":[]}"), - listBlobs(handler, bucket, null) + listBlobs(handler, bucket, null, null) ); final var body = randomAlphaOfLength(50); @@ -85,14 +93,14 @@ public void testSimpleObjectOperations() { assertEquals(new TestHttpResponse(RestStatus.OK, Strings.format(""" {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"50",\ - "generation":"1"}],"prefixes":[]}""", bucket, blobName, blobName)), listBlobs(handler, bucket, null)); + "generation":"1"}],"prefixes":[]}""", bucket, blobName, blobName)), listBlobs(handler, bucket, null, null)); assertEquals(new TestHttpResponse(RestStatus.OK, Strings.format(""" {"kind":"storage#objects","items":[{"kind":"storage#object","bucket":"%s","name":"%s","id":"%s","size":"50",\ - "generation":"1"}],"prefixes":[]}""", bucket, blobName, blobName)), listBlobs(handler, bucket, "path/")); + "generation":"1"}],"prefixes":[]}""", bucket, blobName, blobName)), listBlobs(handler, bucket, "path/", null)); assertEquals(new TestHttpResponse(RestStatus.OK, """ - {"kind":"storage#objects","items":[],"prefixes":[]}"""), listBlobs(handler, bucket, "some/other/path")); + {"kind":"storage#objects","items":[],"prefixes":[]}"""), listBlobs(handler, bucket, "some/other/path", null)); assertEquals( new TestHttpResponse(RestStatus.OK, """ @@ -129,7 +137,7 @@ public void testSimpleObjectOperations() { ); assertEquals(new TestHttpResponse(RestStatus.OK, """ - {"kind":"storage#objects","items":[],"prefixes":[]}"""), listBlobs(handler, bucket, "path/")); + {"kind":"storage#objects","items":[],"prefixes":[]}"""), listBlobs(handler, bucket, "path/", null)); } public void testGetWithBytesRange() { @@ -422,6 +430,114 @@ public void testIfGenerationMatch_GetObject() { ); } + public void testListObjectsWithPrefix() { + final var bucket = randomIdentifier(); + final var handler = new GoogleCloudStorageHttpHandler(bucket); + + final int numberOfFiles = randomIntBetween(1, 100); + final int numberWithMatchingPrefix = randomIntBetween(0, numberOfFiles); + final String prefix = randomIdentifier(); + + // Create expected state + for (int i = 0; i < numberOfFiles; i++) { + final String blobName; + if (i < numberWithMatchingPrefix) { + blobName = prefix + "blob_name_" + i; + } else { + final String nonMatchingPrefix = randomValueOtherThan(prefix, ESTestCase::randomIdentifier); + blobName = nonMatchingPrefix + "blob_name_" + i; + } + assertEquals( + RestStatus.OK, + executeUpload(handler, bucket, blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus() + ); + } + + TestHttpResponse response = listBlobs(handler, bucket, prefix, null); + assertEquals(RestStatus.OK, response.restStatus()); + + XContentTestUtils.JsonMapView jsonMapView = XContentTestUtils.createJsonMapView( + new ByteArrayInputStream(BytesReference.toBytes(response.body())) + ); + assertEquals(numberWithMatchingPrefix, ((List) jsonMapView.get("items")).size()); + } + + public void testListObjectsWithPrefixAndDelimiter() { + final var bucket = randomIdentifier(); + final var handler = new GoogleCloudStorageHttpHandler(bucket); + final var delimiter = randomFrom("/", ".", "+", "\\"); + final var prefix = randomBoolean() ? "" : randomIdentifier() + delimiter; + + final int numberOfFiles = randomIntBetween(1, 100); + final int numberWithDelimiter = randomIntBetween(0, numberOfFiles); + + // Create expected state + final Set topLevelDirectories = new HashSet<>(); + for (int i = 0; i < numberOfFiles; i++) { + final String blobName; + if (i < numberWithDelimiter) { + final String directory = randomAlphaOfLength(3); + blobName = directory + delimiter + "blob_name_" + i; + topLevelDirectories.add(directory + delimiter); + } else { + blobName = randomIdentifier() + "_blob_name_" + i; + } + assertEquals( + RestStatus.OK, + executeUpload(handler, bucket, prefix + blobName, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus() + ); + } + + final TestHttpResponse response = listBlobs(handler, bucket, prefix, delimiter); + assertEquals(RestStatus.OK, response.restStatus()); + + XContentTestUtils.JsonMapView jsonMapView = XContentTestUtils.createJsonMapView( + new ByteArrayInputStream(BytesReference.toBytes(response.body())) + ); + assertEquals(numberOfFiles - numberWithDelimiter, ((List) jsonMapView.get("items")).size()); + assertEquals( + topLevelDirectories.stream().map(d -> prefix + d).collect(Collectors.toSet()), + new HashSet<>(jsonMapView.get("prefixes")) + ); + } + + /** + * Tests the example from The docs + */ + public void testListObjectsExampleFromDocumentation() { + final var bucket = randomIdentifier(); + final var handler = new GoogleCloudStorageHttpHandler(bucket); + + Stream.of("a/b", "a/c", "d", "e", "e/f", "e/g/h") + .forEach( + path -> assertEquals( + RestStatus.OK, + executeUpload(handler, bucket, path, randomBytesReference(randomIntBetween(100, 5_000)), null).restStatus() + ) + ); + + TestHttpResponse response = listBlobs(handler, bucket, null, "/"); + assertEquals(RestStatus.OK, response.restStatus()); + XContentTestUtils.JsonMapView jsonMapView = XContentTestUtils.createJsonMapView( + new ByteArrayInputStream(BytesReference.toBytes(response.body())) + ); + assertEquals( + Set.of("d", "e"), + ((List) jsonMapView.get("items")).stream().map(i -> ((Map) i).get("name")).collect(Collectors.toSet()) + ); + assertEquals(Set.of("a/", "e/"), new HashSet<>(jsonMapView.get("prefixes"))); + + response = listBlobs(handler, bucket, "e/", "/"); + assertEquals(RestStatus.OK, response.restStatus()); + jsonMapView = XContentTestUtils.createJsonMapView(new ByteArrayInputStream(BytesReference.toBytes(response.body()))); + assertEquals( + Set.of("e/f"), + ((List) jsonMapView.get("items")).stream().map(i -> ((Map) i).get("name")).collect(Collectors.toSet()) + ); + // note this differs from the example, but third party test indicates this is what we get back + assertEquals(Set.of("e/g/"), new HashSet<>(jsonMapView.get("prefixes"))); + } + private static TestHttpResponse executeUpload( GoogleCloudStorageHttpHandler handler, String bucket, @@ -449,9 +565,8 @@ private static TestHttpResponse executeResumableUpload( "POST", "/upload/storage/v1/b/" + bucket - + "/?uploadType=resumable&name=" - + blobName - + (ifGenerationMatch != null ? "&ifGenerationMatch=" + ifGenerationMatch : "") + + "/" + + generateQueryString("uploadType", "resumable", "name", blobName, "ifGenerationMatch", ifGenerationMatch) ); final var locationHeader = createUploadResponse.headers.getFirst("Location"); final var sessionURI = locationHeader.substring(locationHeader.indexOf(HOST) + HOST.length()); @@ -476,10 +591,7 @@ private static TestHttpResponse executeMultipartUpload( return handleRequest( handler, "POST", - "/upload/storage/v1/b/" - + bucket - + "/?uploadType=multipart" - + (ifGenerationMatch != null ? "&ifGenerationMatch=" + ifGenerationMatch : ""), + "/upload/storage/v1/b/" + bucket + "/" + generateQueryString("uploadType", "multipart", "ifGenerationMatch", ifGenerationMatch), createGzipCompressedMultipartUploadBody(bucket, blobName, bytes) ); } @@ -494,11 +606,7 @@ private static TestHttpResponse getBlobContents( return handleRequest( handler, "GET", - "/download/storage/v1/b/" - + bucket - + "/o/" - + blobName - + (ifGenerationMatch != null ? "?ifGenerationMatch=" + ifGenerationMatch : ""), + "/download/storage/v1/b/" + bucket + "/o/" + blobName + generateQueryString("ifGenerationMatch", ifGenerationMatch), BytesArray.EMPTY, range != null ? rangeHeader(range.start(), range.end()) : TestHttpExchange.EMPTY_HEADERS ); @@ -513,23 +621,23 @@ private static TestHttpResponse getBlobMetadata( return handleRequest( handler, "GET", - "/storage/v1/b/" + bucket + "/o/" + blobName + (ifGenerationMatch != null ? "?ifGenerationMatch=" + ifGenerationMatch : "") + "/storage/v1/b/" + bucket + "/o/" + blobName + generateQueryString("ifGenerationMatch", ifGenerationMatch) ); } private static long getCurrentGeneration(GoogleCloudStorageHttpHandler handler, String bucket, String blobName) { - TestHttpResponse blobMetadata = getBlobMetadata(handler, bucket, blobName, null); + final TestHttpResponse blobMetadata = getBlobMetadata(handler, bucket, blobName, null); assertEquals(RestStatus.OK, blobMetadata.restStatus()); - Matcher matcher = GENERATION_PATTERN.matcher(blobMetadata.body.utf8ToString()); + final Matcher matcher = GENERATION_PATTERN.matcher(blobMetadata.body.utf8ToString()); assertTrue(matcher.find()); return Long.parseLong(matcher.group(1)); } - private static TestHttpResponse listBlobs(GoogleCloudStorageHttpHandler handler, String bucket, String prefix) { + private static TestHttpResponse listBlobs(GoogleCloudStorageHttpHandler handler, String bucket, String prefix, String delimiter) { return handleRequest( handler, "GET", - "/storage/v1/b/" + bucket + "/o" + (prefix != null ? "?prefix=" + URLEncoder.encode(prefix, StandardCharsets.UTF_8) : "") + "/storage/v1/b/" + bucket + "/o" + generateQueryString("prefix", prefix, "delimiter", delimiter) ); } @@ -551,7 +659,7 @@ private record TestHttpResponse(int status, BytesReference body, Headers headers } RestStatus restStatus() { - return Objects.requireNonNull(RestStatus.fromCode(status)); + return requireNonNull(RestStatus.fromCode(status)); } @Override @@ -601,7 +709,7 @@ private static TestHttpResponse handleRequest( fail(e); } assertNotEquals(0, httpExchange.getResponseCode()); - var responseHeaders = new Headers(); + final var responseHeaders = new Headers(); httpExchange.getResponseHeaders().forEach((header, values) -> { // com.sun.net.httpserver.Headers.Headers() normalize keys if ("Range".equals(header) || "Content-range".equals(header) || "Location".equals(header)) { @@ -611,6 +719,35 @@ private static TestHttpResponse handleRequest( return new TestHttpResponse(httpExchange.getResponseCode(), httpExchange.getResponseBodyContents(), responseHeaders); } + /** + * Generate a query string for the given parameters + * + * @param parameters The query parameters as alternating key, value pairs + * @return The query string including all parameters with a non-null value (e.g. + */ + public static String generateQueryString(Object... parameters) { + if (parameters.length % 2 != 0) { + final String message = "Parameters must be represented as alternating key, value pairs"; + assert false : message; + throw new IllegalArgumentException(message); + } + final StringBuilder builder = new StringBuilder(); + for (int i = 0; i < parameters.length; i += 2) { + final String key = String.valueOf(requireNonNull(parameters[i], "Parameter names must be non-null strings")); + final Object value = parameters[i + 1]; + if (value != null) { + if (builder.isEmpty() == false) { + builder.append("&"); + } + builder.append(key).append("=").append(URLEncoder.encode(String.valueOf(value), StandardCharsets.UTF_8)); + } + } + if (builder.isEmpty() == false) { + return "?" + builder; + } + return ""; + } + private static Headers contentRangeHeader(@Nullable Integer startInclusive, @Nullable Integer endInclusive, @Nullable Integer limit) { final String rangeString = startInclusive != null && endInclusive != null ? startInclusive + "-" + endInclusive : "*"; final String limitString = limit == null ? "*" : limit.toString(); diff --git a/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java b/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java index 21b43636222f9..0321550736660 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/search/SearchResponseUtils.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponse.Clusters; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Response; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -22,6 +23,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.xcontent.XContentParserUtils; @@ -76,6 +78,157 @@ public enum SearchResponseUtils { ; + public static SearchResponseBuilder response() { + return new SearchResponseBuilder(); + } + + public static SearchResponseBuilder response(SearchHits hits) { + return new SearchResponseBuilder().searchHits(hits).numReducePhases(1).shards(1, 1, 0).tookInMillis(100); + } + + public static SearchResponse successfulResponse(SearchHits hits) { + return response(hits).build(); + } + + public static SearchResponse emptyWithTotalHits( + String scrollId, + int totalShards, + int successfulShards, + int skippedShards, + long tookInMillis, + ShardSearchFailure[] shardFailures, + SearchResponse.Clusters clusters + ) { + return new SearchResponse( + SearchHits.EMPTY_WITH_TOTAL_HITS, + null, + null, + false, + null, + null, + 1, + scrollId, + totalShards, + successfulShards, + skippedShards, + tookInMillis, + shardFailures, + clusters + ); + } + + public static class SearchResponseBuilder { + private SearchHits searchHits = SearchHits.empty(Lucene.TOTAL_HITS_EQUAL_TO_ZERO, Float.NaN); + private InternalAggregations aggregations; + private Suggest suggest; + private boolean timedOut; + private Boolean terminatedEarly; + private SearchProfileResults profileResults; + private int numReducePhases; + private String scrollId; + private int totalShards; + private int successfulShards; + private int skippedShards; + private long tookInMillis; + private List shardFailures; + private Clusters clusters = Clusters.EMPTY; + private BytesReference pointInTimeId; + + private SearchResponseBuilder() {} + + public SearchResponseBuilder searchHits(SearchHits searchHits) { + this.searchHits = searchHits; + return this; + } + + public SearchResponseBuilder aggregations(InternalAggregations aggregations) { + this.aggregations = aggregations; + return this; + } + + public SearchResponseBuilder suggest(Suggest suggest) { + this.suggest = suggest; + return this; + } + + public SearchResponseBuilder timedOut(boolean timedOut) { + this.timedOut = timedOut; + return this; + } + + public SearchResponseBuilder terminatedEarly(Boolean terminatedEarly) { + this.terminatedEarly = terminatedEarly; + return this; + } + + public SearchResponseBuilder profileResults(SearchProfileResults profileResults) { + this.profileResults = profileResults; + return this; + } + + public SearchResponseBuilder numReducePhases(int numReducePhases) { + this.numReducePhases = numReducePhases; + return this; + } + + public SearchResponseBuilder scrollId(String scrollId) { + this.scrollId = scrollId; + return this; + } + + public SearchResponseBuilder shards(int total, int successful, int skipped) { + this.totalShards = total; + this.successfulShards = successful; + this.skippedShards = skipped; + return this; + } + + public SearchResponseBuilder tookInMillis(long tookInMillis) { + this.tookInMillis = tookInMillis; + return this; + } + + public SearchResponseBuilder shardFailures(ShardSearchFailure... failures) { + shardFailures = List.of(failures); + return this; + } + + public SearchResponseBuilder shardFailures(List failures) { + shardFailures = List.copyOf(failures); + return this; + } + + public SearchResponseBuilder clusters(Clusters clusters) { + this.clusters = clusters; + return this; + } + + public SearchResponseBuilder pointInTimeId(BytesReference pointInTimeId) { + this.pointInTimeId = pointInTimeId; + return this; + } + + public SearchResponse build() { + return new SearchResponse( + searchHits, + aggregations, + suggest, + timedOut, + terminatedEarly, + profileResults, + numReducePhases, + scrollId, + totalShards, + successfulShards, + skippedShards, + tookInMillis, + shardFailures == null ? ShardSearchFailure.EMPTY_ARRAY : shardFailures.toArray(ShardSearchFailure[]::new), + clusters, + pointInTimeId + ); + } + } + // All fields on the root level of the parsed SearchHit are interpreted as metadata fields // public because we use it in a completion suggestion option @SuppressWarnings("unchecked") @@ -110,33 +263,6 @@ public static SearchResponse responseAsSearchResponse(Response searchResponse) t } } - public static SearchResponse emptyWithTotalHits( - String scrollId, - int totalShards, - int successfulShards, - int skippedShards, - long tookInMillis, - ShardSearchFailure[] shardFailures, - SearchResponse.Clusters clusters - ) { - return new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - null, - null, - false, - null, - null, - 1, - scrollId, - totalShards, - successfulShards, - skippedShards, - tookInMillis, - shardFailures, - clusters - ); - } - public static SearchResponse parseSearchResponse(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); parser.nextToken(); diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/traces-apm@pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/traces-apm@pipeline.yaml index f79c07ff40ac0..476cddee48639 100644 --- a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/traces-apm@pipeline.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/traces-apm@pipeline.yaml @@ -30,13 +30,16 @@ processors: field: ["event.duration"] ignore_failure: true ignore_missing: true -- set: - if: ctx.event?.outcome == 'success' - field: event.success_count - value: 1 - set: if: ctx.event?.outcome == 'failure' field: event.success_count value: 0 +- set: + if: ctx.event?.outcome == 'success' + field: event.success_count + value: 1 +- script: + if: ctx.event?.outcome == 'success' && ctx[ctx.processor?.event]?.representative_count != null + source: ctx.event.success_count = ctx[ctx.processor?.event]?.representative_count; - pipeline: name: apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml index 9484f577583eb..beca2e5890bc0 100644 --- a/x-pack/plugin/apm-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin apm-data. This must be increased whenever an existing template or # pipeline is changed, in order for it to be updated on Elasticsearch upgrade. -version: 12 +version: 13 component-templates: # Data lifecycle. diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_ingest.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_ingest.yml index ea7f948abf0b8..af9b2335afe4a 100644 --- a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_ingest.yml +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_ingest.yml @@ -80,7 +80,35 @@ setup: - '{"@timestamp": "2017-06-22", "event": {"outcome": "unknown"}}' - create: {} - - '{"@timestamp": "2017-06-22", "event": {"outcome": "success"}}' + - '{ + "@timestamp": "2017-06-22", + "processor": {"event": "transaction"}, + "event": {"outcome": "success"}, + "transaction": {"representative_count": 2} + }' + + - create: {} + - '{ + "@timestamp": "2017-06-22", + "processor": {"event": "span"}, + "event": {"outcome": "success"}, + "span": {"representative_count": 3} + }' + + - create: {} + - '{ + "@timestamp": "2017-06-22", + "processor": {"event": "span"}, + "event": {"outcome": "success"}, + "span": {"representative_count": null} + }' + + - create: {} + - '{ + "@timestamp": "2017-06-22", + "processor": {"event": "transaction"}, + "event": {"outcome": "success"} + }' - create: {} - '{"@timestamp": "2017-06-22", "event": {"outcome": "failure"}}' @@ -92,8 +120,11 @@ setup: index: traces-apm-testing body: fields: ["event.success_count"] - - length: { hits.hits: 4 } + - length: { hits.hits: 7 } - match: { hits.hits.0.fields: null } - match: { hits.hits.1.fields: null } - - match: { hits.hits.2.fields: {"event.success_count": [1]} } - - match: { hits.hits.3.fields: {"event.success_count": [0]} } + - match: { hits.hits.2.fields: {"event.success_count": [2]} } + - match: { hits.hits.3.fields: {"event.success_count": [3]} } + - match: { hits.hits.4.fields: {"event.success_count": [1]} } + - match: { hits.hits.5.fields: {"event.success_count": [1]} } + - match: { hits.hits.6.fields: {"event.success_count": [0]} } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java index cf08cdbf09367..ab1b6189c0133 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchTaskTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.InternalAggregations; @@ -472,22 +473,10 @@ private static SearchResponse newSearchResponse( int skippedShards, ShardSearchFailure... failures ) { - return new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - InternalAggregations.EMPTY, - null, - false, - null, - null, - 1, - null, - totalShards, - successfulShards, - skippedShards, - 100, - failures, - SearchResponse.Clusters.EMPTY - ); + return SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS) + .shards(totalShards, successfulShards, skippedShards) + .shardFailures(failures) + .build(); } private static void assertCompletionListeners( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbeddingSparse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbedding.java similarity index 58% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbeddingSparse.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbedding.java index 37bf92e0dbfce..e723a3b4f8f60 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbeddingSparse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbedding.java @@ -7,12 +7,8 @@ package org.elasticsearch.xpack.core.inference.results; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.inference.ChunkedInference; -import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContent; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.ml.search.WeightedToken; import java.io.IOException; import java.util.ArrayList; @@ -21,7 +17,7 @@ import static org.elasticsearch.xpack.core.inference.results.TextEmbeddingUtils.validateInputSizeAgainstEmbeddings; -public record ChunkedInferenceEmbeddingSparse(List chunks) implements ChunkedInference { +public record ChunkedInferenceEmbedding(List chunks) implements ChunkedInference { public static List listOf(List inputs, SparseEmbeddingResults sparseEmbeddingResults) { validateInputSizeAgainstEmbeddings(inputs, sparseEmbeddingResults.embeddings().size()); @@ -29,9 +25,9 @@ public static List listOf(List inputs, SparseEmbedding var results = new ArrayList(inputs.size()); for (int i = 0; i < inputs.size(); i++) { results.add( - new ChunkedInferenceEmbeddingSparse( + new ChunkedInferenceEmbedding( List.of( - new SparseEmbeddingChunk( + new SparseEmbeddingResults.Chunk( sparseEmbeddingResults.embeddings().get(i).tokens(), inputs.get(i), new TextOffset(0, inputs.get(i).length()) @@ -47,21 +43,9 @@ public static List listOf(List inputs, SparseEmbedding @Override public Iterator chunksAsMatchedTextAndByteReference(XContent xcontent) throws IOException { var asChunk = new ArrayList(); - for (var chunk : chunks) { - asChunk.add(new Chunk(chunk.matchedText(), chunk.offset(), toBytesReference(xcontent, chunk.weightedTokens()))); + for (var chunk : chunks()) { + asChunk.add(chunk.toChunk(xcontent)); } return asChunk.iterator(); } - - private static BytesReference toBytesReference(XContent xContent, List tokens) throws IOException { - XContentBuilder b = XContentBuilder.builder(xContent); - b.startObject(); - for (var weightedToken : tokens) { - weightedToken.toXContent(b, ToXContent.EMPTY_PARAMS); - } - b.endObject(); - return BytesReference.bytes(b); - } - - public record SparseEmbeddingChunk(List weightedTokens, String matchedText, TextOffset offset) {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbeddingByte.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbeddingByte.java deleted file mode 100644 index c2f70b0be2916..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbeddingByte.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.inference.results; - -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.inference.ChunkedInference; -import org.elasticsearch.xcontent.XContent; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -public record ChunkedInferenceEmbeddingByte(List chunks) implements ChunkedInference { - - @Override - public Iterator chunksAsMatchedTextAndByteReference(XContent xcontent) throws IOException { - var asChunk = new ArrayList(); - for (var chunk : chunks) { - asChunk.add(new Chunk(chunk.matchedText(), chunk.offset(), toBytesReference(xcontent, chunk.embedding()))); - } - return asChunk.iterator(); - } - - /** - * Serialises the {@code value} array, according to the provided {@link XContent}, into a {@link BytesReference}. - */ - private static BytesReference toBytesReference(XContent xContent, byte[] value) throws IOException { - XContentBuilder builder = XContentBuilder.builder(xContent); - builder.startArray(); - for (byte v : value) { - builder.value(v); - } - builder.endArray(); - return BytesReference.bytes(builder); - } - - public record ByteEmbeddingChunk(byte[] embedding, String matchedText, TextOffset offset) {} -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbeddingFloat.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbeddingFloat.java deleted file mode 100644 index 651d135b761dd..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/ChunkedInferenceEmbeddingFloat.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.inference.results; - -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.inference.ChunkedInference; -import org.elasticsearch.xcontent.XContent; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -public record ChunkedInferenceEmbeddingFloat(List chunks) implements ChunkedInference { - - @Override - public Iterator chunksAsMatchedTextAndByteReference(XContent xcontent) throws IOException { - var asChunk = new ArrayList(); - for (var chunk : chunks) { - asChunk.add(new Chunk(chunk.matchedText(), chunk.offset(), toBytesReference(xcontent, chunk.embedding()))); - } - return asChunk.iterator(); - } - - /** - * Serialises the {@code value} array, according to the provided {@link XContent}, into a {@link BytesReference}. - */ - private static BytesReference toBytesReference(XContent xContent, float[] value) throws IOException { - XContentBuilder b = XContentBuilder.builder(xContent); - b.startArray(); - for (float v : value) { - b.value(v); - } - b.endArray(); - return BytesReference.bytes(b); - } - - public record FloatEmbeddingChunk(float[] embedding, String matchedText, TextOffset offset) {} -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingResults.java new file mode 100644 index 0000000000000..c6f4c6915024b --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingResults.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.inference.results; + +import org.elasticsearch.inference.ChunkedInference; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xcontent.XContent; + +import java.io.IOException; +import java.util.List; + +/** + * The results of a call to the inference service that contains embeddings (sparse or dense). + * A call to the inference service may contain multiple input texts, so this results may + * contain multiple results. + */ +public interface EmbeddingResults> + extends + InferenceServiceResults { + + /** + * A resulting embedding together with its input text. + */ + interface Chunk { + ChunkedInference.Chunk toChunk(XContent xcontent) throws IOException; + + String matchedText(); + + ChunkedInference.TextOffset offset(); + } + + /** + * A resulting embedding for one of the input texts to the inference service. + */ + interface Embedding { + /** + * Combines the resulting embedding with the input into a chunk. + */ + C toChunk(String text, ChunkedInference.TextOffset offset); + } + + /** + * The resulting list of embeddings for the input texts to the inference service. + */ + List embeddings(); +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceByteEmbedding.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceByteEmbedding.java deleted file mode 100644 index 7d7176a9a5a51..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceByteEmbedding.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - * - * this file was contributed to by a generative AI - */ - -package org.elasticsearch.xpack.core.inference.results; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -public record InferenceByteEmbedding(byte[] values) implements Writeable, ToXContentObject, EmbeddingInt { - public static final String EMBEDDING = "embedding"; - - public InferenceByteEmbedding(StreamInput in) throws IOException { - this(in.readByteArray()); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeByteArray(values); - } - - public static InferenceByteEmbedding of(List embeddingValuesList) { - byte[] embeddingValues = new byte[embeddingValuesList.size()]; - for (int i = 0; i < embeddingValuesList.size(); i++) { - embeddingValues[i] = embeddingValuesList.get(i); - } - return new InferenceByteEmbedding(embeddingValues); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - - builder.startArray(EMBEDDING); - for (byte value : values) { - builder.value(value); - } - builder.endArray(); - - builder.endObject(); - return builder; - } - - @Override - public String toString() { - return Strings.toString(this); - } - - float[] toFloatArray() { - float[] floatArray = new float[values.length]; - for (int i = 0; i < values.length; i++) { - floatArray[i] = ((Byte) values[i]).floatValue(); - } - return floatArray; - } - - double[] toDoubleArray() { - double[] doubleArray = new double[values.length]; - for (int i = 0; i < values.length; i++) { - doubleArray[i] = ((Byte) values[i]).doubleValue(); - } - return doubleArray; - } - - @Override - public int getSize() { - return values().length; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InferenceByteEmbedding embedding = (InferenceByteEmbedding) o; - return Arrays.equals(values, embedding.values); - } - - @Override - public int hashCode() { - return Arrays.hashCode(values); - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java deleted file mode 100644 index 1ae54220508c5..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingByteResults.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - * - * this file was contributed to by a generative AI - */ - -package org.elasticsearch.xpack.core.inference.results; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; -import org.elasticsearch.inference.InferenceResults; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * Writes a text embedding result in the follow json format - * { - * "text_embedding_bytes": [ - * { - * "embedding": [ - * 23 - * ] - * }, - * { - * "embedding": [ - * -23 - * ] - * } - * ] - * } - */ -public record InferenceTextEmbeddingByteResults(List embeddings) implements InferenceServiceResults, TextEmbedding { - public static final String NAME = "text_embedding_service_byte_results"; - public static final String TEXT_EMBEDDING_BYTES = "text_embedding_bytes"; - - public InferenceTextEmbeddingByteResults(StreamInput in) throws IOException { - this(in.readCollectionAsList(InferenceByteEmbedding::new)); - } - - @Override - public int getFirstEmbeddingSize() { - return TextEmbeddingUtils.getFirstEmbeddingSize(new ArrayList<>(embeddings)); - } - - @Override - public Iterator toXContentChunked(ToXContent.Params params) { - return ChunkedToXContentHelper.array(TEXT_EMBEDDING_BYTES, embeddings.iterator()); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeCollection(embeddings); - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public List transformToCoordinationFormat() { - return embeddings.stream() - .map(embedding -> new MlTextEmbeddingResults(TEXT_EMBEDDING_BYTES, embedding.toDoubleArray(), false)) - .toList(); - } - - @Override - @SuppressWarnings("deprecation") - public List transformToLegacyFormat() { - var legacyEmbedding = new LegacyTextEmbeddingResults( - embeddings.stream().map(embedding -> new LegacyTextEmbeddingResults.Embedding(embedding.toFloatArray())).toList() - ); - - return List.of(legacyEmbedding); - } - - public Map asMap() { - Map map = new LinkedHashMap<>(); - map.put(TEXT_EMBEDDING_BYTES, embeddings); - - return map; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InferenceTextEmbeddingByteResults that = (InferenceTextEmbeddingByteResults) o; - return Objects.equals(embeddings, that.embeddings); - } - - @Override - public int hashCode() { - return Objects.hash(embeddings); - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/LegacyTextEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/LegacyTextEmbeddingResults.java index 84a0928cae0d8..60bbeb624b532 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/LegacyTextEmbeddingResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/LegacyTextEmbeddingResults.java @@ -44,7 +44,7 @@ * * Legacy text embedding results represents what was returned prior to the * {@link org.elasticsearch.TransportVersions#V_8_12_0} version. - * @deprecated use {@link InferenceTextEmbeddingFloatResults} instead + * @deprecated use {@link TextEmbeddingFloatResults} instead */ @Deprecated public record LegacyTextEmbeddingResults(List embeddings) implements InferenceResults { @@ -114,8 +114,8 @@ public int hashCode() { return Objects.hash(embeddings); } - public InferenceTextEmbeddingFloatResults transformToTextEmbeddingResults() { - return new InferenceTextEmbeddingFloatResults(this); + public TextEmbeddingFloatResults transformToTextEmbeddingResults() { + return new TextEmbeddingFloatResults(this); } public record Embedding(float[] values) implements Writeable, ToXContentObject { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java index dd8229c604ecb..894e8c6c97bfd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/SparseEmbeddingResults.java @@ -9,16 +9,18 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.InferenceResults; -import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; @@ -33,13 +35,15 @@ import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig.DEFAULT_RESULTS_FIELD; -public record SparseEmbeddingResults(List embeddings) implements InferenceServiceResults { +public record SparseEmbeddingResults(List embeddings) + implements + EmbeddingResults { public static final String NAME = "sparse_embedding_results"; public static final String SPARSE_EMBEDDING = TaskType.SPARSE_EMBEDDING.toString(); public SparseEmbeddingResults(StreamInput in) throws IOException { - this(in.readCollectionAsList(Embedding::new)); + this(in.readCollectionAsList(SparseEmbeddingResults.Embedding::new)); } public static SparseEmbeddingResults of(List results) { @@ -47,7 +51,9 @@ public static SparseEmbeddingResults of(List results for (InferenceResults result : results) { if (result instanceof TextExpansionResults expansionResults) { - embeddings.add(Embedding.create(expansionResults.getWeightedTokens(), expansionResults.isTruncated())); + embeddings.add( + SparseEmbeddingResults.Embedding.create(expansionResults.getWeightedTokens(), expansionResults.isTruncated()) + ); } else if (result instanceof org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults errorResult) { if (errorResult.getException() instanceof ElasticsearchStatusException statusException) { throw statusException; @@ -87,7 +93,7 @@ public void writeTo(StreamOutput out) throws IOException { public Map asMap() { Map map = new LinkedHashMap<>(); - var embeddingList = embeddings.stream().map(Embedding::asMap).toList(); + var embeddingList = embeddings.stream().map(SparseEmbeddingResults.Embedding::asMap).toList(); map.put(SPARSE_EMBEDDING, embeddingList); return map; @@ -114,7 +120,11 @@ public List transformToLegacyFormat() { .toList(); } - public record Embedding(List tokens, boolean isTruncated) implements Writeable, ToXContentObject { + public record Embedding(List tokens, boolean isTruncated) + implements + Writeable, + ToXContentObject, + EmbeddingResults.Embedding { public static final String EMBEDDING = "embedding"; public static final String IS_TRUNCATED = "is_truncated"; @@ -163,5 +173,29 @@ public Map asMap() { public String toString() { return Strings.toString(this); } + + @Override + public Chunk toChunk(String text, ChunkedInference.TextOffset offset) { + return new Chunk(tokens, text, offset); + } + } + + public record Chunk(List weightedTokens, String matchedText, ChunkedInference.TextOffset offset) + implements + EmbeddingResults.Chunk { + + public ChunkedInference.Chunk toChunk(XContent xcontent) throws IOException { + return new ChunkedInference.Chunk(matchedText, offset, toBytesReference(xcontent, weightedTokens)); + } + + private static BytesReference toBytesReference(XContent xContent, List tokens) throws IOException { + XContentBuilder b = XContentBuilder.builder(xContent); + b.startObject(); + for (var weightedToken : tokens) { + weightedToken.toXContent(b, ToXContent.EMPTY_PARAMS); + } + b.endObject(); + return BytesReference.bytes(b); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingBitResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingBitResults.java similarity index 83% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingBitResults.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingBitResults.java index 887c07558ab71..ba4a770b04840 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingBitResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingBitResults.java @@ -13,12 +13,10 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.inference.InferenceResults; -import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import java.io.IOException; -import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -42,17 +40,22 @@ * ] * } */ -public record InferenceTextEmbeddingBitResults(List embeddings) implements InferenceServiceResults, TextEmbedding { +public record TextEmbeddingBitResults(List embeddings) + implements + TextEmbeddingResults { public static final String NAME = "text_embedding_service_bit_results"; public static final String TEXT_EMBEDDING_BITS = "text_embedding_bits"; - public InferenceTextEmbeddingBitResults(StreamInput in) throws IOException { - this(in.readCollectionAsList(InferenceByteEmbedding::new)); + public TextEmbeddingBitResults(StreamInput in) throws IOException { + this(in.readCollectionAsList(TextEmbeddingByteResults.Embedding::new)); } @Override public int getFirstEmbeddingSize() { - return TextEmbeddingUtils.getFirstEmbeddingSize(new ArrayList<>(embeddings)); + if (embeddings.isEmpty()) { + throw new IllegalStateException("Embeddings list is empty"); + } + return embeddings.getFirst().values().length; } @Override @@ -98,7 +101,7 @@ public Map asMap() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - InferenceTextEmbeddingBitResults that = (InferenceTextEmbeddingBitResults) o; + TextEmbeddingBitResults that = (TextEmbeddingBitResults) o; return Objects.equals(embeddings, that.embeddings); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingByteResults.java new file mode 100644 index 0000000000000..f8268d7bd4683 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingByteResults.java @@ -0,0 +1,214 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * this file was contributed to by a generative AI + */ + +package org.elasticsearch.xpack.core.inference.results; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.inference.ChunkedInference; +import org.elasticsearch.inference.InferenceResults; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Writes a text embedding result in the follow json format + * { + * "text_embedding_bytes": [ + * { + * "embedding": [ + * 23 + * ] + * }, + * { + * "embedding": [ + * -23 + * ] + * } + * ] + * } + */ +public record TextEmbeddingByteResults(List embeddings) + implements + TextEmbeddingResults { + public static final String NAME = "text_embedding_service_byte_results"; + public static final String TEXT_EMBEDDING_BYTES = "text_embedding_bytes"; + + public TextEmbeddingByteResults(StreamInput in) throws IOException { + this(in.readCollectionAsList(TextEmbeddingByteResults.Embedding::new)); + } + + @Override + public int getFirstEmbeddingSize() { + if (embeddings.isEmpty()) { + throw new IllegalStateException("Embeddings list is empty"); + } + return embeddings.getFirst().values().length; + } + + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContentHelper.array(TEXT_EMBEDDING_BYTES, embeddings.iterator()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(embeddings); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public List transformToCoordinationFormat() { + return embeddings.stream() + .map(embedding -> new MlTextEmbeddingResults(TEXT_EMBEDDING_BYTES, embedding.toDoubleArray(), false)) + .toList(); + } + + @Override + @SuppressWarnings("deprecation") + public List transformToLegacyFormat() { + var legacyEmbedding = new LegacyTextEmbeddingResults( + embeddings.stream().map(embedding -> new LegacyTextEmbeddingResults.Embedding(embedding.toFloatArray())).toList() + ); + + return List.of(legacyEmbedding); + } + + public Map asMap() { + Map map = new LinkedHashMap<>(); + map.put(TEXT_EMBEDDING_BYTES, embeddings); + + return map; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TextEmbeddingByteResults that = (TextEmbeddingByteResults) o; + return Objects.equals(embeddings, that.embeddings); + } + + @Override + public int hashCode() { + return Objects.hash(embeddings); + } + + public record Embedding(byte[] values) implements Writeable, ToXContentObject, EmbeddingResults.Embedding { + public static final String EMBEDDING = "embedding"; + + public Embedding(StreamInput in) throws IOException { + this(in.readByteArray()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeByteArray(values); + } + + public static Embedding of(List embeddingValuesList) { + byte[] embeddingValues = new byte[embeddingValuesList.size()]; + for (int i = 0; i < embeddingValuesList.size(); i++) { + embeddingValues[i] = embeddingValuesList.get(i); + } + return new Embedding(embeddingValues); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder.startArray(EMBEDDING); + for (byte value : values) { + builder.value(value); + } + builder.endArray(); + + builder.endObject(); + return builder; + } + + @Override + public String toString() { + return Strings.toString(this); + } + + float[] toFloatArray() { + float[] floatArray = new float[values.length]; + for (int i = 0; i < values.length; i++) { + floatArray[i] = ((Byte) values[i]).floatValue(); + } + return floatArray; + } + + double[] toDoubleArray() { + double[] doubleArray = new double[values.length]; + for (int i = 0; i < values.length; i++) { + doubleArray[i] = ((Byte) values[i]).doubleValue(); + } + return doubleArray; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Embedding embedding = (Embedding) o; + return Arrays.equals(values, embedding.values); + } + + @Override + public int hashCode() { + return Arrays.hashCode(values); + } + + @Override + public Chunk toChunk(String text, ChunkedInference.TextOffset offset) { + return new Chunk(values, text, offset); + } + } + + /** + * Serialises the {@code value} array, according to the provided {@link XContent}, into a {@link BytesReference}. + */ + public record Chunk(byte[] embedding, String matchedText, ChunkedInference.TextOffset offset) implements EmbeddingResults.Chunk { + + public ChunkedInference.Chunk toChunk(XContent xcontent) throws IOException { + return new ChunkedInference.Chunk(matchedText, offset, toBytesReference(xcontent, embedding)); + } + + private static BytesReference toBytesReference(XContent xContent, byte[] value) throws IOException { + XContentBuilder builder = XContentBuilder.builder(xContent); + builder.startArray(); + for (byte v : value) { + builder.value(v); + } + builder.endArray(); + return BytesReference.bytes(builder); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingFloatResults.java similarity index 70% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingFloatResults.java index 9f9bdfec7cfae..cef381982b447 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceTextEmbeddingFloatResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingFloatResults.java @@ -11,16 +11,18 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.InferenceResults; -import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; @@ -51,32 +53,31 @@ * ] * } */ -public record InferenceTextEmbeddingFloatResults(List embeddings) +public record TextEmbeddingFloatResults(List embeddings) implements - InferenceServiceResults, - TextEmbedding { + TextEmbeddingResults { public static final String NAME = "text_embedding_service_results"; public static final String TEXT_EMBEDDING = TaskType.TEXT_EMBEDDING.toString(); - public InferenceTextEmbeddingFloatResults(StreamInput in) throws IOException { - this(in.readCollectionAsList(InferenceFloatEmbedding::new)); + public TextEmbeddingFloatResults(StreamInput in) throws IOException { + this(in.readCollectionAsList(TextEmbeddingFloatResults.Embedding::new)); } @SuppressWarnings("deprecation") - InferenceTextEmbeddingFloatResults(LegacyTextEmbeddingResults legacyTextEmbeddingResults) { + TextEmbeddingFloatResults(LegacyTextEmbeddingResults legacyTextEmbeddingResults) { this( legacyTextEmbeddingResults.embeddings() .stream() - .map(embedding -> new InferenceFloatEmbedding(embedding.values())) + .map(embedding -> new Embedding(embedding.values())) .collect(Collectors.toList()) ); } - public static InferenceTextEmbeddingFloatResults of(List results) { - List embeddings = new ArrayList<>(results.size()); + public static TextEmbeddingFloatResults of(List results) { + List embeddings = new ArrayList<>(results.size()); for (InferenceResults result : results) { if (result instanceof MlTextEmbeddingResults embeddingResult) { - embeddings.add(InferenceFloatEmbedding.of(embeddingResult)); + embeddings.add(TextEmbeddingFloatResults.Embedding.of(embeddingResult)); } else if (result instanceof org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults errorResult) { if (errorResult.getException() instanceof ElasticsearchStatusException statusException) { throw statusException; @@ -93,12 +94,15 @@ public static InferenceTextEmbeddingFloatResults of(List(embeddings)); + if (embeddings.isEmpty()) { + throw new IllegalStateException("Embeddings list is empty"); + } + return embeddings.getFirst().values().length; } @Override @@ -142,7 +146,7 @@ public Map asMap() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - InferenceTextEmbeddingFloatResults that = (InferenceTextEmbeddingFloatResults) o; + TextEmbeddingFloatResults that = (TextEmbeddingFloatResults) o; return Objects.equals(embeddings, that.embeddings); } @@ -151,29 +155,24 @@ public int hashCode() { return Objects.hash(embeddings); } - public record InferenceFloatEmbedding(float[] values) implements Writeable, ToXContentObject, EmbeddingInt { + public record Embedding(float[] values) implements Writeable, ToXContentObject, EmbeddingResults.Embedding { public static final String EMBEDDING = "embedding"; - public InferenceFloatEmbedding(StreamInput in) throws IOException { + public Embedding(StreamInput in) throws IOException { this(in.readFloatArray()); } - public static InferenceFloatEmbedding of(MlTextEmbeddingResults embeddingResult) { + public static Embedding of(MlTextEmbeddingResults embeddingResult) { float[] embeddingAsArray = embeddingResult.getInferenceAsFloat(); - return new InferenceFloatEmbedding(embeddingAsArray); + return new Embedding(embeddingAsArray); } - public static InferenceFloatEmbedding of(List embeddingValuesList) { + public static Embedding of(List embeddingValuesList) { float[] embeddingValues = new float[embeddingValuesList.size()]; for (int i = 0; i < embeddingValuesList.size(); i++) { embeddingValues[i] = embeddingValuesList.get(i); } - return new InferenceFloatEmbedding(embeddingValues); - } - - @Override - public int getSize() { - return values.length; + return new Embedding(embeddingValues); } @Override @@ -212,7 +211,7 @@ private double[] asDoubleArray() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - InferenceFloatEmbedding embedding = (InferenceFloatEmbedding) o; + Embedding embedding = (Embedding) o; return Arrays.equals(values, embedding.values); } @@ -220,5 +219,30 @@ public boolean equals(Object o) { public int hashCode() { return Arrays.hashCode(values); } + + @Override + public Chunk toChunk(String text, ChunkedInference.TextOffset offset) { + return new Chunk(values, text, offset); + } + } + + public record Chunk(float[] embedding, String matchedText, ChunkedInference.TextOffset offset) implements EmbeddingResults.Chunk { + + public ChunkedInference.Chunk toChunk(XContent xcontent) throws IOException { + return new ChunkedInference.Chunk(matchedText, offset, toBytesReference(xcontent, embedding)); + } + + /** + * Serialises the {@code value} array, according to the provided {@link XContent}, into a {@link BytesReference}. + */ + private static BytesReference toBytesReference(XContent xContent, float[] value) throws IOException { + XContentBuilder b = XContentBuilder.builder(xContent); + b.startArray(); + for (float v : value) { + b.value(v); + } + b.endArray(); + return BytesReference.bytes(b); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbedding.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java similarity index 78% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbedding.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java index a185c2938223e..4caeea4930fd2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbedding.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingResults.java @@ -7,7 +7,9 @@ package org.elasticsearch.xpack.core.inference.results; -public interface TextEmbedding { +public interface TextEmbeddingResults> + extends + EmbeddingResults { /** * Returns the first text embedding entry in the result list's array size. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingUtils.java index cb69f1e403e9c..0ba8102b1dab7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/TextEmbeddingUtils.java @@ -13,20 +13,6 @@ public class TextEmbeddingUtils { - /** - * Returns the first text embedding entry's array size. - * @param embeddings the list of embeddings - * @return the size of the text embedding - * @throws IllegalStateException if the list of embeddings is empty - */ - public static int getFirstEmbeddingSize(List embeddings) throws IllegalStateException { - if (embeddings.isEmpty()) { - throw new IllegalStateException("Embeddings list is empty"); - } - - return embeddings.get(0).getSize(); - } - /** * Throws an exception if the number of elements in the input text list is different than the results in text embedding * response. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java index 9a0b17b22369c..a34c17cfee42b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.rollover.LazyRolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.bulk.TransportBulkAction; @@ -213,6 +214,7 @@ public class InternalUsers { TransportCloseIndexAction.NAME, TransportCreateIndexAction.TYPE.name(), TransportClusterSearchShardsAction.TYPE.name(), + GetSettingsAction.NAME, TransportUpdateSettingsAction.TYPE.name(), RefreshAction.NAME, ReindexAction.NAME, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java index 76b668a87cff5..6f0c5fba2cf5a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexing/AsyncTwoPhaseIndexerTests.java @@ -13,10 +13,10 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.TestThreadPool; @@ -113,25 +113,7 @@ protected void doNextSearch(long waitTimeInNanos, ActionListener return; } - ActionListener.respondAndRelease( - nextPhase, - new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - null, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ) - ); + ActionListener.respondAndRelease(nextPhase, SearchResponseUtils.successfulResponse(SearchHits.EMPTY_WITH_TOTAL_HITS)); } @Override @@ -264,25 +246,7 @@ protected void doNextSearch(long waitTimeInNanos, ActionListener awaitForLatch(); } - ActionListener.respondAndRelease( - nextPhase, - new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - null, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ) - ); + ActionListener.respondAndRelease(nextPhase, SearchResponseUtils.successfulResponse(SearchHits.EMPTY_WITH_TOTAL_HITS)); } @Override diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java index 7125dfd45eaff..693d4cb55bcac 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java @@ -6,12 +6,12 @@ */ package org.elasticsearch.xpack.enrich; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import static org.elasticsearch.action.support.ActionTestUtils.assertNoFailureListener; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -122,18 +123,10 @@ public void testComputeIfAbsent() throws InterruptedException { searchResponseActionListener.onResponse(searchResponse); searchResponse.decRef(); queriedDatabaseLatch.countDown(); - }, new ActionListener<>() { - @Override - public void onResponse(List> response) { - assertThat(response, equalTo(searchResponseMap)); - notifiedOfResultLatch.countDown(); - } - - @Override - public void onFailure(Exception e) { - fail(e); - } - }); + }, assertNoFailureListener(response -> { + assertThat(response, equalTo(searchResponseMap)); + notifiedOfResultLatch.countDown(); + })); assertThat(queriedDatabaseLatch.await(5, TimeUnit.SECONDS), equalTo(true)); assertThat(notifiedOfResultLatch.await(5, TimeUnit.SECONDS), equalTo(true)); EnrichStatsAction.Response.CacheStats cacheStats = enrichCache.getStats(randomAlphaOfLength(10)); @@ -149,17 +142,7 @@ public void onFailure(Exception e) { CountDownLatch notifiedOfResultLatch = new CountDownLatch(1); enrichCache.computeIfAbsent("policy1-1", "1", 1, (searchResponseActionListener) -> { fail("Expected no call to the database because item should have been in the cache"); - }, new ActionListener<>() { - @Override - public void onResponse(List> maps) { - notifiedOfResultLatch.countDown(); - } - - @Override - public void onFailure(Exception e) { - fail(e); - } - }); + }, assertNoFailureListener(r -> notifiedOfResultLatch.countDown())); assertThat(notifiedOfResultLatch.await(5, TimeUnit.SECONDS), equalTo(true)); EnrichStatsAction.Response.CacheStats cacheStats = enrichCache.getStats(randomAlphaOfLength(10)); assertThat(cacheStats.count(), equalTo(1L)); @@ -180,22 +163,7 @@ private SearchResponse convertToSearchResponse(List> searchRespon } }).toArray(SearchHit[]::new); SearchHits hits = SearchHits.unpooled(hitArray, null, 0); - return new SearchResponse( - hits, - null, - null, - false, - false, - null, - 1, - null, - 5, - 4, - 0, - randomLong(), - null, - SearchResponse.Clusters.EMPTY - ); + return SearchResponseUtils.response(hits).shards(5, 4, 0).build(); } private BytesReference convertMapToJson(Map simpleMap) throws IOException { diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java index f10c8e4e41c93..67c88a024e740 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java @@ -11,8 +11,6 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; @@ -25,9 +23,7 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.search.profile.SearchProfileResults; -import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; @@ -257,22 +253,7 @@ protected void requestCounter[0]++; ActionListener.respondAndRelease( listener, - (Response) new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - InternalAggregations.EMPTY, - new Suggest(Collections.emptyList()), - false, - false, - new SearchProfileResults(Collections.emptyMap()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) + (Response) SearchResponseUtils.successfulResponse(SearchHits.EMPTY_WITH_TOTAL_HITS) ); } }; diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/CoordinatorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/CoordinatorTests.java index db523546e13bf..d9b2237bdf4cf 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/CoordinatorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/action/CoordinatorTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.enrich.action; import org.apache.logging.log4j.util.BiConsumer; -import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; @@ -17,13 +16,13 @@ import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.single.shard.SingleShardRequest; import org.elasticsearch.client.internal.ElasticsearchClient; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -371,22 +370,7 @@ public void testReduce() { } private static SearchResponse emptySearchResponse() { - return new SearchResponse( - SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); + return SearchResponseUtils.successfulResponse(SearchHits.empty(Lucene.TOTAL_HITS_EQUAL_TO_ZERO, Float.NaN)); } private class MockLookupFunction implements BiConsumer> { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java index abd928b04a9c7..e8046114ae503 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/ImplicitTiebreakerTests.java @@ -12,12 +12,12 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchResponse.Clusters; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.SearchSortValues; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; @@ -82,10 +82,7 @@ public void query(QueryRequest r, ActionListener l) { ) ); SearchHits searchHits = SearchHits.unpooled(new SearchHit[] { searchHit }, new TotalHits(1, Relation.EQUAL_TO), 0.0f); - ActionListener.respondAndRelease( - l, - new SearchResponse(searchHits, null, null, false, false, null, 0, null, 0, 1, 0, 0, null, Clusters.EMPTY) - ); + ActionListener.respondAndRelease(l, SearchResponseUtils.successfulResponse(searchHits)); } @Override diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java index f6aa851b2fff0..d07de96f09e55 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceSpecTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchResponse.Clusters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.document.DocumentField; @@ -22,6 +21,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence; @@ -220,10 +220,7 @@ public void query(QueryRequest r, ActionListener l) { new TotalHits(eah.hits.size(), Relation.EQUAL_TO), 0.0f ); - ActionListener.respondAndRelease( - l, - new SearchResponse(searchHits, null, null, false, false, null, 0, null, 0, 1, 0, 0, null, Clusters.EMPTY) - ); + ActionListener.respondAndRelease(l, SearchResponseUtils.successfulResponse(searchHits)); } @Override diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java index 58448d981fcca..ef4402f9e9413 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchResponse.Clusters; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.common.ParsingException; @@ -44,6 +43,7 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.SearchSortValues; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -114,10 +114,7 @@ public void query(QueryRequest r, ActionListener l) { new SearchSortValues(new Long[] { (long) ordinal, 1L }, new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW }) ); SearchHits searchHits = SearchHits.unpooled(new SearchHit[] { searchHit }, new TotalHits(1, Relation.EQUAL_TO), 0.0f); - ActionListener.respondAndRelease( - l, - new SearchResponse(searchHits, null, null, false, false, null, 0, null, 0, 1, 0, 0, null, Clusters.EMPTY) - ); + ActionListener.respondAndRelease(l, SearchResponseUtils.successfulResponse(searchHits)); } @Override diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java index 7ebc08a5a3ffe..47c927398f95e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java @@ -660,8 +660,6 @@ private static StringBuilder trimOrPad(StringBuilder buffer) { private static double scaledFloat(String value, String factor) { double scalingFactor = Double.parseDouble(factor); - // this extra division introduces extra imprecision in the following multiplication, but this is how ScaledFloatFieldMapper works. - double scalingFactorInverse = 1d / scalingFactor; - return new BigDecimal(value).multiply(BigDecimal.valueOf(scalingFactor)).longValue() * scalingFactorInverse; + return new BigDecimal(value).multiply(BigDecimal.valueOf(scalingFactor)).longValue() / scalingFactor; } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec index 9177fcbcd2afb..8c186484b7361 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec @@ -281,3 +281,111 @@ languages:integer| emp_no:integer|eval:keyword null |10020 |languages is null null |10021 |languages is null ; + +caseWithMixedNumericValue +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", 1.0, gender == "M", 2, 3.0) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 2.0 +10006 | F | 1.0 +10007 | F | 1.0 +10008 | M | 2.0 +10009 | F | 1.0 +10010 | null | 3.0 +; + +caseWithMixedNumericValueWithNull +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", 1.0, gender == "M", 2, null) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 2.0 +10006 | F | 1.0 +10007 | F | 1.0 +10008 | M | 2.0 +10009 | F | 1.0 +10010 | null | null +; + +caseWithMixedNumericField +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", height, gender == "M", salary, languages) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 63528.0 +10006 | F | 1.56 +10007 | F | 1.7 +10008 | M | 43906.0 +10009 | F | 1.85 +10010 | null | 4.0 +; + +caseWithMixedNumericFieldWithNull +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", height, gender == "M", salary, null) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 63528.0 +10006 | F | 1.56 +10007 | F | 1.7 +10008 | M | 43906.0 +10009 | F | 1.85 +10010 | null | null +; + +caseWithMixedNumericFieldWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", salary_change, gender == "M", salary, languages) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 63528.0 +10006 | F | -3.9 +10007 | F | [-7.06, 0.57, 1.99] +10008 | M | 43906.0 +10009 | F | null +10010 | null | 4.0 +; + +caseWithMixedNumericFieldWithNullWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| WHERE emp_no >= 10005 AND emp_no <= 10010 +| EVAL g = case(gender == "F", salary_change, gender == "M", salary, null) +| KEEP emp_no, gender, g +| SORT emp_no +; + +emp_no:integer | gender:keyword | g:double +10005 | M | 63528.0 +10006 | F | -3.9 +10007 | F | [-7.06, 0.57, 1.99] +10008 | M | 43906.0 +10009 | F | null +10010 | null | null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec index 3505b52e5599e..5e7e78fd95b7c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/floats.csv-spec @@ -8,35 +8,39 @@ a:double | b:double ; inDouble +required_capability: fix_precision_of_scaled_float_fields from employees | keep emp_no, height, height.float, height.half_float, height.scaled_float | where height in (2.03, 2.0299999713897705, 2.029296875, 2.0300000000000002) | sort emp_no; emp_no:integer |height:double |height.float:double |height.half_float:double |height.scaled_float:double -10001 |2.03 |2.0299999713897705 |2.029296875 |2.0300000000000002 -10090 |2.03 |2.0299999713897705 |2.029296875 |2.0300000000000002 +10001 |2.03 |2.0299999713897705 |2.029296875 |2.03 +10090 |2.03 |2.0299999713897705 |2.029296875 |2.03 ; inFloat +required_capability: fix_precision_of_scaled_float_fields from employees | keep emp_no, height, height.float, height.half_float, height.scaled_float | where height.float in (2.03, 2.0299999713897705, 2.029296875, 2.0300000000000002) | sort emp_no; emp_no:integer |height:double |height.float:double |height.half_float:double |height.scaled_float:double -10001 |2.03 |2.0299999713897705 |2.029296875 |2.0300000000000002 -10090 |2.03 |2.0299999713897705 |2.029296875 |2.0300000000000002 +10001 |2.03 |2.0299999713897705 |2.029296875 |2.03 +10090 |2.03 |2.0299999713897705 |2.029296875 |2.03 ; inHalfFloat +required_capability: fix_precision_of_scaled_float_fields from employees | keep emp_no, height, height.float, height.half_float, height.scaled_float | where height.half_float in (2.03, 2.0299999713897705, 2.029296875, 2.0300000000000002) | sort emp_no; emp_no:integer |height:double |height.float:double |height.half_float:double |height.scaled_float:double -10001 |2.03 |2.0299999713897705 |2.029296875 |2.0300000000000002 -10090 |2.03 |2.0299999713897705 |2.029296875 |2.0300000000000002 +10001 |2.03 |2.0299999713897705 |2.029296875 |2.03 +10090 |2.03 |2.0299999713897705 |2.029296875 |2.03 ; inScaledFloat +required_capability: fix_precision_of_scaled_float_fields from employees | keep emp_no, height, height.float, height.half_float, height.scaled_float | where height.scaled_float in (2.03, 2.0299999713897705, 2.029296875, 2.0300000000000002) | sort emp_no; emp_no:integer |height:double |height.float:double |height.half_float:double |height.scaled_float:double -10001 |2.03 |2.0299999713897705 |2.029296875 |2.0300000000000002 -10090 |2.03 |2.0299999713897705 |2.029296875 |2.0300000000000002 +10001 |2.03 |2.0299999713897705 |2.029296875 |2.03 +10090 |2.03 |2.0299999713897705 |2.029296875 |2.03 ; convertFromDouble diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec index 6bc534a9fd918..aa3fbdeb6baa8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/keep.csv-spec @@ -266,10 +266,11 @@ height:double | languages.long:long | still_hired:boolean ; simpleEvalWithSortAndLimitOne +required_capability: fix_precision_of_scaled_float_fields from employees | eval x = languages + 7 | sort x, avg_worked_seconds | limit 1; avg_worked_seconds:long | birth_date:date | emp_no:integer | first_name:keyword | gender:keyword | height:double | height.float:double | height.half_float:double | height.scaled_float:double | hire_date:date | is_rehired:boolean | job_positions:keyword | languages:integer | languages.byte:integer | languages.long:long | languages.short:integer | last_name:keyword | salary:integer | salary_change:double | salary_change.int:integer | salary_change.keyword:keyword |salary_change.long:long | still_hired:boolean | x:integer -208374744 |1956-11-14T00:00:00.000Z|10033 |null |M |1.63 |1.6299999952316284|1.6298828125 |1.6300000000000001 |1987-03-18T00:00:00.000Z|true |null |1 |1 |1 |1 |Merlo |70011 |null |null |null |null |false |8 +208374744 |1956-11-14T00:00:00.000Z|10033 |null |M |1.63 |1.6299999952316284|1.6298828125 |1.63 |1987-03-18T00:00:00.000Z|true |null |1 |1 |1 |1 |Merlo |70011 |null |null |null |null |false |8 ; evalOfAverageValue diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec index b2b4f15860484..56486b8954abe 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec @@ -1583,3 +1583,95 @@ emp_no: integer | x:date | y:date 10001 | 2024-11-03 | 2024-11-06 10002 | 2024-11-03 | 2024-11-06 ; + +greatestWithMixedNumericValues +required_capability: mixed_numeric_types_in_case_greatest_least +ROW g1=GREATEST(10.0, 5.0, 1, -100.1, 0, 1234, -10000), g2=GREATEST(10.0, 5, 1, -100.1, null); + +g1:double |g2:double +1234 |null +; + +leastWithMixedNumericValues +required_capability: mixed_numeric_types_in_case_greatest_least +ROW l1=LEAST(10.0, 5.0, 1, -100.1, 0, 1234, -10000), l2=LEAST(10.0, 5, 1, -100.1, null); + +l1:double |l2:double +-10000 |null +; + +greatestWithMixedNumericFields +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| EVAL g1 = GREATEST(height, salary, languages), g2 = GREATEST(height, salary, languages, null) +| KEEP emp_no, g1, g2 +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | g1:double | g2:double +10001 | 57305.0 | null +10002 | 56371.0 | null +10003 | 61805.0 | null +; + +leastWithMixedNumericFields +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| EVAL l1 = LEAST(height, salary, languages), l2 = LEAST(height, salary, languages, null) +| KEEP emp_no, l1, l2 +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | l1:double | l2:double +10001 | 2.0 | null +10002 | 2.08 | null +10003 | 1.83 | null +; + +greatestWithMixedNumericValuesWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +ROW g1=GREATEST([10.0, 4], 1), g2=GREATEST([10.0, 4], 1, null); + +g1:double |g2:double +10 |null +; + +leastWithMixedNumericValuesWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +ROW l1=LEAST([10.0, 4], 1), l2=LEAST([10.0, 4], 1, null); + +l1:double |l2:double +1 |null +; + +greatestWithMixedNumericFieldsWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| EVAL g1 = GREATEST(salary_change, salary, languages), g2 = GREATEST(salary_change, salary, languages, null) +| KEEP emp_no, g1, g2 +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | g1:double | g2:double +10001 | 57305.0 | null +10002 | 56371.0 | null +10003 | 61805.0 | null +; + +leastWithMixedNumericFieldsWithMV +required_capability: mixed_numeric_types_in_case_greatest_least +FROM employees +| EVAL l1 = LEAST(salary_change, salary, languages), l2 = LEAST(salary_change, salary, languages, null) +| KEEP emp_no, l1, l2 +| SORT emp_no +| LIMIT 3 +; + +emp_no:integer | l1:double | l2:double +10001 | 1.19 | null +10002 | -7.23 | null +10003 | 4.0 | null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec index 1b4c1f0bc2b6c..20ce3ecc5a396 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mv_expand.csv-spec @@ -319,10 +319,11 @@ a | a //see https://github.com/elastic/elasticsearch/issues/103331 keepStarMvExpand#[skip:-8.12.99] +required_capability: fix_precision_of_scaled_float_fields from employees | where emp_no == 10001 | keep * | mv_expand first_name; avg_worked_seconds:long | birth_date:date | emp_no:integer | first_name:keyword | gender:keyword | height:double | height.float:double | height.half_float:double | height.scaled_float:double | hire_date:date | is_rehired:boolean | job_positions:keyword | languages:integer | languages.byte:integer | languages.long:long | languages.short:integer | last_name:keyword | salary:integer | salary_change:double | salary_change.int:integer | salary_change.keyword:keyword | salary_change.long:long | still_hired:boolean -268728049 | 1953-09-02T00:00:00.000Z | 10001 | Georgi | M | 2.03 | 2.0299999713897705 | 2.029296875 | 2.0300000000000002 | 1986-06-26T00:00:00.000Z | [false, true] | [Accountant, Senior Python Developer] | 2 | 2 | 2 | 2 | Facello | 57305 | 1.19 | 1 | 1.19 | 1 | true +268728049 | 1953-09-02T00:00:00.000Z | 10001 | Georgi | M | 2.03 | 2.0299999713897705 | 2.029296875 | 2.03 | 1986-06-26T00:00:00.000Z | [false, true] | [Accountant, Senior Python Developer] | 2 | 2 | 2 | 2 | Facello | 57305 | 1.19 | 1 | 1.19 | 1 | true ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec index ca4c627cae749..21e878bdc6be1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/rename.csv-spec @@ -159,11 +159,12 @@ y:integer | x:date ; renameIntertwinedWithSort +required_capability: fix_precision_of_scaled_float_fields FROM employees | eval x = salary | rename x as y | rename y as x | sort x | rename x as y | limit 10; avg_worked_seconds:l | birth_date:date | emp_no:i | first_name:s | gender:s | height:d | height.float:d | height.half_float:d | height.scaled_float:d| hire_date:date | is_rehired:bool | job_positions:s | languages:i | languages.byte:i | languages.long:l | languages.short:i | last_name:s | salary:i | salary_change:d | salary_change.int:i | salary_change.keyword:s | salary_change.long:l | still_hired:bool | y:i -390266432 | 1959-08-19T00:00:00.000Z | 10015 | Guoxiang | null | 1.66 | 1.659999966621399 | 1.66015625 | 1.6600000000000001 | 1987-07-02T00:00:00.000Z | [false, false, false, true]| [Head Human Resources, Junior Developer, Principal Support Engineer, Support Engineer] | 5 | 5 | 5 | 5 | Nooteboom | 25324 | [12.4, 14.25] | [12, 14] | [12.40, 14.25] | [12, 14] | true | 25324 +390266432 | 1959-08-19T00:00:00.000Z | 10015 | Guoxiang | null | 1.66 | 1.659999966621399 | 1.66015625 | 1.66 | 1987-07-02T00:00:00.000Z | [false, false, false, true]| [Head Human Resources, Junior Developer, Principal Support Engineer, Support Engineer] | 5 | 5 | 5 | 5 | Nooteboom | 25324 | [12.4, 14.25] | [12, 14] | [12.40, 14.25] | [12, 14] | true | 25324 203838153 | 1953-02-08T00:00:00.000Z | 10035 | null | M | 1.81 | 1.809999942779541 | 1.8095703125 | 1.81 | 1988-09-05T00:00:00.000Z | false | [Data Scientist, Senior Python Developer] | 5 | 5 | 5 | 5 | Chappelet | 25945 | [-6.58, -2.54] | [-6, -2] | [-2.54, -6.58] | [-6, -2] | false | 25945 313407352 | 1964-10-18T00:00:00.000Z | 10092 | Valdiodio | F | 1.75 | 1.75 | 1.75 | 1.75 | 1989-09-22T00:00:00.000Z | [false, false, true, true] | [Accountant, Junior Developer] | 1 | 1 | 1 | 1 | Niizuma | 25976 | [-6.77, 0.39, 8.3, 8.78] | [-6, 0, 8, 8] | [-6.77, 0.39, 8.30,8.78] | [-6, 0, 8, 8] | false | 25976 248451647 | null | 10048 | Florian | M | 2.0 | 2.0 | 2.0 | 2.0 | 1985-02-24T00:00:00.000Z | [true, true] | Internship | 3 | 3 | 3 | 3 | Syrotiuk | 26436 | null | null | null | null | false | 26436 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_count_distinct.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_count_distinct.csv-spec index b4f6a701ec272..d89a0759d5d50 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_count_distinct.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_count_distinct.csv-spec @@ -52,6 +52,7 @@ h:long ; countDistinctOfScaledFloat +required_capability: fix_precision_of_scaled_float_fields // float becomes double until https://github.com/elastic/elasticsearch-internal/issues/724 from employees | stats h = count_distinct(height.scaled_float); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 091baafe293d2..cffb1f950fc6f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -734,10 +734,11 @@ null ; convertFromFloats +required_capability: fix_precision_of_scaled_float_fields from employees | sort emp_no| eval double = to_string(height), float = to_string(height.float), scaled_float = to_string(height.scaled_float), half_float = to_string(height.half_float) | keep emp_no, double, float, scaled_float, half_float, height | limit 2; emp_no:integer |double:keyword |float:keyword |scaled_float:keyword |half_float:keyword |height:double -10001 |2.03 |2.0299999713897705|2.0300000000000002 |2.029296875 |2.03 +10001 |2.03 |2.0299999713897705|2.03 |2.029296875 |2.03 10002 |2.08 |2.0799999237060547|2.08 |2.080078125 |2.08 ; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterCancellationIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterCancellationIT.java index 57041d3d6a06f..df27145d488a0 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterCancellationIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterCancellationIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.action; -import org.elasticsearch.Build; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction; @@ -287,44 +286,4 @@ public void testCancelSkipUnavailable() throws Exception { Exception error = expectThrows(Exception.class, requestFuture::actionGet); assertThat(error, instanceOf(TaskCancelledException.class)); } - - // Check that closing remote node with skip_unavailable=true produces partial - public void testCloseSkipUnavailable() throws Exception { - // We are using delay() here because closing cluster while inside pause fields doesn't seem to produce clean closure - assumeTrue("Only snapshot builds have delay()", Build.current().isSnapshot()); - createRemoteIndex(between(1000, 5000)); - createLocalIndex(10); - EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest(); - request.query(""" - FROM test*,cluster-a:test* METADATA _index - | EVAL cluster=MV_FIRST(SPLIT(_index, ":")) - | WHERE CASE(cluster == "cluster-a", delay(1ms), true) - | STATS total = sum(const) | LIMIT 1 - """); - request.pragmas(randomPragmas()); - var requestFuture = client().execute(EsqlQueryAction.INSTANCE, request); - assertTrue(SimplePauseFieldPlugin.startEmitting.await(30, TimeUnit.SECONDS)); - SimplePauseFieldPlugin.allowEmitting.countDown(); - cluster(REMOTE_CLUSTER).close(); - try (var resp = requestFuture.actionGet()) { - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertNotNull(executionInfo); - assertThat(executionInfo.isPartial(), equalTo(true)); - - List> values = getValuesList(resp); - assertThat(values.get(0).size(), equalTo(1)); - // We can't be sure of the exact value here as we don't know if any data from remote came in, but all local data should be there - assertThat((long) values.get(0).get(0), greaterThanOrEqualTo(10L)); - - EsqlExecutionInfo.Cluster cluster = executionInfo.getCluster(REMOTE_CLUSTER); - EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); - - assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); - assertThat(localCluster.getSuccessfulShards(), equalTo(1)); - - assertThat(cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.PARTIAL)); - assertThat(cluster.getSuccessfulShards(), equalTo(0)); - assertThat(cluster.getFailures().size(), equalTo(1)); - } - } } diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index 6556cbba5c5f9..ed49ffdef8ae4 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -195,14 +195,12 @@ IN: 'in'; IS: 'is'; LAST : 'last'; LIKE: 'like'; -LP : '('; NOT : 'not'; NULL : 'null'; NULLS : 'nulls'; OR : 'or'; PARAM: '?'; RLIKE: 'rlike'; -RP : ')'; TRUE : 'true'; EQ : '=='; @@ -223,8 +221,6 @@ LEFT_BRACES : '{'; RIGHT_BRACES : '}'; NESTED_WHERE : WHERE -> type(WHERE); -NESTED_SORT : {this.isDevVersion()}? SORT -> type(SORT); -NESTED_LIMIT : {this.isDevVersion()}? LIMIT -> type(LIMIT); NAMED_OR_POSITIONAL_PARAM : PARAM (LETTER | UNDERSCORE) UNQUOTED_ID_BODY* @@ -239,6 +235,9 @@ NAMED_OR_POSITIONAL_PARAM OPENING_BRACKET : '[' -> pushMode(EXPRESSION_MODE), pushMode(EXPRESSION_MODE); CLOSING_BRACKET : ']' -> popMode, popMode; +LP : '(' -> pushMode(EXPRESSION_MODE), pushMode(EXPRESSION_MODE); +RP : ')' -> popMode, popMode; + UNQUOTED_IDENTIFIER : LETTER UNQUOTED_ID_BODY* // only allow @ at beginning of identifier to keep the option to allow @ as infix operator in the future @@ -678,8 +677,8 @@ INSIST_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN); // mode FORK_MODE; FORK_LP : LP -> type(LP), pushMode(DEFAULT_MODE); -FORK_RP : RP -> type(RP), popMode; FORK_PIPE : PIPE -> type(PIPE), popMode; + FORK_WS : WS -> channel(HIDDEN); FORK_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN); FORK_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN); diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens index 7756d406adee8..2ea8b68922934 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens @@ -47,32 +47,32 @@ IN=46 IS=47 LAST=48 LIKE=49 -LP=50 -NOT=51 -NULL=52 -NULLS=53 -OR=54 -PARAM=55 -RLIKE=56 -RP=57 -TRUE=58 -EQ=59 -CIEQ=60 -NEQ=61 -LT=62 -LTE=63 -GT=64 -GTE=65 -PLUS=66 -MINUS=67 -ASTERISK=68 -SLASH=69 -PERCENT=70 -LEFT_BRACES=71 -RIGHT_BRACES=72 -NAMED_OR_POSITIONAL_PARAM=73 -OPENING_BRACKET=74 -CLOSING_BRACKET=75 +NOT=50 +NULL=51 +NULLS=52 +OR=53 +PARAM=54 +RLIKE=55 +TRUE=56 +EQ=57 +CIEQ=58 +NEQ=59 +LT=60 +LTE=61 +GT=62 +GTE=63 +PLUS=64 +MINUS=65 +ASTERISK=66 +SLASH=67 +PERCENT=68 +LEFT_BRACES=69 +RIGHT_BRACES=70 +NAMED_OR_POSITIONAL_PARAM=71 +OPENING_BRACKET=72 +CLOSING_BRACKET=73 +LP=74 +RP=75 UNQUOTED_IDENTIFIER=76 QUOTED_IDENTIFIER=77 EXPR_LINE_COMMENT=78 @@ -173,30 +173,29 @@ FORK_MULTILINE_COMMENT=142 'is'=47 'last'=48 'like'=49 -'('=50 -'not'=51 -'null'=52 -'nulls'=53 -'or'=54 -'?'=55 -'rlike'=56 -')'=57 -'true'=58 -'=='=59 -'=~'=60 -'!='=61 -'<'=62 -'<='=63 -'>'=64 -'>='=65 -'+'=66 -'-'=67 -'*'=68 -'/'=69 -'%'=70 -'{'=71 -'}'=72 -']'=75 +'not'=50 +'null'=51 +'nulls'=52 +'or'=53 +'?'=54 +'rlike'=55 +'true'=56 +'=='=57 +'=~'=58 +'!='=59 +'<'=60 +'<='=61 +'>'=62 +'>='=63 +'+'=64 +'-'=65 +'*'=66 +'/'=67 +'%'=68 +'{'=69 +'}'=70 +']'=73 +')'=75 'metadata'=84 'as'=93 'on'=97 diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens index 7756d406adee8..2ea8b68922934 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens @@ -47,32 +47,32 @@ IN=46 IS=47 LAST=48 LIKE=49 -LP=50 -NOT=51 -NULL=52 -NULLS=53 -OR=54 -PARAM=55 -RLIKE=56 -RP=57 -TRUE=58 -EQ=59 -CIEQ=60 -NEQ=61 -LT=62 -LTE=63 -GT=64 -GTE=65 -PLUS=66 -MINUS=67 -ASTERISK=68 -SLASH=69 -PERCENT=70 -LEFT_BRACES=71 -RIGHT_BRACES=72 -NAMED_OR_POSITIONAL_PARAM=73 -OPENING_BRACKET=74 -CLOSING_BRACKET=75 +NOT=50 +NULL=51 +NULLS=52 +OR=53 +PARAM=54 +RLIKE=55 +TRUE=56 +EQ=57 +CIEQ=58 +NEQ=59 +LT=60 +LTE=61 +GT=62 +GTE=63 +PLUS=64 +MINUS=65 +ASTERISK=66 +SLASH=67 +PERCENT=68 +LEFT_BRACES=69 +RIGHT_BRACES=70 +NAMED_OR_POSITIONAL_PARAM=71 +OPENING_BRACKET=72 +CLOSING_BRACKET=73 +LP=74 +RP=75 UNQUOTED_IDENTIFIER=76 QUOTED_IDENTIFIER=77 EXPR_LINE_COMMENT=78 @@ -173,30 +173,29 @@ FORK_MULTILINE_COMMENT=142 'is'=47 'last'=48 'like'=49 -'('=50 -'not'=51 -'null'=52 -'nulls'=53 -'or'=54 -'?'=55 -'rlike'=56 -')'=57 -'true'=58 -'=='=59 -'=~'=60 -'!='=61 -'<'=62 -'<='=63 -'>'=64 -'>='=65 -'+'=66 -'-'=67 -'*'=68 -'/'=69 -'%'=70 -'{'=71 -'}'=72 -']'=75 +'not'=50 +'null'=51 +'nulls'=52 +'or'=53 +'?'=54 +'rlike'=55 +'true'=56 +'=='=57 +'=~'=58 +'!='=59 +'<'=60 +'<='=61 +'>'=62 +'>='=63 +'+'=64 +'-'=65 +'*'=66 +'/'=67 +'%'=68 +'{'=69 +'}'=70 +']'=73 +')'=75 'metadata'=84 'as'=93 'on'=97 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 434429f073b6b..980e5402ea560 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -408,6 +408,12 @@ public enum Cap { */ FIX_PARSING_LARGE_NEGATIVE_NUMBERS, + /** + * Fix precision of scaled_float field values retrieved from stored source + * see Slight inconsistency in ESQL using scaled_float field #122547 + */ + FIX_PRECISION_OF_SCALED_FLOAT_FIELDS, + /** * Fix the status code returned when trying to run count_distinct on the _source type (which is not supported). * see count_distinct(_source) returns a 500 response @@ -834,7 +840,12 @@ public enum Cap { /** * Support for FORK command */ - FORK(Build.current().isSnapshot()); + FORK(Build.current().isSnapshot()), + + /** + * Allow mixed numeric types in conditional functions - case, greatest and least + */ + MIXED_NUMERIC_TYPES_IN_CASE_GREATEST_LEAST; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 589826501423b..c8df71a0e1753 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -52,6 +52,9 @@ import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Least; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FoldablesConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; @@ -1314,7 +1317,7 @@ private static Expression processIn(In in) { } private static boolean canCastMixedNumericTypes(org.elasticsearch.xpack.esql.core.expression.function.Function f) { - return f instanceof Coalesce; + return f instanceof Coalesce || f instanceof Case || f instanceof Greatest || f instanceof Least; } private static boolean canCastNumeric(DataType from, DataType to) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp index 3f1b0baf6808b..f2afedee8ac01 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp @@ -49,14 +49,12 @@ null 'is' 'last' 'like' -'(' 'not' 'null' 'nulls' 'or' '?' 'rlike' -')' 'true' '==' '=~' @@ -76,6 +74,8 @@ null null ']' null +')' +null null null null @@ -194,14 +194,12 @@ IN IS LAST LIKE -LP NOT NULL NULLS OR PARAM RLIKE -RP TRUE EQ CIEQ @@ -220,6 +218,8 @@ RIGHT_BRACES NAMED_OR_POSITIONAL_PARAM OPENING_BRACKET CLOSING_BRACKET +LP +RP UNQUOTED_IDENTIFIER QUOTED_IDENTIFIER EXPR_LINE_COMMENT @@ -348,14 +348,12 @@ IN IS LAST LIKE -LP NOT NULL NULLS OR PARAM RLIKE -RP TRUE EQ CIEQ @@ -372,11 +370,11 @@ PERCENT LEFT_BRACES RIGHT_BRACES NESTED_WHERE -NESTED_SORT -NESTED_LIMIT NAMED_OR_POSITIONAL_PARAM OPENING_BRACKET CLOSING_BRACKET +LP +RP UNQUOTED_IDENTIFIER QUOTED_ID QUOTED_IDENTIFIER @@ -527,7 +525,6 @@ INSIST_WS INSIST_LINE_COMMENT INSIST_MULTILINE_COMMENT FORK_LP -FORK_RP FORK_PIPE FORK_WS FORK_LINE_COMMENT @@ -559,4 +556,4 @@ INSIST_MODE FORK_MODE atn: -[4, 0, 142, 1799, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 2, 198, 7, 198, 2, 199, 7, 199, 2, 200, 7, 200, 2, 201, 7, 201, 2, 202, 7, 202, 2, 203, 7, 203, 2, 204, 7, 204, 2, 205, 7, 205, 2, 206, 7, 206, 2, 207, 7, 207, 2, 208, 7, 208, 2, 209, 7, 209, 2, 210, 7, 210, 2, 211, 7, 211, 2, 212, 7, 212, 2, 213, 7, 213, 2, 214, 7, 214, 2, 215, 7, 215, 2, 216, 7, 216, 2, 217, 7, 217, 2, 218, 7, 218, 2, 219, 7, 219, 2, 220, 7, 220, 2, 221, 7, 221, 2, 222, 7, 222, 2, 223, 7, 223, 2, 224, 7, 224, 2, 225, 7, 225, 2, 226, 7, 226, 2, 227, 7, 227, 2, 228, 7, 228, 2, 229, 7, 229, 2, 230, 7, 230, 2, 231, 7, 231, 2, 232, 7, 232, 2, 233, 7, 233, 2, 234, 7, 234, 2, 235, 7, 235, 2, 236, 7, 236, 2, 237, 7, 237, 2, 238, 7, 238, 2, 239, 7, 239, 2, 240, 7, 240, 2, 241, 7, 241, 2, 242, 7, 242, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 4, 26, 744, 8, 26, 11, 26, 12, 26, 745, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 754, 8, 27, 10, 27, 12, 27, 757, 9, 27, 1, 27, 3, 27, 760, 8, 27, 1, 27, 3, 27, 763, 8, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 5, 28, 772, 8, 28, 10, 28, 12, 28, 775, 9, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 29, 4, 29, 783, 8, 29, 11, 29, 12, 29, 784, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 35, 1, 35, 3, 35, 804, 8, 35, 1, 35, 4, 35, 807, 8, 35, 11, 35, 12, 35, 808, 1, 36, 1, 36, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 3, 38, 818, 8, 38, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 3, 40, 825, 8, 40, 1, 41, 1, 41, 1, 41, 5, 41, 830, 8, 41, 10, 41, 12, 41, 833, 9, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 5, 41, 841, 8, 41, 10, 41, 12, 41, 844, 9, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 3, 41, 851, 8, 41, 1, 41, 3, 41, 854, 8, 41, 3, 41, 856, 8, 41, 1, 42, 4, 42, 859, 8, 42, 11, 42, 12, 42, 860, 1, 43, 4, 43, 864, 8, 43, 11, 43, 12, 43, 865, 1, 43, 1, 43, 5, 43, 870, 8, 43, 10, 43, 12, 43, 873, 9, 43, 1, 43, 1, 43, 4, 43, 877, 8, 43, 11, 43, 12, 43, 878, 1, 43, 4, 43, 882, 8, 43, 11, 43, 12, 43, 883, 1, 43, 1, 43, 5, 43, 888, 8, 43, 10, 43, 12, 43, 891, 9, 43, 3, 43, 893, 8, 43, 1, 43, 1, 43, 1, 43, 1, 43, 4, 43, 899, 8, 43, 11, 43, 12, 43, 900, 1, 43, 1, 43, 3, 43, 905, 8, 43, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 69, 1, 70, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 76, 1, 76, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 3, 85, 1047, 8, 85, 1, 85, 5, 85, 1050, 8, 85, 10, 85, 12, 85, 1053, 9, 85, 1, 85, 1, 85, 4, 85, 1057, 8, 85, 11, 85, 12, 85, 1058, 3, 85, 1061, 8, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 5, 88, 1075, 8, 88, 10, 88, 12, 88, 1078, 9, 88, 1, 88, 1, 88, 3, 88, 1082, 8, 88, 1, 88, 4, 88, 1085, 8, 88, 11, 88, 12, 88, 1086, 3, 88, 1089, 8, 88, 1, 89, 1, 89, 4, 89, 1093, 8, 89, 11, 89, 12, 89, 1094, 1, 89, 1, 89, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 105, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 3, 106, 1172, 8, 106, 1, 107, 4, 107, 1175, 8, 107, 11, 107, 12, 107, 1176, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 3, 118, 1224, 8, 118, 1, 119, 1, 119, 3, 119, 1228, 8, 119, 1, 119, 5, 119, 1231, 8, 119, 10, 119, 12, 119, 1234, 9, 119, 1, 119, 1, 119, 3, 119, 1238, 8, 119, 1, 119, 4, 119, 1241, 8, 119, 11, 119, 12, 119, 1242, 3, 119, 1245, 8, 119, 1, 120, 1, 120, 4, 120, 1249, 8, 120, 11, 120, 12, 120, 1250, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 140, 4, 140, 1334, 8, 140, 11, 140, 12, 140, 1335, 1, 140, 1, 140, 3, 140, 1340, 8, 140, 1, 140, 4, 140, 1343, 8, 140, 11, 140, 12, 140, 1344, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 173, 4, 173, 1486, 8, 173, 11, 173, 12, 173, 1487, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 197, 1, 197, 1, 198, 1, 198, 1, 198, 1, 198, 1, 198, 1, 198, 1, 198, 1, 198, 1, 198, 1, 199, 1, 199, 1, 199, 1, 199, 1, 200, 1, 200, 1, 200, 1, 200, 1, 201, 1, 201, 1, 201, 1, 201, 1, 202, 1, 202, 1, 202, 1, 202, 1, 203, 1, 203, 1, 203, 1, 203, 1, 204, 1, 204, 1, 204, 1, 204, 1, 205, 1, 205, 1, 205, 1, 205, 1, 206, 1, 206, 1, 206, 1, 206, 1, 207, 1, 207, 1, 207, 1, 207, 1, 207, 1, 208, 1, 208, 1, 208, 1, 208, 1, 208, 1, 208, 1, 209, 1, 209, 1, 209, 1, 209, 1, 209, 1, 209, 1, 210, 1, 210, 1, 210, 1, 210, 1, 211, 1, 211, 1, 211, 1, 211, 1, 212, 1, 212, 1, 212, 1, 212, 1, 213, 1, 213, 1, 213, 1, 213, 1, 213, 1, 213, 1, 214, 1, 214, 1, 214, 1, 214, 1, 214, 1, 214, 1, 215, 1, 215, 1, 215, 1, 215, 1, 216, 1, 216, 1, 216, 1, 216, 1, 217, 1, 217, 1, 217, 1, 217, 1, 218, 1, 218, 1, 218, 1, 218, 1, 218, 1, 218, 1, 219, 1, 219, 1, 219, 1, 219, 1, 219, 1, 219, 1, 220, 1, 220, 1, 220, 1, 220, 1, 220, 1, 220, 1, 221, 1, 221, 1, 221, 1, 221, 1, 221, 1, 222, 1, 222, 1, 222, 1, 222, 1, 222, 1, 223, 1, 223, 1, 223, 1, 223, 1, 224, 1, 224, 1, 224, 1, 224, 1, 225, 1, 225, 1, 225, 1, 225, 1, 226, 1, 226, 1, 226, 1, 226, 1, 227, 1, 227, 1, 227, 1, 227, 1, 228, 1, 228, 1, 228, 1, 228, 1, 229, 1, 229, 1, 229, 1, 229, 1, 230, 1, 230, 1, 230, 1, 230, 1, 231, 1, 231, 1, 231, 1, 231, 1, 232, 1, 232, 1, 232, 1, 232, 1, 232, 1, 233, 1, 233, 1, 233, 1, 233, 1, 234, 1, 234, 1, 234, 1, 234, 1, 235, 1, 235, 1, 235, 1, 235, 1, 236, 1, 236, 1, 236, 1, 236, 1, 237, 1, 237, 1, 237, 1, 237, 1, 237, 1, 238, 1, 238, 1, 238, 1, 238, 1, 238, 1, 239, 1, 239, 1, 239, 1, 239, 1, 239, 1, 240, 1, 240, 1, 240, 1, 240, 1, 241, 1, 241, 1, 241, 1, 241, 1, 242, 1, 242, 1, 242, 1, 242, 2, 773, 842, 0, 243, 19, 1, 21, 2, 23, 3, 25, 4, 27, 5, 29, 6, 31, 7, 33, 8, 35, 9, 37, 10, 39, 11, 41, 12, 43, 13, 45, 14, 47, 15, 49, 16, 51, 17, 53, 18, 55, 19, 57, 20, 59, 21, 61, 22, 63, 23, 65, 24, 67, 25, 69, 26, 71, 27, 73, 28, 75, 29, 77, 30, 79, 31, 81, 0, 83, 0, 85, 0, 87, 0, 89, 0, 91, 0, 93, 0, 95, 0, 97, 0, 99, 0, 101, 32, 103, 33, 105, 34, 107, 35, 109, 36, 111, 37, 113, 38, 115, 39, 117, 40, 119, 41, 121, 42, 123, 43, 125, 44, 127, 45, 129, 46, 131, 47, 133, 48, 135, 49, 137, 50, 139, 51, 141, 52, 143, 53, 145, 54, 147, 55, 149, 56, 151, 57, 153, 58, 155, 59, 157, 60, 159, 61, 161, 62, 163, 63, 165, 64, 167, 65, 169, 66, 171, 67, 173, 68, 175, 69, 177, 70, 179, 71, 181, 72, 183, 0, 185, 0, 187, 0, 189, 73, 191, 74, 193, 75, 195, 76, 197, 0, 199, 77, 201, 78, 203, 79, 205, 80, 207, 0, 209, 0, 211, 81, 213, 82, 215, 83, 217, 0, 219, 0, 221, 0, 223, 0, 225, 0, 227, 0, 229, 84, 231, 0, 233, 85, 235, 0, 237, 0, 239, 86, 241, 87, 243, 88, 245, 0, 247, 0, 249, 0, 251, 0, 253, 0, 255, 0, 257, 0, 259, 89, 261, 90, 263, 91, 265, 92, 267, 0, 269, 0, 271, 0, 273, 0, 275, 0, 277, 0, 279, 93, 281, 0, 283, 94, 285, 95, 287, 96, 289, 0, 291, 0, 293, 97, 295, 98, 297, 0, 299, 99, 301, 0, 303, 100, 305, 101, 307, 102, 309, 0, 311, 0, 313, 0, 315, 0, 317, 0, 319, 0, 321, 0, 323, 0, 325, 0, 327, 103, 329, 104, 331, 105, 333, 0, 335, 0, 337, 0, 339, 0, 341, 0, 343, 0, 345, 106, 347, 107, 349, 108, 351, 0, 353, 109, 355, 110, 357, 111, 359, 112, 361, 0, 363, 0, 365, 113, 367, 114, 369, 115, 371, 116, 373, 0, 375, 0, 377, 0, 379, 0, 381, 0, 383, 0, 385, 0, 387, 117, 389, 118, 391, 119, 393, 0, 395, 0, 397, 0, 399, 0, 401, 120, 403, 121, 405, 122, 407, 0, 409, 123, 411, 0, 413, 0, 415, 124, 417, 0, 419, 0, 421, 0, 423, 0, 425, 0, 427, 125, 429, 126, 431, 127, 433, 0, 435, 0, 437, 0, 439, 128, 441, 129, 443, 130, 445, 0, 447, 0, 449, 131, 451, 132, 453, 133, 455, 0, 457, 0, 459, 0, 461, 0, 463, 0, 465, 0, 467, 0, 469, 0, 471, 0, 473, 0, 475, 0, 477, 134, 479, 135, 481, 136, 483, 0, 485, 0, 487, 137, 489, 138, 491, 139, 493, 0, 495, 0, 497, 0, 499, 140, 501, 141, 503, 142, 19, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 36, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 2, 0, 74, 74, 106, 106, 1823, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 0, 63, 1, 0, 0, 0, 0, 65, 1, 0, 0, 0, 0, 67, 1, 0, 0, 0, 0, 69, 1, 0, 0, 0, 0, 71, 1, 0, 0, 0, 0, 73, 1, 0, 0, 0, 0, 75, 1, 0, 0, 0, 0, 77, 1, 0, 0, 0, 1, 79, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 171, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 1, 181, 1, 0, 0, 0, 1, 183, 1, 0, 0, 0, 1, 185, 1, 0, 0, 0, 1, 187, 1, 0, 0, 0, 1, 189, 1, 0, 0, 0, 1, 191, 1, 0, 0, 0, 1, 193, 1, 0, 0, 0, 1, 195, 1, 0, 0, 0, 1, 199, 1, 0, 0, 0, 1, 201, 1, 0, 0, 0, 1, 203, 1, 0, 0, 0, 1, 205, 1, 0, 0, 0, 2, 207, 1, 0, 0, 0, 2, 209, 1, 0, 0, 0, 2, 211, 1, 0, 0, 0, 2, 213, 1, 0, 0, 0, 2, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 3, 219, 1, 0, 0, 0, 3, 221, 1, 0, 0, 0, 3, 223, 1, 0, 0, 0, 3, 225, 1, 0, 0, 0, 3, 227, 1, 0, 0, 0, 3, 229, 1, 0, 0, 0, 3, 233, 1, 0, 0, 0, 3, 235, 1, 0, 0, 0, 3, 237, 1, 0, 0, 0, 3, 239, 1, 0, 0, 0, 3, 241, 1, 0, 0, 0, 3, 243, 1, 0, 0, 0, 4, 245, 1, 0, 0, 0, 4, 247, 1, 0, 0, 0, 4, 249, 1, 0, 0, 0, 4, 251, 1, 0, 0, 0, 4, 253, 1, 0, 0, 0, 4, 259, 1, 0, 0, 0, 4, 261, 1, 0, 0, 0, 4, 263, 1, 0, 0, 0, 4, 265, 1, 0, 0, 0, 5, 267, 1, 0, 0, 0, 5, 269, 1, 0, 0, 0, 5, 271, 1, 0, 0, 0, 5, 273, 1, 0, 0, 0, 5, 275, 1, 0, 0, 0, 5, 277, 1, 0, 0, 0, 5, 279, 1, 0, 0, 0, 5, 281, 1, 0, 0, 0, 5, 283, 1, 0, 0, 0, 5, 285, 1, 0, 0, 0, 5, 287, 1, 0, 0, 0, 6, 289, 1, 0, 0, 0, 6, 291, 1, 0, 0, 0, 6, 293, 1, 0, 0, 0, 6, 295, 1, 0, 0, 0, 6, 299, 1, 0, 0, 0, 6, 301, 1, 0, 0, 0, 6, 303, 1, 0, 0, 0, 6, 305, 1, 0, 0, 0, 6, 307, 1, 0, 0, 0, 7, 309, 1, 0, 0, 0, 7, 311, 1, 0, 0, 0, 7, 313, 1, 0, 0, 0, 7, 315, 1, 0, 0, 0, 7, 317, 1, 0, 0, 0, 7, 319, 1, 0, 0, 0, 7, 321, 1, 0, 0, 0, 7, 323, 1, 0, 0, 0, 7, 325, 1, 0, 0, 0, 7, 327, 1, 0, 0, 0, 7, 329, 1, 0, 0, 0, 7, 331, 1, 0, 0, 0, 8, 333, 1, 0, 0, 0, 8, 335, 1, 0, 0, 0, 8, 337, 1, 0, 0, 0, 8, 339, 1, 0, 0, 0, 8, 341, 1, 0, 0, 0, 8, 343, 1, 0, 0, 0, 8, 345, 1, 0, 0, 0, 8, 347, 1, 0, 0, 0, 8, 349, 1, 0, 0, 0, 9, 351, 1, 0, 0, 0, 9, 353, 1, 0, 0, 0, 9, 355, 1, 0, 0, 0, 9, 357, 1, 0, 0, 0, 9, 359, 1, 0, 0, 0, 10, 361, 1, 0, 0, 0, 10, 363, 1, 0, 0, 0, 10, 365, 1, 0, 0, 0, 10, 367, 1, 0, 0, 0, 10, 369, 1, 0, 0, 0, 10, 371, 1, 0, 0, 0, 11, 373, 1, 0, 0, 0, 11, 375, 1, 0, 0, 0, 11, 377, 1, 0, 0, 0, 11, 379, 1, 0, 0, 0, 11, 381, 1, 0, 0, 0, 11, 383, 1, 0, 0, 0, 11, 385, 1, 0, 0, 0, 11, 387, 1, 0, 0, 0, 11, 389, 1, 0, 0, 0, 11, 391, 1, 0, 0, 0, 12, 393, 1, 0, 0, 0, 12, 395, 1, 0, 0, 0, 12, 397, 1, 0, 0, 0, 12, 399, 1, 0, 0, 0, 12, 401, 1, 0, 0, 0, 12, 403, 1, 0, 0, 0, 12, 405, 1, 0, 0, 0, 13, 407, 1, 0, 0, 0, 13, 409, 1, 0, 0, 0, 13, 411, 1, 0, 0, 0, 13, 413, 1, 0, 0, 0, 13, 415, 1, 0, 0, 0, 13, 417, 1, 0, 0, 0, 13, 419, 1, 0, 0, 0, 13, 421, 1, 0, 0, 0, 13, 423, 1, 0, 0, 0, 13, 425, 1, 0, 0, 0, 13, 427, 1, 0, 0, 0, 13, 429, 1, 0, 0, 0, 13, 431, 1, 0, 0, 0, 14, 433, 1, 0, 0, 0, 14, 435, 1, 0, 0, 0, 14, 437, 1, 0, 0, 0, 14, 439, 1, 0, 0, 0, 14, 441, 1, 0, 0, 0, 14, 443, 1, 0, 0, 0, 15, 445, 1, 0, 0, 0, 15, 447, 1, 0, 0, 0, 15, 449, 1, 0, 0, 0, 15, 451, 1, 0, 0, 0, 15, 453, 1, 0, 0, 0, 15, 455, 1, 0, 0, 0, 15, 457, 1, 0, 0, 0, 15, 459, 1, 0, 0, 0, 15, 461, 1, 0, 0, 0, 16, 463, 1, 0, 0, 0, 16, 465, 1, 0, 0, 0, 16, 467, 1, 0, 0, 0, 16, 469, 1, 0, 0, 0, 16, 471, 1, 0, 0, 0, 16, 473, 1, 0, 0, 0, 16, 475, 1, 0, 0, 0, 16, 477, 1, 0, 0, 0, 16, 479, 1, 0, 0, 0, 16, 481, 1, 0, 0, 0, 17, 483, 1, 0, 0, 0, 17, 485, 1, 0, 0, 0, 17, 487, 1, 0, 0, 0, 17, 489, 1, 0, 0, 0, 17, 491, 1, 0, 0, 0, 18, 493, 1, 0, 0, 0, 18, 495, 1, 0, 0, 0, 18, 497, 1, 0, 0, 0, 18, 499, 1, 0, 0, 0, 18, 501, 1, 0, 0, 0, 18, 503, 1, 0, 0, 0, 19, 505, 1, 0, 0, 0, 21, 515, 1, 0, 0, 0, 23, 522, 1, 0, 0, 0, 25, 531, 1, 0, 0, 0, 27, 538, 1, 0, 0, 0, 29, 548, 1, 0, 0, 0, 31, 555, 1, 0, 0, 0, 33, 562, 1, 0, 0, 0, 35, 569, 1, 0, 0, 0, 37, 577, 1, 0, 0, 0, 39, 589, 1, 0, 0, 0, 41, 598, 1, 0, 0, 0, 43, 604, 1, 0, 0, 0, 45, 611, 1, 0, 0, 0, 47, 618, 1, 0, 0, 0, 49, 626, 1, 0, 0, 0, 51, 634, 1, 0, 0, 0, 53, 643, 1, 0, 0, 0, 55, 659, 1, 0, 0, 0, 57, 674, 1, 0, 0, 0, 59, 686, 1, 0, 0, 0, 61, 698, 1, 0, 0, 0, 63, 709, 1, 0, 0, 0, 65, 717, 1, 0, 0, 0, 67, 725, 1, 0, 0, 0, 69, 734, 1, 0, 0, 0, 71, 743, 1, 0, 0, 0, 73, 749, 1, 0, 0, 0, 75, 766, 1, 0, 0, 0, 77, 782, 1, 0, 0, 0, 79, 788, 1, 0, 0, 0, 81, 792, 1, 0, 0, 0, 83, 794, 1, 0, 0, 0, 85, 796, 1, 0, 0, 0, 87, 799, 1, 0, 0, 0, 89, 801, 1, 0, 0, 0, 91, 810, 1, 0, 0, 0, 93, 812, 1, 0, 0, 0, 95, 817, 1, 0, 0, 0, 97, 819, 1, 0, 0, 0, 99, 824, 1, 0, 0, 0, 101, 855, 1, 0, 0, 0, 103, 858, 1, 0, 0, 0, 105, 904, 1, 0, 0, 0, 107, 906, 1, 0, 0, 0, 109, 909, 1, 0, 0, 0, 111, 913, 1, 0, 0, 0, 113, 917, 1, 0, 0, 0, 115, 919, 1, 0, 0, 0, 117, 922, 1, 0, 0, 0, 119, 924, 1, 0, 0, 0, 121, 926, 1, 0, 0, 0, 123, 931, 1, 0, 0, 0, 125, 933, 1, 0, 0, 0, 127, 939, 1, 0, 0, 0, 129, 945, 1, 0, 0, 0, 131, 948, 1, 0, 0, 0, 133, 951, 1, 0, 0, 0, 135, 956, 1, 0, 0, 0, 137, 961, 1, 0, 0, 0, 139, 963, 1, 0, 0, 0, 141, 967, 1, 0, 0, 0, 143, 972, 1, 0, 0, 0, 145, 978, 1, 0, 0, 0, 147, 981, 1, 0, 0, 0, 149, 983, 1, 0, 0, 0, 151, 989, 1, 0, 0, 0, 153, 991, 1, 0, 0, 0, 155, 996, 1, 0, 0, 0, 157, 999, 1, 0, 0, 0, 159, 1002, 1, 0, 0, 0, 161, 1005, 1, 0, 0, 0, 163, 1007, 1, 0, 0, 0, 165, 1010, 1, 0, 0, 0, 167, 1012, 1, 0, 0, 0, 169, 1015, 1, 0, 0, 0, 171, 1017, 1, 0, 0, 0, 173, 1019, 1, 0, 0, 0, 175, 1021, 1, 0, 0, 0, 177, 1023, 1, 0, 0, 0, 179, 1025, 1, 0, 0, 0, 181, 1027, 1, 0, 0, 0, 183, 1029, 1, 0, 0, 0, 185, 1033, 1, 0, 0, 0, 187, 1038, 1, 0, 0, 0, 189, 1060, 1, 0, 0, 0, 191, 1062, 1, 0, 0, 0, 193, 1067, 1, 0, 0, 0, 195, 1088, 1, 0, 0, 0, 197, 1090, 1, 0, 0, 0, 199, 1098, 1, 0, 0, 0, 201, 1100, 1, 0, 0, 0, 203, 1104, 1, 0, 0, 0, 205, 1108, 1, 0, 0, 0, 207, 1112, 1, 0, 0, 0, 209, 1117, 1, 0, 0, 0, 211, 1122, 1, 0, 0, 0, 213, 1126, 1, 0, 0, 0, 215, 1130, 1, 0, 0, 0, 217, 1134, 1, 0, 0, 0, 219, 1139, 1, 0, 0, 0, 221, 1143, 1, 0, 0, 0, 223, 1147, 1, 0, 0, 0, 225, 1151, 1, 0, 0, 0, 227, 1155, 1, 0, 0, 0, 229, 1159, 1, 0, 0, 0, 231, 1171, 1, 0, 0, 0, 233, 1174, 1, 0, 0, 0, 235, 1178, 1, 0, 0, 0, 237, 1182, 1, 0, 0, 0, 239, 1186, 1, 0, 0, 0, 241, 1190, 1, 0, 0, 0, 243, 1194, 1, 0, 0, 0, 245, 1198, 1, 0, 0, 0, 247, 1203, 1, 0, 0, 0, 249, 1207, 1, 0, 0, 0, 251, 1211, 1, 0, 0, 0, 253, 1215, 1, 0, 0, 0, 255, 1223, 1, 0, 0, 0, 257, 1244, 1, 0, 0, 0, 259, 1248, 1, 0, 0, 0, 261, 1252, 1, 0, 0, 0, 263, 1256, 1, 0, 0, 0, 265, 1260, 1, 0, 0, 0, 267, 1264, 1, 0, 0, 0, 269, 1269, 1, 0, 0, 0, 271, 1273, 1, 0, 0, 0, 273, 1277, 1, 0, 0, 0, 275, 1281, 1, 0, 0, 0, 277, 1285, 1, 0, 0, 0, 279, 1289, 1, 0, 0, 0, 281, 1292, 1, 0, 0, 0, 283, 1296, 1, 0, 0, 0, 285, 1300, 1, 0, 0, 0, 287, 1304, 1, 0, 0, 0, 289, 1308, 1, 0, 0, 0, 291, 1313, 1, 0, 0, 0, 293, 1318, 1, 0, 0, 0, 295, 1323, 1, 0, 0, 0, 297, 1330, 1, 0, 0, 0, 299, 1339, 1, 0, 0, 0, 301, 1346, 1, 0, 0, 0, 303, 1350, 1, 0, 0, 0, 305, 1354, 1, 0, 0, 0, 307, 1358, 1, 0, 0, 0, 309, 1362, 1, 0, 0, 0, 311, 1368, 1, 0, 0, 0, 313, 1372, 1, 0, 0, 0, 315, 1376, 1, 0, 0, 0, 317, 1380, 1, 0, 0, 0, 319, 1384, 1, 0, 0, 0, 321, 1388, 1, 0, 0, 0, 323, 1392, 1, 0, 0, 0, 325, 1396, 1, 0, 0, 0, 327, 1400, 1, 0, 0, 0, 329, 1404, 1, 0, 0, 0, 331, 1408, 1, 0, 0, 0, 333, 1412, 1, 0, 0, 0, 335, 1417, 1, 0, 0, 0, 337, 1421, 1, 0, 0, 0, 339, 1425, 1, 0, 0, 0, 341, 1429, 1, 0, 0, 0, 343, 1433, 1, 0, 0, 0, 345, 1437, 1, 0, 0, 0, 347, 1441, 1, 0, 0, 0, 349, 1445, 1, 0, 0, 0, 351, 1449, 1, 0, 0, 0, 353, 1454, 1, 0, 0, 0, 355, 1459, 1, 0, 0, 0, 357, 1463, 1, 0, 0, 0, 359, 1467, 1, 0, 0, 0, 361, 1471, 1, 0, 0, 0, 363, 1476, 1, 0, 0, 0, 365, 1485, 1, 0, 0, 0, 367, 1489, 1, 0, 0, 0, 369, 1493, 1, 0, 0, 0, 371, 1497, 1, 0, 0, 0, 373, 1501, 1, 0, 0, 0, 375, 1506, 1, 0, 0, 0, 377, 1510, 1, 0, 0, 0, 379, 1514, 1, 0, 0, 0, 381, 1518, 1, 0, 0, 0, 383, 1523, 1, 0, 0, 0, 385, 1527, 1, 0, 0, 0, 387, 1531, 1, 0, 0, 0, 389, 1535, 1, 0, 0, 0, 391, 1539, 1, 0, 0, 0, 393, 1543, 1, 0, 0, 0, 395, 1549, 1, 0, 0, 0, 397, 1553, 1, 0, 0, 0, 399, 1557, 1, 0, 0, 0, 401, 1561, 1, 0, 0, 0, 403, 1565, 1, 0, 0, 0, 405, 1569, 1, 0, 0, 0, 407, 1573, 1, 0, 0, 0, 409, 1578, 1, 0, 0, 0, 411, 1583, 1, 0, 0, 0, 413, 1587, 1, 0, 0, 0, 415, 1593, 1, 0, 0, 0, 417, 1602, 1, 0, 0, 0, 419, 1606, 1, 0, 0, 0, 421, 1610, 1, 0, 0, 0, 423, 1614, 1, 0, 0, 0, 425, 1618, 1, 0, 0, 0, 427, 1622, 1, 0, 0, 0, 429, 1626, 1, 0, 0, 0, 431, 1630, 1, 0, 0, 0, 433, 1634, 1, 0, 0, 0, 435, 1639, 1, 0, 0, 0, 437, 1645, 1, 0, 0, 0, 439, 1651, 1, 0, 0, 0, 441, 1655, 1, 0, 0, 0, 443, 1659, 1, 0, 0, 0, 445, 1663, 1, 0, 0, 0, 447, 1669, 1, 0, 0, 0, 449, 1675, 1, 0, 0, 0, 451, 1679, 1, 0, 0, 0, 453, 1683, 1, 0, 0, 0, 455, 1687, 1, 0, 0, 0, 457, 1693, 1, 0, 0, 0, 459, 1699, 1, 0, 0, 0, 461, 1705, 1, 0, 0, 0, 463, 1710, 1, 0, 0, 0, 465, 1715, 1, 0, 0, 0, 467, 1719, 1, 0, 0, 0, 469, 1723, 1, 0, 0, 0, 471, 1727, 1, 0, 0, 0, 473, 1731, 1, 0, 0, 0, 475, 1735, 1, 0, 0, 0, 477, 1739, 1, 0, 0, 0, 479, 1743, 1, 0, 0, 0, 481, 1747, 1, 0, 0, 0, 483, 1751, 1, 0, 0, 0, 485, 1756, 1, 0, 0, 0, 487, 1760, 1, 0, 0, 0, 489, 1764, 1, 0, 0, 0, 491, 1768, 1, 0, 0, 0, 493, 1772, 1, 0, 0, 0, 495, 1777, 1, 0, 0, 0, 497, 1782, 1, 0, 0, 0, 499, 1787, 1, 0, 0, 0, 501, 1791, 1, 0, 0, 0, 503, 1795, 1, 0, 0, 0, 505, 506, 7, 0, 0, 0, 506, 507, 7, 1, 0, 0, 507, 508, 7, 2, 0, 0, 508, 509, 7, 2, 0, 0, 509, 510, 7, 3, 0, 0, 510, 511, 7, 4, 0, 0, 511, 512, 7, 5, 0, 0, 512, 513, 1, 0, 0, 0, 513, 514, 6, 0, 0, 0, 514, 20, 1, 0, 0, 0, 515, 516, 7, 0, 0, 0, 516, 517, 7, 6, 0, 0, 517, 518, 7, 7, 0, 0, 518, 519, 7, 8, 0, 0, 519, 520, 1, 0, 0, 0, 520, 521, 6, 1, 1, 0, 521, 22, 1, 0, 0, 0, 522, 523, 7, 3, 0, 0, 523, 524, 7, 9, 0, 0, 524, 525, 7, 6, 0, 0, 525, 526, 7, 1, 0, 0, 526, 527, 7, 4, 0, 0, 527, 528, 7, 10, 0, 0, 528, 529, 1, 0, 0, 0, 529, 530, 6, 2, 2, 0, 530, 24, 1, 0, 0, 0, 531, 532, 7, 3, 0, 0, 532, 533, 7, 11, 0, 0, 533, 534, 7, 12, 0, 0, 534, 535, 7, 13, 0, 0, 535, 536, 1, 0, 0, 0, 536, 537, 6, 3, 0, 0, 537, 26, 1, 0, 0, 0, 538, 539, 7, 3, 0, 0, 539, 540, 7, 14, 0, 0, 540, 541, 7, 8, 0, 0, 541, 542, 7, 13, 0, 0, 542, 543, 7, 12, 0, 0, 543, 544, 7, 1, 0, 0, 544, 545, 7, 9, 0, 0, 545, 546, 1, 0, 0, 0, 546, 547, 6, 4, 3, 0, 547, 28, 1, 0, 0, 0, 548, 549, 7, 15, 0, 0, 549, 550, 7, 6, 0, 0, 550, 551, 7, 7, 0, 0, 551, 552, 7, 16, 0, 0, 552, 553, 1, 0, 0, 0, 553, 554, 6, 5, 4, 0, 554, 30, 1, 0, 0, 0, 555, 556, 7, 17, 0, 0, 556, 557, 7, 6, 0, 0, 557, 558, 7, 7, 0, 0, 558, 559, 7, 18, 0, 0, 559, 560, 1, 0, 0, 0, 560, 561, 6, 6, 0, 0, 561, 32, 1, 0, 0, 0, 562, 563, 7, 18, 0, 0, 563, 564, 7, 3, 0, 0, 564, 565, 7, 3, 0, 0, 565, 566, 7, 8, 0, 0, 566, 567, 1, 0, 0, 0, 567, 568, 6, 7, 1, 0, 568, 34, 1, 0, 0, 0, 569, 570, 7, 13, 0, 0, 570, 571, 7, 1, 0, 0, 571, 572, 7, 16, 0, 0, 572, 573, 7, 1, 0, 0, 573, 574, 7, 5, 0, 0, 574, 575, 1, 0, 0, 0, 575, 576, 6, 8, 0, 0, 576, 36, 1, 0, 0, 0, 577, 578, 7, 16, 0, 0, 578, 579, 7, 11, 0, 0, 579, 580, 5, 95, 0, 0, 580, 581, 7, 3, 0, 0, 581, 582, 7, 14, 0, 0, 582, 583, 7, 8, 0, 0, 583, 584, 7, 12, 0, 0, 584, 585, 7, 9, 0, 0, 585, 586, 7, 0, 0, 0, 586, 587, 1, 0, 0, 0, 587, 588, 6, 9, 5, 0, 588, 38, 1, 0, 0, 0, 589, 590, 7, 6, 0, 0, 590, 591, 7, 3, 0, 0, 591, 592, 7, 9, 0, 0, 592, 593, 7, 12, 0, 0, 593, 594, 7, 16, 0, 0, 594, 595, 7, 3, 0, 0, 595, 596, 1, 0, 0, 0, 596, 597, 6, 10, 6, 0, 597, 40, 1, 0, 0, 0, 598, 599, 7, 6, 0, 0, 599, 600, 7, 7, 0, 0, 600, 601, 7, 19, 0, 0, 601, 602, 1, 0, 0, 0, 602, 603, 6, 11, 0, 0, 603, 42, 1, 0, 0, 0, 604, 605, 7, 2, 0, 0, 605, 606, 7, 10, 0, 0, 606, 607, 7, 7, 0, 0, 607, 608, 7, 19, 0, 0, 608, 609, 1, 0, 0, 0, 609, 610, 6, 12, 7, 0, 610, 44, 1, 0, 0, 0, 611, 612, 7, 2, 0, 0, 612, 613, 7, 7, 0, 0, 613, 614, 7, 6, 0, 0, 614, 615, 7, 5, 0, 0, 615, 616, 1, 0, 0, 0, 616, 617, 6, 13, 0, 0, 617, 46, 1, 0, 0, 0, 618, 619, 7, 2, 0, 0, 619, 620, 7, 5, 0, 0, 620, 621, 7, 12, 0, 0, 621, 622, 7, 5, 0, 0, 622, 623, 7, 2, 0, 0, 623, 624, 1, 0, 0, 0, 624, 625, 6, 14, 0, 0, 625, 48, 1, 0, 0, 0, 626, 627, 7, 19, 0, 0, 627, 628, 7, 10, 0, 0, 628, 629, 7, 3, 0, 0, 629, 630, 7, 6, 0, 0, 630, 631, 7, 3, 0, 0, 631, 632, 1, 0, 0, 0, 632, 633, 6, 15, 0, 0, 633, 50, 1, 0, 0, 0, 634, 635, 7, 13, 0, 0, 635, 636, 7, 7, 0, 0, 636, 637, 7, 7, 0, 0, 637, 638, 7, 18, 0, 0, 638, 639, 7, 20, 0, 0, 639, 640, 7, 8, 0, 0, 640, 641, 1, 0, 0, 0, 641, 642, 6, 16, 8, 0, 642, 52, 1, 0, 0, 0, 643, 644, 4, 17, 0, 0, 644, 645, 7, 4, 0, 0, 645, 646, 7, 10, 0, 0, 646, 647, 7, 12, 0, 0, 647, 648, 7, 9, 0, 0, 648, 649, 7, 17, 0, 0, 649, 650, 7, 3, 0, 0, 650, 651, 5, 95, 0, 0, 651, 652, 7, 8, 0, 0, 652, 653, 7, 7, 0, 0, 653, 654, 7, 1, 0, 0, 654, 655, 7, 9, 0, 0, 655, 656, 7, 5, 0, 0, 656, 657, 1, 0, 0, 0, 657, 658, 6, 17, 9, 0, 658, 54, 1, 0, 0, 0, 659, 660, 4, 18, 1, 0, 660, 661, 7, 1, 0, 0, 661, 662, 7, 9, 0, 0, 662, 663, 7, 13, 0, 0, 663, 664, 7, 1, 0, 0, 664, 665, 7, 9, 0, 0, 665, 666, 7, 3, 0, 0, 666, 667, 7, 2, 0, 0, 667, 668, 7, 5, 0, 0, 668, 669, 7, 12, 0, 0, 669, 670, 7, 5, 0, 0, 670, 671, 7, 2, 0, 0, 671, 672, 1, 0, 0, 0, 672, 673, 6, 18, 0, 0, 673, 56, 1, 0, 0, 0, 674, 675, 4, 19, 2, 0, 675, 676, 7, 1, 0, 0, 676, 677, 7, 9, 0, 0, 677, 678, 7, 2, 0, 0, 678, 679, 7, 1, 0, 0, 679, 680, 7, 2, 0, 0, 680, 681, 7, 5, 0, 0, 681, 682, 5, 95, 0, 0, 682, 683, 5, 128020, 0, 0, 683, 684, 1, 0, 0, 0, 684, 685, 6, 19, 1, 0, 685, 58, 1, 0, 0, 0, 686, 687, 4, 20, 3, 0, 687, 688, 7, 13, 0, 0, 688, 689, 7, 7, 0, 0, 689, 690, 7, 7, 0, 0, 690, 691, 7, 18, 0, 0, 691, 692, 7, 20, 0, 0, 692, 693, 7, 8, 0, 0, 693, 694, 5, 95, 0, 0, 694, 695, 5, 128020, 0, 0, 695, 696, 1, 0, 0, 0, 696, 697, 6, 20, 10, 0, 697, 60, 1, 0, 0, 0, 698, 699, 4, 21, 4, 0, 699, 700, 7, 16, 0, 0, 700, 701, 7, 3, 0, 0, 701, 702, 7, 5, 0, 0, 702, 703, 7, 6, 0, 0, 703, 704, 7, 1, 0, 0, 704, 705, 7, 4, 0, 0, 705, 706, 7, 2, 0, 0, 706, 707, 1, 0, 0, 0, 707, 708, 6, 21, 11, 0, 708, 62, 1, 0, 0, 0, 709, 710, 4, 22, 5, 0, 710, 711, 7, 15, 0, 0, 711, 712, 7, 20, 0, 0, 712, 713, 7, 13, 0, 0, 713, 714, 7, 13, 0, 0, 714, 715, 1, 0, 0, 0, 715, 716, 6, 22, 8, 0, 716, 64, 1, 0, 0, 0, 717, 718, 4, 23, 6, 0, 718, 719, 7, 13, 0, 0, 719, 720, 7, 3, 0, 0, 720, 721, 7, 15, 0, 0, 721, 722, 7, 5, 0, 0, 722, 723, 1, 0, 0, 0, 723, 724, 6, 23, 8, 0, 724, 66, 1, 0, 0, 0, 725, 726, 4, 24, 7, 0, 726, 727, 7, 6, 0, 0, 727, 728, 7, 1, 0, 0, 728, 729, 7, 17, 0, 0, 729, 730, 7, 10, 0, 0, 730, 731, 7, 5, 0, 0, 731, 732, 1, 0, 0, 0, 732, 733, 6, 24, 8, 0, 733, 68, 1, 0, 0, 0, 734, 735, 4, 25, 8, 0, 735, 736, 7, 15, 0, 0, 736, 737, 7, 7, 0, 0, 737, 738, 7, 6, 0, 0, 738, 739, 7, 18, 0, 0, 739, 740, 1, 0, 0, 0, 740, 741, 6, 25, 12, 0, 741, 70, 1, 0, 0, 0, 742, 744, 8, 21, 0, 0, 743, 742, 1, 0, 0, 0, 744, 745, 1, 0, 0, 0, 745, 743, 1, 0, 0, 0, 745, 746, 1, 0, 0, 0, 746, 747, 1, 0, 0, 0, 747, 748, 6, 26, 0, 0, 748, 72, 1, 0, 0, 0, 749, 750, 5, 47, 0, 0, 750, 751, 5, 47, 0, 0, 751, 755, 1, 0, 0, 0, 752, 754, 8, 22, 0, 0, 753, 752, 1, 0, 0, 0, 754, 757, 1, 0, 0, 0, 755, 753, 1, 0, 0, 0, 755, 756, 1, 0, 0, 0, 756, 759, 1, 0, 0, 0, 757, 755, 1, 0, 0, 0, 758, 760, 5, 13, 0, 0, 759, 758, 1, 0, 0, 0, 759, 760, 1, 0, 0, 0, 760, 762, 1, 0, 0, 0, 761, 763, 5, 10, 0, 0, 762, 761, 1, 0, 0, 0, 762, 763, 1, 0, 0, 0, 763, 764, 1, 0, 0, 0, 764, 765, 6, 27, 13, 0, 765, 74, 1, 0, 0, 0, 766, 767, 5, 47, 0, 0, 767, 768, 5, 42, 0, 0, 768, 773, 1, 0, 0, 0, 769, 772, 3, 75, 28, 0, 770, 772, 9, 0, 0, 0, 771, 769, 1, 0, 0, 0, 771, 770, 1, 0, 0, 0, 772, 775, 1, 0, 0, 0, 773, 774, 1, 0, 0, 0, 773, 771, 1, 0, 0, 0, 774, 776, 1, 0, 0, 0, 775, 773, 1, 0, 0, 0, 776, 777, 5, 42, 0, 0, 777, 778, 5, 47, 0, 0, 778, 779, 1, 0, 0, 0, 779, 780, 6, 28, 13, 0, 780, 76, 1, 0, 0, 0, 781, 783, 7, 23, 0, 0, 782, 781, 1, 0, 0, 0, 783, 784, 1, 0, 0, 0, 784, 782, 1, 0, 0, 0, 784, 785, 1, 0, 0, 0, 785, 786, 1, 0, 0, 0, 786, 787, 6, 29, 13, 0, 787, 78, 1, 0, 0, 0, 788, 789, 5, 124, 0, 0, 789, 790, 1, 0, 0, 0, 790, 791, 6, 30, 14, 0, 791, 80, 1, 0, 0, 0, 792, 793, 7, 24, 0, 0, 793, 82, 1, 0, 0, 0, 794, 795, 7, 25, 0, 0, 795, 84, 1, 0, 0, 0, 796, 797, 5, 92, 0, 0, 797, 798, 7, 26, 0, 0, 798, 86, 1, 0, 0, 0, 799, 800, 8, 27, 0, 0, 800, 88, 1, 0, 0, 0, 801, 803, 7, 3, 0, 0, 802, 804, 7, 28, 0, 0, 803, 802, 1, 0, 0, 0, 803, 804, 1, 0, 0, 0, 804, 806, 1, 0, 0, 0, 805, 807, 3, 81, 31, 0, 806, 805, 1, 0, 0, 0, 807, 808, 1, 0, 0, 0, 808, 806, 1, 0, 0, 0, 808, 809, 1, 0, 0, 0, 809, 90, 1, 0, 0, 0, 810, 811, 5, 64, 0, 0, 811, 92, 1, 0, 0, 0, 812, 813, 5, 96, 0, 0, 813, 94, 1, 0, 0, 0, 814, 818, 8, 29, 0, 0, 815, 816, 5, 96, 0, 0, 816, 818, 5, 96, 0, 0, 817, 814, 1, 0, 0, 0, 817, 815, 1, 0, 0, 0, 818, 96, 1, 0, 0, 0, 819, 820, 5, 95, 0, 0, 820, 98, 1, 0, 0, 0, 821, 825, 3, 83, 32, 0, 822, 825, 3, 81, 31, 0, 823, 825, 3, 97, 39, 0, 824, 821, 1, 0, 0, 0, 824, 822, 1, 0, 0, 0, 824, 823, 1, 0, 0, 0, 825, 100, 1, 0, 0, 0, 826, 831, 5, 34, 0, 0, 827, 830, 3, 85, 33, 0, 828, 830, 3, 87, 34, 0, 829, 827, 1, 0, 0, 0, 829, 828, 1, 0, 0, 0, 830, 833, 1, 0, 0, 0, 831, 829, 1, 0, 0, 0, 831, 832, 1, 0, 0, 0, 832, 834, 1, 0, 0, 0, 833, 831, 1, 0, 0, 0, 834, 856, 5, 34, 0, 0, 835, 836, 5, 34, 0, 0, 836, 837, 5, 34, 0, 0, 837, 838, 5, 34, 0, 0, 838, 842, 1, 0, 0, 0, 839, 841, 8, 22, 0, 0, 840, 839, 1, 0, 0, 0, 841, 844, 1, 0, 0, 0, 842, 843, 1, 0, 0, 0, 842, 840, 1, 0, 0, 0, 843, 845, 1, 0, 0, 0, 844, 842, 1, 0, 0, 0, 845, 846, 5, 34, 0, 0, 846, 847, 5, 34, 0, 0, 847, 848, 5, 34, 0, 0, 848, 850, 1, 0, 0, 0, 849, 851, 5, 34, 0, 0, 850, 849, 1, 0, 0, 0, 850, 851, 1, 0, 0, 0, 851, 853, 1, 0, 0, 0, 852, 854, 5, 34, 0, 0, 853, 852, 1, 0, 0, 0, 853, 854, 1, 0, 0, 0, 854, 856, 1, 0, 0, 0, 855, 826, 1, 0, 0, 0, 855, 835, 1, 0, 0, 0, 856, 102, 1, 0, 0, 0, 857, 859, 3, 81, 31, 0, 858, 857, 1, 0, 0, 0, 859, 860, 1, 0, 0, 0, 860, 858, 1, 0, 0, 0, 860, 861, 1, 0, 0, 0, 861, 104, 1, 0, 0, 0, 862, 864, 3, 81, 31, 0, 863, 862, 1, 0, 0, 0, 864, 865, 1, 0, 0, 0, 865, 863, 1, 0, 0, 0, 865, 866, 1, 0, 0, 0, 866, 867, 1, 0, 0, 0, 867, 871, 3, 123, 52, 0, 868, 870, 3, 81, 31, 0, 869, 868, 1, 0, 0, 0, 870, 873, 1, 0, 0, 0, 871, 869, 1, 0, 0, 0, 871, 872, 1, 0, 0, 0, 872, 905, 1, 0, 0, 0, 873, 871, 1, 0, 0, 0, 874, 876, 3, 123, 52, 0, 875, 877, 3, 81, 31, 0, 876, 875, 1, 0, 0, 0, 877, 878, 1, 0, 0, 0, 878, 876, 1, 0, 0, 0, 878, 879, 1, 0, 0, 0, 879, 905, 1, 0, 0, 0, 880, 882, 3, 81, 31, 0, 881, 880, 1, 0, 0, 0, 882, 883, 1, 0, 0, 0, 883, 881, 1, 0, 0, 0, 883, 884, 1, 0, 0, 0, 884, 892, 1, 0, 0, 0, 885, 889, 3, 123, 52, 0, 886, 888, 3, 81, 31, 0, 887, 886, 1, 0, 0, 0, 888, 891, 1, 0, 0, 0, 889, 887, 1, 0, 0, 0, 889, 890, 1, 0, 0, 0, 890, 893, 1, 0, 0, 0, 891, 889, 1, 0, 0, 0, 892, 885, 1, 0, 0, 0, 892, 893, 1, 0, 0, 0, 893, 894, 1, 0, 0, 0, 894, 895, 3, 89, 35, 0, 895, 905, 1, 0, 0, 0, 896, 898, 3, 123, 52, 0, 897, 899, 3, 81, 31, 0, 898, 897, 1, 0, 0, 0, 899, 900, 1, 0, 0, 0, 900, 898, 1, 0, 0, 0, 900, 901, 1, 0, 0, 0, 901, 902, 1, 0, 0, 0, 902, 903, 3, 89, 35, 0, 903, 905, 1, 0, 0, 0, 904, 863, 1, 0, 0, 0, 904, 874, 1, 0, 0, 0, 904, 881, 1, 0, 0, 0, 904, 896, 1, 0, 0, 0, 905, 106, 1, 0, 0, 0, 906, 907, 7, 30, 0, 0, 907, 908, 7, 31, 0, 0, 908, 108, 1, 0, 0, 0, 909, 910, 7, 12, 0, 0, 910, 911, 7, 9, 0, 0, 911, 912, 7, 0, 0, 0, 912, 110, 1, 0, 0, 0, 913, 914, 7, 12, 0, 0, 914, 915, 7, 2, 0, 0, 915, 916, 7, 4, 0, 0, 916, 112, 1, 0, 0, 0, 917, 918, 5, 61, 0, 0, 918, 114, 1, 0, 0, 0, 919, 920, 5, 58, 0, 0, 920, 921, 5, 58, 0, 0, 921, 116, 1, 0, 0, 0, 922, 923, 5, 58, 0, 0, 923, 118, 1, 0, 0, 0, 924, 925, 5, 44, 0, 0, 925, 120, 1, 0, 0, 0, 926, 927, 7, 0, 0, 0, 927, 928, 7, 3, 0, 0, 928, 929, 7, 2, 0, 0, 929, 930, 7, 4, 0, 0, 930, 122, 1, 0, 0, 0, 931, 932, 5, 46, 0, 0, 932, 124, 1, 0, 0, 0, 933, 934, 7, 15, 0, 0, 934, 935, 7, 12, 0, 0, 935, 936, 7, 13, 0, 0, 936, 937, 7, 2, 0, 0, 937, 938, 7, 3, 0, 0, 938, 126, 1, 0, 0, 0, 939, 940, 7, 15, 0, 0, 940, 941, 7, 1, 0, 0, 941, 942, 7, 6, 0, 0, 942, 943, 7, 2, 0, 0, 943, 944, 7, 5, 0, 0, 944, 128, 1, 0, 0, 0, 945, 946, 7, 1, 0, 0, 946, 947, 7, 9, 0, 0, 947, 130, 1, 0, 0, 0, 948, 949, 7, 1, 0, 0, 949, 950, 7, 2, 0, 0, 950, 132, 1, 0, 0, 0, 951, 952, 7, 13, 0, 0, 952, 953, 7, 12, 0, 0, 953, 954, 7, 2, 0, 0, 954, 955, 7, 5, 0, 0, 955, 134, 1, 0, 0, 0, 956, 957, 7, 13, 0, 0, 957, 958, 7, 1, 0, 0, 958, 959, 7, 18, 0, 0, 959, 960, 7, 3, 0, 0, 960, 136, 1, 0, 0, 0, 961, 962, 5, 40, 0, 0, 962, 138, 1, 0, 0, 0, 963, 964, 7, 9, 0, 0, 964, 965, 7, 7, 0, 0, 965, 966, 7, 5, 0, 0, 966, 140, 1, 0, 0, 0, 967, 968, 7, 9, 0, 0, 968, 969, 7, 20, 0, 0, 969, 970, 7, 13, 0, 0, 970, 971, 7, 13, 0, 0, 971, 142, 1, 0, 0, 0, 972, 973, 7, 9, 0, 0, 973, 974, 7, 20, 0, 0, 974, 975, 7, 13, 0, 0, 975, 976, 7, 13, 0, 0, 976, 977, 7, 2, 0, 0, 977, 144, 1, 0, 0, 0, 978, 979, 7, 7, 0, 0, 979, 980, 7, 6, 0, 0, 980, 146, 1, 0, 0, 0, 981, 982, 5, 63, 0, 0, 982, 148, 1, 0, 0, 0, 983, 984, 7, 6, 0, 0, 984, 985, 7, 13, 0, 0, 985, 986, 7, 1, 0, 0, 986, 987, 7, 18, 0, 0, 987, 988, 7, 3, 0, 0, 988, 150, 1, 0, 0, 0, 989, 990, 5, 41, 0, 0, 990, 152, 1, 0, 0, 0, 991, 992, 7, 5, 0, 0, 992, 993, 7, 6, 0, 0, 993, 994, 7, 20, 0, 0, 994, 995, 7, 3, 0, 0, 995, 154, 1, 0, 0, 0, 996, 997, 5, 61, 0, 0, 997, 998, 5, 61, 0, 0, 998, 156, 1, 0, 0, 0, 999, 1000, 5, 61, 0, 0, 1000, 1001, 5, 126, 0, 0, 1001, 158, 1, 0, 0, 0, 1002, 1003, 5, 33, 0, 0, 1003, 1004, 5, 61, 0, 0, 1004, 160, 1, 0, 0, 0, 1005, 1006, 5, 60, 0, 0, 1006, 162, 1, 0, 0, 0, 1007, 1008, 5, 60, 0, 0, 1008, 1009, 5, 61, 0, 0, 1009, 164, 1, 0, 0, 0, 1010, 1011, 5, 62, 0, 0, 1011, 166, 1, 0, 0, 0, 1012, 1013, 5, 62, 0, 0, 1013, 1014, 5, 61, 0, 0, 1014, 168, 1, 0, 0, 0, 1015, 1016, 5, 43, 0, 0, 1016, 170, 1, 0, 0, 0, 1017, 1018, 5, 45, 0, 0, 1018, 172, 1, 0, 0, 0, 1019, 1020, 5, 42, 0, 0, 1020, 174, 1, 0, 0, 0, 1021, 1022, 5, 47, 0, 0, 1022, 176, 1, 0, 0, 0, 1023, 1024, 5, 37, 0, 0, 1024, 178, 1, 0, 0, 0, 1025, 1026, 5, 123, 0, 0, 1026, 180, 1, 0, 0, 0, 1027, 1028, 5, 125, 0, 0, 1028, 182, 1, 0, 0, 0, 1029, 1030, 3, 49, 15, 0, 1030, 1031, 1, 0, 0, 0, 1031, 1032, 6, 82, 15, 0, 1032, 184, 1, 0, 0, 0, 1033, 1034, 4, 83, 9, 0, 1034, 1035, 3, 45, 13, 0, 1035, 1036, 1, 0, 0, 0, 1036, 1037, 6, 83, 16, 0, 1037, 186, 1, 0, 0, 0, 1038, 1039, 4, 84, 10, 0, 1039, 1040, 3, 35, 8, 0, 1040, 1041, 1, 0, 0, 0, 1041, 1042, 6, 84, 17, 0, 1042, 188, 1, 0, 0, 0, 1043, 1046, 3, 147, 64, 0, 1044, 1047, 3, 83, 32, 0, 1045, 1047, 3, 97, 39, 0, 1046, 1044, 1, 0, 0, 0, 1046, 1045, 1, 0, 0, 0, 1047, 1051, 1, 0, 0, 0, 1048, 1050, 3, 99, 40, 0, 1049, 1048, 1, 0, 0, 0, 1050, 1053, 1, 0, 0, 0, 1051, 1049, 1, 0, 0, 0, 1051, 1052, 1, 0, 0, 0, 1052, 1061, 1, 0, 0, 0, 1053, 1051, 1, 0, 0, 0, 1054, 1056, 3, 147, 64, 0, 1055, 1057, 3, 81, 31, 0, 1056, 1055, 1, 0, 0, 0, 1057, 1058, 1, 0, 0, 0, 1058, 1056, 1, 0, 0, 0, 1058, 1059, 1, 0, 0, 0, 1059, 1061, 1, 0, 0, 0, 1060, 1043, 1, 0, 0, 0, 1060, 1054, 1, 0, 0, 0, 1061, 190, 1, 0, 0, 0, 1062, 1063, 5, 91, 0, 0, 1063, 1064, 1, 0, 0, 0, 1064, 1065, 6, 86, 0, 0, 1065, 1066, 6, 86, 0, 0, 1066, 192, 1, 0, 0, 0, 1067, 1068, 5, 93, 0, 0, 1068, 1069, 1, 0, 0, 0, 1069, 1070, 6, 87, 14, 0, 1070, 1071, 6, 87, 14, 0, 1071, 194, 1, 0, 0, 0, 1072, 1076, 3, 83, 32, 0, 1073, 1075, 3, 99, 40, 0, 1074, 1073, 1, 0, 0, 0, 1075, 1078, 1, 0, 0, 0, 1076, 1074, 1, 0, 0, 0, 1076, 1077, 1, 0, 0, 0, 1077, 1089, 1, 0, 0, 0, 1078, 1076, 1, 0, 0, 0, 1079, 1082, 3, 97, 39, 0, 1080, 1082, 3, 91, 36, 0, 1081, 1079, 1, 0, 0, 0, 1081, 1080, 1, 0, 0, 0, 1082, 1084, 1, 0, 0, 0, 1083, 1085, 3, 99, 40, 0, 1084, 1083, 1, 0, 0, 0, 1085, 1086, 1, 0, 0, 0, 1086, 1084, 1, 0, 0, 0, 1086, 1087, 1, 0, 0, 0, 1087, 1089, 1, 0, 0, 0, 1088, 1072, 1, 0, 0, 0, 1088, 1081, 1, 0, 0, 0, 1089, 196, 1, 0, 0, 0, 1090, 1092, 3, 93, 37, 0, 1091, 1093, 3, 95, 38, 0, 1092, 1091, 1, 0, 0, 0, 1093, 1094, 1, 0, 0, 0, 1094, 1092, 1, 0, 0, 0, 1094, 1095, 1, 0, 0, 0, 1095, 1096, 1, 0, 0, 0, 1096, 1097, 3, 93, 37, 0, 1097, 198, 1, 0, 0, 0, 1098, 1099, 3, 197, 89, 0, 1099, 200, 1, 0, 0, 0, 1100, 1101, 3, 73, 27, 0, 1101, 1102, 1, 0, 0, 0, 1102, 1103, 6, 91, 13, 0, 1103, 202, 1, 0, 0, 0, 1104, 1105, 3, 75, 28, 0, 1105, 1106, 1, 0, 0, 0, 1106, 1107, 6, 92, 13, 0, 1107, 204, 1, 0, 0, 0, 1108, 1109, 3, 77, 29, 0, 1109, 1110, 1, 0, 0, 0, 1110, 1111, 6, 93, 13, 0, 1111, 206, 1, 0, 0, 0, 1112, 1113, 3, 191, 86, 0, 1113, 1114, 1, 0, 0, 0, 1114, 1115, 6, 94, 18, 0, 1115, 1116, 6, 94, 19, 0, 1116, 208, 1, 0, 0, 0, 1117, 1118, 3, 79, 30, 0, 1118, 1119, 1, 0, 0, 0, 1119, 1120, 6, 95, 20, 0, 1120, 1121, 6, 95, 14, 0, 1121, 210, 1, 0, 0, 0, 1122, 1123, 3, 77, 29, 0, 1123, 1124, 1, 0, 0, 0, 1124, 1125, 6, 96, 13, 0, 1125, 212, 1, 0, 0, 0, 1126, 1127, 3, 73, 27, 0, 1127, 1128, 1, 0, 0, 0, 1128, 1129, 6, 97, 13, 0, 1129, 214, 1, 0, 0, 0, 1130, 1131, 3, 75, 28, 0, 1131, 1132, 1, 0, 0, 0, 1132, 1133, 6, 98, 13, 0, 1133, 216, 1, 0, 0, 0, 1134, 1135, 3, 79, 30, 0, 1135, 1136, 1, 0, 0, 0, 1136, 1137, 6, 99, 20, 0, 1137, 1138, 6, 99, 14, 0, 1138, 218, 1, 0, 0, 0, 1139, 1140, 3, 191, 86, 0, 1140, 1141, 1, 0, 0, 0, 1141, 1142, 6, 100, 18, 0, 1142, 220, 1, 0, 0, 0, 1143, 1144, 3, 193, 87, 0, 1144, 1145, 1, 0, 0, 0, 1145, 1146, 6, 101, 21, 0, 1146, 222, 1, 0, 0, 0, 1147, 1148, 3, 117, 49, 0, 1148, 1149, 1, 0, 0, 0, 1149, 1150, 6, 102, 22, 0, 1150, 224, 1, 0, 0, 0, 1151, 1152, 3, 119, 50, 0, 1152, 1153, 1, 0, 0, 0, 1153, 1154, 6, 103, 23, 0, 1154, 226, 1, 0, 0, 0, 1155, 1156, 3, 113, 47, 0, 1156, 1157, 1, 0, 0, 0, 1157, 1158, 6, 104, 24, 0, 1158, 228, 1, 0, 0, 0, 1159, 1160, 7, 16, 0, 0, 1160, 1161, 7, 3, 0, 0, 1161, 1162, 7, 5, 0, 0, 1162, 1163, 7, 12, 0, 0, 1163, 1164, 7, 0, 0, 0, 1164, 1165, 7, 12, 0, 0, 1165, 1166, 7, 5, 0, 0, 1166, 1167, 7, 12, 0, 0, 1167, 230, 1, 0, 0, 0, 1168, 1172, 8, 32, 0, 0, 1169, 1170, 5, 47, 0, 0, 1170, 1172, 8, 33, 0, 0, 1171, 1168, 1, 0, 0, 0, 1171, 1169, 1, 0, 0, 0, 1172, 232, 1, 0, 0, 0, 1173, 1175, 3, 231, 106, 0, 1174, 1173, 1, 0, 0, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1174, 1, 0, 0, 0, 1176, 1177, 1, 0, 0, 0, 1177, 234, 1, 0, 0, 0, 1178, 1179, 3, 233, 107, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 108, 25, 0, 1181, 236, 1, 0, 0, 0, 1182, 1183, 3, 101, 41, 0, 1183, 1184, 1, 0, 0, 0, 1184, 1185, 6, 109, 26, 0, 1185, 238, 1, 0, 0, 0, 1186, 1187, 3, 73, 27, 0, 1187, 1188, 1, 0, 0, 0, 1188, 1189, 6, 110, 13, 0, 1189, 240, 1, 0, 0, 0, 1190, 1191, 3, 75, 28, 0, 1191, 1192, 1, 0, 0, 0, 1192, 1193, 6, 111, 13, 0, 1193, 242, 1, 0, 0, 0, 1194, 1195, 3, 77, 29, 0, 1195, 1196, 1, 0, 0, 0, 1196, 1197, 6, 112, 13, 0, 1197, 244, 1, 0, 0, 0, 1198, 1199, 3, 79, 30, 0, 1199, 1200, 1, 0, 0, 0, 1200, 1201, 6, 113, 20, 0, 1201, 1202, 6, 113, 14, 0, 1202, 246, 1, 0, 0, 0, 1203, 1204, 3, 123, 52, 0, 1204, 1205, 1, 0, 0, 0, 1205, 1206, 6, 114, 27, 0, 1206, 248, 1, 0, 0, 0, 1207, 1208, 3, 119, 50, 0, 1208, 1209, 1, 0, 0, 0, 1209, 1210, 6, 115, 23, 0, 1210, 250, 1, 0, 0, 0, 1211, 1212, 3, 147, 64, 0, 1212, 1213, 1, 0, 0, 0, 1213, 1214, 6, 116, 28, 0, 1214, 252, 1, 0, 0, 0, 1215, 1216, 3, 189, 85, 0, 1216, 1217, 1, 0, 0, 0, 1217, 1218, 6, 117, 29, 0, 1218, 254, 1, 0, 0, 0, 1219, 1224, 3, 83, 32, 0, 1220, 1224, 3, 81, 31, 0, 1221, 1224, 3, 97, 39, 0, 1222, 1224, 3, 173, 77, 0, 1223, 1219, 1, 0, 0, 0, 1223, 1220, 1, 0, 0, 0, 1223, 1221, 1, 0, 0, 0, 1223, 1222, 1, 0, 0, 0, 1224, 256, 1, 0, 0, 0, 1225, 1228, 3, 83, 32, 0, 1226, 1228, 3, 173, 77, 0, 1227, 1225, 1, 0, 0, 0, 1227, 1226, 1, 0, 0, 0, 1228, 1232, 1, 0, 0, 0, 1229, 1231, 3, 255, 118, 0, 1230, 1229, 1, 0, 0, 0, 1231, 1234, 1, 0, 0, 0, 1232, 1230, 1, 0, 0, 0, 1232, 1233, 1, 0, 0, 0, 1233, 1245, 1, 0, 0, 0, 1234, 1232, 1, 0, 0, 0, 1235, 1238, 3, 97, 39, 0, 1236, 1238, 3, 91, 36, 0, 1237, 1235, 1, 0, 0, 0, 1237, 1236, 1, 0, 0, 0, 1238, 1240, 1, 0, 0, 0, 1239, 1241, 3, 255, 118, 0, 1240, 1239, 1, 0, 0, 0, 1241, 1242, 1, 0, 0, 0, 1242, 1240, 1, 0, 0, 0, 1242, 1243, 1, 0, 0, 0, 1243, 1245, 1, 0, 0, 0, 1244, 1227, 1, 0, 0, 0, 1244, 1237, 1, 0, 0, 0, 1245, 258, 1, 0, 0, 0, 1246, 1249, 3, 257, 119, 0, 1247, 1249, 3, 197, 89, 0, 1248, 1246, 1, 0, 0, 0, 1248, 1247, 1, 0, 0, 0, 1249, 1250, 1, 0, 0, 0, 1250, 1248, 1, 0, 0, 0, 1250, 1251, 1, 0, 0, 0, 1251, 260, 1, 0, 0, 0, 1252, 1253, 3, 73, 27, 0, 1253, 1254, 1, 0, 0, 0, 1254, 1255, 6, 121, 13, 0, 1255, 262, 1, 0, 0, 0, 1256, 1257, 3, 75, 28, 0, 1257, 1258, 1, 0, 0, 0, 1258, 1259, 6, 122, 13, 0, 1259, 264, 1, 0, 0, 0, 1260, 1261, 3, 77, 29, 0, 1261, 1262, 1, 0, 0, 0, 1262, 1263, 6, 123, 13, 0, 1263, 266, 1, 0, 0, 0, 1264, 1265, 3, 79, 30, 0, 1265, 1266, 1, 0, 0, 0, 1266, 1267, 6, 124, 20, 0, 1267, 1268, 6, 124, 14, 0, 1268, 268, 1, 0, 0, 0, 1269, 1270, 3, 113, 47, 0, 1270, 1271, 1, 0, 0, 0, 1271, 1272, 6, 125, 24, 0, 1272, 270, 1, 0, 0, 0, 1273, 1274, 3, 119, 50, 0, 1274, 1275, 1, 0, 0, 0, 1275, 1276, 6, 126, 23, 0, 1276, 272, 1, 0, 0, 0, 1277, 1278, 3, 123, 52, 0, 1278, 1279, 1, 0, 0, 0, 1279, 1280, 6, 127, 27, 0, 1280, 274, 1, 0, 0, 0, 1281, 1282, 3, 147, 64, 0, 1282, 1283, 1, 0, 0, 0, 1283, 1284, 6, 128, 28, 0, 1284, 276, 1, 0, 0, 0, 1285, 1286, 3, 189, 85, 0, 1286, 1287, 1, 0, 0, 0, 1287, 1288, 6, 129, 29, 0, 1288, 278, 1, 0, 0, 0, 1289, 1290, 7, 12, 0, 0, 1290, 1291, 7, 2, 0, 0, 1291, 280, 1, 0, 0, 0, 1292, 1293, 3, 259, 120, 0, 1293, 1294, 1, 0, 0, 0, 1294, 1295, 6, 131, 30, 0, 1295, 282, 1, 0, 0, 0, 1296, 1297, 3, 73, 27, 0, 1297, 1298, 1, 0, 0, 0, 1298, 1299, 6, 132, 13, 0, 1299, 284, 1, 0, 0, 0, 1300, 1301, 3, 75, 28, 0, 1301, 1302, 1, 0, 0, 0, 1302, 1303, 6, 133, 13, 0, 1303, 286, 1, 0, 0, 0, 1304, 1305, 3, 77, 29, 0, 1305, 1306, 1, 0, 0, 0, 1306, 1307, 6, 134, 13, 0, 1307, 288, 1, 0, 0, 0, 1308, 1309, 3, 79, 30, 0, 1309, 1310, 1, 0, 0, 0, 1310, 1311, 6, 135, 20, 0, 1311, 1312, 6, 135, 14, 0, 1312, 290, 1, 0, 0, 0, 1313, 1314, 3, 191, 86, 0, 1314, 1315, 1, 0, 0, 0, 1315, 1316, 6, 136, 18, 0, 1316, 1317, 6, 136, 31, 0, 1317, 292, 1, 0, 0, 0, 1318, 1319, 7, 7, 0, 0, 1319, 1320, 7, 9, 0, 0, 1320, 1321, 1, 0, 0, 0, 1321, 1322, 6, 137, 32, 0, 1322, 294, 1, 0, 0, 0, 1323, 1324, 7, 19, 0, 0, 1324, 1325, 7, 1, 0, 0, 1325, 1326, 7, 5, 0, 0, 1326, 1327, 7, 10, 0, 0, 1327, 1328, 1, 0, 0, 0, 1328, 1329, 6, 138, 32, 0, 1329, 296, 1, 0, 0, 0, 1330, 1331, 8, 34, 0, 0, 1331, 298, 1, 0, 0, 0, 1332, 1334, 3, 297, 139, 0, 1333, 1332, 1, 0, 0, 0, 1334, 1335, 1, 0, 0, 0, 1335, 1333, 1, 0, 0, 0, 1335, 1336, 1, 0, 0, 0, 1336, 1337, 1, 0, 0, 0, 1337, 1338, 3, 117, 49, 0, 1338, 1340, 1, 0, 0, 0, 1339, 1333, 1, 0, 0, 0, 1339, 1340, 1, 0, 0, 0, 1340, 1342, 1, 0, 0, 0, 1341, 1343, 3, 297, 139, 0, 1342, 1341, 1, 0, 0, 0, 1343, 1344, 1, 0, 0, 0, 1344, 1342, 1, 0, 0, 0, 1344, 1345, 1, 0, 0, 0, 1345, 300, 1, 0, 0, 0, 1346, 1347, 3, 299, 140, 0, 1347, 1348, 1, 0, 0, 0, 1348, 1349, 6, 141, 33, 0, 1349, 302, 1, 0, 0, 0, 1350, 1351, 3, 73, 27, 0, 1351, 1352, 1, 0, 0, 0, 1352, 1353, 6, 142, 13, 0, 1353, 304, 1, 0, 0, 0, 1354, 1355, 3, 75, 28, 0, 1355, 1356, 1, 0, 0, 0, 1356, 1357, 6, 143, 13, 0, 1357, 306, 1, 0, 0, 0, 1358, 1359, 3, 77, 29, 0, 1359, 1360, 1, 0, 0, 0, 1360, 1361, 6, 144, 13, 0, 1361, 308, 1, 0, 0, 0, 1362, 1363, 3, 79, 30, 0, 1363, 1364, 1, 0, 0, 0, 1364, 1365, 6, 145, 20, 0, 1365, 1366, 6, 145, 14, 0, 1366, 1367, 6, 145, 14, 0, 1367, 310, 1, 0, 0, 0, 1368, 1369, 3, 113, 47, 0, 1369, 1370, 1, 0, 0, 0, 1370, 1371, 6, 146, 24, 0, 1371, 312, 1, 0, 0, 0, 1372, 1373, 3, 119, 50, 0, 1373, 1374, 1, 0, 0, 0, 1374, 1375, 6, 147, 23, 0, 1375, 314, 1, 0, 0, 0, 1376, 1377, 3, 123, 52, 0, 1377, 1378, 1, 0, 0, 0, 1378, 1379, 6, 148, 27, 0, 1379, 316, 1, 0, 0, 0, 1380, 1381, 3, 295, 138, 0, 1381, 1382, 1, 0, 0, 0, 1382, 1383, 6, 149, 34, 0, 1383, 318, 1, 0, 0, 0, 1384, 1385, 3, 259, 120, 0, 1385, 1386, 1, 0, 0, 0, 1386, 1387, 6, 150, 30, 0, 1387, 320, 1, 0, 0, 0, 1388, 1389, 3, 199, 90, 0, 1389, 1390, 1, 0, 0, 0, 1390, 1391, 6, 151, 35, 0, 1391, 322, 1, 0, 0, 0, 1392, 1393, 3, 147, 64, 0, 1393, 1394, 1, 0, 0, 0, 1394, 1395, 6, 152, 28, 0, 1395, 324, 1, 0, 0, 0, 1396, 1397, 3, 189, 85, 0, 1397, 1398, 1, 0, 0, 0, 1398, 1399, 6, 153, 29, 0, 1399, 326, 1, 0, 0, 0, 1400, 1401, 3, 73, 27, 0, 1401, 1402, 1, 0, 0, 0, 1402, 1403, 6, 154, 13, 0, 1403, 328, 1, 0, 0, 0, 1404, 1405, 3, 75, 28, 0, 1405, 1406, 1, 0, 0, 0, 1406, 1407, 6, 155, 13, 0, 1407, 330, 1, 0, 0, 0, 1408, 1409, 3, 77, 29, 0, 1409, 1410, 1, 0, 0, 0, 1410, 1411, 6, 156, 13, 0, 1411, 332, 1, 0, 0, 0, 1412, 1413, 3, 79, 30, 0, 1413, 1414, 1, 0, 0, 0, 1414, 1415, 6, 157, 20, 0, 1415, 1416, 6, 157, 14, 0, 1416, 334, 1, 0, 0, 0, 1417, 1418, 3, 123, 52, 0, 1418, 1419, 1, 0, 0, 0, 1419, 1420, 6, 158, 27, 0, 1420, 336, 1, 0, 0, 0, 1421, 1422, 3, 147, 64, 0, 1422, 1423, 1, 0, 0, 0, 1423, 1424, 6, 159, 28, 0, 1424, 338, 1, 0, 0, 0, 1425, 1426, 3, 189, 85, 0, 1426, 1427, 1, 0, 0, 0, 1427, 1428, 6, 160, 29, 0, 1428, 340, 1, 0, 0, 0, 1429, 1430, 3, 199, 90, 0, 1430, 1431, 1, 0, 0, 0, 1431, 1432, 6, 161, 35, 0, 1432, 342, 1, 0, 0, 0, 1433, 1434, 3, 195, 88, 0, 1434, 1435, 1, 0, 0, 0, 1435, 1436, 6, 162, 36, 0, 1436, 344, 1, 0, 0, 0, 1437, 1438, 3, 73, 27, 0, 1438, 1439, 1, 0, 0, 0, 1439, 1440, 6, 163, 13, 0, 1440, 346, 1, 0, 0, 0, 1441, 1442, 3, 75, 28, 0, 1442, 1443, 1, 0, 0, 0, 1443, 1444, 6, 164, 13, 0, 1444, 348, 1, 0, 0, 0, 1445, 1446, 3, 77, 29, 0, 1446, 1447, 1, 0, 0, 0, 1447, 1448, 6, 165, 13, 0, 1448, 350, 1, 0, 0, 0, 1449, 1450, 3, 79, 30, 0, 1450, 1451, 1, 0, 0, 0, 1451, 1452, 6, 166, 20, 0, 1452, 1453, 6, 166, 14, 0, 1453, 352, 1, 0, 0, 0, 1454, 1455, 7, 1, 0, 0, 1455, 1456, 7, 9, 0, 0, 1456, 1457, 7, 15, 0, 0, 1457, 1458, 7, 7, 0, 0, 1458, 354, 1, 0, 0, 0, 1459, 1460, 3, 73, 27, 0, 1460, 1461, 1, 0, 0, 0, 1461, 1462, 6, 168, 13, 0, 1462, 356, 1, 0, 0, 0, 1463, 1464, 3, 75, 28, 0, 1464, 1465, 1, 0, 0, 0, 1465, 1466, 6, 169, 13, 0, 1466, 358, 1, 0, 0, 0, 1467, 1468, 3, 77, 29, 0, 1468, 1469, 1, 0, 0, 0, 1469, 1470, 6, 170, 13, 0, 1470, 360, 1, 0, 0, 0, 1471, 1472, 3, 193, 87, 0, 1472, 1473, 1, 0, 0, 0, 1473, 1474, 6, 171, 21, 0, 1474, 1475, 6, 171, 14, 0, 1475, 362, 1, 0, 0, 0, 1476, 1477, 3, 117, 49, 0, 1477, 1478, 1, 0, 0, 0, 1478, 1479, 6, 172, 22, 0, 1479, 364, 1, 0, 0, 0, 1480, 1486, 3, 91, 36, 0, 1481, 1486, 3, 81, 31, 0, 1482, 1486, 3, 123, 52, 0, 1483, 1486, 3, 83, 32, 0, 1484, 1486, 3, 97, 39, 0, 1485, 1480, 1, 0, 0, 0, 1485, 1481, 1, 0, 0, 0, 1485, 1482, 1, 0, 0, 0, 1485, 1483, 1, 0, 0, 0, 1485, 1484, 1, 0, 0, 0, 1486, 1487, 1, 0, 0, 0, 1487, 1485, 1, 0, 0, 0, 1487, 1488, 1, 0, 0, 0, 1488, 366, 1, 0, 0, 0, 1489, 1490, 3, 73, 27, 0, 1490, 1491, 1, 0, 0, 0, 1491, 1492, 6, 174, 13, 0, 1492, 368, 1, 0, 0, 0, 1493, 1494, 3, 75, 28, 0, 1494, 1495, 1, 0, 0, 0, 1495, 1496, 6, 175, 13, 0, 1496, 370, 1, 0, 0, 0, 1497, 1498, 3, 77, 29, 0, 1498, 1499, 1, 0, 0, 0, 1499, 1500, 6, 176, 13, 0, 1500, 372, 1, 0, 0, 0, 1501, 1502, 3, 79, 30, 0, 1502, 1503, 1, 0, 0, 0, 1503, 1504, 6, 177, 20, 0, 1504, 1505, 6, 177, 14, 0, 1505, 374, 1, 0, 0, 0, 1506, 1507, 3, 117, 49, 0, 1507, 1508, 1, 0, 0, 0, 1508, 1509, 6, 178, 22, 0, 1509, 376, 1, 0, 0, 0, 1510, 1511, 3, 119, 50, 0, 1511, 1512, 1, 0, 0, 0, 1512, 1513, 6, 179, 23, 0, 1513, 378, 1, 0, 0, 0, 1514, 1515, 3, 123, 52, 0, 1515, 1516, 1, 0, 0, 0, 1516, 1517, 6, 180, 27, 0, 1517, 380, 1, 0, 0, 0, 1518, 1519, 3, 293, 137, 0, 1519, 1520, 1, 0, 0, 0, 1520, 1521, 6, 181, 37, 0, 1521, 1522, 6, 181, 38, 0, 1522, 382, 1, 0, 0, 0, 1523, 1524, 3, 233, 107, 0, 1524, 1525, 1, 0, 0, 0, 1525, 1526, 6, 182, 25, 0, 1526, 384, 1, 0, 0, 0, 1527, 1528, 3, 101, 41, 0, 1528, 1529, 1, 0, 0, 0, 1529, 1530, 6, 183, 26, 0, 1530, 386, 1, 0, 0, 0, 1531, 1532, 3, 73, 27, 0, 1532, 1533, 1, 0, 0, 0, 1533, 1534, 6, 184, 13, 0, 1534, 388, 1, 0, 0, 0, 1535, 1536, 3, 75, 28, 0, 1536, 1537, 1, 0, 0, 0, 1537, 1538, 6, 185, 13, 0, 1538, 390, 1, 0, 0, 0, 1539, 1540, 3, 77, 29, 0, 1540, 1541, 1, 0, 0, 0, 1541, 1542, 6, 186, 13, 0, 1542, 392, 1, 0, 0, 0, 1543, 1544, 3, 79, 30, 0, 1544, 1545, 1, 0, 0, 0, 1545, 1546, 6, 187, 20, 0, 1546, 1547, 6, 187, 14, 0, 1547, 1548, 6, 187, 14, 0, 1548, 394, 1, 0, 0, 0, 1549, 1550, 3, 119, 50, 0, 1550, 1551, 1, 0, 0, 0, 1551, 1552, 6, 188, 23, 0, 1552, 396, 1, 0, 0, 0, 1553, 1554, 3, 123, 52, 0, 1554, 1555, 1, 0, 0, 0, 1555, 1556, 6, 189, 27, 0, 1556, 398, 1, 0, 0, 0, 1557, 1558, 3, 259, 120, 0, 1558, 1559, 1, 0, 0, 0, 1559, 1560, 6, 190, 30, 0, 1560, 400, 1, 0, 0, 0, 1561, 1562, 3, 73, 27, 0, 1562, 1563, 1, 0, 0, 0, 1563, 1564, 6, 191, 13, 0, 1564, 402, 1, 0, 0, 0, 1565, 1566, 3, 75, 28, 0, 1566, 1567, 1, 0, 0, 0, 1567, 1568, 6, 192, 13, 0, 1568, 404, 1, 0, 0, 0, 1569, 1570, 3, 77, 29, 0, 1570, 1571, 1, 0, 0, 0, 1571, 1572, 6, 193, 13, 0, 1572, 406, 1, 0, 0, 0, 1573, 1574, 3, 79, 30, 0, 1574, 1575, 1, 0, 0, 0, 1575, 1576, 6, 194, 20, 0, 1576, 1577, 6, 194, 14, 0, 1577, 408, 1, 0, 0, 0, 1578, 1579, 7, 35, 0, 0, 1579, 1580, 7, 7, 0, 0, 1580, 1581, 7, 1, 0, 0, 1581, 1582, 7, 9, 0, 0, 1582, 410, 1, 0, 0, 0, 1583, 1584, 3, 279, 130, 0, 1584, 1585, 1, 0, 0, 0, 1585, 1586, 6, 196, 39, 0, 1586, 412, 1, 0, 0, 0, 1587, 1588, 3, 293, 137, 0, 1588, 1589, 1, 0, 0, 0, 1589, 1590, 6, 197, 37, 0, 1590, 1591, 6, 197, 14, 0, 1591, 1592, 6, 197, 0, 0, 1592, 414, 1, 0, 0, 0, 1593, 1594, 7, 20, 0, 0, 1594, 1595, 7, 2, 0, 0, 1595, 1596, 7, 1, 0, 0, 1596, 1597, 7, 9, 0, 0, 1597, 1598, 7, 17, 0, 0, 1598, 1599, 1, 0, 0, 0, 1599, 1600, 6, 198, 14, 0, 1600, 1601, 6, 198, 0, 0, 1601, 416, 1, 0, 0, 0, 1602, 1603, 3, 233, 107, 0, 1603, 1604, 1, 0, 0, 0, 1604, 1605, 6, 199, 25, 0, 1605, 418, 1, 0, 0, 0, 1606, 1607, 3, 101, 41, 0, 1607, 1608, 1, 0, 0, 0, 1608, 1609, 6, 200, 26, 0, 1609, 420, 1, 0, 0, 0, 1610, 1611, 3, 117, 49, 0, 1611, 1612, 1, 0, 0, 0, 1612, 1613, 6, 201, 22, 0, 1613, 422, 1, 0, 0, 0, 1614, 1615, 3, 195, 88, 0, 1615, 1616, 1, 0, 0, 0, 1616, 1617, 6, 202, 36, 0, 1617, 424, 1, 0, 0, 0, 1618, 1619, 3, 199, 90, 0, 1619, 1620, 1, 0, 0, 0, 1620, 1621, 6, 203, 35, 0, 1621, 426, 1, 0, 0, 0, 1622, 1623, 3, 73, 27, 0, 1623, 1624, 1, 0, 0, 0, 1624, 1625, 6, 204, 13, 0, 1625, 428, 1, 0, 0, 0, 1626, 1627, 3, 75, 28, 0, 1627, 1628, 1, 0, 0, 0, 1628, 1629, 6, 205, 13, 0, 1629, 430, 1, 0, 0, 0, 1630, 1631, 3, 77, 29, 0, 1631, 1632, 1, 0, 0, 0, 1632, 1633, 6, 206, 13, 0, 1633, 432, 1, 0, 0, 0, 1634, 1635, 3, 79, 30, 0, 1635, 1636, 1, 0, 0, 0, 1636, 1637, 6, 207, 20, 0, 1637, 1638, 6, 207, 14, 0, 1638, 434, 1, 0, 0, 0, 1639, 1640, 3, 233, 107, 0, 1640, 1641, 1, 0, 0, 0, 1641, 1642, 6, 208, 25, 0, 1642, 1643, 6, 208, 14, 0, 1643, 1644, 6, 208, 40, 0, 1644, 436, 1, 0, 0, 0, 1645, 1646, 3, 101, 41, 0, 1646, 1647, 1, 0, 0, 0, 1647, 1648, 6, 209, 26, 0, 1648, 1649, 6, 209, 14, 0, 1649, 1650, 6, 209, 40, 0, 1650, 438, 1, 0, 0, 0, 1651, 1652, 3, 73, 27, 0, 1652, 1653, 1, 0, 0, 0, 1653, 1654, 6, 210, 13, 0, 1654, 440, 1, 0, 0, 0, 1655, 1656, 3, 75, 28, 0, 1656, 1657, 1, 0, 0, 0, 1657, 1658, 6, 211, 13, 0, 1658, 442, 1, 0, 0, 0, 1659, 1660, 3, 77, 29, 0, 1660, 1661, 1, 0, 0, 0, 1661, 1662, 6, 212, 13, 0, 1662, 444, 1, 0, 0, 0, 1663, 1664, 3, 117, 49, 0, 1664, 1665, 1, 0, 0, 0, 1665, 1666, 6, 213, 22, 0, 1666, 1667, 6, 213, 14, 0, 1667, 1668, 6, 213, 11, 0, 1668, 446, 1, 0, 0, 0, 1669, 1670, 3, 119, 50, 0, 1670, 1671, 1, 0, 0, 0, 1671, 1672, 6, 214, 23, 0, 1672, 1673, 6, 214, 14, 0, 1673, 1674, 6, 214, 11, 0, 1674, 448, 1, 0, 0, 0, 1675, 1676, 3, 73, 27, 0, 1676, 1677, 1, 0, 0, 0, 1677, 1678, 6, 215, 13, 0, 1678, 450, 1, 0, 0, 0, 1679, 1680, 3, 75, 28, 0, 1680, 1681, 1, 0, 0, 0, 1681, 1682, 6, 216, 13, 0, 1682, 452, 1, 0, 0, 0, 1683, 1684, 3, 77, 29, 0, 1684, 1685, 1, 0, 0, 0, 1685, 1686, 6, 217, 13, 0, 1686, 454, 1, 0, 0, 0, 1687, 1688, 3, 199, 90, 0, 1688, 1689, 1, 0, 0, 0, 1689, 1690, 6, 218, 14, 0, 1690, 1691, 6, 218, 0, 0, 1691, 1692, 6, 218, 35, 0, 1692, 456, 1, 0, 0, 0, 1693, 1694, 3, 195, 88, 0, 1694, 1695, 1, 0, 0, 0, 1695, 1696, 6, 219, 14, 0, 1696, 1697, 6, 219, 0, 0, 1697, 1698, 6, 219, 36, 0, 1698, 458, 1, 0, 0, 0, 1699, 1700, 3, 107, 44, 0, 1700, 1701, 1, 0, 0, 0, 1701, 1702, 6, 220, 14, 0, 1702, 1703, 6, 220, 0, 0, 1703, 1704, 6, 220, 41, 0, 1704, 460, 1, 0, 0, 0, 1705, 1706, 3, 79, 30, 0, 1706, 1707, 1, 0, 0, 0, 1707, 1708, 6, 221, 20, 0, 1708, 1709, 6, 221, 14, 0, 1709, 462, 1, 0, 0, 0, 1710, 1711, 3, 79, 30, 0, 1711, 1712, 1, 0, 0, 0, 1712, 1713, 6, 222, 20, 0, 1713, 1714, 6, 222, 14, 0, 1714, 464, 1, 0, 0, 0, 1715, 1716, 3, 293, 137, 0, 1716, 1717, 1, 0, 0, 0, 1717, 1718, 6, 223, 37, 0, 1718, 466, 1, 0, 0, 0, 1719, 1720, 3, 279, 130, 0, 1720, 1721, 1, 0, 0, 0, 1721, 1722, 6, 224, 39, 0, 1722, 468, 1, 0, 0, 0, 1723, 1724, 3, 123, 52, 0, 1724, 1725, 1, 0, 0, 0, 1725, 1726, 6, 225, 27, 0, 1726, 470, 1, 0, 0, 0, 1727, 1728, 3, 119, 50, 0, 1728, 1729, 1, 0, 0, 0, 1729, 1730, 6, 226, 23, 0, 1730, 472, 1, 0, 0, 0, 1731, 1732, 3, 199, 90, 0, 1732, 1733, 1, 0, 0, 0, 1733, 1734, 6, 227, 35, 0, 1734, 474, 1, 0, 0, 0, 1735, 1736, 3, 195, 88, 0, 1736, 1737, 1, 0, 0, 0, 1737, 1738, 6, 228, 36, 0, 1738, 476, 1, 0, 0, 0, 1739, 1740, 3, 73, 27, 0, 1740, 1741, 1, 0, 0, 0, 1741, 1742, 6, 229, 13, 0, 1742, 478, 1, 0, 0, 0, 1743, 1744, 3, 75, 28, 0, 1744, 1745, 1, 0, 0, 0, 1745, 1746, 6, 230, 13, 0, 1746, 480, 1, 0, 0, 0, 1747, 1748, 3, 77, 29, 0, 1748, 1749, 1, 0, 0, 0, 1749, 1750, 6, 231, 13, 0, 1750, 482, 1, 0, 0, 0, 1751, 1752, 3, 79, 30, 0, 1752, 1753, 1, 0, 0, 0, 1753, 1754, 6, 232, 20, 0, 1754, 1755, 6, 232, 14, 0, 1755, 484, 1, 0, 0, 0, 1756, 1757, 3, 195, 88, 0, 1757, 1758, 1, 0, 0, 0, 1758, 1759, 6, 233, 36, 0, 1759, 486, 1, 0, 0, 0, 1760, 1761, 3, 77, 29, 0, 1761, 1762, 1, 0, 0, 0, 1762, 1763, 6, 234, 13, 0, 1763, 488, 1, 0, 0, 0, 1764, 1765, 3, 73, 27, 0, 1765, 1766, 1, 0, 0, 0, 1766, 1767, 6, 235, 13, 0, 1767, 490, 1, 0, 0, 0, 1768, 1769, 3, 75, 28, 0, 1769, 1770, 1, 0, 0, 0, 1770, 1771, 6, 236, 13, 0, 1771, 492, 1, 0, 0, 0, 1772, 1773, 3, 137, 59, 0, 1773, 1774, 1, 0, 0, 0, 1774, 1775, 6, 237, 42, 0, 1775, 1776, 6, 237, 19, 0, 1776, 494, 1, 0, 0, 0, 1777, 1778, 3, 151, 66, 0, 1778, 1779, 1, 0, 0, 0, 1779, 1780, 6, 238, 43, 0, 1780, 1781, 6, 238, 14, 0, 1781, 496, 1, 0, 0, 0, 1782, 1783, 3, 79, 30, 0, 1783, 1784, 1, 0, 0, 0, 1784, 1785, 6, 239, 20, 0, 1785, 1786, 6, 239, 14, 0, 1786, 498, 1, 0, 0, 0, 1787, 1788, 3, 77, 29, 0, 1788, 1789, 1, 0, 0, 0, 1789, 1790, 6, 240, 13, 0, 1790, 500, 1, 0, 0, 0, 1791, 1792, 3, 73, 27, 0, 1792, 1793, 1, 0, 0, 0, 1793, 1794, 6, 241, 13, 0, 1794, 502, 1, 0, 0, 0, 1795, 1796, 3, 75, 28, 0, 1796, 1797, 1, 0, 0, 0, 1797, 1798, 6, 242, 13, 0, 1798, 504, 1, 0, 0, 0, 69, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 745, 755, 759, 762, 771, 773, 784, 803, 808, 817, 824, 829, 831, 842, 850, 853, 855, 860, 865, 871, 878, 883, 889, 892, 900, 904, 1046, 1051, 1058, 1060, 1076, 1081, 1086, 1088, 1094, 1171, 1176, 1223, 1227, 1232, 1237, 1242, 1244, 1248, 1250, 1335, 1339, 1344, 1485, 1487, 44, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 13, 0, 5, 16, 0, 5, 11, 0, 5, 14, 0, 5, 18, 0, 0, 1, 0, 4, 0, 0, 7, 16, 0, 7, 14, 0, 7, 9, 0, 7, 74, 0, 5, 0, 0, 7, 31, 0, 7, 75, 0, 7, 40, 0, 7, 41, 0, 7, 38, 0, 7, 85, 0, 7, 32, 0, 7, 43, 0, 7, 55, 0, 7, 73, 0, 7, 89, 0, 5, 10, 0, 5, 7, 0, 7, 99, 0, 7, 98, 0, 7, 77, 0, 7, 76, 0, 7, 97, 0, 5, 12, 0, 7, 93, 0, 5, 15, 0, 7, 35, 0, 7, 50, 0, 7, 57, 0] \ No newline at end of file +[4, 0, 142, 1784, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 2, 196, 7, 196, 2, 197, 7, 197, 2, 198, 7, 198, 2, 199, 7, 199, 2, 200, 7, 200, 2, 201, 7, 201, 2, 202, 7, 202, 2, 203, 7, 203, 2, 204, 7, 204, 2, 205, 7, 205, 2, 206, 7, 206, 2, 207, 7, 207, 2, 208, 7, 208, 2, 209, 7, 209, 2, 210, 7, 210, 2, 211, 7, 211, 2, 212, 7, 212, 2, 213, 7, 213, 2, 214, 7, 214, 2, 215, 7, 215, 2, 216, 7, 216, 2, 217, 7, 217, 2, 218, 7, 218, 2, 219, 7, 219, 2, 220, 7, 220, 2, 221, 7, 221, 2, 222, 7, 222, 2, 223, 7, 223, 2, 224, 7, 224, 2, 225, 7, 225, 2, 226, 7, 226, 2, 227, 7, 227, 2, 228, 7, 228, 2, 229, 7, 229, 2, 230, 7, 230, 2, 231, 7, 231, 2, 232, 7, 232, 2, 233, 7, 233, 2, 234, 7, 234, 2, 235, 7, 235, 2, 236, 7, 236, 2, 237, 7, 237, 2, 238, 7, 238, 2, 239, 7, 239, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 4, 26, 738, 8, 26, 11, 26, 12, 26, 739, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 5, 27, 748, 8, 27, 10, 27, 12, 27, 751, 9, 27, 1, 27, 3, 27, 754, 8, 27, 1, 27, 3, 27, 757, 8, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 5, 28, 766, 8, 28, 10, 28, 12, 28, 769, 9, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 28, 1, 29, 4, 29, 777, 8, 29, 11, 29, 12, 29, 778, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 35, 1, 35, 3, 35, 798, 8, 35, 1, 35, 4, 35, 801, 8, 35, 11, 35, 12, 35, 802, 1, 36, 1, 36, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 3, 38, 812, 8, 38, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 3, 40, 819, 8, 40, 1, 41, 1, 41, 1, 41, 5, 41, 824, 8, 41, 10, 41, 12, 41, 827, 9, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 5, 41, 835, 8, 41, 10, 41, 12, 41, 838, 9, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 3, 41, 845, 8, 41, 1, 41, 3, 41, 848, 8, 41, 3, 41, 850, 8, 41, 1, 42, 4, 42, 853, 8, 42, 11, 42, 12, 42, 854, 1, 43, 4, 43, 858, 8, 43, 11, 43, 12, 43, 859, 1, 43, 1, 43, 5, 43, 864, 8, 43, 10, 43, 12, 43, 867, 9, 43, 1, 43, 1, 43, 4, 43, 871, 8, 43, 11, 43, 12, 43, 872, 1, 43, 4, 43, 876, 8, 43, 11, 43, 12, 43, 877, 1, 43, 1, 43, 5, 43, 882, 8, 43, 10, 43, 12, 43, 885, 9, 43, 3, 43, 887, 8, 43, 1, 43, 1, 43, 1, 43, 1, 43, 4, 43, 893, 8, 43, 11, 43, 12, 43, 894, 1, 43, 1, 43, 3, 43, 899, 8, 43, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 75, 1, 75, 1, 76, 1, 76, 1, 77, 1, 77, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 3, 81, 1027, 8, 81, 1, 81, 5, 81, 1030, 8, 81, 10, 81, 12, 81, 1033, 9, 81, 1, 81, 1, 81, 4, 81, 1037, 8, 81, 11, 81, 12, 81, 1038, 3, 81, 1041, 8, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 5, 86, 1065, 8, 86, 10, 86, 12, 86, 1068, 9, 86, 1, 86, 1, 86, 3, 86, 1072, 8, 86, 1, 86, 4, 86, 1075, 8, 86, 11, 86, 12, 86, 1076, 3, 86, 1079, 8, 86, 1, 87, 1, 87, 4, 87, 1083, 8, 87, 11, 87, 12, 87, 1084, 1, 87, 1, 87, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 1, 96, 1, 97, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 103, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 3, 104, 1162, 8, 104, 1, 105, 4, 105, 1165, 8, 105, 11, 105, 12, 105, 1166, 1, 106, 1, 106, 1, 106, 1, 106, 1, 107, 1, 107, 1, 107, 1, 107, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 3, 116, 1214, 8, 116, 1, 117, 1, 117, 3, 117, 1218, 8, 117, 1, 117, 5, 117, 1221, 8, 117, 10, 117, 12, 117, 1224, 9, 117, 1, 117, 1, 117, 3, 117, 1228, 8, 117, 1, 117, 4, 117, 1231, 8, 117, 11, 117, 12, 117, 1232, 3, 117, 1235, 8, 117, 1, 118, 1, 118, 4, 118, 1239, 8, 118, 11, 118, 12, 118, 1240, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 125, 1, 125, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 138, 4, 138, 1324, 8, 138, 11, 138, 12, 138, 1325, 1, 138, 1, 138, 3, 138, 1330, 8, 138, 1, 138, 4, 138, 1333, 8, 138, 11, 138, 12, 138, 1334, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 171, 4, 171, 1476, 8, 171, 11, 171, 12, 171, 1477, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 196, 1, 197, 1, 197, 1, 197, 1, 197, 1, 198, 1, 198, 1, 198, 1, 198, 1, 199, 1, 199, 1, 199, 1, 199, 1, 200, 1, 200, 1, 200, 1, 200, 1, 201, 1, 201, 1, 201, 1, 201, 1, 202, 1, 202, 1, 202, 1, 202, 1, 203, 1, 203, 1, 203, 1, 203, 1, 204, 1, 204, 1, 204, 1, 204, 1, 205, 1, 205, 1, 205, 1, 205, 1, 205, 1, 206, 1, 206, 1, 206, 1, 206, 1, 206, 1, 206, 1, 207, 1, 207, 1, 207, 1, 207, 1, 207, 1, 207, 1, 208, 1, 208, 1, 208, 1, 208, 1, 209, 1, 209, 1, 209, 1, 209, 1, 210, 1, 210, 1, 210, 1, 210, 1, 211, 1, 211, 1, 211, 1, 211, 1, 211, 1, 211, 1, 212, 1, 212, 1, 212, 1, 212, 1, 212, 1, 212, 1, 213, 1, 213, 1, 213, 1, 213, 1, 214, 1, 214, 1, 214, 1, 214, 1, 215, 1, 215, 1, 215, 1, 215, 1, 216, 1, 216, 1, 216, 1, 216, 1, 216, 1, 216, 1, 217, 1, 217, 1, 217, 1, 217, 1, 217, 1, 217, 1, 218, 1, 218, 1, 218, 1, 218, 1, 218, 1, 218, 1, 219, 1, 219, 1, 219, 1, 219, 1, 219, 1, 220, 1, 220, 1, 220, 1, 220, 1, 220, 1, 221, 1, 221, 1, 221, 1, 221, 1, 222, 1, 222, 1, 222, 1, 222, 1, 223, 1, 223, 1, 223, 1, 223, 1, 224, 1, 224, 1, 224, 1, 224, 1, 225, 1, 225, 1, 225, 1, 225, 1, 226, 1, 226, 1, 226, 1, 226, 1, 227, 1, 227, 1, 227, 1, 227, 1, 228, 1, 228, 1, 228, 1, 228, 1, 229, 1, 229, 1, 229, 1, 229, 1, 230, 1, 230, 1, 230, 1, 230, 1, 230, 1, 231, 1, 231, 1, 231, 1, 231, 1, 232, 1, 232, 1, 232, 1, 232, 1, 233, 1, 233, 1, 233, 1, 233, 1, 234, 1, 234, 1, 234, 1, 234, 1, 235, 1, 235, 1, 235, 1, 235, 1, 235, 1, 236, 1, 236, 1, 236, 1, 236, 1, 236, 1, 237, 1, 237, 1, 237, 1, 237, 1, 238, 1, 238, 1, 238, 1, 238, 1, 239, 1, 239, 1, 239, 1, 239, 2, 767, 836, 0, 240, 19, 1, 21, 2, 23, 3, 25, 4, 27, 5, 29, 6, 31, 7, 33, 8, 35, 9, 37, 10, 39, 11, 41, 12, 43, 13, 45, 14, 47, 15, 49, 16, 51, 17, 53, 18, 55, 19, 57, 20, 59, 21, 61, 22, 63, 23, 65, 24, 67, 25, 69, 26, 71, 27, 73, 28, 75, 29, 77, 30, 79, 31, 81, 0, 83, 0, 85, 0, 87, 0, 89, 0, 91, 0, 93, 0, 95, 0, 97, 0, 99, 0, 101, 32, 103, 33, 105, 34, 107, 35, 109, 36, 111, 37, 113, 38, 115, 39, 117, 40, 119, 41, 121, 42, 123, 43, 125, 44, 127, 45, 129, 46, 131, 47, 133, 48, 135, 49, 137, 50, 139, 51, 141, 52, 143, 53, 145, 54, 147, 55, 149, 56, 151, 57, 153, 58, 155, 59, 157, 60, 159, 61, 161, 62, 163, 63, 165, 64, 167, 65, 169, 66, 171, 67, 173, 68, 175, 69, 177, 70, 179, 0, 181, 71, 183, 72, 185, 73, 187, 74, 189, 75, 191, 76, 193, 0, 195, 77, 197, 78, 199, 79, 201, 80, 203, 0, 205, 0, 207, 81, 209, 82, 211, 83, 213, 0, 215, 0, 217, 0, 219, 0, 221, 0, 223, 0, 225, 84, 227, 0, 229, 85, 231, 0, 233, 0, 235, 86, 237, 87, 239, 88, 241, 0, 243, 0, 245, 0, 247, 0, 249, 0, 251, 0, 253, 0, 255, 89, 257, 90, 259, 91, 261, 92, 263, 0, 265, 0, 267, 0, 269, 0, 271, 0, 273, 0, 275, 93, 277, 0, 279, 94, 281, 95, 283, 96, 285, 0, 287, 0, 289, 97, 291, 98, 293, 0, 295, 99, 297, 0, 299, 100, 301, 101, 303, 102, 305, 0, 307, 0, 309, 0, 311, 0, 313, 0, 315, 0, 317, 0, 319, 0, 321, 0, 323, 103, 325, 104, 327, 105, 329, 0, 331, 0, 333, 0, 335, 0, 337, 0, 339, 0, 341, 106, 343, 107, 345, 108, 347, 0, 349, 109, 351, 110, 353, 111, 355, 112, 357, 0, 359, 0, 361, 113, 363, 114, 365, 115, 367, 116, 369, 0, 371, 0, 373, 0, 375, 0, 377, 0, 379, 0, 381, 0, 383, 117, 385, 118, 387, 119, 389, 0, 391, 0, 393, 0, 395, 0, 397, 120, 399, 121, 401, 122, 403, 0, 405, 123, 407, 0, 409, 0, 411, 124, 413, 0, 415, 0, 417, 0, 419, 0, 421, 0, 423, 125, 425, 126, 427, 127, 429, 0, 431, 0, 433, 0, 435, 128, 437, 129, 439, 130, 441, 0, 443, 0, 445, 131, 447, 132, 449, 133, 451, 0, 453, 0, 455, 0, 457, 0, 459, 0, 461, 0, 463, 0, 465, 0, 467, 0, 469, 0, 471, 0, 473, 134, 475, 135, 477, 136, 479, 0, 481, 0, 483, 137, 485, 138, 487, 139, 489, 0, 491, 0, 493, 140, 495, 141, 497, 142, 19, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 36, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 2, 0, 74, 74, 106, 106, 1808, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 0, 63, 1, 0, 0, 0, 0, 65, 1, 0, 0, 0, 0, 67, 1, 0, 0, 0, 0, 69, 1, 0, 0, 0, 0, 71, 1, 0, 0, 0, 0, 73, 1, 0, 0, 0, 0, 75, 1, 0, 0, 0, 0, 77, 1, 0, 0, 0, 1, 79, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 171, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 1, 181, 1, 0, 0, 0, 1, 183, 1, 0, 0, 0, 1, 185, 1, 0, 0, 0, 1, 187, 1, 0, 0, 0, 1, 189, 1, 0, 0, 0, 1, 191, 1, 0, 0, 0, 1, 195, 1, 0, 0, 0, 1, 197, 1, 0, 0, 0, 1, 199, 1, 0, 0, 0, 1, 201, 1, 0, 0, 0, 2, 203, 1, 0, 0, 0, 2, 205, 1, 0, 0, 0, 2, 207, 1, 0, 0, 0, 2, 209, 1, 0, 0, 0, 2, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 3, 219, 1, 0, 0, 0, 3, 221, 1, 0, 0, 0, 3, 223, 1, 0, 0, 0, 3, 225, 1, 0, 0, 0, 3, 229, 1, 0, 0, 0, 3, 231, 1, 0, 0, 0, 3, 233, 1, 0, 0, 0, 3, 235, 1, 0, 0, 0, 3, 237, 1, 0, 0, 0, 3, 239, 1, 0, 0, 0, 4, 241, 1, 0, 0, 0, 4, 243, 1, 0, 0, 0, 4, 245, 1, 0, 0, 0, 4, 247, 1, 0, 0, 0, 4, 249, 1, 0, 0, 0, 4, 255, 1, 0, 0, 0, 4, 257, 1, 0, 0, 0, 4, 259, 1, 0, 0, 0, 4, 261, 1, 0, 0, 0, 5, 263, 1, 0, 0, 0, 5, 265, 1, 0, 0, 0, 5, 267, 1, 0, 0, 0, 5, 269, 1, 0, 0, 0, 5, 271, 1, 0, 0, 0, 5, 273, 1, 0, 0, 0, 5, 275, 1, 0, 0, 0, 5, 277, 1, 0, 0, 0, 5, 279, 1, 0, 0, 0, 5, 281, 1, 0, 0, 0, 5, 283, 1, 0, 0, 0, 6, 285, 1, 0, 0, 0, 6, 287, 1, 0, 0, 0, 6, 289, 1, 0, 0, 0, 6, 291, 1, 0, 0, 0, 6, 295, 1, 0, 0, 0, 6, 297, 1, 0, 0, 0, 6, 299, 1, 0, 0, 0, 6, 301, 1, 0, 0, 0, 6, 303, 1, 0, 0, 0, 7, 305, 1, 0, 0, 0, 7, 307, 1, 0, 0, 0, 7, 309, 1, 0, 0, 0, 7, 311, 1, 0, 0, 0, 7, 313, 1, 0, 0, 0, 7, 315, 1, 0, 0, 0, 7, 317, 1, 0, 0, 0, 7, 319, 1, 0, 0, 0, 7, 321, 1, 0, 0, 0, 7, 323, 1, 0, 0, 0, 7, 325, 1, 0, 0, 0, 7, 327, 1, 0, 0, 0, 8, 329, 1, 0, 0, 0, 8, 331, 1, 0, 0, 0, 8, 333, 1, 0, 0, 0, 8, 335, 1, 0, 0, 0, 8, 337, 1, 0, 0, 0, 8, 339, 1, 0, 0, 0, 8, 341, 1, 0, 0, 0, 8, 343, 1, 0, 0, 0, 8, 345, 1, 0, 0, 0, 9, 347, 1, 0, 0, 0, 9, 349, 1, 0, 0, 0, 9, 351, 1, 0, 0, 0, 9, 353, 1, 0, 0, 0, 9, 355, 1, 0, 0, 0, 10, 357, 1, 0, 0, 0, 10, 359, 1, 0, 0, 0, 10, 361, 1, 0, 0, 0, 10, 363, 1, 0, 0, 0, 10, 365, 1, 0, 0, 0, 10, 367, 1, 0, 0, 0, 11, 369, 1, 0, 0, 0, 11, 371, 1, 0, 0, 0, 11, 373, 1, 0, 0, 0, 11, 375, 1, 0, 0, 0, 11, 377, 1, 0, 0, 0, 11, 379, 1, 0, 0, 0, 11, 381, 1, 0, 0, 0, 11, 383, 1, 0, 0, 0, 11, 385, 1, 0, 0, 0, 11, 387, 1, 0, 0, 0, 12, 389, 1, 0, 0, 0, 12, 391, 1, 0, 0, 0, 12, 393, 1, 0, 0, 0, 12, 395, 1, 0, 0, 0, 12, 397, 1, 0, 0, 0, 12, 399, 1, 0, 0, 0, 12, 401, 1, 0, 0, 0, 13, 403, 1, 0, 0, 0, 13, 405, 1, 0, 0, 0, 13, 407, 1, 0, 0, 0, 13, 409, 1, 0, 0, 0, 13, 411, 1, 0, 0, 0, 13, 413, 1, 0, 0, 0, 13, 415, 1, 0, 0, 0, 13, 417, 1, 0, 0, 0, 13, 419, 1, 0, 0, 0, 13, 421, 1, 0, 0, 0, 13, 423, 1, 0, 0, 0, 13, 425, 1, 0, 0, 0, 13, 427, 1, 0, 0, 0, 14, 429, 1, 0, 0, 0, 14, 431, 1, 0, 0, 0, 14, 433, 1, 0, 0, 0, 14, 435, 1, 0, 0, 0, 14, 437, 1, 0, 0, 0, 14, 439, 1, 0, 0, 0, 15, 441, 1, 0, 0, 0, 15, 443, 1, 0, 0, 0, 15, 445, 1, 0, 0, 0, 15, 447, 1, 0, 0, 0, 15, 449, 1, 0, 0, 0, 15, 451, 1, 0, 0, 0, 15, 453, 1, 0, 0, 0, 15, 455, 1, 0, 0, 0, 15, 457, 1, 0, 0, 0, 16, 459, 1, 0, 0, 0, 16, 461, 1, 0, 0, 0, 16, 463, 1, 0, 0, 0, 16, 465, 1, 0, 0, 0, 16, 467, 1, 0, 0, 0, 16, 469, 1, 0, 0, 0, 16, 471, 1, 0, 0, 0, 16, 473, 1, 0, 0, 0, 16, 475, 1, 0, 0, 0, 16, 477, 1, 0, 0, 0, 17, 479, 1, 0, 0, 0, 17, 481, 1, 0, 0, 0, 17, 483, 1, 0, 0, 0, 17, 485, 1, 0, 0, 0, 17, 487, 1, 0, 0, 0, 18, 489, 1, 0, 0, 0, 18, 491, 1, 0, 0, 0, 18, 493, 1, 0, 0, 0, 18, 495, 1, 0, 0, 0, 18, 497, 1, 0, 0, 0, 19, 499, 1, 0, 0, 0, 21, 509, 1, 0, 0, 0, 23, 516, 1, 0, 0, 0, 25, 525, 1, 0, 0, 0, 27, 532, 1, 0, 0, 0, 29, 542, 1, 0, 0, 0, 31, 549, 1, 0, 0, 0, 33, 556, 1, 0, 0, 0, 35, 563, 1, 0, 0, 0, 37, 571, 1, 0, 0, 0, 39, 583, 1, 0, 0, 0, 41, 592, 1, 0, 0, 0, 43, 598, 1, 0, 0, 0, 45, 605, 1, 0, 0, 0, 47, 612, 1, 0, 0, 0, 49, 620, 1, 0, 0, 0, 51, 628, 1, 0, 0, 0, 53, 637, 1, 0, 0, 0, 55, 653, 1, 0, 0, 0, 57, 668, 1, 0, 0, 0, 59, 680, 1, 0, 0, 0, 61, 692, 1, 0, 0, 0, 63, 703, 1, 0, 0, 0, 65, 711, 1, 0, 0, 0, 67, 719, 1, 0, 0, 0, 69, 728, 1, 0, 0, 0, 71, 737, 1, 0, 0, 0, 73, 743, 1, 0, 0, 0, 75, 760, 1, 0, 0, 0, 77, 776, 1, 0, 0, 0, 79, 782, 1, 0, 0, 0, 81, 786, 1, 0, 0, 0, 83, 788, 1, 0, 0, 0, 85, 790, 1, 0, 0, 0, 87, 793, 1, 0, 0, 0, 89, 795, 1, 0, 0, 0, 91, 804, 1, 0, 0, 0, 93, 806, 1, 0, 0, 0, 95, 811, 1, 0, 0, 0, 97, 813, 1, 0, 0, 0, 99, 818, 1, 0, 0, 0, 101, 849, 1, 0, 0, 0, 103, 852, 1, 0, 0, 0, 105, 898, 1, 0, 0, 0, 107, 900, 1, 0, 0, 0, 109, 903, 1, 0, 0, 0, 111, 907, 1, 0, 0, 0, 113, 911, 1, 0, 0, 0, 115, 913, 1, 0, 0, 0, 117, 916, 1, 0, 0, 0, 119, 918, 1, 0, 0, 0, 121, 920, 1, 0, 0, 0, 123, 925, 1, 0, 0, 0, 125, 927, 1, 0, 0, 0, 127, 933, 1, 0, 0, 0, 129, 939, 1, 0, 0, 0, 131, 942, 1, 0, 0, 0, 133, 945, 1, 0, 0, 0, 135, 950, 1, 0, 0, 0, 137, 955, 1, 0, 0, 0, 139, 959, 1, 0, 0, 0, 141, 964, 1, 0, 0, 0, 143, 970, 1, 0, 0, 0, 145, 973, 1, 0, 0, 0, 147, 975, 1, 0, 0, 0, 149, 981, 1, 0, 0, 0, 151, 986, 1, 0, 0, 0, 153, 989, 1, 0, 0, 0, 155, 992, 1, 0, 0, 0, 157, 995, 1, 0, 0, 0, 159, 997, 1, 0, 0, 0, 161, 1000, 1, 0, 0, 0, 163, 1002, 1, 0, 0, 0, 165, 1005, 1, 0, 0, 0, 167, 1007, 1, 0, 0, 0, 169, 1009, 1, 0, 0, 0, 171, 1011, 1, 0, 0, 0, 173, 1013, 1, 0, 0, 0, 175, 1015, 1, 0, 0, 0, 177, 1017, 1, 0, 0, 0, 179, 1019, 1, 0, 0, 0, 181, 1040, 1, 0, 0, 0, 183, 1042, 1, 0, 0, 0, 185, 1047, 1, 0, 0, 0, 187, 1052, 1, 0, 0, 0, 189, 1057, 1, 0, 0, 0, 191, 1078, 1, 0, 0, 0, 193, 1080, 1, 0, 0, 0, 195, 1088, 1, 0, 0, 0, 197, 1090, 1, 0, 0, 0, 199, 1094, 1, 0, 0, 0, 201, 1098, 1, 0, 0, 0, 203, 1102, 1, 0, 0, 0, 205, 1107, 1, 0, 0, 0, 207, 1112, 1, 0, 0, 0, 209, 1116, 1, 0, 0, 0, 211, 1120, 1, 0, 0, 0, 213, 1124, 1, 0, 0, 0, 215, 1129, 1, 0, 0, 0, 217, 1133, 1, 0, 0, 0, 219, 1137, 1, 0, 0, 0, 221, 1141, 1, 0, 0, 0, 223, 1145, 1, 0, 0, 0, 225, 1149, 1, 0, 0, 0, 227, 1161, 1, 0, 0, 0, 229, 1164, 1, 0, 0, 0, 231, 1168, 1, 0, 0, 0, 233, 1172, 1, 0, 0, 0, 235, 1176, 1, 0, 0, 0, 237, 1180, 1, 0, 0, 0, 239, 1184, 1, 0, 0, 0, 241, 1188, 1, 0, 0, 0, 243, 1193, 1, 0, 0, 0, 245, 1197, 1, 0, 0, 0, 247, 1201, 1, 0, 0, 0, 249, 1205, 1, 0, 0, 0, 251, 1213, 1, 0, 0, 0, 253, 1234, 1, 0, 0, 0, 255, 1238, 1, 0, 0, 0, 257, 1242, 1, 0, 0, 0, 259, 1246, 1, 0, 0, 0, 261, 1250, 1, 0, 0, 0, 263, 1254, 1, 0, 0, 0, 265, 1259, 1, 0, 0, 0, 267, 1263, 1, 0, 0, 0, 269, 1267, 1, 0, 0, 0, 271, 1271, 1, 0, 0, 0, 273, 1275, 1, 0, 0, 0, 275, 1279, 1, 0, 0, 0, 277, 1282, 1, 0, 0, 0, 279, 1286, 1, 0, 0, 0, 281, 1290, 1, 0, 0, 0, 283, 1294, 1, 0, 0, 0, 285, 1298, 1, 0, 0, 0, 287, 1303, 1, 0, 0, 0, 289, 1308, 1, 0, 0, 0, 291, 1313, 1, 0, 0, 0, 293, 1320, 1, 0, 0, 0, 295, 1329, 1, 0, 0, 0, 297, 1336, 1, 0, 0, 0, 299, 1340, 1, 0, 0, 0, 301, 1344, 1, 0, 0, 0, 303, 1348, 1, 0, 0, 0, 305, 1352, 1, 0, 0, 0, 307, 1358, 1, 0, 0, 0, 309, 1362, 1, 0, 0, 0, 311, 1366, 1, 0, 0, 0, 313, 1370, 1, 0, 0, 0, 315, 1374, 1, 0, 0, 0, 317, 1378, 1, 0, 0, 0, 319, 1382, 1, 0, 0, 0, 321, 1386, 1, 0, 0, 0, 323, 1390, 1, 0, 0, 0, 325, 1394, 1, 0, 0, 0, 327, 1398, 1, 0, 0, 0, 329, 1402, 1, 0, 0, 0, 331, 1407, 1, 0, 0, 0, 333, 1411, 1, 0, 0, 0, 335, 1415, 1, 0, 0, 0, 337, 1419, 1, 0, 0, 0, 339, 1423, 1, 0, 0, 0, 341, 1427, 1, 0, 0, 0, 343, 1431, 1, 0, 0, 0, 345, 1435, 1, 0, 0, 0, 347, 1439, 1, 0, 0, 0, 349, 1444, 1, 0, 0, 0, 351, 1449, 1, 0, 0, 0, 353, 1453, 1, 0, 0, 0, 355, 1457, 1, 0, 0, 0, 357, 1461, 1, 0, 0, 0, 359, 1466, 1, 0, 0, 0, 361, 1475, 1, 0, 0, 0, 363, 1479, 1, 0, 0, 0, 365, 1483, 1, 0, 0, 0, 367, 1487, 1, 0, 0, 0, 369, 1491, 1, 0, 0, 0, 371, 1496, 1, 0, 0, 0, 373, 1500, 1, 0, 0, 0, 375, 1504, 1, 0, 0, 0, 377, 1508, 1, 0, 0, 0, 379, 1513, 1, 0, 0, 0, 381, 1517, 1, 0, 0, 0, 383, 1521, 1, 0, 0, 0, 385, 1525, 1, 0, 0, 0, 387, 1529, 1, 0, 0, 0, 389, 1533, 1, 0, 0, 0, 391, 1539, 1, 0, 0, 0, 393, 1543, 1, 0, 0, 0, 395, 1547, 1, 0, 0, 0, 397, 1551, 1, 0, 0, 0, 399, 1555, 1, 0, 0, 0, 401, 1559, 1, 0, 0, 0, 403, 1563, 1, 0, 0, 0, 405, 1568, 1, 0, 0, 0, 407, 1573, 1, 0, 0, 0, 409, 1577, 1, 0, 0, 0, 411, 1583, 1, 0, 0, 0, 413, 1592, 1, 0, 0, 0, 415, 1596, 1, 0, 0, 0, 417, 1600, 1, 0, 0, 0, 419, 1604, 1, 0, 0, 0, 421, 1608, 1, 0, 0, 0, 423, 1612, 1, 0, 0, 0, 425, 1616, 1, 0, 0, 0, 427, 1620, 1, 0, 0, 0, 429, 1624, 1, 0, 0, 0, 431, 1629, 1, 0, 0, 0, 433, 1635, 1, 0, 0, 0, 435, 1641, 1, 0, 0, 0, 437, 1645, 1, 0, 0, 0, 439, 1649, 1, 0, 0, 0, 441, 1653, 1, 0, 0, 0, 443, 1659, 1, 0, 0, 0, 445, 1665, 1, 0, 0, 0, 447, 1669, 1, 0, 0, 0, 449, 1673, 1, 0, 0, 0, 451, 1677, 1, 0, 0, 0, 453, 1683, 1, 0, 0, 0, 455, 1689, 1, 0, 0, 0, 457, 1695, 1, 0, 0, 0, 459, 1700, 1, 0, 0, 0, 461, 1705, 1, 0, 0, 0, 463, 1709, 1, 0, 0, 0, 465, 1713, 1, 0, 0, 0, 467, 1717, 1, 0, 0, 0, 469, 1721, 1, 0, 0, 0, 471, 1725, 1, 0, 0, 0, 473, 1729, 1, 0, 0, 0, 475, 1733, 1, 0, 0, 0, 477, 1737, 1, 0, 0, 0, 479, 1741, 1, 0, 0, 0, 481, 1746, 1, 0, 0, 0, 483, 1750, 1, 0, 0, 0, 485, 1754, 1, 0, 0, 0, 487, 1758, 1, 0, 0, 0, 489, 1762, 1, 0, 0, 0, 491, 1767, 1, 0, 0, 0, 493, 1772, 1, 0, 0, 0, 495, 1776, 1, 0, 0, 0, 497, 1780, 1, 0, 0, 0, 499, 500, 7, 0, 0, 0, 500, 501, 7, 1, 0, 0, 501, 502, 7, 2, 0, 0, 502, 503, 7, 2, 0, 0, 503, 504, 7, 3, 0, 0, 504, 505, 7, 4, 0, 0, 505, 506, 7, 5, 0, 0, 506, 507, 1, 0, 0, 0, 507, 508, 6, 0, 0, 0, 508, 20, 1, 0, 0, 0, 509, 510, 7, 0, 0, 0, 510, 511, 7, 6, 0, 0, 511, 512, 7, 7, 0, 0, 512, 513, 7, 8, 0, 0, 513, 514, 1, 0, 0, 0, 514, 515, 6, 1, 1, 0, 515, 22, 1, 0, 0, 0, 516, 517, 7, 3, 0, 0, 517, 518, 7, 9, 0, 0, 518, 519, 7, 6, 0, 0, 519, 520, 7, 1, 0, 0, 520, 521, 7, 4, 0, 0, 521, 522, 7, 10, 0, 0, 522, 523, 1, 0, 0, 0, 523, 524, 6, 2, 2, 0, 524, 24, 1, 0, 0, 0, 525, 526, 7, 3, 0, 0, 526, 527, 7, 11, 0, 0, 527, 528, 7, 12, 0, 0, 528, 529, 7, 13, 0, 0, 529, 530, 1, 0, 0, 0, 530, 531, 6, 3, 0, 0, 531, 26, 1, 0, 0, 0, 532, 533, 7, 3, 0, 0, 533, 534, 7, 14, 0, 0, 534, 535, 7, 8, 0, 0, 535, 536, 7, 13, 0, 0, 536, 537, 7, 12, 0, 0, 537, 538, 7, 1, 0, 0, 538, 539, 7, 9, 0, 0, 539, 540, 1, 0, 0, 0, 540, 541, 6, 4, 3, 0, 541, 28, 1, 0, 0, 0, 542, 543, 7, 15, 0, 0, 543, 544, 7, 6, 0, 0, 544, 545, 7, 7, 0, 0, 545, 546, 7, 16, 0, 0, 546, 547, 1, 0, 0, 0, 547, 548, 6, 5, 4, 0, 548, 30, 1, 0, 0, 0, 549, 550, 7, 17, 0, 0, 550, 551, 7, 6, 0, 0, 551, 552, 7, 7, 0, 0, 552, 553, 7, 18, 0, 0, 553, 554, 1, 0, 0, 0, 554, 555, 6, 6, 0, 0, 555, 32, 1, 0, 0, 0, 556, 557, 7, 18, 0, 0, 557, 558, 7, 3, 0, 0, 558, 559, 7, 3, 0, 0, 559, 560, 7, 8, 0, 0, 560, 561, 1, 0, 0, 0, 561, 562, 6, 7, 1, 0, 562, 34, 1, 0, 0, 0, 563, 564, 7, 13, 0, 0, 564, 565, 7, 1, 0, 0, 565, 566, 7, 16, 0, 0, 566, 567, 7, 1, 0, 0, 567, 568, 7, 5, 0, 0, 568, 569, 1, 0, 0, 0, 569, 570, 6, 8, 0, 0, 570, 36, 1, 0, 0, 0, 571, 572, 7, 16, 0, 0, 572, 573, 7, 11, 0, 0, 573, 574, 5, 95, 0, 0, 574, 575, 7, 3, 0, 0, 575, 576, 7, 14, 0, 0, 576, 577, 7, 8, 0, 0, 577, 578, 7, 12, 0, 0, 578, 579, 7, 9, 0, 0, 579, 580, 7, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 582, 6, 9, 5, 0, 582, 38, 1, 0, 0, 0, 583, 584, 7, 6, 0, 0, 584, 585, 7, 3, 0, 0, 585, 586, 7, 9, 0, 0, 586, 587, 7, 12, 0, 0, 587, 588, 7, 16, 0, 0, 588, 589, 7, 3, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 6, 10, 6, 0, 591, 40, 1, 0, 0, 0, 592, 593, 7, 6, 0, 0, 593, 594, 7, 7, 0, 0, 594, 595, 7, 19, 0, 0, 595, 596, 1, 0, 0, 0, 596, 597, 6, 11, 0, 0, 597, 42, 1, 0, 0, 0, 598, 599, 7, 2, 0, 0, 599, 600, 7, 10, 0, 0, 600, 601, 7, 7, 0, 0, 601, 602, 7, 19, 0, 0, 602, 603, 1, 0, 0, 0, 603, 604, 6, 12, 7, 0, 604, 44, 1, 0, 0, 0, 605, 606, 7, 2, 0, 0, 606, 607, 7, 7, 0, 0, 607, 608, 7, 6, 0, 0, 608, 609, 7, 5, 0, 0, 609, 610, 1, 0, 0, 0, 610, 611, 6, 13, 0, 0, 611, 46, 1, 0, 0, 0, 612, 613, 7, 2, 0, 0, 613, 614, 7, 5, 0, 0, 614, 615, 7, 12, 0, 0, 615, 616, 7, 5, 0, 0, 616, 617, 7, 2, 0, 0, 617, 618, 1, 0, 0, 0, 618, 619, 6, 14, 0, 0, 619, 48, 1, 0, 0, 0, 620, 621, 7, 19, 0, 0, 621, 622, 7, 10, 0, 0, 622, 623, 7, 3, 0, 0, 623, 624, 7, 6, 0, 0, 624, 625, 7, 3, 0, 0, 625, 626, 1, 0, 0, 0, 626, 627, 6, 15, 0, 0, 627, 50, 1, 0, 0, 0, 628, 629, 7, 13, 0, 0, 629, 630, 7, 7, 0, 0, 630, 631, 7, 7, 0, 0, 631, 632, 7, 18, 0, 0, 632, 633, 7, 20, 0, 0, 633, 634, 7, 8, 0, 0, 634, 635, 1, 0, 0, 0, 635, 636, 6, 16, 8, 0, 636, 52, 1, 0, 0, 0, 637, 638, 4, 17, 0, 0, 638, 639, 7, 4, 0, 0, 639, 640, 7, 10, 0, 0, 640, 641, 7, 12, 0, 0, 641, 642, 7, 9, 0, 0, 642, 643, 7, 17, 0, 0, 643, 644, 7, 3, 0, 0, 644, 645, 5, 95, 0, 0, 645, 646, 7, 8, 0, 0, 646, 647, 7, 7, 0, 0, 647, 648, 7, 1, 0, 0, 648, 649, 7, 9, 0, 0, 649, 650, 7, 5, 0, 0, 650, 651, 1, 0, 0, 0, 651, 652, 6, 17, 9, 0, 652, 54, 1, 0, 0, 0, 653, 654, 4, 18, 1, 0, 654, 655, 7, 1, 0, 0, 655, 656, 7, 9, 0, 0, 656, 657, 7, 13, 0, 0, 657, 658, 7, 1, 0, 0, 658, 659, 7, 9, 0, 0, 659, 660, 7, 3, 0, 0, 660, 661, 7, 2, 0, 0, 661, 662, 7, 5, 0, 0, 662, 663, 7, 12, 0, 0, 663, 664, 7, 5, 0, 0, 664, 665, 7, 2, 0, 0, 665, 666, 1, 0, 0, 0, 666, 667, 6, 18, 0, 0, 667, 56, 1, 0, 0, 0, 668, 669, 4, 19, 2, 0, 669, 670, 7, 1, 0, 0, 670, 671, 7, 9, 0, 0, 671, 672, 7, 2, 0, 0, 672, 673, 7, 1, 0, 0, 673, 674, 7, 2, 0, 0, 674, 675, 7, 5, 0, 0, 675, 676, 5, 95, 0, 0, 676, 677, 5, 128020, 0, 0, 677, 678, 1, 0, 0, 0, 678, 679, 6, 19, 1, 0, 679, 58, 1, 0, 0, 0, 680, 681, 4, 20, 3, 0, 681, 682, 7, 13, 0, 0, 682, 683, 7, 7, 0, 0, 683, 684, 7, 7, 0, 0, 684, 685, 7, 18, 0, 0, 685, 686, 7, 20, 0, 0, 686, 687, 7, 8, 0, 0, 687, 688, 5, 95, 0, 0, 688, 689, 5, 128020, 0, 0, 689, 690, 1, 0, 0, 0, 690, 691, 6, 20, 10, 0, 691, 60, 1, 0, 0, 0, 692, 693, 4, 21, 4, 0, 693, 694, 7, 16, 0, 0, 694, 695, 7, 3, 0, 0, 695, 696, 7, 5, 0, 0, 696, 697, 7, 6, 0, 0, 697, 698, 7, 1, 0, 0, 698, 699, 7, 4, 0, 0, 699, 700, 7, 2, 0, 0, 700, 701, 1, 0, 0, 0, 701, 702, 6, 21, 11, 0, 702, 62, 1, 0, 0, 0, 703, 704, 4, 22, 5, 0, 704, 705, 7, 15, 0, 0, 705, 706, 7, 20, 0, 0, 706, 707, 7, 13, 0, 0, 707, 708, 7, 13, 0, 0, 708, 709, 1, 0, 0, 0, 709, 710, 6, 22, 8, 0, 710, 64, 1, 0, 0, 0, 711, 712, 4, 23, 6, 0, 712, 713, 7, 13, 0, 0, 713, 714, 7, 3, 0, 0, 714, 715, 7, 15, 0, 0, 715, 716, 7, 5, 0, 0, 716, 717, 1, 0, 0, 0, 717, 718, 6, 23, 8, 0, 718, 66, 1, 0, 0, 0, 719, 720, 4, 24, 7, 0, 720, 721, 7, 6, 0, 0, 721, 722, 7, 1, 0, 0, 722, 723, 7, 17, 0, 0, 723, 724, 7, 10, 0, 0, 724, 725, 7, 5, 0, 0, 725, 726, 1, 0, 0, 0, 726, 727, 6, 24, 8, 0, 727, 68, 1, 0, 0, 0, 728, 729, 4, 25, 8, 0, 729, 730, 7, 15, 0, 0, 730, 731, 7, 7, 0, 0, 731, 732, 7, 6, 0, 0, 732, 733, 7, 18, 0, 0, 733, 734, 1, 0, 0, 0, 734, 735, 6, 25, 12, 0, 735, 70, 1, 0, 0, 0, 736, 738, 8, 21, 0, 0, 737, 736, 1, 0, 0, 0, 738, 739, 1, 0, 0, 0, 739, 737, 1, 0, 0, 0, 739, 740, 1, 0, 0, 0, 740, 741, 1, 0, 0, 0, 741, 742, 6, 26, 0, 0, 742, 72, 1, 0, 0, 0, 743, 744, 5, 47, 0, 0, 744, 745, 5, 47, 0, 0, 745, 749, 1, 0, 0, 0, 746, 748, 8, 22, 0, 0, 747, 746, 1, 0, 0, 0, 748, 751, 1, 0, 0, 0, 749, 747, 1, 0, 0, 0, 749, 750, 1, 0, 0, 0, 750, 753, 1, 0, 0, 0, 751, 749, 1, 0, 0, 0, 752, 754, 5, 13, 0, 0, 753, 752, 1, 0, 0, 0, 753, 754, 1, 0, 0, 0, 754, 756, 1, 0, 0, 0, 755, 757, 5, 10, 0, 0, 756, 755, 1, 0, 0, 0, 756, 757, 1, 0, 0, 0, 757, 758, 1, 0, 0, 0, 758, 759, 6, 27, 13, 0, 759, 74, 1, 0, 0, 0, 760, 761, 5, 47, 0, 0, 761, 762, 5, 42, 0, 0, 762, 767, 1, 0, 0, 0, 763, 766, 3, 75, 28, 0, 764, 766, 9, 0, 0, 0, 765, 763, 1, 0, 0, 0, 765, 764, 1, 0, 0, 0, 766, 769, 1, 0, 0, 0, 767, 768, 1, 0, 0, 0, 767, 765, 1, 0, 0, 0, 768, 770, 1, 0, 0, 0, 769, 767, 1, 0, 0, 0, 770, 771, 5, 42, 0, 0, 771, 772, 5, 47, 0, 0, 772, 773, 1, 0, 0, 0, 773, 774, 6, 28, 13, 0, 774, 76, 1, 0, 0, 0, 775, 777, 7, 23, 0, 0, 776, 775, 1, 0, 0, 0, 777, 778, 1, 0, 0, 0, 778, 776, 1, 0, 0, 0, 778, 779, 1, 0, 0, 0, 779, 780, 1, 0, 0, 0, 780, 781, 6, 29, 13, 0, 781, 78, 1, 0, 0, 0, 782, 783, 5, 124, 0, 0, 783, 784, 1, 0, 0, 0, 784, 785, 6, 30, 14, 0, 785, 80, 1, 0, 0, 0, 786, 787, 7, 24, 0, 0, 787, 82, 1, 0, 0, 0, 788, 789, 7, 25, 0, 0, 789, 84, 1, 0, 0, 0, 790, 791, 5, 92, 0, 0, 791, 792, 7, 26, 0, 0, 792, 86, 1, 0, 0, 0, 793, 794, 8, 27, 0, 0, 794, 88, 1, 0, 0, 0, 795, 797, 7, 3, 0, 0, 796, 798, 7, 28, 0, 0, 797, 796, 1, 0, 0, 0, 797, 798, 1, 0, 0, 0, 798, 800, 1, 0, 0, 0, 799, 801, 3, 81, 31, 0, 800, 799, 1, 0, 0, 0, 801, 802, 1, 0, 0, 0, 802, 800, 1, 0, 0, 0, 802, 803, 1, 0, 0, 0, 803, 90, 1, 0, 0, 0, 804, 805, 5, 64, 0, 0, 805, 92, 1, 0, 0, 0, 806, 807, 5, 96, 0, 0, 807, 94, 1, 0, 0, 0, 808, 812, 8, 29, 0, 0, 809, 810, 5, 96, 0, 0, 810, 812, 5, 96, 0, 0, 811, 808, 1, 0, 0, 0, 811, 809, 1, 0, 0, 0, 812, 96, 1, 0, 0, 0, 813, 814, 5, 95, 0, 0, 814, 98, 1, 0, 0, 0, 815, 819, 3, 83, 32, 0, 816, 819, 3, 81, 31, 0, 817, 819, 3, 97, 39, 0, 818, 815, 1, 0, 0, 0, 818, 816, 1, 0, 0, 0, 818, 817, 1, 0, 0, 0, 819, 100, 1, 0, 0, 0, 820, 825, 5, 34, 0, 0, 821, 824, 3, 85, 33, 0, 822, 824, 3, 87, 34, 0, 823, 821, 1, 0, 0, 0, 823, 822, 1, 0, 0, 0, 824, 827, 1, 0, 0, 0, 825, 823, 1, 0, 0, 0, 825, 826, 1, 0, 0, 0, 826, 828, 1, 0, 0, 0, 827, 825, 1, 0, 0, 0, 828, 850, 5, 34, 0, 0, 829, 830, 5, 34, 0, 0, 830, 831, 5, 34, 0, 0, 831, 832, 5, 34, 0, 0, 832, 836, 1, 0, 0, 0, 833, 835, 8, 22, 0, 0, 834, 833, 1, 0, 0, 0, 835, 838, 1, 0, 0, 0, 836, 837, 1, 0, 0, 0, 836, 834, 1, 0, 0, 0, 837, 839, 1, 0, 0, 0, 838, 836, 1, 0, 0, 0, 839, 840, 5, 34, 0, 0, 840, 841, 5, 34, 0, 0, 841, 842, 5, 34, 0, 0, 842, 844, 1, 0, 0, 0, 843, 845, 5, 34, 0, 0, 844, 843, 1, 0, 0, 0, 844, 845, 1, 0, 0, 0, 845, 847, 1, 0, 0, 0, 846, 848, 5, 34, 0, 0, 847, 846, 1, 0, 0, 0, 847, 848, 1, 0, 0, 0, 848, 850, 1, 0, 0, 0, 849, 820, 1, 0, 0, 0, 849, 829, 1, 0, 0, 0, 850, 102, 1, 0, 0, 0, 851, 853, 3, 81, 31, 0, 852, 851, 1, 0, 0, 0, 853, 854, 1, 0, 0, 0, 854, 852, 1, 0, 0, 0, 854, 855, 1, 0, 0, 0, 855, 104, 1, 0, 0, 0, 856, 858, 3, 81, 31, 0, 857, 856, 1, 0, 0, 0, 858, 859, 1, 0, 0, 0, 859, 857, 1, 0, 0, 0, 859, 860, 1, 0, 0, 0, 860, 861, 1, 0, 0, 0, 861, 865, 3, 123, 52, 0, 862, 864, 3, 81, 31, 0, 863, 862, 1, 0, 0, 0, 864, 867, 1, 0, 0, 0, 865, 863, 1, 0, 0, 0, 865, 866, 1, 0, 0, 0, 866, 899, 1, 0, 0, 0, 867, 865, 1, 0, 0, 0, 868, 870, 3, 123, 52, 0, 869, 871, 3, 81, 31, 0, 870, 869, 1, 0, 0, 0, 871, 872, 1, 0, 0, 0, 872, 870, 1, 0, 0, 0, 872, 873, 1, 0, 0, 0, 873, 899, 1, 0, 0, 0, 874, 876, 3, 81, 31, 0, 875, 874, 1, 0, 0, 0, 876, 877, 1, 0, 0, 0, 877, 875, 1, 0, 0, 0, 877, 878, 1, 0, 0, 0, 878, 886, 1, 0, 0, 0, 879, 883, 3, 123, 52, 0, 880, 882, 3, 81, 31, 0, 881, 880, 1, 0, 0, 0, 882, 885, 1, 0, 0, 0, 883, 881, 1, 0, 0, 0, 883, 884, 1, 0, 0, 0, 884, 887, 1, 0, 0, 0, 885, 883, 1, 0, 0, 0, 886, 879, 1, 0, 0, 0, 886, 887, 1, 0, 0, 0, 887, 888, 1, 0, 0, 0, 888, 889, 3, 89, 35, 0, 889, 899, 1, 0, 0, 0, 890, 892, 3, 123, 52, 0, 891, 893, 3, 81, 31, 0, 892, 891, 1, 0, 0, 0, 893, 894, 1, 0, 0, 0, 894, 892, 1, 0, 0, 0, 894, 895, 1, 0, 0, 0, 895, 896, 1, 0, 0, 0, 896, 897, 3, 89, 35, 0, 897, 899, 1, 0, 0, 0, 898, 857, 1, 0, 0, 0, 898, 868, 1, 0, 0, 0, 898, 875, 1, 0, 0, 0, 898, 890, 1, 0, 0, 0, 899, 106, 1, 0, 0, 0, 900, 901, 7, 30, 0, 0, 901, 902, 7, 31, 0, 0, 902, 108, 1, 0, 0, 0, 903, 904, 7, 12, 0, 0, 904, 905, 7, 9, 0, 0, 905, 906, 7, 0, 0, 0, 906, 110, 1, 0, 0, 0, 907, 908, 7, 12, 0, 0, 908, 909, 7, 2, 0, 0, 909, 910, 7, 4, 0, 0, 910, 112, 1, 0, 0, 0, 911, 912, 5, 61, 0, 0, 912, 114, 1, 0, 0, 0, 913, 914, 5, 58, 0, 0, 914, 915, 5, 58, 0, 0, 915, 116, 1, 0, 0, 0, 916, 917, 5, 58, 0, 0, 917, 118, 1, 0, 0, 0, 918, 919, 5, 44, 0, 0, 919, 120, 1, 0, 0, 0, 920, 921, 7, 0, 0, 0, 921, 922, 7, 3, 0, 0, 922, 923, 7, 2, 0, 0, 923, 924, 7, 4, 0, 0, 924, 122, 1, 0, 0, 0, 925, 926, 5, 46, 0, 0, 926, 124, 1, 0, 0, 0, 927, 928, 7, 15, 0, 0, 928, 929, 7, 12, 0, 0, 929, 930, 7, 13, 0, 0, 930, 931, 7, 2, 0, 0, 931, 932, 7, 3, 0, 0, 932, 126, 1, 0, 0, 0, 933, 934, 7, 15, 0, 0, 934, 935, 7, 1, 0, 0, 935, 936, 7, 6, 0, 0, 936, 937, 7, 2, 0, 0, 937, 938, 7, 5, 0, 0, 938, 128, 1, 0, 0, 0, 939, 940, 7, 1, 0, 0, 940, 941, 7, 9, 0, 0, 941, 130, 1, 0, 0, 0, 942, 943, 7, 1, 0, 0, 943, 944, 7, 2, 0, 0, 944, 132, 1, 0, 0, 0, 945, 946, 7, 13, 0, 0, 946, 947, 7, 12, 0, 0, 947, 948, 7, 2, 0, 0, 948, 949, 7, 5, 0, 0, 949, 134, 1, 0, 0, 0, 950, 951, 7, 13, 0, 0, 951, 952, 7, 1, 0, 0, 952, 953, 7, 18, 0, 0, 953, 954, 7, 3, 0, 0, 954, 136, 1, 0, 0, 0, 955, 956, 7, 9, 0, 0, 956, 957, 7, 7, 0, 0, 957, 958, 7, 5, 0, 0, 958, 138, 1, 0, 0, 0, 959, 960, 7, 9, 0, 0, 960, 961, 7, 20, 0, 0, 961, 962, 7, 13, 0, 0, 962, 963, 7, 13, 0, 0, 963, 140, 1, 0, 0, 0, 964, 965, 7, 9, 0, 0, 965, 966, 7, 20, 0, 0, 966, 967, 7, 13, 0, 0, 967, 968, 7, 13, 0, 0, 968, 969, 7, 2, 0, 0, 969, 142, 1, 0, 0, 0, 970, 971, 7, 7, 0, 0, 971, 972, 7, 6, 0, 0, 972, 144, 1, 0, 0, 0, 973, 974, 5, 63, 0, 0, 974, 146, 1, 0, 0, 0, 975, 976, 7, 6, 0, 0, 976, 977, 7, 13, 0, 0, 977, 978, 7, 1, 0, 0, 978, 979, 7, 18, 0, 0, 979, 980, 7, 3, 0, 0, 980, 148, 1, 0, 0, 0, 981, 982, 7, 5, 0, 0, 982, 983, 7, 6, 0, 0, 983, 984, 7, 20, 0, 0, 984, 985, 7, 3, 0, 0, 985, 150, 1, 0, 0, 0, 986, 987, 5, 61, 0, 0, 987, 988, 5, 61, 0, 0, 988, 152, 1, 0, 0, 0, 989, 990, 5, 61, 0, 0, 990, 991, 5, 126, 0, 0, 991, 154, 1, 0, 0, 0, 992, 993, 5, 33, 0, 0, 993, 994, 5, 61, 0, 0, 994, 156, 1, 0, 0, 0, 995, 996, 5, 60, 0, 0, 996, 158, 1, 0, 0, 0, 997, 998, 5, 60, 0, 0, 998, 999, 5, 61, 0, 0, 999, 160, 1, 0, 0, 0, 1000, 1001, 5, 62, 0, 0, 1001, 162, 1, 0, 0, 0, 1002, 1003, 5, 62, 0, 0, 1003, 1004, 5, 61, 0, 0, 1004, 164, 1, 0, 0, 0, 1005, 1006, 5, 43, 0, 0, 1006, 166, 1, 0, 0, 0, 1007, 1008, 5, 45, 0, 0, 1008, 168, 1, 0, 0, 0, 1009, 1010, 5, 42, 0, 0, 1010, 170, 1, 0, 0, 0, 1011, 1012, 5, 47, 0, 0, 1012, 172, 1, 0, 0, 0, 1013, 1014, 5, 37, 0, 0, 1014, 174, 1, 0, 0, 0, 1015, 1016, 5, 123, 0, 0, 1016, 176, 1, 0, 0, 0, 1017, 1018, 5, 125, 0, 0, 1018, 178, 1, 0, 0, 0, 1019, 1020, 3, 49, 15, 0, 1020, 1021, 1, 0, 0, 0, 1021, 1022, 6, 80, 15, 0, 1022, 180, 1, 0, 0, 0, 1023, 1026, 3, 145, 63, 0, 1024, 1027, 3, 83, 32, 0, 1025, 1027, 3, 97, 39, 0, 1026, 1024, 1, 0, 0, 0, 1026, 1025, 1, 0, 0, 0, 1027, 1031, 1, 0, 0, 0, 1028, 1030, 3, 99, 40, 0, 1029, 1028, 1, 0, 0, 0, 1030, 1033, 1, 0, 0, 0, 1031, 1029, 1, 0, 0, 0, 1031, 1032, 1, 0, 0, 0, 1032, 1041, 1, 0, 0, 0, 1033, 1031, 1, 0, 0, 0, 1034, 1036, 3, 145, 63, 0, 1035, 1037, 3, 81, 31, 0, 1036, 1035, 1, 0, 0, 0, 1037, 1038, 1, 0, 0, 0, 1038, 1036, 1, 0, 0, 0, 1038, 1039, 1, 0, 0, 0, 1039, 1041, 1, 0, 0, 0, 1040, 1023, 1, 0, 0, 0, 1040, 1034, 1, 0, 0, 0, 1041, 182, 1, 0, 0, 0, 1042, 1043, 5, 91, 0, 0, 1043, 1044, 1, 0, 0, 0, 1044, 1045, 6, 82, 0, 0, 1045, 1046, 6, 82, 0, 0, 1046, 184, 1, 0, 0, 0, 1047, 1048, 5, 93, 0, 0, 1048, 1049, 1, 0, 0, 0, 1049, 1050, 6, 83, 14, 0, 1050, 1051, 6, 83, 14, 0, 1051, 186, 1, 0, 0, 0, 1052, 1053, 5, 40, 0, 0, 1053, 1054, 1, 0, 0, 0, 1054, 1055, 6, 84, 0, 0, 1055, 1056, 6, 84, 0, 0, 1056, 188, 1, 0, 0, 0, 1057, 1058, 5, 41, 0, 0, 1058, 1059, 1, 0, 0, 0, 1059, 1060, 6, 85, 14, 0, 1060, 1061, 6, 85, 14, 0, 1061, 190, 1, 0, 0, 0, 1062, 1066, 3, 83, 32, 0, 1063, 1065, 3, 99, 40, 0, 1064, 1063, 1, 0, 0, 0, 1065, 1068, 1, 0, 0, 0, 1066, 1064, 1, 0, 0, 0, 1066, 1067, 1, 0, 0, 0, 1067, 1079, 1, 0, 0, 0, 1068, 1066, 1, 0, 0, 0, 1069, 1072, 3, 97, 39, 0, 1070, 1072, 3, 91, 36, 0, 1071, 1069, 1, 0, 0, 0, 1071, 1070, 1, 0, 0, 0, 1072, 1074, 1, 0, 0, 0, 1073, 1075, 3, 99, 40, 0, 1074, 1073, 1, 0, 0, 0, 1075, 1076, 1, 0, 0, 0, 1076, 1074, 1, 0, 0, 0, 1076, 1077, 1, 0, 0, 0, 1077, 1079, 1, 0, 0, 0, 1078, 1062, 1, 0, 0, 0, 1078, 1071, 1, 0, 0, 0, 1079, 192, 1, 0, 0, 0, 1080, 1082, 3, 93, 37, 0, 1081, 1083, 3, 95, 38, 0, 1082, 1081, 1, 0, 0, 0, 1083, 1084, 1, 0, 0, 0, 1084, 1082, 1, 0, 0, 0, 1084, 1085, 1, 0, 0, 0, 1085, 1086, 1, 0, 0, 0, 1086, 1087, 3, 93, 37, 0, 1087, 194, 1, 0, 0, 0, 1088, 1089, 3, 193, 87, 0, 1089, 196, 1, 0, 0, 0, 1090, 1091, 3, 73, 27, 0, 1091, 1092, 1, 0, 0, 0, 1092, 1093, 6, 89, 13, 0, 1093, 198, 1, 0, 0, 0, 1094, 1095, 3, 75, 28, 0, 1095, 1096, 1, 0, 0, 0, 1096, 1097, 6, 90, 13, 0, 1097, 200, 1, 0, 0, 0, 1098, 1099, 3, 77, 29, 0, 1099, 1100, 1, 0, 0, 0, 1100, 1101, 6, 91, 13, 0, 1101, 202, 1, 0, 0, 0, 1102, 1103, 3, 183, 82, 0, 1103, 1104, 1, 0, 0, 0, 1104, 1105, 6, 92, 16, 0, 1105, 1106, 6, 92, 17, 0, 1106, 204, 1, 0, 0, 0, 1107, 1108, 3, 79, 30, 0, 1108, 1109, 1, 0, 0, 0, 1109, 1110, 6, 93, 18, 0, 1110, 1111, 6, 93, 14, 0, 1111, 206, 1, 0, 0, 0, 1112, 1113, 3, 77, 29, 0, 1113, 1114, 1, 0, 0, 0, 1114, 1115, 6, 94, 13, 0, 1115, 208, 1, 0, 0, 0, 1116, 1117, 3, 73, 27, 0, 1117, 1118, 1, 0, 0, 0, 1118, 1119, 6, 95, 13, 0, 1119, 210, 1, 0, 0, 0, 1120, 1121, 3, 75, 28, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 96, 13, 0, 1123, 212, 1, 0, 0, 0, 1124, 1125, 3, 79, 30, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 97, 18, 0, 1127, 1128, 6, 97, 14, 0, 1128, 214, 1, 0, 0, 0, 1129, 1130, 3, 183, 82, 0, 1130, 1131, 1, 0, 0, 0, 1131, 1132, 6, 98, 16, 0, 1132, 216, 1, 0, 0, 0, 1133, 1134, 3, 185, 83, 0, 1134, 1135, 1, 0, 0, 0, 1135, 1136, 6, 99, 19, 0, 1136, 218, 1, 0, 0, 0, 1137, 1138, 3, 117, 49, 0, 1138, 1139, 1, 0, 0, 0, 1139, 1140, 6, 100, 20, 0, 1140, 220, 1, 0, 0, 0, 1141, 1142, 3, 119, 50, 0, 1142, 1143, 1, 0, 0, 0, 1143, 1144, 6, 101, 21, 0, 1144, 222, 1, 0, 0, 0, 1145, 1146, 3, 113, 47, 0, 1146, 1147, 1, 0, 0, 0, 1147, 1148, 6, 102, 22, 0, 1148, 224, 1, 0, 0, 0, 1149, 1150, 7, 16, 0, 0, 1150, 1151, 7, 3, 0, 0, 1151, 1152, 7, 5, 0, 0, 1152, 1153, 7, 12, 0, 0, 1153, 1154, 7, 0, 0, 0, 1154, 1155, 7, 12, 0, 0, 1155, 1156, 7, 5, 0, 0, 1156, 1157, 7, 12, 0, 0, 1157, 226, 1, 0, 0, 0, 1158, 1162, 8, 32, 0, 0, 1159, 1160, 5, 47, 0, 0, 1160, 1162, 8, 33, 0, 0, 1161, 1158, 1, 0, 0, 0, 1161, 1159, 1, 0, 0, 0, 1162, 228, 1, 0, 0, 0, 1163, 1165, 3, 227, 104, 0, 1164, 1163, 1, 0, 0, 0, 1165, 1166, 1, 0, 0, 0, 1166, 1164, 1, 0, 0, 0, 1166, 1167, 1, 0, 0, 0, 1167, 230, 1, 0, 0, 0, 1168, 1169, 3, 229, 105, 0, 1169, 1170, 1, 0, 0, 0, 1170, 1171, 6, 106, 23, 0, 1171, 232, 1, 0, 0, 0, 1172, 1173, 3, 101, 41, 0, 1173, 1174, 1, 0, 0, 0, 1174, 1175, 6, 107, 24, 0, 1175, 234, 1, 0, 0, 0, 1176, 1177, 3, 73, 27, 0, 1177, 1178, 1, 0, 0, 0, 1178, 1179, 6, 108, 13, 0, 1179, 236, 1, 0, 0, 0, 1180, 1181, 3, 75, 28, 0, 1181, 1182, 1, 0, 0, 0, 1182, 1183, 6, 109, 13, 0, 1183, 238, 1, 0, 0, 0, 1184, 1185, 3, 77, 29, 0, 1185, 1186, 1, 0, 0, 0, 1186, 1187, 6, 110, 13, 0, 1187, 240, 1, 0, 0, 0, 1188, 1189, 3, 79, 30, 0, 1189, 1190, 1, 0, 0, 0, 1190, 1191, 6, 111, 18, 0, 1191, 1192, 6, 111, 14, 0, 1192, 242, 1, 0, 0, 0, 1193, 1194, 3, 123, 52, 0, 1194, 1195, 1, 0, 0, 0, 1195, 1196, 6, 112, 25, 0, 1196, 244, 1, 0, 0, 0, 1197, 1198, 3, 119, 50, 0, 1198, 1199, 1, 0, 0, 0, 1199, 1200, 6, 113, 21, 0, 1200, 246, 1, 0, 0, 0, 1201, 1202, 3, 145, 63, 0, 1202, 1203, 1, 0, 0, 0, 1203, 1204, 6, 114, 26, 0, 1204, 248, 1, 0, 0, 0, 1205, 1206, 3, 181, 81, 0, 1206, 1207, 1, 0, 0, 0, 1207, 1208, 6, 115, 27, 0, 1208, 250, 1, 0, 0, 0, 1209, 1214, 3, 83, 32, 0, 1210, 1214, 3, 81, 31, 0, 1211, 1214, 3, 97, 39, 0, 1212, 1214, 3, 169, 75, 0, 1213, 1209, 1, 0, 0, 0, 1213, 1210, 1, 0, 0, 0, 1213, 1211, 1, 0, 0, 0, 1213, 1212, 1, 0, 0, 0, 1214, 252, 1, 0, 0, 0, 1215, 1218, 3, 83, 32, 0, 1216, 1218, 3, 169, 75, 0, 1217, 1215, 1, 0, 0, 0, 1217, 1216, 1, 0, 0, 0, 1218, 1222, 1, 0, 0, 0, 1219, 1221, 3, 251, 116, 0, 1220, 1219, 1, 0, 0, 0, 1221, 1224, 1, 0, 0, 0, 1222, 1220, 1, 0, 0, 0, 1222, 1223, 1, 0, 0, 0, 1223, 1235, 1, 0, 0, 0, 1224, 1222, 1, 0, 0, 0, 1225, 1228, 3, 97, 39, 0, 1226, 1228, 3, 91, 36, 0, 1227, 1225, 1, 0, 0, 0, 1227, 1226, 1, 0, 0, 0, 1228, 1230, 1, 0, 0, 0, 1229, 1231, 3, 251, 116, 0, 1230, 1229, 1, 0, 0, 0, 1231, 1232, 1, 0, 0, 0, 1232, 1230, 1, 0, 0, 0, 1232, 1233, 1, 0, 0, 0, 1233, 1235, 1, 0, 0, 0, 1234, 1217, 1, 0, 0, 0, 1234, 1227, 1, 0, 0, 0, 1235, 254, 1, 0, 0, 0, 1236, 1239, 3, 253, 117, 0, 1237, 1239, 3, 193, 87, 0, 1238, 1236, 1, 0, 0, 0, 1238, 1237, 1, 0, 0, 0, 1239, 1240, 1, 0, 0, 0, 1240, 1238, 1, 0, 0, 0, 1240, 1241, 1, 0, 0, 0, 1241, 256, 1, 0, 0, 0, 1242, 1243, 3, 73, 27, 0, 1243, 1244, 1, 0, 0, 0, 1244, 1245, 6, 119, 13, 0, 1245, 258, 1, 0, 0, 0, 1246, 1247, 3, 75, 28, 0, 1247, 1248, 1, 0, 0, 0, 1248, 1249, 6, 120, 13, 0, 1249, 260, 1, 0, 0, 0, 1250, 1251, 3, 77, 29, 0, 1251, 1252, 1, 0, 0, 0, 1252, 1253, 6, 121, 13, 0, 1253, 262, 1, 0, 0, 0, 1254, 1255, 3, 79, 30, 0, 1255, 1256, 1, 0, 0, 0, 1256, 1257, 6, 122, 18, 0, 1257, 1258, 6, 122, 14, 0, 1258, 264, 1, 0, 0, 0, 1259, 1260, 3, 113, 47, 0, 1260, 1261, 1, 0, 0, 0, 1261, 1262, 6, 123, 22, 0, 1262, 266, 1, 0, 0, 0, 1263, 1264, 3, 119, 50, 0, 1264, 1265, 1, 0, 0, 0, 1265, 1266, 6, 124, 21, 0, 1266, 268, 1, 0, 0, 0, 1267, 1268, 3, 123, 52, 0, 1268, 1269, 1, 0, 0, 0, 1269, 1270, 6, 125, 25, 0, 1270, 270, 1, 0, 0, 0, 1271, 1272, 3, 145, 63, 0, 1272, 1273, 1, 0, 0, 0, 1273, 1274, 6, 126, 26, 0, 1274, 272, 1, 0, 0, 0, 1275, 1276, 3, 181, 81, 0, 1276, 1277, 1, 0, 0, 0, 1277, 1278, 6, 127, 27, 0, 1278, 274, 1, 0, 0, 0, 1279, 1280, 7, 12, 0, 0, 1280, 1281, 7, 2, 0, 0, 1281, 276, 1, 0, 0, 0, 1282, 1283, 3, 255, 118, 0, 1283, 1284, 1, 0, 0, 0, 1284, 1285, 6, 129, 28, 0, 1285, 278, 1, 0, 0, 0, 1286, 1287, 3, 73, 27, 0, 1287, 1288, 1, 0, 0, 0, 1288, 1289, 6, 130, 13, 0, 1289, 280, 1, 0, 0, 0, 1290, 1291, 3, 75, 28, 0, 1291, 1292, 1, 0, 0, 0, 1292, 1293, 6, 131, 13, 0, 1293, 282, 1, 0, 0, 0, 1294, 1295, 3, 77, 29, 0, 1295, 1296, 1, 0, 0, 0, 1296, 1297, 6, 132, 13, 0, 1297, 284, 1, 0, 0, 0, 1298, 1299, 3, 79, 30, 0, 1299, 1300, 1, 0, 0, 0, 1300, 1301, 6, 133, 18, 0, 1301, 1302, 6, 133, 14, 0, 1302, 286, 1, 0, 0, 0, 1303, 1304, 3, 183, 82, 0, 1304, 1305, 1, 0, 0, 0, 1305, 1306, 6, 134, 16, 0, 1306, 1307, 6, 134, 29, 0, 1307, 288, 1, 0, 0, 0, 1308, 1309, 7, 7, 0, 0, 1309, 1310, 7, 9, 0, 0, 1310, 1311, 1, 0, 0, 0, 1311, 1312, 6, 135, 30, 0, 1312, 290, 1, 0, 0, 0, 1313, 1314, 7, 19, 0, 0, 1314, 1315, 7, 1, 0, 0, 1315, 1316, 7, 5, 0, 0, 1316, 1317, 7, 10, 0, 0, 1317, 1318, 1, 0, 0, 0, 1318, 1319, 6, 136, 30, 0, 1319, 292, 1, 0, 0, 0, 1320, 1321, 8, 34, 0, 0, 1321, 294, 1, 0, 0, 0, 1322, 1324, 3, 293, 137, 0, 1323, 1322, 1, 0, 0, 0, 1324, 1325, 1, 0, 0, 0, 1325, 1323, 1, 0, 0, 0, 1325, 1326, 1, 0, 0, 0, 1326, 1327, 1, 0, 0, 0, 1327, 1328, 3, 117, 49, 0, 1328, 1330, 1, 0, 0, 0, 1329, 1323, 1, 0, 0, 0, 1329, 1330, 1, 0, 0, 0, 1330, 1332, 1, 0, 0, 0, 1331, 1333, 3, 293, 137, 0, 1332, 1331, 1, 0, 0, 0, 1333, 1334, 1, 0, 0, 0, 1334, 1332, 1, 0, 0, 0, 1334, 1335, 1, 0, 0, 0, 1335, 296, 1, 0, 0, 0, 1336, 1337, 3, 295, 138, 0, 1337, 1338, 1, 0, 0, 0, 1338, 1339, 6, 139, 31, 0, 1339, 298, 1, 0, 0, 0, 1340, 1341, 3, 73, 27, 0, 1341, 1342, 1, 0, 0, 0, 1342, 1343, 6, 140, 13, 0, 1343, 300, 1, 0, 0, 0, 1344, 1345, 3, 75, 28, 0, 1345, 1346, 1, 0, 0, 0, 1346, 1347, 6, 141, 13, 0, 1347, 302, 1, 0, 0, 0, 1348, 1349, 3, 77, 29, 0, 1349, 1350, 1, 0, 0, 0, 1350, 1351, 6, 142, 13, 0, 1351, 304, 1, 0, 0, 0, 1352, 1353, 3, 79, 30, 0, 1353, 1354, 1, 0, 0, 0, 1354, 1355, 6, 143, 18, 0, 1355, 1356, 6, 143, 14, 0, 1356, 1357, 6, 143, 14, 0, 1357, 306, 1, 0, 0, 0, 1358, 1359, 3, 113, 47, 0, 1359, 1360, 1, 0, 0, 0, 1360, 1361, 6, 144, 22, 0, 1361, 308, 1, 0, 0, 0, 1362, 1363, 3, 119, 50, 0, 1363, 1364, 1, 0, 0, 0, 1364, 1365, 6, 145, 21, 0, 1365, 310, 1, 0, 0, 0, 1366, 1367, 3, 123, 52, 0, 1367, 1368, 1, 0, 0, 0, 1368, 1369, 6, 146, 25, 0, 1369, 312, 1, 0, 0, 0, 1370, 1371, 3, 291, 136, 0, 1371, 1372, 1, 0, 0, 0, 1372, 1373, 6, 147, 32, 0, 1373, 314, 1, 0, 0, 0, 1374, 1375, 3, 255, 118, 0, 1375, 1376, 1, 0, 0, 0, 1376, 1377, 6, 148, 28, 0, 1377, 316, 1, 0, 0, 0, 1378, 1379, 3, 195, 88, 0, 1379, 1380, 1, 0, 0, 0, 1380, 1381, 6, 149, 33, 0, 1381, 318, 1, 0, 0, 0, 1382, 1383, 3, 145, 63, 0, 1383, 1384, 1, 0, 0, 0, 1384, 1385, 6, 150, 26, 0, 1385, 320, 1, 0, 0, 0, 1386, 1387, 3, 181, 81, 0, 1387, 1388, 1, 0, 0, 0, 1388, 1389, 6, 151, 27, 0, 1389, 322, 1, 0, 0, 0, 1390, 1391, 3, 73, 27, 0, 1391, 1392, 1, 0, 0, 0, 1392, 1393, 6, 152, 13, 0, 1393, 324, 1, 0, 0, 0, 1394, 1395, 3, 75, 28, 0, 1395, 1396, 1, 0, 0, 0, 1396, 1397, 6, 153, 13, 0, 1397, 326, 1, 0, 0, 0, 1398, 1399, 3, 77, 29, 0, 1399, 1400, 1, 0, 0, 0, 1400, 1401, 6, 154, 13, 0, 1401, 328, 1, 0, 0, 0, 1402, 1403, 3, 79, 30, 0, 1403, 1404, 1, 0, 0, 0, 1404, 1405, 6, 155, 18, 0, 1405, 1406, 6, 155, 14, 0, 1406, 330, 1, 0, 0, 0, 1407, 1408, 3, 123, 52, 0, 1408, 1409, 1, 0, 0, 0, 1409, 1410, 6, 156, 25, 0, 1410, 332, 1, 0, 0, 0, 1411, 1412, 3, 145, 63, 0, 1412, 1413, 1, 0, 0, 0, 1413, 1414, 6, 157, 26, 0, 1414, 334, 1, 0, 0, 0, 1415, 1416, 3, 181, 81, 0, 1416, 1417, 1, 0, 0, 0, 1417, 1418, 6, 158, 27, 0, 1418, 336, 1, 0, 0, 0, 1419, 1420, 3, 195, 88, 0, 1420, 1421, 1, 0, 0, 0, 1421, 1422, 6, 159, 33, 0, 1422, 338, 1, 0, 0, 0, 1423, 1424, 3, 191, 86, 0, 1424, 1425, 1, 0, 0, 0, 1425, 1426, 6, 160, 34, 0, 1426, 340, 1, 0, 0, 0, 1427, 1428, 3, 73, 27, 0, 1428, 1429, 1, 0, 0, 0, 1429, 1430, 6, 161, 13, 0, 1430, 342, 1, 0, 0, 0, 1431, 1432, 3, 75, 28, 0, 1432, 1433, 1, 0, 0, 0, 1433, 1434, 6, 162, 13, 0, 1434, 344, 1, 0, 0, 0, 1435, 1436, 3, 77, 29, 0, 1436, 1437, 1, 0, 0, 0, 1437, 1438, 6, 163, 13, 0, 1438, 346, 1, 0, 0, 0, 1439, 1440, 3, 79, 30, 0, 1440, 1441, 1, 0, 0, 0, 1441, 1442, 6, 164, 18, 0, 1442, 1443, 6, 164, 14, 0, 1443, 348, 1, 0, 0, 0, 1444, 1445, 7, 1, 0, 0, 1445, 1446, 7, 9, 0, 0, 1446, 1447, 7, 15, 0, 0, 1447, 1448, 7, 7, 0, 0, 1448, 350, 1, 0, 0, 0, 1449, 1450, 3, 73, 27, 0, 1450, 1451, 1, 0, 0, 0, 1451, 1452, 6, 166, 13, 0, 1452, 352, 1, 0, 0, 0, 1453, 1454, 3, 75, 28, 0, 1454, 1455, 1, 0, 0, 0, 1455, 1456, 6, 167, 13, 0, 1456, 354, 1, 0, 0, 0, 1457, 1458, 3, 77, 29, 0, 1458, 1459, 1, 0, 0, 0, 1459, 1460, 6, 168, 13, 0, 1460, 356, 1, 0, 0, 0, 1461, 1462, 3, 185, 83, 0, 1462, 1463, 1, 0, 0, 0, 1463, 1464, 6, 169, 19, 0, 1464, 1465, 6, 169, 14, 0, 1465, 358, 1, 0, 0, 0, 1466, 1467, 3, 117, 49, 0, 1467, 1468, 1, 0, 0, 0, 1468, 1469, 6, 170, 20, 0, 1469, 360, 1, 0, 0, 0, 1470, 1476, 3, 91, 36, 0, 1471, 1476, 3, 81, 31, 0, 1472, 1476, 3, 123, 52, 0, 1473, 1476, 3, 83, 32, 0, 1474, 1476, 3, 97, 39, 0, 1475, 1470, 1, 0, 0, 0, 1475, 1471, 1, 0, 0, 0, 1475, 1472, 1, 0, 0, 0, 1475, 1473, 1, 0, 0, 0, 1475, 1474, 1, 0, 0, 0, 1476, 1477, 1, 0, 0, 0, 1477, 1475, 1, 0, 0, 0, 1477, 1478, 1, 0, 0, 0, 1478, 362, 1, 0, 0, 0, 1479, 1480, 3, 73, 27, 0, 1480, 1481, 1, 0, 0, 0, 1481, 1482, 6, 172, 13, 0, 1482, 364, 1, 0, 0, 0, 1483, 1484, 3, 75, 28, 0, 1484, 1485, 1, 0, 0, 0, 1485, 1486, 6, 173, 13, 0, 1486, 366, 1, 0, 0, 0, 1487, 1488, 3, 77, 29, 0, 1488, 1489, 1, 0, 0, 0, 1489, 1490, 6, 174, 13, 0, 1490, 368, 1, 0, 0, 0, 1491, 1492, 3, 79, 30, 0, 1492, 1493, 1, 0, 0, 0, 1493, 1494, 6, 175, 18, 0, 1494, 1495, 6, 175, 14, 0, 1495, 370, 1, 0, 0, 0, 1496, 1497, 3, 117, 49, 0, 1497, 1498, 1, 0, 0, 0, 1498, 1499, 6, 176, 20, 0, 1499, 372, 1, 0, 0, 0, 1500, 1501, 3, 119, 50, 0, 1501, 1502, 1, 0, 0, 0, 1502, 1503, 6, 177, 21, 0, 1503, 374, 1, 0, 0, 0, 1504, 1505, 3, 123, 52, 0, 1505, 1506, 1, 0, 0, 0, 1506, 1507, 6, 178, 25, 0, 1507, 376, 1, 0, 0, 0, 1508, 1509, 3, 289, 135, 0, 1509, 1510, 1, 0, 0, 0, 1510, 1511, 6, 179, 35, 0, 1511, 1512, 6, 179, 36, 0, 1512, 378, 1, 0, 0, 0, 1513, 1514, 3, 229, 105, 0, 1514, 1515, 1, 0, 0, 0, 1515, 1516, 6, 180, 23, 0, 1516, 380, 1, 0, 0, 0, 1517, 1518, 3, 101, 41, 0, 1518, 1519, 1, 0, 0, 0, 1519, 1520, 6, 181, 24, 0, 1520, 382, 1, 0, 0, 0, 1521, 1522, 3, 73, 27, 0, 1522, 1523, 1, 0, 0, 0, 1523, 1524, 6, 182, 13, 0, 1524, 384, 1, 0, 0, 0, 1525, 1526, 3, 75, 28, 0, 1526, 1527, 1, 0, 0, 0, 1527, 1528, 6, 183, 13, 0, 1528, 386, 1, 0, 0, 0, 1529, 1530, 3, 77, 29, 0, 1530, 1531, 1, 0, 0, 0, 1531, 1532, 6, 184, 13, 0, 1532, 388, 1, 0, 0, 0, 1533, 1534, 3, 79, 30, 0, 1534, 1535, 1, 0, 0, 0, 1535, 1536, 6, 185, 18, 0, 1536, 1537, 6, 185, 14, 0, 1537, 1538, 6, 185, 14, 0, 1538, 390, 1, 0, 0, 0, 1539, 1540, 3, 119, 50, 0, 1540, 1541, 1, 0, 0, 0, 1541, 1542, 6, 186, 21, 0, 1542, 392, 1, 0, 0, 0, 1543, 1544, 3, 123, 52, 0, 1544, 1545, 1, 0, 0, 0, 1545, 1546, 6, 187, 25, 0, 1546, 394, 1, 0, 0, 0, 1547, 1548, 3, 255, 118, 0, 1548, 1549, 1, 0, 0, 0, 1549, 1550, 6, 188, 28, 0, 1550, 396, 1, 0, 0, 0, 1551, 1552, 3, 73, 27, 0, 1552, 1553, 1, 0, 0, 0, 1553, 1554, 6, 189, 13, 0, 1554, 398, 1, 0, 0, 0, 1555, 1556, 3, 75, 28, 0, 1556, 1557, 1, 0, 0, 0, 1557, 1558, 6, 190, 13, 0, 1558, 400, 1, 0, 0, 0, 1559, 1560, 3, 77, 29, 0, 1560, 1561, 1, 0, 0, 0, 1561, 1562, 6, 191, 13, 0, 1562, 402, 1, 0, 0, 0, 1563, 1564, 3, 79, 30, 0, 1564, 1565, 1, 0, 0, 0, 1565, 1566, 6, 192, 18, 0, 1566, 1567, 6, 192, 14, 0, 1567, 404, 1, 0, 0, 0, 1568, 1569, 7, 35, 0, 0, 1569, 1570, 7, 7, 0, 0, 1570, 1571, 7, 1, 0, 0, 1571, 1572, 7, 9, 0, 0, 1572, 406, 1, 0, 0, 0, 1573, 1574, 3, 275, 128, 0, 1574, 1575, 1, 0, 0, 0, 1575, 1576, 6, 194, 37, 0, 1576, 408, 1, 0, 0, 0, 1577, 1578, 3, 289, 135, 0, 1578, 1579, 1, 0, 0, 0, 1579, 1580, 6, 195, 35, 0, 1580, 1581, 6, 195, 14, 0, 1581, 1582, 6, 195, 0, 0, 1582, 410, 1, 0, 0, 0, 1583, 1584, 7, 20, 0, 0, 1584, 1585, 7, 2, 0, 0, 1585, 1586, 7, 1, 0, 0, 1586, 1587, 7, 9, 0, 0, 1587, 1588, 7, 17, 0, 0, 1588, 1589, 1, 0, 0, 0, 1589, 1590, 6, 196, 14, 0, 1590, 1591, 6, 196, 0, 0, 1591, 412, 1, 0, 0, 0, 1592, 1593, 3, 229, 105, 0, 1593, 1594, 1, 0, 0, 0, 1594, 1595, 6, 197, 23, 0, 1595, 414, 1, 0, 0, 0, 1596, 1597, 3, 101, 41, 0, 1597, 1598, 1, 0, 0, 0, 1598, 1599, 6, 198, 24, 0, 1599, 416, 1, 0, 0, 0, 1600, 1601, 3, 117, 49, 0, 1601, 1602, 1, 0, 0, 0, 1602, 1603, 6, 199, 20, 0, 1603, 418, 1, 0, 0, 0, 1604, 1605, 3, 191, 86, 0, 1605, 1606, 1, 0, 0, 0, 1606, 1607, 6, 200, 34, 0, 1607, 420, 1, 0, 0, 0, 1608, 1609, 3, 195, 88, 0, 1609, 1610, 1, 0, 0, 0, 1610, 1611, 6, 201, 33, 0, 1611, 422, 1, 0, 0, 0, 1612, 1613, 3, 73, 27, 0, 1613, 1614, 1, 0, 0, 0, 1614, 1615, 6, 202, 13, 0, 1615, 424, 1, 0, 0, 0, 1616, 1617, 3, 75, 28, 0, 1617, 1618, 1, 0, 0, 0, 1618, 1619, 6, 203, 13, 0, 1619, 426, 1, 0, 0, 0, 1620, 1621, 3, 77, 29, 0, 1621, 1622, 1, 0, 0, 0, 1622, 1623, 6, 204, 13, 0, 1623, 428, 1, 0, 0, 0, 1624, 1625, 3, 79, 30, 0, 1625, 1626, 1, 0, 0, 0, 1626, 1627, 6, 205, 18, 0, 1627, 1628, 6, 205, 14, 0, 1628, 430, 1, 0, 0, 0, 1629, 1630, 3, 229, 105, 0, 1630, 1631, 1, 0, 0, 0, 1631, 1632, 6, 206, 23, 0, 1632, 1633, 6, 206, 14, 0, 1633, 1634, 6, 206, 38, 0, 1634, 432, 1, 0, 0, 0, 1635, 1636, 3, 101, 41, 0, 1636, 1637, 1, 0, 0, 0, 1637, 1638, 6, 207, 24, 0, 1638, 1639, 6, 207, 14, 0, 1639, 1640, 6, 207, 38, 0, 1640, 434, 1, 0, 0, 0, 1641, 1642, 3, 73, 27, 0, 1642, 1643, 1, 0, 0, 0, 1643, 1644, 6, 208, 13, 0, 1644, 436, 1, 0, 0, 0, 1645, 1646, 3, 75, 28, 0, 1646, 1647, 1, 0, 0, 0, 1647, 1648, 6, 209, 13, 0, 1648, 438, 1, 0, 0, 0, 1649, 1650, 3, 77, 29, 0, 1650, 1651, 1, 0, 0, 0, 1651, 1652, 6, 210, 13, 0, 1652, 440, 1, 0, 0, 0, 1653, 1654, 3, 117, 49, 0, 1654, 1655, 1, 0, 0, 0, 1655, 1656, 6, 211, 20, 0, 1656, 1657, 6, 211, 14, 0, 1657, 1658, 6, 211, 11, 0, 1658, 442, 1, 0, 0, 0, 1659, 1660, 3, 119, 50, 0, 1660, 1661, 1, 0, 0, 0, 1661, 1662, 6, 212, 21, 0, 1662, 1663, 6, 212, 14, 0, 1663, 1664, 6, 212, 11, 0, 1664, 444, 1, 0, 0, 0, 1665, 1666, 3, 73, 27, 0, 1666, 1667, 1, 0, 0, 0, 1667, 1668, 6, 213, 13, 0, 1668, 446, 1, 0, 0, 0, 1669, 1670, 3, 75, 28, 0, 1670, 1671, 1, 0, 0, 0, 1671, 1672, 6, 214, 13, 0, 1672, 448, 1, 0, 0, 0, 1673, 1674, 3, 77, 29, 0, 1674, 1675, 1, 0, 0, 0, 1675, 1676, 6, 215, 13, 0, 1676, 450, 1, 0, 0, 0, 1677, 1678, 3, 195, 88, 0, 1678, 1679, 1, 0, 0, 0, 1679, 1680, 6, 216, 14, 0, 1680, 1681, 6, 216, 0, 0, 1681, 1682, 6, 216, 33, 0, 1682, 452, 1, 0, 0, 0, 1683, 1684, 3, 191, 86, 0, 1684, 1685, 1, 0, 0, 0, 1685, 1686, 6, 217, 14, 0, 1686, 1687, 6, 217, 0, 0, 1687, 1688, 6, 217, 34, 0, 1688, 454, 1, 0, 0, 0, 1689, 1690, 3, 107, 44, 0, 1690, 1691, 1, 0, 0, 0, 1691, 1692, 6, 218, 14, 0, 1692, 1693, 6, 218, 0, 0, 1693, 1694, 6, 218, 39, 0, 1694, 456, 1, 0, 0, 0, 1695, 1696, 3, 79, 30, 0, 1696, 1697, 1, 0, 0, 0, 1697, 1698, 6, 219, 18, 0, 1698, 1699, 6, 219, 14, 0, 1699, 458, 1, 0, 0, 0, 1700, 1701, 3, 79, 30, 0, 1701, 1702, 1, 0, 0, 0, 1702, 1703, 6, 220, 18, 0, 1703, 1704, 6, 220, 14, 0, 1704, 460, 1, 0, 0, 0, 1705, 1706, 3, 289, 135, 0, 1706, 1707, 1, 0, 0, 0, 1707, 1708, 6, 221, 35, 0, 1708, 462, 1, 0, 0, 0, 1709, 1710, 3, 275, 128, 0, 1710, 1711, 1, 0, 0, 0, 1711, 1712, 6, 222, 37, 0, 1712, 464, 1, 0, 0, 0, 1713, 1714, 3, 123, 52, 0, 1714, 1715, 1, 0, 0, 0, 1715, 1716, 6, 223, 25, 0, 1716, 466, 1, 0, 0, 0, 1717, 1718, 3, 119, 50, 0, 1718, 1719, 1, 0, 0, 0, 1719, 1720, 6, 224, 21, 0, 1720, 468, 1, 0, 0, 0, 1721, 1722, 3, 195, 88, 0, 1722, 1723, 1, 0, 0, 0, 1723, 1724, 6, 225, 33, 0, 1724, 470, 1, 0, 0, 0, 1725, 1726, 3, 191, 86, 0, 1726, 1727, 1, 0, 0, 0, 1727, 1728, 6, 226, 34, 0, 1728, 472, 1, 0, 0, 0, 1729, 1730, 3, 73, 27, 0, 1730, 1731, 1, 0, 0, 0, 1731, 1732, 6, 227, 13, 0, 1732, 474, 1, 0, 0, 0, 1733, 1734, 3, 75, 28, 0, 1734, 1735, 1, 0, 0, 0, 1735, 1736, 6, 228, 13, 0, 1736, 476, 1, 0, 0, 0, 1737, 1738, 3, 77, 29, 0, 1738, 1739, 1, 0, 0, 0, 1739, 1740, 6, 229, 13, 0, 1740, 478, 1, 0, 0, 0, 1741, 1742, 3, 79, 30, 0, 1742, 1743, 1, 0, 0, 0, 1743, 1744, 6, 230, 18, 0, 1744, 1745, 6, 230, 14, 0, 1745, 480, 1, 0, 0, 0, 1746, 1747, 3, 191, 86, 0, 1747, 1748, 1, 0, 0, 0, 1748, 1749, 6, 231, 34, 0, 1749, 482, 1, 0, 0, 0, 1750, 1751, 3, 77, 29, 0, 1751, 1752, 1, 0, 0, 0, 1752, 1753, 6, 232, 13, 0, 1753, 484, 1, 0, 0, 0, 1754, 1755, 3, 73, 27, 0, 1755, 1756, 1, 0, 0, 0, 1756, 1757, 6, 233, 13, 0, 1757, 486, 1, 0, 0, 0, 1758, 1759, 3, 75, 28, 0, 1759, 1760, 1, 0, 0, 0, 1760, 1761, 6, 234, 13, 0, 1761, 488, 1, 0, 0, 0, 1762, 1763, 3, 187, 84, 0, 1763, 1764, 1, 0, 0, 0, 1764, 1765, 6, 235, 40, 0, 1765, 1766, 6, 235, 17, 0, 1766, 490, 1, 0, 0, 0, 1767, 1768, 3, 79, 30, 0, 1768, 1769, 1, 0, 0, 0, 1769, 1770, 6, 236, 18, 0, 1770, 1771, 6, 236, 14, 0, 1771, 492, 1, 0, 0, 0, 1772, 1773, 3, 77, 29, 0, 1773, 1774, 1, 0, 0, 0, 1774, 1775, 6, 237, 13, 0, 1775, 494, 1, 0, 0, 0, 1776, 1777, 3, 73, 27, 0, 1777, 1778, 1, 0, 0, 0, 1778, 1779, 6, 238, 13, 0, 1779, 496, 1, 0, 0, 0, 1780, 1781, 3, 75, 28, 0, 1781, 1782, 1, 0, 0, 0, 1782, 1783, 6, 239, 13, 0, 1783, 498, 1, 0, 0, 0, 69, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 739, 749, 753, 756, 765, 767, 778, 797, 802, 811, 818, 823, 825, 836, 844, 847, 849, 854, 859, 865, 872, 877, 883, 886, 894, 898, 1026, 1031, 1038, 1040, 1066, 1071, 1076, 1078, 1084, 1161, 1166, 1213, 1217, 1222, 1227, 1232, 1234, 1238, 1240, 1325, 1329, 1334, 1475, 1477, 41, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 13, 0, 5, 16, 0, 5, 11, 0, 5, 14, 0, 5, 18, 0, 0, 1, 0, 4, 0, 0, 7, 16, 0, 7, 72, 0, 5, 0, 0, 7, 31, 0, 7, 73, 0, 7, 40, 0, 7, 41, 0, 7, 38, 0, 7, 85, 0, 7, 32, 0, 7, 43, 0, 7, 54, 0, 7, 71, 0, 7, 89, 0, 5, 10, 0, 5, 7, 0, 7, 99, 0, 7, 98, 0, 7, 77, 0, 7, 76, 0, 7, 97, 0, 5, 12, 0, 7, 93, 0, 5, 15, 0, 7, 35, 0, 7, 74, 0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index 08c17791963c2..16e0455c6b459 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -32,21 +32,21 @@ public class EsqlBaseLexer extends LexerConfig { DEV_FORK=26, UNKNOWN_CMD=27, LINE_COMMENT=28, MULTILINE_COMMENT=29, WS=30, PIPE=31, QUOTED_STRING=32, INTEGER_LITERAL=33, DECIMAL_LITERAL=34, BY=35, AND=36, ASC=37, ASSIGN=38, CAST_OP=39, COLON=40, COMMA=41, DESC=42, DOT=43, - FALSE=44, FIRST=45, IN=46, IS=47, LAST=48, LIKE=49, LP=50, NOT=51, NULL=52, - NULLS=53, OR=54, PARAM=55, RLIKE=56, RP=57, TRUE=58, EQ=59, CIEQ=60, NEQ=61, - LT=62, LTE=63, GT=64, GTE=65, PLUS=66, MINUS=67, ASTERISK=68, SLASH=69, - PERCENT=70, LEFT_BRACES=71, RIGHT_BRACES=72, NAMED_OR_POSITIONAL_PARAM=73, - OPENING_BRACKET=74, CLOSING_BRACKET=75, UNQUOTED_IDENTIFIER=76, QUOTED_IDENTIFIER=77, - EXPR_LINE_COMMENT=78, EXPR_MULTILINE_COMMENT=79, EXPR_WS=80, EXPLAIN_WS=81, - EXPLAIN_LINE_COMMENT=82, EXPLAIN_MULTILINE_COMMENT=83, METADATA=84, UNQUOTED_SOURCE=85, - FROM_LINE_COMMENT=86, FROM_MULTILINE_COMMENT=87, FROM_WS=88, ID_PATTERN=89, - PROJECT_LINE_COMMENT=90, PROJECT_MULTILINE_COMMENT=91, PROJECT_WS=92, - AS=93, RENAME_LINE_COMMENT=94, RENAME_MULTILINE_COMMENT=95, RENAME_WS=96, - ON=97, WITH=98, ENRICH_POLICY_NAME=99, ENRICH_LINE_COMMENT=100, ENRICH_MULTILINE_COMMENT=101, - ENRICH_WS=102, ENRICH_FIELD_LINE_COMMENT=103, ENRICH_FIELD_MULTILINE_COMMENT=104, - ENRICH_FIELD_WS=105, MVEXPAND_LINE_COMMENT=106, MVEXPAND_MULTILINE_COMMENT=107, - MVEXPAND_WS=108, INFO=109, SHOW_LINE_COMMENT=110, SHOW_MULTILINE_COMMENT=111, - SHOW_WS=112, SETTING=113, SETTING_LINE_COMMENT=114, SETTTING_MULTILINE_COMMENT=115, + FALSE=44, FIRST=45, IN=46, IS=47, LAST=48, LIKE=49, NOT=50, NULL=51, NULLS=52, + OR=53, PARAM=54, RLIKE=55, TRUE=56, EQ=57, CIEQ=58, NEQ=59, LT=60, LTE=61, + GT=62, GTE=63, PLUS=64, MINUS=65, ASTERISK=66, SLASH=67, PERCENT=68, LEFT_BRACES=69, + RIGHT_BRACES=70, NAMED_OR_POSITIONAL_PARAM=71, OPENING_BRACKET=72, CLOSING_BRACKET=73, + LP=74, RP=75, UNQUOTED_IDENTIFIER=76, QUOTED_IDENTIFIER=77, EXPR_LINE_COMMENT=78, + EXPR_MULTILINE_COMMENT=79, EXPR_WS=80, EXPLAIN_WS=81, EXPLAIN_LINE_COMMENT=82, + EXPLAIN_MULTILINE_COMMENT=83, METADATA=84, UNQUOTED_SOURCE=85, FROM_LINE_COMMENT=86, + FROM_MULTILINE_COMMENT=87, FROM_WS=88, ID_PATTERN=89, PROJECT_LINE_COMMENT=90, + PROJECT_MULTILINE_COMMENT=91, PROJECT_WS=92, AS=93, RENAME_LINE_COMMENT=94, + RENAME_MULTILINE_COMMENT=95, RENAME_WS=96, ON=97, WITH=98, ENRICH_POLICY_NAME=99, + ENRICH_LINE_COMMENT=100, ENRICH_MULTILINE_COMMENT=101, ENRICH_WS=102, + ENRICH_FIELD_LINE_COMMENT=103, ENRICH_FIELD_MULTILINE_COMMENT=104, ENRICH_FIELD_WS=105, + MVEXPAND_LINE_COMMENT=106, MVEXPAND_MULTILINE_COMMENT=107, MVEXPAND_WS=108, + INFO=109, SHOW_LINE_COMMENT=110, SHOW_MULTILINE_COMMENT=111, SHOW_WS=112, + SETTING=113, SETTING_LINE_COMMENT=114, SETTTING_MULTILINE_COMMENT=115, SETTING_WS=116, LOOKUP_LINE_COMMENT=117, LOOKUP_MULTILINE_COMMENT=118, LOOKUP_WS=119, LOOKUP_FIELD_LINE_COMMENT=120, LOOKUP_FIELD_MULTILINE_COMMENT=121, LOOKUP_FIELD_WS=122, JOIN=123, USING=124, JOIN_LINE_COMMENT=125, JOIN_MULTILINE_COMMENT=126, @@ -82,18 +82,17 @@ private static String[] makeRuleNames() { "BACKQUOTE", "BACKQUOTE_BLOCK", "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COLON", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", - "LIKE", "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", - "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", - "SLASH", "PERCENT", "LEFT_BRACES", "RIGHT_BRACES", "NESTED_WHERE", "NESTED_SORT", - "NESTED_LIMIT", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", - "UNQUOTED_IDENTIFIER", "QUOTED_ID", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", - "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_OPENING_BRACKET", "EXPLAIN_PIPE", - "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", "FROM_PIPE", - "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", "FROM_COLON", "FROM_COMMA", - "FROM_ASSIGN", "METADATA", "UNQUOTED_SOURCE_PART", "UNQUOTED_SOURCE", - "FROM_UNQUOTED_SOURCE", "FROM_QUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", - "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", "PROJECT_PARAM", - "PROJECT_NAMED_OR_POSITIONAL_PARAM", "UNQUOTED_ID_BODY_WITH_PATTERN", + "LIKE", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "TRUE", "EQ", + "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", + "SLASH", "PERCENT", "LEFT_BRACES", "RIGHT_BRACES", "NESTED_WHERE", "NAMED_OR_POSITIONAL_PARAM", + "OPENING_BRACKET", "CLOSING_BRACKET", "LP", "RP", "UNQUOTED_IDENTIFIER", + "QUOTED_ID", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", + "EXPR_WS", "EXPLAIN_OPENING_BRACKET", "EXPLAIN_PIPE", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", + "EXPLAIN_MULTILINE_COMMENT", "FROM_PIPE", "FROM_OPENING_BRACKET", "FROM_CLOSING_BRACKET", + "FROM_COLON", "FROM_COMMA", "FROM_ASSIGN", "METADATA", "UNQUOTED_SOURCE_PART", + "UNQUOTED_SOURCE", "FROM_UNQUOTED_SOURCE", "FROM_QUOTED_SOURCE", "FROM_LINE_COMMENT", + "FROM_MULTILINE_COMMENT", "FROM_WS", "PROJECT_PIPE", "PROJECT_DOT", "PROJECT_COMMA", + "PROJECT_PARAM", "PROJECT_NAMED_OR_POSITIONAL_PARAM", "UNQUOTED_ID_BODY_WITH_PATTERN", "UNQUOTED_ID_PATTERN", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "RENAME_PIPE", "RENAME_ASSIGN", "RENAME_COMMA", "RENAME_DOT", "RENAME_PARAM", "RENAME_NAMED_OR_POSITIONAL_PARAM", "AS", "RENAME_ID_PATTERN", @@ -125,8 +124,8 @@ private static String[] makeRuleNames() { "CHANGE_POINT_COMMA", "CHANGE_POINT_QUOTED_IDENTIFIER", "CHANGE_POINT_UNQUOTED_IDENTIFIER", "CHANGE_POINT_LINE_COMMENT", "CHANGE_POINT_MULTILINE_COMMENT", "CHANGE_POINT_WS", "INSIST_PIPE", "INSIST_IDENTIFIER", "INSIST_WS", "INSIST_LINE_COMMENT", - "INSIST_MULTILINE_COMMENT", "FORK_LP", "FORK_RP", "FORK_PIPE", "FORK_WS", - "FORK_LINE_COMMENT", "FORK_MULTILINE_COMMENT" + "INSIST_MULTILINE_COMMENT", "FORK_LP", "FORK_PIPE", "FORK_WS", "FORK_LINE_COMMENT", + "FORK_MULTILINE_COMMENT" }; } public static final String[] ruleNames = makeRuleNames(); @@ -138,10 +137,10 @@ private static String[] makeLiteralNames() { "'sort'", "'stats'", "'where'", "'lookup'", null, null, null, null, null, null, null, null, null, null, null, null, null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "':'", "','", "'desc'", "'.'", - "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", "'('", "'not'", - "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", "'true'", "'=='", - "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", - "'%'", "'{'", "'}'", null, null, "']'", null, null, null, null, null, + "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", "'not'", "'null'", + "'nulls'", "'or'", "'?'", "'rlike'", "'true'", "'=='", "'=~'", "'!='", + "'<'", "'<='", "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", "'{'", + "'}'", null, null, "']'", null, "')'", null, null, null, null, null, null, null, null, "'metadata'", null, null, null, null, null, null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, null, null, null, null, null, null, null, null, "'info'", null, null, null, null, @@ -158,17 +157,17 @@ private static String[] makeSymbolicNames() { "DEV_FORK", "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COLON", "COMMA", "DESC", "DOT", "FALSE", - "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", "NULL", "NULLS", "OR", - "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", - "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "LEFT_BRACES", - "RIGHT_BRACES", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", - "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", - "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", - "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", - "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", - "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", - "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", "ENRICH_LINE_COMMENT", - "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", + "FIRST", "IN", "IS", "LAST", "LIKE", "NOT", "NULL", "NULLS", "OR", "PARAM", + "RLIKE", "TRUE", "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", + "MINUS", "ASTERISK", "SLASH", "PERCENT", "LEFT_BRACES", "RIGHT_BRACES", + "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", "LP", + "RP", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", + "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", + "EXPLAIN_MULTILINE_COMMENT", "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", + "FROM_MULTILINE_COMMENT", "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", + "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", + "RENAME_MULTILINE_COMMENT", "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", + "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", "SHOW_WS", "SETTING", "SETTING_LINE_COMMENT", @@ -262,10 +261,6 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { return DEV_JOIN_RIGHT_sempred((RuleContext)_localctx, predIndex); case 25: return DEV_FORK_sempred((RuleContext)_localctx, predIndex); - case 83: - return NESTED_SORT_sempred((RuleContext)_localctx, predIndex); - case 84: - return NESTED_LIMIT_sempred((RuleContext)_localctx, predIndex); } return true; } @@ -332,23 +327,9 @@ private boolean DEV_FORK_sempred(RuleContext _localctx, int predIndex) { } return true; } - private boolean NESTED_SORT_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 9: - return this.isDevVersion(); - } - return true; - } - private boolean NESTED_LIMIT_sempred(RuleContext _localctx, int predIndex) { - switch (predIndex) { - case 10: - return this.isDevVersion(); - } - return true; - } public static final String _serializedATN = - "\u0004\u0000\u008e\u0707\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ + "\u0004\u0000\u008e\u06f8\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ "\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff"+ @@ -420,7 +401,6 @@ private boolean NESTED_LIMIT_sempred(RuleContext _localctx, int predIndex) { "\u0002\u00e7\u0007\u00e7\u0002\u00e8\u0007\u00e8\u0002\u00e9\u0007\u00e9"+ "\u0002\u00ea\u0007\u00ea\u0002\u00eb\u0007\u00eb\u0002\u00ec\u0007\u00ec"+ "\u0002\u00ed\u0007\u00ed\u0002\u00ee\u0007\u00ee\u0002\u00ef\u0007\u00ef"+ - "\u0002\u00f0\u0007\u00f0\u0002\u00f1\u0007\u00f1\u0002\u00f2\u0007\u00f2"+ "\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000"+ "\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001"+ "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0002"+ @@ -458,1071 +438,1061 @@ private boolean NESTED_LIMIT_sempred(RuleContext _localctx, int predIndex) { "\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018"+ "\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019"+ "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019"+ - "\u0001\u0019\u0001\u001a\u0004\u001a\u02e8\b\u001a\u000b\u001a\f\u001a"+ - "\u02e9\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001"+ - "\u001b\u0005\u001b\u02f2\b\u001b\n\u001b\f\u001b\u02f5\t\u001b\u0001\u001b"+ - "\u0003\u001b\u02f8\b\u001b\u0001\u001b\u0003\u001b\u02fb\b\u001b\u0001"+ + "\u0001\u0019\u0001\u001a\u0004\u001a\u02e2\b\u001a\u000b\u001a\f\u001a"+ + "\u02e3\u0001\u001a\u0001\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0001"+ + "\u001b\u0005\u001b\u02ec\b\u001b\n\u001b\f\u001b\u02ef\t\u001b\u0001\u001b"+ + "\u0003\u001b\u02f2\b\u001b\u0001\u001b\u0003\u001b\u02f5\b\u001b\u0001"+ "\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001"+ - "\u001c\u0005\u001c\u0304\b\u001c\n\u001c\f\u001c\u0307\t\u001c\u0001\u001c"+ + "\u001c\u0005\u001c\u02fe\b\u001c\n\u001c\f\u001c\u0301\t\u001c\u0001\u001c"+ "\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001d\u0004\u001d"+ - "\u030f\b\u001d\u000b\u001d\f\u001d\u0310\u0001\u001d\u0001\u001d\u0001"+ + "\u0309\b\u001d\u000b\u001d\f\u001d\u030a\u0001\u001d\u0001\u001d\u0001"+ "\u001e\u0001\u001e\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001"+ - " \u0001 \u0001!\u0001!\u0001!\u0001\"\u0001\"\u0001#\u0001#\u0003#\u0324"+ - "\b#\u0001#\u0004#\u0327\b#\u000b#\f#\u0328\u0001$\u0001$\u0001%\u0001"+ - "%\u0001&\u0001&\u0001&\u0003&\u0332\b&\u0001\'\u0001\'\u0001(\u0001(\u0001"+ - "(\u0003(\u0339\b(\u0001)\u0001)\u0001)\u0005)\u033e\b)\n)\f)\u0341\t)"+ - "\u0001)\u0001)\u0001)\u0001)\u0001)\u0001)\u0005)\u0349\b)\n)\f)\u034c"+ - "\t)\u0001)\u0001)\u0001)\u0001)\u0001)\u0003)\u0353\b)\u0001)\u0003)\u0356"+ - "\b)\u0003)\u0358\b)\u0001*\u0004*\u035b\b*\u000b*\f*\u035c\u0001+\u0004"+ - "+\u0360\b+\u000b+\f+\u0361\u0001+\u0001+\u0005+\u0366\b+\n+\f+\u0369\t"+ - "+\u0001+\u0001+\u0004+\u036d\b+\u000b+\f+\u036e\u0001+\u0004+\u0372\b"+ - "+\u000b+\f+\u0373\u0001+\u0001+\u0005+\u0378\b+\n+\f+\u037b\t+\u0003+"+ - "\u037d\b+\u0001+\u0001+\u0001+\u0001+\u0004+\u0383\b+\u000b+\f+\u0384"+ - "\u0001+\u0001+\u0003+\u0389\b+\u0001,\u0001,\u0001,\u0001-\u0001-\u0001"+ + " \u0001 \u0001!\u0001!\u0001!\u0001\"\u0001\"\u0001#\u0001#\u0003#\u031e"+ + "\b#\u0001#\u0004#\u0321\b#\u000b#\f#\u0322\u0001$\u0001$\u0001%\u0001"+ + "%\u0001&\u0001&\u0001&\u0003&\u032c\b&\u0001\'\u0001\'\u0001(\u0001(\u0001"+ + "(\u0003(\u0333\b(\u0001)\u0001)\u0001)\u0005)\u0338\b)\n)\f)\u033b\t)"+ + "\u0001)\u0001)\u0001)\u0001)\u0001)\u0001)\u0005)\u0343\b)\n)\f)\u0346"+ + "\t)\u0001)\u0001)\u0001)\u0001)\u0001)\u0003)\u034d\b)\u0001)\u0003)\u0350"+ + "\b)\u0003)\u0352\b)\u0001*\u0004*\u0355\b*\u000b*\f*\u0356\u0001+\u0004"+ + "+\u035a\b+\u000b+\f+\u035b\u0001+\u0001+\u0005+\u0360\b+\n+\f+\u0363\t"+ + "+\u0001+\u0001+\u0004+\u0367\b+\u000b+\f+\u0368\u0001+\u0004+\u036c\b"+ + "+\u000b+\f+\u036d\u0001+\u0001+\u0005+\u0372\b+\n+\f+\u0375\t+\u0003+"+ + "\u0377\b+\u0001+\u0001+\u0001+\u0001+\u0004+\u037d\b+\u000b+\f+\u037e"+ + "\u0001+\u0001+\u0003+\u0383\b+\u0001,\u0001,\u0001,\u0001-\u0001-\u0001"+ "-\u0001-\u0001.\u0001.\u0001.\u0001.\u0001/\u0001/\u00010\u00010\u0001"+ "0\u00011\u00011\u00012\u00012\u00013\u00013\u00013\u00013\u00013\u0001"+ "4\u00014\u00015\u00015\u00015\u00015\u00015\u00015\u00016\u00016\u0001"+ "6\u00016\u00016\u00016\u00017\u00017\u00017\u00018\u00018\u00018\u0001"+ "9\u00019\u00019\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0001:\u0001"+ - ";\u0001;\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001=\u0001"+ - "=\u0001>\u0001>\u0001>\u0001>\u0001>\u0001>\u0001?\u0001?\u0001?\u0001"+ - "@\u0001@\u0001A\u0001A\u0001A\u0001A\u0001A\u0001A\u0001B\u0001B\u0001"+ - "C\u0001C\u0001C\u0001C\u0001C\u0001D\u0001D\u0001D\u0001E\u0001E\u0001"+ - "E\u0001F\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001H\u0001I\u0001"+ - "I\u0001J\u0001J\u0001J\u0001K\u0001K\u0001L\u0001L\u0001M\u0001M\u0001"+ - "N\u0001N\u0001O\u0001O\u0001P\u0001P\u0001Q\u0001Q\u0001R\u0001R\u0001"+ - "R\u0001R\u0001S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001"+ - "T\u0001T\u0001U\u0001U\u0001U\u0003U\u0417\bU\u0001U\u0005U\u041a\bU\n"+ - "U\fU\u041d\tU\u0001U\u0001U\u0004U\u0421\bU\u000bU\fU\u0422\u0003U\u0425"+ - "\bU\u0001V\u0001V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001W\u0001W\u0001"+ - "W\u0001X\u0001X\u0005X\u0433\bX\nX\fX\u0436\tX\u0001X\u0001X\u0003X\u043a"+ - "\bX\u0001X\u0004X\u043d\bX\u000bX\fX\u043e\u0003X\u0441\bX\u0001Y\u0001"+ - "Y\u0004Y\u0445\bY\u000bY\fY\u0446\u0001Y\u0001Y\u0001Z\u0001Z\u0001[\u0001"+ - "[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001"+ - "]\u0001^\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001_\u0001"+ - "_\u0001`\u0001`\u0001`\u0001`\u0001a\u0001a\u0001a\u0001a\u0001b\u0001"+ - "b\u0001b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001"+ - "d\u0001d\u0001e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001"+ - "g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001i\u0001i\u0001"+ - "i\u0001i\u0001i\u0001i\u0001i\u0001i\u0001i\u0001j\u0001j\u0001j\u0003"+ - "j\u0494\bj\u0001k\u0004k\u0497\bk\u000bk\fk\u0498\u0001l\u0001l\u0001"+ - "l\u0001l\u0001m\u0001m\u0001m\u0001m\u0001n\u0001n\u0001n\u0001n\u0001"+ - "o\u0001o\u0001o\u0001o\u0001p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001"+ - "q\u0001q\u0001q\u0001r\u0001r\u0001r\u0001r\u0001s\u0001s\u0001s\u0001"+ - "s\u0001t\u0001t\u0001t\u0001t\u0001u\u0001u\u0001u\u0001u\u0001v\u0001"+ - "v\u0001v\u0001v\u0003v\u04c8\bv\u0001w\u0001w\u0003w\u04cc\bw\u0001w\u0005"+ - "w\u04cf\bw\nw\fw\u04d2\tw\u0001w\u0001w\u0003w\u04d6\bw\u0001w\u0004w"+ - "\u04d9\bw\u000bw\fw\u04da\u0003w\u04dd\bw\u0001x\u0001x\u0004x\u04e1\b"+ - "x\u000bx\fx\u04e2\u0001y\u0001y\u0001y\u0001y\u0001z\u0001z\u0001z\u0001"+ - "z\u0001{\u0001{\u0001{\u0001{\u0001|\u0001|\u0001|\u0001|\u0001|\u0001"+ - "}\u0001}\u0001}\u0001}\u0001~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f"+ - "\u0001\u007f\u0001\u007f\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0080"+ - "\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0081\u0001\u0082\u0001\u0082"+ - "\u0001\u0082\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084"+ - "\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085"+ - "\u0001\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0087"+ + ";\u0001;\u0001;\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001"+ + "=\u0001=\u0001=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001?\u0001"+ + "@\u0001@\u0001@\u0001@\u0001@\u0001@\u0001A\u0001A\u0001A\u0001A\u0001"+ + "A\u0001B\u0001B\u0001B\u0001C\u0001C\u0001C\u0001D\u0001D\u0001D\u0001"+ + "E\u0001E\u0001F\u0001F\u0001F\u0001G\u0001G\u0001H\u0001H\u0001H\u0001"+ + "I\u0001I\u0001J\u0001J\u0001K\u0001K\u0001L\u0001L\u0001M\u0001M\u0001"+ + "N\u0001N\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001Q\u0001Q\u0001"+ + "Q\u0003Q\u0403\bQ\u0001Q\u0005Q\u0406\bQ\nQ\fQ\u0409\tQ\u0001Q\u0001Q"+ + "\u0004Q\u040d\bQ\u000bQ\fQ\u040e\u0003Q\u0411\bQ\u0001R\u0001R\u0001R"+ + "\u0001R\u0001R\u0001S\u0001S\u0001S\u0001S\u0001S\u0001T\u0001T\u0001"+ + "T\u0001T\u0001T\u0001U\u0001U\u0001U\u0001U\u0001U\u0001V\u0001V\u0005"+ + "V\u0429\bV\nV\fV\u042c\tV\u0001V\u0001V\u0003V\u0430\bV\u0001V\u0004V"+ + "\u0433\bV\u000bV\fV\u0434\u0003V\u0437\bV\u0001W\u0001W\u0004W\u043b\b"+ + "W\u000bW\fW\u043c\u0001W\u0001W\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001"+ + "Y\u0001Z\u0001Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001"+ + "\\\u0001\\\u0001\\\u0001\\\u0001]\u0001]\u0001]\u0001]\u0001]\u0001^\u0001"+ + "^\u0001^\u0001^\u0001_\u0001_\u0001_\u0001_\u0001`\u0001`\u0001`\u0001"+ + "`\u0001a\u0001a\u0001a\u0001a\u0001a\u0001b\u0001b\u0001b\u0001b\u0001"+ + "c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001d\u0001e\u0001e\u0001"+ + "e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001g\u0001g\u0001g\u0001g\u0001"+ + "g\u0001g\u0001g\u0001g\u0001g\u0001h\u0001h\u0001h\u0003h\u048a\bh\u0001"+ + "i\u0004i\u048d\bi\u000bi\fi\u048e\u0001j\u0001j\u0001j\u0001j\u0001k\u0001"+ + "k\u0001k\u0001k\u0001l\u0001l\u0001l\u0001l\u0001m\u0001m\u0001m\u0001"+ + "m\u0001n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001o\u0001"+ + "p\u0001p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001"+ + "r\u0001r\u0001s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001t\u0003"+ + "t\u04be\bt\u0001u\u0001u\u0003u\u04c2\bu\u0001u\u0005u\u04c5\bu\nu\fu"+ + "\u04c8\tu\u0001u\u0001u\u0003u\u04cc\bu\u0001u\u0004u\u04cf\bu\u000bu"+ + "\fu\u04d0\u0003u\u04d3\bu\u0001v\u0001v\u0004v\u04d7\bv\u000bv\fv\u04d8"+ + "\u0001w\u0001w\u0001w\u0001w\u0001x\u0001x\u0001x\u0001x\u0001y\u0001"+ + "y\u0001y\u0001y\u0001z\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001"+ + "{\u0001{\u0001|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001}\u0001}\u0001"+ + "~\u0001~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f"+ + "\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0081\u0001\u0081\u0001\u0081"+ + "\u0001\u0081\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0083"+ + "\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084\u0001\u0084\u0001\u0084"+ + "\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085\u0001\u0085"+ + "\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0087"+ "\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001\u0088"+ - "\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089"+ - "\u0001\u0089\u0001\u0089\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008a"+ - "\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001\u008c"+ - "\u0004\u008c\u0536\b\u008c\u000b\u008c\f\u008c\u0537\u0001\u008c\u0001"+ - "\u008c\u0003\u008c\u053c\b\u008c\u0001\u008c\u0004\u008c\u053f\b\u008c"+ - "\u000b\u008c\f\u008c\u0540\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d"+ - "\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f"+ + "\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0088\u0001\u0089"+ + "\u0001\u0089\u0001\u008a\u0004\u008a\u052c\b\u008a\u000b\u008a\f\u008a"+ + "\u052d\u0001\u008a\u0001\u008a\u0003\u008a\u0532\b\u008a\u0001\u008a\u0004"+ + "\u008a\u0535\b\u008a\u000b\u008a\f\u008a\u0536\u0001\u008b\u0001\u008b"+ + "\u0001\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c"+ + "\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001\u008e"+ + "\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f"+ "\u0001\u008f\u0001\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090"+ - "\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091"+ - "\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0093\u0001\u0093"+ - "\u0001\u0093\u0001\u0093\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0094"+ - "\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0095\u0001\u0096\u0001\u0096"+ - "\u0001\u0096\u0001\u0096\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0097"+ - "\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0099\u0001\u0099"+ - "\u0001\u0099\u0001\u0099\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a"+ - "\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c"+ - "\u0001\u009c\u0001\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d"+ - "\u0001\u009d\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009f"+ - "\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u00a0\u0001\u00a0\u0001\u00a0"+ - "\u0001\u00a0\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a2"+ - "\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a3\u0001\u00a3\u0001\u00a3"+ - "\u0001\u00a3\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a5"+ + "\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092"+ + "\u0001\u0092\u0001\u0092\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093"+ + "\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001\u0095"+ + "\u0001\u0095\u0001\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096"+ + "\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098"+ + "\u0001\u0098\u0001\u0098\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099"+ + "\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009b\u0001\u009b"+ + "\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c"+ + "\u0001\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009e"+ + "\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001\u009f"+ + "\u0001\u009f\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a1"+ + "\u0001\u00a1\u0001\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2"+ + "\u0001\u00a2\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4"+ + "\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a5\u0001\u00a5"+ "\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001\u00a6\u0001\u00a6\u0001\u00a6"+ - "\u0001\u00a6\u0001\u00a6\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a7"+ - "\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a9"+ - "\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00aa\u0001\u00aa\u0001\u00aa"+ - "\u0001\u00aa\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ab"+ - "\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ad\u0001\u00ad"+ - "\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0004\u00ad\u05ce\b\u00ad\u000b\u00ad"+ - "\f\u00ad\u05cf\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00af"+ - "\u0001\u00af\u0001\u00af\u0001\u00af\u0001\u00b0\u0001\u00b0\u0001\u00b0"+ - "\u0001\u00b0\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1"+ - "\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b3\u0001\u00b3"+ - "\u0001\u00b3\u0001\u00b3\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b4"+ - "\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b6"+ - "\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b7\u0001\u00b7\u0001\u00b7"+ - "\u0001\u00b7\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9"+ - "\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001\u00ba\u0001\u00ba"+ - "\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb"+ - "\u0001\u00bb\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd"+ - "\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00be\u0001\u00be\u0001\u00be"+ - "\u0001\u00be\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0"+ - "\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001\u00c1"+ - "\u0001\u00c1\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2"+ - "\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c4"+ - "\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c5\u0001\u00c5\u0001\u00c5"+ - "\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001\u00c6\u0001\u00c6\u0001\u00c6"+ - "\u0001\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c6"+ - "\u0001\u00c7\u0001\u00c7\u0001\u00c7\u0001\u00c7\u0001\u00c8\u0001\u00c8"+ - "\u0001\u00c8\u0001\u00c8\u0001\u00c9\u0001\u00c9\u0001\u00c9\u0001\u00c9"+ - "\u0001\u00ca\u0001\u00ca\u0001\u00ca\u0001\u00ca\u0001\u00cb\u0001\u00cb"+ - "\u0001\u00cb\u0001\u00cb\u0001\u00cc\u0001\u00cc\u0001\u00cc\u0001\u00cc"+ - "\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00ce\u0001\u00ce"+ - "\u0001\u00ce\u0001\u00ce\u0001\u00cf\u0001\u00cf\u0001\u00cf\u0001\u00cf"+ - "\u0001\u00cf\u0001\u00d0\u0001\u00d0\u0001\u00d0\u0001\u00d0\u0001\u00d0"+ - "\u0001\u00d0\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001\u00d1"+ - "\u0001\u00d1\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d3"+ - "\u0001\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d4\u0001\u00d4\u0001\u00d4"+ - "\u0001\u00d4\u0001\u00d5\u0001\u00d5\u0001\u00d5\u0001\u00d5\u0001\u00d5"+ - "\u0001\u00d5\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001\u00d6"+ - "\u0001\u00d6\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d8"+ - "\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d9\u0001\u00d9\u0001\u00d9"+ - "\u0001\u00d9\u0001\u00da\u0001\u00da\u0001\u00da\u0001\u00da\u0001\u00da"+ - "\u0001\u00da\u0001\u00db\u0001\u00db\u0001\u00db\u0001\u00db\u0001\u00db"+ - "\u0001\u00db\u0001\u00dc\u0001\u00dc\u0001\u00dc\u0001\u00dc\u0001\u00dc"+ - "\u0001\u00dc\u0001\u00dd\u0001\u00dd\u0001\u00dd\u0001\u00dd\u0001\u00dd"+ - "\u0001\u00de\u0001\u00de\u0001\u00de\u0001\u00de\u0001\u00de\u0001\u00df"+ - "\u0001\u00df\u0001\u00df\u0001\u00df\u0001\u00e0\u0001\u00e0\u0001\u00e0"+ - "\u0001\u00e0\u0001\u00e1\u0001\u00e1\u0001\u00e1\u0001\u00e1\u0001\u00e2"+ - "\u0001\u00e2\u0001\u00e2\u0001\u00e2\u0001\u00e3\u0001\u00e3\u0001\u00e3"+ - "\u0001\u00e3\u0001\u00e4\u0001\u00e4\u0001\u00e4\u0001\u00e4\u0001\u00e5"+ - "\u0001\u00e5\u0001\u00e5\u0001\u00e5\u0001\u00e6\u0001\u00e6\u0001\u00e6"+ - "\u0001\u00e6\u0001\u00e7\u0001\u00e7\u0001\u00e7\u0001\u00e7\u0001\u00e8"+ - "\u0001\u00e8\u0001\u00e8\u0001\u00e8\u0001\u00e8\u0001\u00e9\u0001\u00e9"+ - "\u0001\u00e9\u0001\u00e9\u0001\u00ea\u0001\u00ea\u0001\u00ea\u0001\u00ea"+ - "\u0001\u00eb\u0001\u00eb\u0001\u00eb\u0001\u00eb\u0001\u00ec\u0001\u00ec"+ - "\u0001\u00ec\u0001\u00ec\u0001\u00ed\u0001\u00ed\u0001\u00ed\u0001\u00ed"+ - "\u0001\u00ed\u0001\u00ee\u0001\u00ee\u0001\u00ee\u0001\u00ee\u0001\u00ee"+ - "\u0001\u00ef\u0001\u00ef\u0001\u00ef\u0001\u00ef\u0001\u00ef\u0001\u00f0"+ - "\u0001\u00f0\u0001\u00f0\u0001\u00f0\u0001\u00f1\u0001\u00f1\u0001\u00f1"+ - "\u0001\u00f1\u0001\u00f2\u0001\u00f2\u0001\u00f2\u0001\u00f2\u0002\u0305"+ - "\u034a\u0000\u00f3\u0013\u0001\u0015\u0002\u0017\u0003\u0019\u0004\u001b"+ - "\u0005\u001d\u0006\u001f\u0007!\b#\t%\n\'\u000b)\f+\r-\u000e/\u000f1\u0010"+ - "3\u00115\u00127\u00139\u0014;\u0015=\u0016?\u0017A\u0018C\u0019E\u001a"+ - "G\u001bI\u001cK\u001dM\u001eO\u001fQ\u0000S\u0000U\u0000W\u0000Y\u0000"+ - "[\u0000]\u0000_\u0000a\u0000c\u0000e g!i\"k#m$o%q&s\'u(w)y*{+},\u007f"+ - "-\u0081.\u0083/\u00850\u00871\u00892\u008b3\u008d4\u008f5\u00916\u0093"+ - "7\u00958\u00979\u0099:\u009b;\u009d<\u009f=\u00a1>\u00a3?\u00a5@\u00a7"+ - "A\u00a9B\u00abC\u00adD\u00afE\u00b1F\u00b3G\u00b5H\u00b7\u0000\u00b9\u0000"+ - "\u00bb\u0000\u00bdI\u00bfJ\u00c1K\u00c3L\u00c5\u0000\u00c7M\u00c9N\u00cb"+ - "O\u00cdP\u00cf\u0000\u00d1\u0000\u00d3Q\u00d5R\u00d7S\u00d9\u0000\u00db"+ - "\u0000\u00dd\u0000\u00df\u0000\u00e1\u0000\u00e3\u0000\u00e5T\u00e7\u0000"+ - "\u00e9U\u00eb\u0000\u00ed\u0000\u00efV\u00f1W\u00f3X\u00f5\u0000\u00f7"+ - "\u0000\u00f9\u0000\u00fb\u0000\u00fd\u0000\u00ff\u0000\u0101\u0000\u0103"+ - "Y\u0105Z\u0107[\u0109\\\u010b\u0000\u010d\u0000\u010f\u0000\u0111\u0000"+ - "\u0113\u0000\u0115\u0000\u0117]\u0119\u0000\u011b^\u011d_\u011f`\u0121"+ - "\u0000\u0123\u0000\u0125a\u0127b\u0129\u0000\u012bc\u012d\u0000\u012f"+ - "d\u0131e\u0133f\u0135\u0000\u0137\u0000\u0139\u0000\u013b\u0000\u013d"+ - "\u0000\u013f\u0000\u0141\u0000\u0143\u0000\u0145\u0000\u0147g\u0149h\u014b"+ - "i\u014d\u0000\u014f\u0000\u0151\u0000\u0153\u0000\u0155\u0000\u0157\u0000"+ - "\u0159j\u015bk\u015dl\u015f\u0000\u0161m\u0163n\u0165o\u0167p\u0169\u0000"+ - "\u016b\u0000\u016dq\u016fr\u0171s\u0173t\u0175\u0000\u0177\u0000\u0179"+ - "\u0000\u017b\u0000\u017d\u0000\u017f\u0000\u0181\u0000\u0183u\u0185v\u0187"+ - "w\u0189\u0000\u018b\u0000\u018d\u0000\u018f\u0000\u0191x\u0193y\u0195"+ - "z\u0197\u0000\u0199{\u019b\u0000\u019d\u0000\u019f|\u01a1\u0000\u01a3"+ - "\u0000\u01a5\u0000\u01a7\u0000\u01a9\u0000\u01ab}\u01ad~\u01af\u007f\u01b1"+ - "\u0000\u01b3\u0000\u01b5\u0000\u01b7\u0080\u01b9\u0081\u01bb\u0082\u01bd"+ - "\u0000\u01bf\u0000\u01c1\u0083\u01c3\u0084\u01c5\u0085\u01c7\u0000\u01c9"+ - "\u0000\u01cb\u0000\u01cd\u0000\u01cf\u0000\u01d1\u0000\u01d3\u0000\u01d5"+ - "\u0000\u01d7\u0000\u01d9\u0000\u01db\u0000\u01dd\u0086\u01df\u0087\u01e1"+ - "\u0088\u01e3\u0000\u01e5\u0000\u01e7\u0089\u01e9\u008a\u01eb\u008b\u01ed"+ - "\u0000\u01ef\u0000\u01f1\u0000\u01f3\u008c\u01f5\u008d\u01f7\u008e\u0013"+ - "\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e"+ - "\u000f\u0010\u0011\u0012$\u0002\u0000DDdd\u0002\u0000IIii\u0002\u0000"+ - "SSss\u0002\u0000EEee\u0002\u0000CCcc\u0002\u0000TTtt\u0002\u0000RRrr\u0002"+ - "\u0000OOoo\u0002\u0000PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002\u0000"+ - "VVvv\u0002\u0000AAaa\u0002\u0000LLll\u0002\u0000XXxx\u0002\u0000FFff\u0002"+ - "\u0000MMmm\u0002\u0000GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002\u0000"+ - "UUuu\u0006\u0000\t\n\r\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r"+ - "\r \u0001\u000009\u0002\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004\u0000"+ - "\n\n\r\r\"\"\\\\\u0002\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002\u0000"+ - "YYyy\u000b\u0000\t\n\r\r \"\",,//::==[[]]||\u0002\u0000**//\u000b\u0000"+ - "\t\n\r\r \"#,,//::<<>?\\\\||\u0002\u0000JJjj\u071f\u0000\u0013\u0001"+ - "\u0000\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001"+ - "\u0000\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001"+ - "\u0000\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001"+ - "\u0000\u0000\u0000\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001\u0000\u0000"+ - "\u0000\u0000%\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000\u0000\u0000"+ - "\u0000)\u0001\u0000\u0000\u0000\u0000+\u0001\u0000\u0000\u0000\u0000-"+ - "\u0001\u0000\u0000\u0000\u0000/\u0001\u0000\u0000\u0000\u00001\u0001\u0000"+ - "\u0000\u0000\u00003\u0001\u0000\u0000\u0000\u00005\u0001\u0000\u0000\u0000"+ - "\u00007\u0001\u0000\u0000\u0000\u00009\u0001\u0000\u0000\u0000\u0000;"+ - "\u0001\u0000\u0000\u0000\u0000=\u0001\u0000\u0000\u0000\u0000?\u0001\u0000"+ - "\u0000\u0000\u0000A\u0001\u0000\u0000\u0000\u0000C\u0001\u0000\u0000\u0000"+ - "\u0000E\u0001\u0000\u0000\u0000\u0000G\u0001\u0000\u0000\u0000\u0000I"+ - "\u0001\u0000\u0000\u0000\u0000K\u0001\u0000\u0000\u0000\u0000M\u0001\u0000"+ - "\u0000\u0000\u0001O\u0001\u0000\u0000\u0000\u0001e\u0001\u0000\u0000\u0000"+ - "\u0001g\u0001\u0000\u0000\u0000\u0001i\u0001\u0000\u0000\u0000\u0001k"+ - "\u0001\u0000\u0000\u0000\u0001m\u0001\u0000\u0000\u0000\u0001o\u0001\u0000"+ - "\u0000\u0000\u0001q\u0001\u0000\u0000\u0000\u0001s\u0001\u0000\u0000\u0000"+ - "\u0001u\u0001\u0000\u0000\u0000\u0001w\u0001\u0000\u0000\u0000\u0001y"+ - "\u0001\u0000\u0000\u0000\u0001{\u0001\u0000\u0000\u0000\u0001}\u0001\u0000"+ - "\u0000\u0000\u0001\u007f\u0001\u0000\u0000\u0000\u0001\u0081\u0001\u0000"+ - "\u0000\u0000\u0001\u0083\u0001\u0000\u0000\u0000\u0001\u0085\u0001\u0000"+ - "\u0000\u0000\u0001\u0087\u0001\u0000\u0000\u0000\u0001\u0089\u0001\u0000"+ - "\u0000\u0000\u0001\u008b\u0001\u0000\u0000\u0000\u0001\u008d\u0001\u0000"+ - "\u0000\u0000\u0001\u008f\u0001\u0000\u0000\u0000\u0001\u0091\u0001\u0000"+ - "\u0000\u0000\u0001\u0093\u0001\u0000\u0000\u0000\u0001\u0095\u0001\u0000"+ - "\u0000\u0000\u0001\u0097\u0001\u0000\u0000\u0000\u0001\u0099\u0001\u0000"+ - "\u0000\u0000\u0001\u009b\u0001\u0000\u0000\u0000\u0001\u009d\u0001\u0000"+ - "\u0000\u0000\u0001\u009f\u0001\u0000\u0000\u0000\u0001\u00a1\u0001\u0000"+ - "\u0000\u0000\u0001\u00a3\u0001\u0000\u0000\u0000\u0001\u00a5\u0001\u0000"+ - "\u0000\u0000\u0001\u00a7\u0001\u0000\u0000\u0000\u0001\u00a9\u0001\u0000"+ - "\u0000\u0000\u0001\u00ab\u0001\u0000\u0000\u0000\u0001\u00ad\u0001\u0000"+ - "\u0000\u0000\u0001\u00af\u0001\u0000\u0000\u0000\u0001\u00b1\u0001\u0000"+ - "\u0000\u0000\u0001\u00b3\u0001\u0000\u0000\u0000\u0001\u00b5\u0001\u0000"+ - "\u0000\u0000\u0001\u00b7\u0001\u0000\u0000\u0000\u0001\u00b9\u0001\u0000"+ - "\u0000\u0000\u0001\u00bb\u0001\u0000\u0000\u0000\u0001\u00bd\u0001\u0000"+ - "\u0000\u0000\u0001\u00bf\u0001\u0000\u0000\u0000\u0001\u00c1\u0001\u0000"+ - "\u0000\u0000\u0001\u00c3\u0001\u0000\u0000\u0000\u0001\u00c7\u0001\u0000"+ - "\u0000\u0000\u0001\u00c9\u0001\u0000\u0000\u0000\u0001\u00cb\u0001\u0000"+ - "\u0000\u0000\u0001\u00cd\u0001\u0000\u0000\u0000\u0002\u00cf\u0001\u0000"+ - "\u0000\u0000\u0002\u00d1\u0001\u0000\u0000\u0000\u0002\u00d3\u0001\u0000"+ - "\u0000\u0000\u0002\u00d5\u0001\u0000\u0000\u0000\u0002\u00d7\u0001\u0000"+ - "\u0000\u0000\u0003\u00d9\u0001\u0000\u0000\u0000\u0003\u00db\u0001\u0000"+ - "\u0000\u0000\u0003\u00dd\u0001\u0000\u0000\u0000\u0003\u00df\u0001\u0000"+ - "\u0000\u0000\u0003\u00e1\u0001\u0000\u0000\u0000\u0003\u00e3\u0001\u0000"+ - "\u0000\u0000\u0003\u00e5\u0001\u0000\u0000\u0000\u0003\u00e9\u0001\u0000"+ - "\u0000\u0000\u0003\u00eb\u0001\u0000\u0000\u0000\u0003\u00ed\u0001\u0000"+ - "\u0000\u0000\u0003\u00ef\u0001\u0000\u0000\u0000\u0003\u00f1\u0001\u0000"+ - "\u0000\u0000\u0003\u00f3\u0001\u0000\u0000\u0000\u0004\u00f5\u0001\u0000"+ - "\u0000\u0000\u0004\u00f7\u0001\u0000\u0000\u0000\u0004\u00f9\u0001\u0000"+ - "\u0000\u0000\u0004\u00fb\u0001\u0000\u0000\u0000\u0004\u00fd\u0001\u0000"+ - "\u0000\u0000\u0004\u0103\u0001\u0000\u0000\u0000\u0004\u0105\u0001\u0000"+ - "\u0000\u0000\u0004\u0107\u0001\u0000\u0000\u0000\u0004\u0109\u0001\u0000"+ - "\u0000\u0000\u0005\u010b\u0001\u0000\u0000\u0000\u0005\u010d\u0001\u0000"+ - "\u0000\u0000\u0005\u010f\u0001\u0000\u0000\u0000\u0005\u0111\u0001\u0000"+ - "\u0000\u0000\u0005\u0113\u0001\u0000\u0000\u0000\u0005\u0115\u0001\u0000"+ - "\u0000\u0000\u0005\u0117\u0001\u0000\u0000\u0000\u0005\u0119\u0001\u0000"+ - "\u0000\u0000\u0005\u011b\u0001\u0000\u0000\u0000\u0005\u011d\u0001\u0000"+ - "\u0000\u0000\u0005\u011f\u0001\u0000\u0000\u0000\u0006\u0121\u0001\u0000"+ - "\u0000\u0000\u0006\u0123\u0001\u0000\u0000\u0000\u0006\u0125\u0001\u0000"+ - "\u0000\u0000\u0006\u0127\u0001\u0000\u0000\u0000\u0006\u012b\u0001\u0000"+ - "\u0000\u0000\u0006\u012d\u0001\u0000\u0000\u0000\u0006\u012f\u0001\u0000"+ - "\u0000\u0000\u0006\u0131\u0001\u0000\u0000\u0000\u0006\u0133\u0001\u0000"+ - "\u0000\u0000\u0007\u0135\u0001\u0000\u0000\u0000\u0007\u0137\u0001\u0000"+ - "\u0000\u0000\u0007\u0139\u0001\u0000\u0000\u0000\u0007\u013b\u0001\u0000"+ - "\u0000\u0000\u0007\u013d\u0001\u0000\u0000\u0000\u0007\u013f\u0001\u0000"+ - "\u0000\u0000\u0007\u0141\u0001\u0000\u0000\u0000\u0007\u0143\u0001\u0000"+ - "\u0000\u0000\u0007\u0145\u0001\u0000\u0000\u0000\u0007\u0147\u0001\u0000"+ - "\u0000\u0000\u0007\u0149\u0001\u0000\u0000\u0000\u0007\u014b\u0001\u0000"+ + "\u0001\u00a6\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8"+ + "\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a9\u0001\u00a9\u0001\u00a9"+ + "\u0001\u00a9\u0001\u00a9\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa"+ + "\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0004\u00ab"+ + "\u05c4\b\u00ab\u000b\u00ab\f\u00ab\u05c5\u0001\u00ac\u0001\u00ac\u0001"+ + "\u00ac\u0001\u00ac\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001"+ + "\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001\u00af\u0001"+ + "\u00af\u0001\u00af\u0001\u00af\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001"+ + "\u00b0\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001"+ + "\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001"+ + "\u00b3\u0001\u00b3\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001"+ + "\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001"+ + "\u00b6\u0001\u00b6\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001"+ + "\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9\u0001\u00b9\u0001"+ + "\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001\u00ba\u0001"+ + "\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001"+ + "\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd\u0001\u00bd\u0001"+ + "\u00bd\u0001\u00bd\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001"+ + "\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001\u00c0\u0001"+ + "\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001"+ + "\u00c1\u0001\u00c1\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001"+ + "\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001"+ + "\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c4\u0001"+ + "\u00c4\u0001\u00c4\u0001\u00c4\u0001\u00c5\u0001\u00c5\u0001\u00c5\u0001"+ + "\u00c5\u0001\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c6\u0001\u00c7\u0001"+ + "\u00c7\u0001\u00c7\u0001\u00c7\u0001\u00c8\u0001\u00c8\u0001\u00c8\u0001"+ + "\u00c8\u0001\u00c9\u0001\u00c9\u0001\u00c9\u0001\u00c9\u0001\u00ca\u0001"+ + "\u00ca\u0001\u00ca\u0001\u00ca\u0001\u00cb\u0001\u00cb\u0001\u00cb\u0001"+ + "\u00cb\u0001\u00cc\u0001\u00cc\u0001\u00cc\u0001\u00cc\u0001\u00cd\u0001"+ + "\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00cd\u0001\u00ce\u0001\u00ce\u0001"+ + "\u00ce\u0001\u00ce\u0001\u00ce\u0001\u00ce\u0001\u00cf\u0001\u00cf\u0001"+ + "\u00cf\u0001\u00cf\u0001\u00cf\u0001\u00cf\u0001\u00d0\u0001\u00d0\u0001"+ + "\u00d0\u0001\u00d0\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001\u00d1\u0001"+ + "\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d2\u0001\u00d3\u0001\u00d3\u0001"+ + "\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d3\u0001\u00d4\u0001\u00d4\u0001"+ + "\u00d4\u0001\u00d4\u0001\u00d4\u0001\u00d4\u0001\u00d5\u0001\u00d5\u0001"+ + "\u00d5\u0001\u00d5\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001\u00d6\u0001"+ + "\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d7\u0001\u00d8\u0001\u00d8\u0001"+ + "\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d8\u0001\u00d9\u0001\u00d9\u0001"+ + "\u00d9\u0001\u00d9\u0001\u00d9\u0001\u00d9\u0001\u00da\u0001\u00da\u0001"+ + "\u00da\u0001\u00da\u0001\u00da\u0001\u00da\u0001\u00db\u0001\u00db\u0001"+ + "\u00db\u0001\u00db\u0001\u00db\u0001\u00dc\u0001\u00dc\u0001\u00dc\u0001"+ + "\u00dc\u0001\u00dc\u0001\u00dd\u0001\u00dd\u0001\u00dd\u0001\u00dd\u0001"+ + "\u00de\u0001\u00de\u0001\u00de\u0001\u00de\u0001\u00df\u0001\u00df\u0001"+ + "\u00df\u0001\u00df\u0001\u00e0\u0001\u00e0\u0001\u00e0\u0001\u00e0\u0001"+ + "\u00e1\u0001\u00e1\u0001\u00e1\u0001\u00e1\u0001\u00e2\u0001\u00e2\u0001"+ + "\u00e2\u0001\u00e2\u0001\u00e3\u0001\u00e3\u0001\u00e3\u0001\u00e3\u0001"+ + "\u00e4\u0001\u00e4\u0001\u00e4\u0001\u00e4\u0001\u00e5\u0001\u00e5\u0001"+ + "\u00e5\u0001\u00e5\u0001\u00e6\u0001\u00e6\u0001\u00e6\u0001\u00e6\u0001"+ + "\u00e6\u0001\u00e7\u0001\u00e7\u0001\u00e7\u0001\u00e7\u0001\u00e8\u0001"+ + "\u00e8\u0001\u00e8\u0001\u00e8\u0001\u00e9\u0001\u00e9\u0001\u00e9\u0001"+ + "\u00e9\u0001\u00ea\u0001\u00ea\u0001\u00ea\u0001\u00ea\u0001\u00eb\u0001"+ + "\u00eb\u0001\u00eb\u0001\u00eb\u0001\u00eb\u0001\u00ec\u0001\u00ec\u0001"+ + "\u00ec\u0001\u00ec\u0001\u00ec\u0001\u00ed\u0001\u00ed\u0001\u00ed\u0001"+ + "\u00ed\u0001\u00ee\u0001\u00ee\u0001\u00ee\u0001\u00ee\u0001\u00ef\u0001"+ + "\u00ef\u0001\u00ef\u0001\u00ef\u0002\u02ff\u0344\u0000\u00f0\u0013\u0001"+ + "\u0015\u0002\u0017\u0003\u0019\u0004\u001b\u0005\u001d\u0006\u001f\u0007"+ + "!\b#\t%\n\'\u000b)\f+\r-\u000e/\u000f1\u00103\u00115\u00127\u00139\u0014"+ + ";\u0015=\u0016?\u0017A\u0018C\u0019E\u001aG\u001bI\u001cK\u001dM\u001e"+ + "O\u001fQ\u0000S\u0000U\u0000W\u0000Y\u0000[\u0000]\u0000_\u0000a\u0000"+ + "c\u0000e g!i\"k#m$o%q&s\'u(w)y*{+},\u007f-\u0081.\u0083/\u00850\u0087"+ + "1\u00892\u008b3\u008d4\u008f5\u00916\u00937\u00958\u00979\u0099:\u009b"+ + ";\u009d<\u009f=\u00a1>\u00a3?\u00a5@\u00a7A\u00a9B\u00abC\u00adD\u00af"+ + "E\u00b1F\u00b3\u0000\u00b5G\u00b7H\u00b9I\u00bbJ\u00bdK\u00bfL\u00c1\u0000"+ + "\u00c3M\u00c5N\u00c7O\u00c9P\u00cb\u0000\u00cd\u0000\u00cfQ\u00d1R\u00d3"+ + "S\u00d5\u0000\u00d7\u0000\u00d9\u0000\u00db\u0000\u00dd\u0000\u00df\u0000"+ + "\u00e1T\u00e3\u0000\u00e5U\u00e7\u0000\u00e9\u0000\u00ebV\u00edW\u00ef"+ + "X\u00f1\u0000\u00f3\u0000\u00f5\u0000\u00f7\u0000\u00f9\u0000\u00fb\u0000"+ + "\u00fd\u0000\u00ffY\u0101Z\u0103[\u0105\\\u0107\u0000\u0109\u0000\u010b"+ + "\u0000\u010d\u0000\u010f\u0000\u0111\u0000\u0113]\u0115\u0000\u0117^\u0119"+ + "_\u011b`\u011d\u0000\u011f\u0000\u0121a\u0123b\u0125\u0000\u0127c\u0129"+ + "\u0000\u012bd\u012de\u012ff\u0131\u0000\u0133\u0000\u0135\u0000\u0137"+ + "\u0000\u0139\u0000\u013b\u0000\u013d\u0000\u013f\u0000\u0141\u0000\u0143"+ + "g\u0145h\u0147i\u0149\u0000\u014b\u0000\u014d\u0000\u014f\u0000\u0151"+ + "\u0000\u0153\u0000\u0155j\u0157k\u0159l\u015b\u0000\u015dm\u015fn\u0161"+ + "o\u0163p\u0165\u0000\u0167\u0000\u0169q\u016br\u016ds\u016ft\u0171\u0000"+ + "\u0173\u0000\u0175\u0000\u0177\u0000\u0179\u0000\u017b\u0000\u017d\u0000"+ + "\u017fu\u0181v\u0183w\u0185\u0000\u0187\u0000\u0189\u0000\u018b\u0000"+ + "\u018dx\u018fy\u0191z\u0193\u0000\u0195{\u0197\u0000\u0199\u0000\u019b"+ + "|\u019d\u0000\u019f\u0000\u01a1\u0000\u01a3\u0000\u01a5\u0000\u01a7}\u01a9"+ + "~\u01ab\u007f\u01ad\u0000\u01af\u0000\u01b1\u0000\u01b3\u0080\u01b5\u0081"+ + "\u01b7\u0082\u01b9\u0000\u01bb\u0000\u01bd\u0083\u01bf\u0084\u01c1\u0085"+ + "\u01c3\u0000\u01c5\u0000\u01c7\u0000\u01c9\u0000\u01cb\u0000\u01cd\u0000"+ + "\u01cf\u0000\u01d1\u0000\u01d3\u0000\u01d5\u0000\u01d7\u0000\u01d9\u0086"+ + "\u01db\u0087\u01dd\u0088\u01df\u0000\u01e1\u0000\u01e3\u0089\u01e5\u008a"+ + "\u01e7\u008b\u01e9\u0000\u01eb\u0000\u01ed\u008c\u01ef\u008d\u01f1\u008e"+ + "\u0013\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r"+ + "\u000e\u000f\u0010\u0011\u0012$\u0002\u0000DDdd\u0002\u0000IIii\u0002"+ + "\u0000SSss\u0002\u0000EEee\u0002\u0000CCcc\u0002\u0000TTtt\u0002\u0000"+ + "RRrr\u0002\u0000OOoo\u0002\u0000PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002"+ + "\u0000VVvv\u0002\u0000AAaa\u0002\u0000LLll\u0002\u0000XXxx\u0002\u0000"+ + "FFff\u0002\u0000MMmm\u0002\u0000GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002"+ + "\u0000UUuu\u0006\u0000\t\n\r\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000"+ + "\t\n\r\r \u0001\u000009\u0002\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004"+ + "\u0000\n\n\r\r\"\"\\\\\u0002\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002"+ + "\u0000YYyy\u000b\u0000\t\n\r\r \"\",,//::==[[]]||\u0002\u0000**//\u000b"+ + "\u0000\t\n\r\r \"#,,//::<<>?\\\\||\u0002\u0000JJjj\u0710\u0000\u0013"+ + "\u0001\u0000\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017"+ + "\u0001\u0000\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b"+ + "\u0001\u0000\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f"+ + "\u0001\u0000\u0000\u0000\u0000!\u0001\u0000\u0000\u0000\u0000#\u0001\u0000"+ + "\u0000\u0000\u0000%\u0001\u0000\u0000\u0000\u0000\'\u0001\u0000\u0000"+ + "\u0000\u0000)\u0001\u0000\u0000\u0000\u0000+\u0001\u0000\u0000\u0000\u0000"+ + "-\u0001\u0000\u0000\u0000\u0000/\u0001\u0000\u0000\u0000\u00001\u0001"+ + "\u0000\u0000\u0000\u00003\u0001\u0000\u0000\u0000\u00005\u0001\u0000\u0000"+ + "\u0000\u00007\u0001\u0000\u0000\u0000\u00009\u0001\u0000\u0000\u0000\u0000"+ + ";\u0001\u0000\u0000\u0000\u0000=\u0001\u0000\u0000\u0000\u0000?\u0001"+ + "\u0000\u0000\u0000\u0000A\u0001\u0000\u0000\u0000\u0000C\u0001\u0000\u0000"+ + "\u0000\u0000E\u0001\u0000\u0000\u0000\u0000G\u0001\u0000\u0000\u0000\u0000"+ + "I\u0001\u0000\u0000\u0000\u0000K\u0001\u0000\u0000\u0000\u0000M\u0001"+ + "\u0000\u0000\u0000\u0001O\u0001\u0000\u0000\u0000\u0001e\u0001\u0000\u0000"+ + "\u0000\u0001g\u0001\u0000\u0000\u0000\u0001i\u0001\u0000\u0000\u0000\u0001"+ + "k\u0001\u0000\u0000\u0000\u0001m\u0001\u0000\u0000\u0000\u0001o\u0001"+ + "\u0000\u0000\u0000\u0001q\u0001\u0000\u0000\u0000\u0001s\u0001\u0000\u0000"+ + "\u0000\u0001u\u0001\u0000\u0000\u0000\u0001w\u0001\u0000\u0000\u0000\u0001"+ + "y\u0001\u0000\u0000\u0000\u0001{\u0001\u0000\u0000\u0000\u0001}\u0001"+ + "\u0000\u0000\u0000\u0001\u007f\u0001\u0000\u0000\u0000\u0001\u0081\u0001"+ + "\u0000\u0000\u0000\u0001\u0083\u0001\u0000\u0000\u0000\u0001\u0085\u0001"+ + "\u0000\u0000\u0000\u0001\u0087\u0001\u0000\u0000\u0000\u0001\u0089\u0001"+ + "\u0000\u0000\u0000\u0001\u008b\u0001\u0000\u0000\u0000\u0001\u008d\u0001"+ + "\u0000\u0000\u0000\u0001\u008f\u0001\u0000\u0000\u0000\u0001\u0091\u0001"+ + "\u0000\u0000\u0000\u0001\u0093\u0001\u0000\u0000\u0000\u0001\u0095\u0001"+ + "\u0000\u0000\u0000\u0001\u0097\u0001\u0000\u0000\u0000\u0001\u0099\u0001"+ + "\u0000\u0000\u0000\u0001\u009b\u0001\u0000\u0000\u0000\u0001\u009d\u0001"+ + "\u0000\u0000\u0000\u0001\u009f\u0001\u0000\u0000\u0000\u0001\u00a1\u0001"+ + "\u0000\u0000\u0000\u0001\u00a3\u0001\u0000\u0000\u0000\u0001\u00a5\u0001"+ + "\u0000\u0000\u0000\u0001\u00a7\u0001\u0000\u0000\u0000\u0001\u00a9\u0001"+ + "\u0000\u0000\u0000\u0001\u00ab\u0001\u0000\u0000\u0000\u0001\u00ad\u0001"+ + "\u0000\u0000\u0000\u0001\u00af\u0001\u0000\u0000\u0000\u0001\u00b1\u0001"+ + "\u0000\u0000\u0000\u0001\u00b3\u0001\u0000\u0000\u0000\u0001\u00b5\u0001"+ + "\u0000\u0000\u0000\u0001\u00b7\u0001\u0000\u0000\u0000\u0001\u00b9\u0001"+ + "\u0000\u0000\u0000\u0001\u00bb\u0001\u0000\u0000\u0000\u0001\u00bd\u0001"+ + "\u0000\u0000\u0000\u0001\u00bf\u0001\u0000\u0000\u0000\u0001\u00c3\u0001"+ + "\u0000\u0000\u0000\u0001\u00c5\u0001\u0000\u0000\u0000\u0001\u00c7\u0001"+ + "\u0000\u0000\u0000\u0001\u00c9\u0001\u0000\u0000\u0000\u0002\u00cb\u0001"+ + "\u0000\u0000\u0000\u0002\u00cd\u0001\u0000\u0000\u0000\u0002\u00cf\u0001"+ + "\u0000\u0000\u0000\u0002\u00d1\u0001\u0000\u0000\u0000\u0002\u00d3\u0001"+ + "\u0000\u0000\u0000\u0003\u00d5\u0001\u0000\u0000\u0000\u0003\u00d7\u0001"+ + "\u0000\u0000\u0000\u0003\u00d9\u0001\u0000\u0000\u0000\u0003\u00db\u0001"+ + "\u0000\u0000\u0000\u0003\u00dd\u0001\u0000\u0000\u0000\u0003\u00df\u0001"+ + "\u0000\u0000\u0000\u0003\u00e1\u0001\u0000\u0000\u0000\u0003\u00e5\u0001"+ + "\u0000\u0000\u0000\u0003\u00e7\u0001\u0000\u0000\u0000\u0003\u00e9\u0001"+ + "\u0000\u0000\u0000\u0003\u00eb\u0001\u0000\u0000\u0000\u0003\u00ed\u0001"+ + "\u0000\u0000\u0000\u0003\u00ef\u0001\u0000\u0000\u0000\u0004\u00f1\u0001"+ + "\u0000\u0000\u0000\u0004\u00f3\u0001\u0000\u0000\u0000\u0004\u00f5\u0001"+ + "\u0000\u0000\u0000\u0004\u00f7\u0001\u0000\u0000\u0000\u0004\u00f9\u0001"+ + "\u0000\u0000\u0000\u0004\u00ff\u0001\u0000\u0000\u0000\u0004\u0101\u0001"+ + "\u0000\u0000\u0000\u0004\u0103\u0001\u0000\u0000\u0000\u0004\u0105\u0001"+ + "\u0000\u0000\u0000\u0005\u0107\u0001\u0000\u0000\u0000\u0005\u0109\u0001"+ + "\u0000\u0000\u0000\u0005\u010b\u0001\u0000\u0000\u0000\u0005\u010d\u0001"+ + "\u0000\u0000\u0000\u0005\u010f\u0001\u0000\u0000\u0000\u0005\u0111\u0001"+ + "\u0000\u0000\u0000\u0005\u0113\u0001\u0000\u0000\u0000\u0005\u0115\u0001"+ + "\u0000\u0000\u0000\u0005\u0117\u0001\u0000\u0000\u0000\u0005\u0119\u0001"+ + "\u0000\u0000\u0000\u0005\u011b\u0001\u0000\u0000\u0000\u0006\u011d\u0001"+ + "\u0000\u0000\u0000\u0006\u011f\u0001\u0000\u0000\u0000\u0006\u0121\u0001"+ + "\u0000\u0000\u0000\u0006\u0123\u0001\u0000\u0000\u0000\u0006\u0127\u0001"+ + "\u0000\u0000\u0000\u0006\u0129\u0001\u0000\u0000\u0000\u0006\u012b\u0001"+ + "\u0000\u0000\u0000\u0006\u012d\u0001\u0000\u0000\u0000\u0006\u012f\u0001"+ + "\u0000\u0000\u0000\u0007\u0131\u0001\u0000\u0000\u0000\u0007\u0133\u0001"+ + "\u0000\u0000\u0000\u0007\u0135\u0001\u0000\u0000\u0000\u0007\u0137\u0001"+ + "\u0000\u0000\u0000\u0007\u0139\u0001\u0000\u0000\u0000\u0007\u013b\u0001"+ + "\u0000\u0000\u0000\u0007\u013d\u0001\u0000\u0000\u0000\u0007\u013f\u0001"+ + "\u0000\u0000\u0000\u0007\u0141\u0001\u0000\u0000\u0000\u0007\u0143\u0001"+ + "\u0000\u0000\u0000\u0007\u0145\u0001\u0000\u0000\u0000\u0007\u0147\u0001"+ + "\u0000\u0000\u0000\b\u0149\u0001\u0000\u0000\u0000\b\u014b\u0001\u0000"+ "\u0000\u0000\b\u014d\u0001\u0000\u0000\u0000\b\u014f\u0001\u0000\u0000"+ "\u0000\b\u0151\u0001\u0000\u0000\u0000\b\u0153\u0001\u0000\u0000\u0000"+ "\b\u0155\u0001\u0000\u0000\u0000\b\u0157\u0001\u0000\u0000\u0000\b\u0159"+ - "\u0001\u0000\u0000\u0000\b\u015b\u0001\u0000\u0000\u0000\b\u015d\u0001"+ + "\u0001\u0000\u0000\u0000\t\u015b\u0001\u0000\u0000\u0000\t\u015d\u0001"+ "\u0000\u0000\u0000\t\u015f\u0001\u0000\u0000\u0000\t\u0161\u0001\u0000"+ - "\u0000\u0000\t\u0163\u0001\u0000\u0000\u0000\t\u0165\u0001\u0000\u0000"+ - "\u0000\t\u0167\u0001\u0000\u0000\u0000\n\u0169\u0001\u0000\u0000\u0000"+ + "\u0000\u0000\t\u0163\u0001\u0000\u0000\u0000\n\u0165\u0001\u0000\u0000"+ + "\u0000\n\u0167\u0001\u0000\u0000\u0000\n\u0169\u0001\u0000\u0000\u0000"+ "\n\u016b\u0001\u0000\u0000\u0000\n\u016d\u0001\u0000\u0000\u0000\n\u016f"+ - "\u0001\u0000\u0000\u0000\n\u0171\u0001\u0000\u0000\u0000\n\u0173\u0001"+ - "\u0000\u0000\u0000\u000b\u0175\u0001\u0000\u0000\u0000\u000b\u0177\u0001"+ - "\u0000\u0000\u0000\u000b\u0179\u0001\u0000\u0000\u0000\u000b\u017b\u0001"+ - "\u0000\u0000\u0000\u000b\u017d\u0001\u0000\u0000\u0000\u000b\u017f\u0001"+ - "\u0000\u0000\u0000\u000b\u0181\u0001\u0000\u0000\u0000\u000b\u0183\u0001"+ - "\u0000\u0000\u0000\u000b\u0185\u0001\u0000\u0000\u0000\u000b\u0187\u0001"+ + "\u0001\u0000\u0000\u0000\u000b\u0171\u0001\u0000\u0000\u0000\u000b\u0173"+ + "\u0001\u0000\u0000\u0000\u000b\u0175\u0001\u0000\u0000\u0000\u000b\u0177"+ + "\u0001\u0000\u0000\u0000\u000b\u0179\u0001\u0000\u0000\u0000\u000b\u017b"+ + "\u0001\u0000\u0000\u0000\u000b\u017d\u0001\u0000\u0000\u0000\u000b\u017f"+ + "\u0001\u0000\u0000\u0000\u000b\u0181\u0001\u0000\u0000\u0000\u000b\u0183"+ + "\u0001\u0000\u0000\u0000\f\u0185\u0001\u0000\u0000\u0000\f\u0187\u0001"+ "\u0000\u0000\u0000\f\u0189\u0001\u0000\u0000\u0000\f\u018b\u0001\u0000"+ "\u0000\u0000\f\u018d\u0001\u0000\u0000\u0000\f\u018f\u0001\u0000\u0000"+ - "\u0000\f\u0191\u0001\u0000\u0000\u0000\f\u0193\u0001\u0000\u0000\u0000"+ - "\f\u0195\u0001\u0000\u0000\u0000\r\u0197\u0001\u0000\u0000\u0000\r\u0199"+ + "\u0000\f\u0191\u0001\u0000\u0000\u0000\r\u0193\u0001\u0000\u0000\u0000"+ + "\r\u0195\u0001\u0000\u0000\u0000\r\u0197\u0001\u0000\u0000\u0000\r\u0199"+ "\u0001\u0000\u0000\u0000\r\u019b\u0001\u0000\u0000\u0000\r\u019d\u0001"+ "\u0000\u0000\u0000\r\u019f\u0001\u0000\u0000\u0000\r\u01a1\u0001\u0000"+ "\u0000\u0000\r\u01a3\u0001\u0000\u0000\u0000\r\u01a5\u0001\u0000\u0000"+ "\u0000\r\u01a7\u0001\u0000\u0000\u0000\r\u01a9\u0001\u0000\u0000\u0000"+ - "\r\u01ab\u0001\u0000\u0000\u0000\r\u01ad\u0001\u0000\u0000\u0000\r\u01af"+ - "\u0001\u0000\u0000\u0000\u000e\u01b1\u0001\u0000\u0000\u0000\u000e\u01b3"+ - "\u0001\u0000\u0000\u0000\u000e\u01b5\u0001\u0000\u0000\u0000\u000e\u01b7"+ - "\u0001\u0000\u0000\u0000\u000e\u01b9\u0001\u0000\u0000\u0000\u000e\u01bb"+ - "\u0001\u0000\u0000\u0000\u000f\u01bd\u0001\u0000\u0000\u0000\u000f\u01bf"+ - "\u0001\u0000\u0000\u0000\u000f\u01c1\u0001\u0000\u0000\u0000\u000f\u01c3"+ - "\u0001\u0000\u0000\u0000\u000f\u01c5\u0001\u0000\u0000\u0000\u000f\u01c7"+ - "\u0001\u0000\u0000\u0000\u000f\u01c9\u0001\u0000\u0000\u0000\u000f\u01cb"+ - "\u0001\u0000\u0000\u0000\u000f\u01cd\u0001\u0000\u0000\u0000\u0010\u01cf"+ - "\u0001\u0000\u0000\u0000\u0010\u01d1\u0001\u0000\u0000\u0000\u0010\u01d3"+ - "\u0001\u0000\u0000\u0000\u0010\u01d5\u0001\u0000\u0000\u0000\u0010\u01d7"+ - "\u0001\u0000\u0000\u0000\u0010\u01d9\u0001\u0000\u0000\u0000\u0010\u01db"+ - "\u0001\u0000\u0000\u0000\u0010\u01dd\u0001\u0000\u0000\u0000\u0010\u01df"+ - "\u0001\u0000\u0000\u0000\u0010\u01e1\u0001\u0000\u0000\u0000\u0011\u01e3"+ - "\u0001\u0000\u0000\u0000\u0011\u01e5\u0001\u0000\u0000\u0000\u0011\u01e7"+ - "\u0001\u0000\u0000\u0000\u0011\u01e9\u0001\u0000\u0000\u0000\u0011\u01eb"+ - "\u0001\u0000\u0000\u0000\u0012\u01ed\u0001\u0000\u0000\u0000\u0012\u01ef"+ - "\u0001\u0000\u0000\u0000\u0012\u01f1\u0001\u0000\u0000\u0000\u0012\u01f3"+ - "\u0001\u0000\u0000\u0000\u0012\u01f5\u0001\u0000\u0000\u0000\u0012\u01f7"+ - "\u0001\u0000\u0000\u0000\u0013\u01f9\u0001\u0000\u0000\u0000\u0015\u0203"+ - "\u0001\u0000\u0000\u0000\u0017\u020a\u0001\u0000\u0000\u0000\u0019\u0213"+ - "\u0001\u0000\u0000\u0000\u001b\u021a\u0001\u0000\u0000\u0000\u001d\u0224"+ - "\u0001\u0000\u0000\u0000\u001f\u022b\u0001\u0000\u0000\u0000!\u0232\u0001"+ - "\u0000\u0000\u0000#\u0239\u0001\u0000\u0000\u0000%\u0241\u0001\u0000\u0000"+ - "\u0000\'\u024d\u0001\u0000\u0000\u0000)\u0256\u0001\u0000\u0000\u0000"+ - "+\u025c\u0001\u0000\u0000\u0000-\u0263\u0001\u0000\u0000\u0000/\u026a"+ - "\u0001\u0000\u0000\u00001\u0272\u0001\u0000\u0000\u00003\u027a\u0001\u0000"+ - "\u0000\u00005\u0283\u0001\u0000\u0000\u00007\u0293\u0001\u0000\u0000\u0000"+ - "9\u02a2\u0001\u0000\u0000\u0000;\u02ae\u0001\u0000\u0000\u0000=\u02ba"+ - "\u0001\u0000\u0000\u0000?\u02c5\u0001\u0000\u0000\u0000A\u02cd\u0001\u0000"+ - "\u0000\u0000C\u02d5\u0001\u0000\u0000\u0000E\u02de\u0001\u0000\u0000\u0000"+ - "G\u02e7\u0001\u0000\u0000\u0000I\u02ed\u0001\u0000\u0000\u0000K\u02fe"+ - "\u0001\u0000\u0000\u0000M\u030e\u0001\u0000\u0000\u0000O\u0314\u0001\u0000"+ - "\u0000\u0000Q\u0318\u0001\u0000\u0000\u0000S\u031a\u0001\u0000\u0000\u0000"+ - "U\u031c\u0001\u0000\u0000\u0000W\u031f\u0001\u0000\u0000\u0000Y\u0321"+ - "\u0001\u0000\u0000\u0000[\u032a\u0001\u0000\u0000\u0000]\u032c\u0001\u0000"+ - "\u0000\u0000_\u0331\u0001\u0000\u0000\u0000a\u0333\u0001\u0000\u0000\u0000"+ - "c\u0338\u0001\u0000\u0000\u0000e\u0357\u0001\u0000\u0000\u0000g\u035a"+ - "\u0001\u0000\u0000\u0000i\u0388\u0001\u0000\u0000\u0000k\u038a\u0001\u0000"+ - "\u0000\u0000m\u038d\u0001\u0000\u0000\u0000o\u0391\u0001\u0000\u0000\u0000"+ - "q\u0395\u0001\u0000\u0000\u0000s\u0397\u0001\u0000\u0000\u0000u\u039a"+ - "\u0001\u0000\u0000\u0000w\u039c\u0001\u0000\u0000\u0000y\u039e\u0001\u0000"+ - "\u0000\u0000{\u03a3\u0001\u0000\u0000\u0000}\u03a5\u0001\u0000\u0000\u0000"+ - "\u007f\u03ab\u0001\u0000\u0000\u0000\u0081\u03b1\u0001\u0000\u0000\u0000"+ - "\u0083\u03b4\u0001\u0000\u0000\u0000\u0085\u03b7\u0001\u0000\u0000\u0000"+ - "\u0087\u03bc\u0001\u0000\u0000\u0000\u0089\u03c1\u0001\u0000\u0000\u0000"+ - "\u008b\u03c3\u0001\u0000\u0000\u0000\u008d\u03c7\u0001\u0000\u0000\u0000"+ - "\u008f\u03cc\u0001\u0000\u0000\u0000\u0091\u03d2\u0001\u0000\u0000\u0000"+ - "\u0093\u03d5\u0001\u0000\u0000\u0000\u0095\u03d7\u0001\u0000\u0000\u0000"+ - "\u0097\u03dd\u0001\u0000\u0000\u0000\u0099\u03df\u0001\u0000\u0000\u0000"+ - "\u009b\u03e4\u0001\u0000\u0000\u0000\u009d\u03e7\u0001\u0000\u0000\u0000"+ - "\u009f\u03ea\u0001\u0000\u0000\u0000\u00a1\u03ed\u0001\u0000\u0000\u0000"+ - "\u00a3\u03ef\u0001\u0000\u0000\u0000\u00a5\u03f2\u0001\u0000\u0000\u0000"+ - "\u00a7\u03f4\u0001\u0000\u0000\u0000\u00a9\u03f7\u0001\u0000\u0000\u0000"+ - "\u00ab\u03f9\u0001\u0000\u0000\u0000\u00ad\u03fb\u0001\u0000\u0000\u0000"+ - "\u00af\u03fd\u0001\u0000\u0000\u0000\u00b1\u03ff\u0001\u0000\u0000\u0000"+ - "\u00b3\u0401\u0001\u0000\u0000\u0000\u00b5\u0403\u0001\u0000\u0000\u0000"+ - "\u00b7\u0405\u0001\u0000\u0000\u0000\u00b9\u0409\u0001\u0000\u0000\u0000"+ - "\u00bb\u040e\u0001\u0000\u0000\u0000\u00bd\u0424\u0001\u0000\u0000\u0000"+ - "\u00bf\u0426\u0001\u0000\u0000\u0000\u00c1\u042b\u0001\u0000\u0000\u0000"+ - "\u00c3\u0440\u0001\u0000\u0000\u0000\u00c5\u0442\u0001\u0000\u0000\u0000"+ - "\u00c7\u044a\u0001\u0000\u0000\u0000\u00c9\u044c\u0001\u0000\u0000\u0000"+ - "\u00cb\u0450\u0001\u0000\u0000\u0000\u00cd\u0454\u0001\u0000\u0000\u0000"+ - "\u00cf\u0458\u0001\u0000\u0000\u0000\u00d1\u045d\u0001\u0000\u0000\u0000"+ - "\u00d3\u0462\u0001\u0000\u0000\u0000\u00d5\u0466\u0001\u0000\u0000\u0000"+ - "\u00d7\u046a\u0001\u0000\u0000\u0000\u00d9\u046e\u0001\u0000\u0000\u0000"+ - "\u00db\u0473\u0001\u0000\u0000\u0000\u00dd\u0477\u0001\u0000\u0000\u0000"+ - "\u00df\u047b\u0001\u0000\u0000\u0000\u00e1\u047f\u0001\u0000\u0000\u0000"+ - "\u00e3\u0483\u0001\u0000\u0000\u0000\u00e5\u0487\u0001\u0000\u0000\u0000"+ - "\u00e7\u0493\u0001\u0000\u0000\u0000\u00e9\u0496\u0001\u0000\u0000\u0000"+ - "\u00eb\u049a\u0001\u0000\u0000\u0000\u00ed\u049e\u0001\u0000\u0000\u0000"+ - "\u00ef\u04a2\u0001\u0000\u0000\u0000\u00f1\u04a6\u0001\u0000\u0000\u0000"+ - "\u00f3\u04aa\u0001\u0000\u0000\u0000\u00f5\u04ae\u0001\u0000\u0000\u0000"+ - "\u00f7\u04b3\u0001\u0000\u0000\u0000\u00f9\u04b7\u0001\u0000\u0000\u0000"+ - "\u00fb\u04bb\u0001\u0000\u0000\u0000\u00fd\u04bf\u0001\u0000\u0000\u0000"+ - "\u00ff\u04c7\u0001\u0000\u0000\u0000\u0101\u04dc\u0001\u0000\u0000\u0000"+ - "\u0103\u04e0\u0001\u0000\u0000\u0000\u0105\u04e4\u0001\u0000\u0000\u0000"+ - "\u0107\u04e8\u0001\u0000\u0000\u0000\u0109\u04ec\u0001\u0000\u0000\u0000"+ - "\u010b\u04f0\u0001\u0000\u0000\u0000\u010d\u04f5\u0001\u0000\u0000\u0000"+ - "\u010f\u04f9\u0001\u0000\u0000\u0000\u0111\u04fd\u0001\u0000\u0000\u0000"+ - "\u0113\u0501\u0001\u0000\u0000\u0000\u0115\u0505\u0001\u0000\u0000\u0000"+ - "\u0117\u0509\u0001\u0000\u0000\u0000\u0119\u050c\u0001\u0000\u0000\u0000"+ - "\u011b\u0510\u0001\u0000\u0000\u0000\u011d\u0514\u0001\u0000\u0000\u0000"+ - "\u011f\u0518\u0001\u0000\u0000\u0000\u0121\u051c\u0001\u0000\u0000\u0000"+ - "\u0123\u0521\u0001\u0000\u0000\u0000\u0125\u0526\u0001\u0000\u0000\u0000"+ - "\u0127\u052b\u0001\u0000\u0000\u0000\u0129\u0532\u0001\u0000\u0000\u0000"+ - "\u012b\u053b\u0001\u0000\u0000\u0000\u012d\u0542\u0001\u0000\u0000\u0000"+ - "\u012f\u0546\u0001\u0000\u0000\u0000\u0131\u054a\u0001\u0000\u0000\u0000"+ - "\u0133\u054e\u0001\u0000\u0000\u0000\u0135\u0552\u0001\u0000\u0000\u0000"+ - "\u0137\u0558\u0001\u0000\u0000\u0000\u0139\u055c\u0001\u0000\u0000\u0000"+ - "\u013b\u0560\u0001\u0000\u0000\u0000\u013d\u0564\u0001\u0000\u0000\u0000"+ - "\u013f\u0568\u0001\u0000\u0000\u0000\u0141\u056c\u0001\u0000\u0000\u0000"+ - "\u0143\u0570\u0001\u0000\u0000\u0000\u0145\u0574\u0001\u0000\u0000\u0000"+ - "\u0147\u0578\u0001\u0000\u0000\u0000\u0149\u057c\u0001\u0000\u0000\u0000"+ - "\u014b\u0580\u0001\u0000\u0000\u0000\u014d\u0584\u0001\u0000\u0000\u0000"+ - "\u014f\u0589\u0001\u0000\u0000\u0000\u0151\u058d\u0001\u0000\u0000\u0000"+ - "\u0153\u0591\u0001\u0000\u0000\u0000\u0155\u0595\u0001\u0000\u0000\u0000"+ - "\u0157\u0599\u0001\u0000\u0000\u0000\u0159\u059d\u0001\u0000\u0000\u0000"+ - "\u015b\u05a1\u0001\u0000\u0000\u0000\u015d\u05a5\u0001\u0000\u0000\u0000"+ - "\u015f\u05a9\u0001\u0000\u0000\u0000\u0161\u05ae\u0001\u0000\u0000\u0000"+ - "\u0163\u05b3\u0001\u0000\u0000\u0000\u0165\u05b7\u0001\u0000\u0000\u0000"+ - "\u0167\u05bb\u0001\u0000\u0000\u0000\u0169\u05bf\u0001\u0000\u0000\u0000"+ - "\u016b\u05c4\u0001\u0000\u0000\u0000\u016d\u05cd\u0001\u0000\u0000\u0000"+ - "\u016f\u05d1\u0001\u0000\u0000\u0000\u0171\u05d5\u0001\u0000\u0000\u0000"+ - "\u0173\u05d9\u0001\u0000\u0000\u0000\u0175\u05dd\u0001\u0000\u0000\u0000"+ - "\u0177\u05e2\u0001\u0000\u0000\u0000\u0179\u05e6\u0001\u0000\u0000\u0000"+ - "\u017b\u05ea\u0001\u0000\u0000\u0000\u017d\u05ee\u0001\u0000\u0000\u0000"+ - "\u017f\u05f3\u0001\u0000\u0000\u0000\u0181\u05f7\u0001\u0000\u0000\u0000"+ - "\u0183\u05fb\u0001\u0000\u0000\u0000\u0185\u05ff\u0001\u0000\u0000\u0000"+ - "\u0187\u0603\u0001\u0000\u0000\u0000\u0189\u0607\u0001\u0000\u0000\u0000"+ - "\u018b\u060d\u0001\u0000\u0000\u0000\u018d\u0611\u0001\u0000\u0000\u0000"+ - "\u018f\u0615\u0001\u0000\u0000\u0000\u0191\u0619\u0001\u0000\u0000\u0000"+ - "\u0193\u061d\u0001\u0000\u0000\u0000\u0195\u0621\u0001\u0000\u0000\u0000"+ - "\u0197\u0625\u0001\u0000\u0000\u0000\u0199\u062a\u0001\u0000\u0000\u0000"+ - "\u019b\u062f\u0001\u0000\u0000\u0000\u019d\u0633\u0001\u0000\u0000\u0000"+ - "\u019f\u0639\u0001\u0000\u0000\u0000\u01a1\u0642\u0001\u0000\u0000\u0000"+ - "\u01a3\u0646\u0001\u0000\u0000\u0000\u01a5\u064a\u0001\u0000\u0000\u0000"+ - "\u01a7\u064e\u0001\u0000\u0000\u0000\u01a9\u0652\u0001\u0000\u0000\u0000"+ - "\u01ab\u0656\u0001\u0000\u0000\u0000\u01ad\u065a\u0001\u0000\u0000\u0000"+ - "\u01af\u065e\u0001\u0000\u0000\u0000\u01b1\u0662\u0001\u0000\u0000\u0000"+ - "\u01b3\u0667\u0001\u0000\u0000\u0000\u01b5\u066d\u0001\u0000\u0000\u0000"+ - "\u01b7\u0673\u0001\u0000\u0000\u0000\u01b9\u0677\u0001\u0000\u0000\u0000"+ - "\u01bb\u067b\u0001\u0000\u0000\u0000\u01bd\u067f\u0001\u0000\u0000\u0000"+ - "\u01bf\u0685\u0001\u0000\u0000\u0000\u01c1\u068b\u0001\u0000\u0000\u0000"+ - "\u01c3\u068f\u0001\u0000\u0000\u0000\u01c5\u0693\u0001\u0000\u0000\u0000"+ - "\u01c7\u0697\u0001\u0000\u0000\u0000\u01c9\u069d\u0001\u0000\u0000\u0000"+ - "\u01cb\u06a3\u0001\u0000\u0000\u0000\u01cd\u06a9\u0001\u0000\u0000\u0000"+ - "\u01cf\u06ae\u0001\u0000\u0000\u0000\u01d1\u06b3\u0001\u0000\u0000\u0000"+ - "\u01d3\u06b7\u0001\u0000\u0000\u0000\u01d5\u06bb\u0001\u0000\u0000\u0000"+ - "\u01d7\u06bf\u0001\u0000\u0000\u0000\u01d9\u06c3\u0001\u0000\u0000\u0000"+ - "\u01db\u06c7\u0001\u0000\u0000\u0000\u01dd\u06cb\u0001\u0000\u0000\u0000"+ - "\u01df\u06cf\u0001\u0000\u0000\u0000\u01e1\u06d3\u0001\u0000\u0000\u0000"+ - "\u01e3\u06d7\u0001\u0000\u0000\u0000\u01e5\u06dc\u0001\u0000\u0000\u0000"+ - "\u01e7\u06e0\u0001\u0000\u0000\u0000\u01e9\u06e4\u0001\u0000\u0000\u0000"+ - "\u01eb\u06e8\u0001\u0000\u0000\u0000\u01ed\u06ec\u0001\u0000\u0000\u0000"+ - "\u01ef\u06f1\u0001\u0000\u0000\u0000\u01f1\u06f6\u0001\u0000\u0000\u0000"+ - "\u01f3\u06fb\u0001\u0000\u0000\u0000\u01f5\u06ff\u0001\u0000\u0000\u0000"+ - "\u01f7\u0703\u0001\u0000\u0000\u0000\u01f9\u01fa\u0007\u0000\u0000\u0000"+ - "\u01fa\u01fb\u0007\u0001\u0000\u0000\u01fb\u01fc\u0007\u0002\u0000\u0000"+ - "\u01fc\u01fd\u0007\u0002\u0000\u0000\u01fd\u01fe\u0007\u0003\u0000\u0000"+ - "\u01fe\u01ff\u0007\u0004\u0000\u0000\u01ff\u0200\u0007\u0005\u0000\u0000"+ - "\u0200\u0201\u0001\u0000\u0000\u0000\u0201\u0202\u0006\u0000\u0000\u0000"+ - "\u0202\u0014\u0001\u0000\u0000\u0000\u0203\u0204\u0007\u0000\u0000\u0000"+ - "\u0204\u0205\u0007\u0006\u0000\u0000\u0205\u0206\u0007\u0007\u0000\u0000"+ - "\u0206\u0207\u0007\b\u0000\u0000\u0207\u0208\u0001\u0000\u0000\u0000\u0208"+ - "\u0209\u0006\u0001\u0001\u0000\u0209\u0016\u0001\u0000\u0000\u0000\u020a"+ - "\u020b\u0007\u0003\u0000\u0000\u020b\u020c\u0007\t\u0000\u0000\u020c\u020d"+ - "\u0007\u0006\u0000\u0000\u020d\u020e\u0007\u0001\u0000\u0000\u020e\u020f"+ - "\u0007\u0004\u0000\u0000\u020f\u0210\u0007\n\u0000\u0000\u0210\u0211\u0001"+ - "\u0000\u0000\u0000\u0211\u0212\u0006\u0002\u0002\u0000\u0212\u0018\u0001"+ - "\u0000\u0000\u0000\u0213\u0214\u0007\u0003\u0000\u0000\u0214\u0215\u0007"+ - "\u000b\u0000\u0000\u0215\u0216\u0007\f\u0000\u0000\u0216\u0217\u0007\r"+ - "\u0000\u0000\u0217\u0218\u0001\u0000\u0000\u0000\u0218\u0219\u0006\u0003"+ - "\u0000\u0000\u0219\u001a\u0001\u0000\u0000\u0000\u021a\u021b\u0007\u0003"+ - "\u0000\u0000\u021b\u021c\u0007\u000e\u0000\u0000\u021c\u021d\u0007\b\u0000"+ - "\u0000\u021d\u021e\u0007\r\u0000\u0000\u021e\u021f\u0007\f\u0000\u0000"+ - "\u021f\u0220\u0007\u0001\u0000\u0000\u0220\u0221\u0007\t\u0000\u0000\u0221"+ - "\u0222\u0001\u0000\u0000\u0000\u0222\u0223\u0006\u0004\u0003\u0000\u0223"+ - "\u001c\u0001\u0000\u0000\u0000\u0224\u0225\u0007\u000f\u0000\u0000\u0225"+ - "\u0226\u0007\u0006\u0000\u0000\u0226\u0227\u0007\u0007\u0000\u0000\u0227"+ - "\u0228\u0007\u0010\u0000\u0000\u0228\u0229\u0001\u0000\u0000\u0000\u0229"+ - "\u022a\u0006\u0005\u0004\u0000\u022a\u001e\u0001\u0000\u0000\u0000\u022b"+ - "\u022c\u0007\u0011\u0000\u0000\u022c\u022d\u0007\u0006\u0000\u0000\u022d"+ - "\u022e\u0007\u0007\u0000\u0000\u022e\u022f\u0007\u0012\u0000\u0000\u022f"+ - "\u0230\u0001\u0000\u0000\u0000\u0230\u0231\u0006\u0006\u0000\u0000\u0231"+ - " \u0001\u0000\u0000\u0000\u0232\u0233\u0007\u0012\u0000\u0000\u0233\u0234"+ - "\u0007\u0003\u0000\u0000\u0234\u0235\u0007\u0003\u0000\u0000\u0235\u0236"+ - "\u0007\b\u0000\u0000\u0236\u0237\u0001\u0000\u0000\u0000\u0237\u0238\u0006"+ - "\u0007\u0001\u0000\u0238\"\u0001\u0000\u0000\u0000\u0239\u023a\u0007\r"+ - "\u0000\u0000\u023a\u023b\u0007\u0001\u0000\u0000\u023b\u023c\u0007\u0010"+ - "\u0000\u0000\u023c\u023d\u0007\u0001\u0000\u0000\u023d\u023e\u0007\u0005"+ - "\u0000\u0000\u023e\u023f\u0001\u0000\u0000\u0000\u023f\u0240\u0006\b\u0000"+ - "\u0000\u0240$\u0001\u0000\u0000\u0000\u0241\u0242\u0007\u0010\u0000\u0000"+ - "\u0242\u0243\u0007\u000b\u0000\u0000\u0243\u0244\u0005_\u0000\u0000\u0244"+ - "\u0245\u0007\u0003\u0000\u0000\u0245\u0246\u0007\u000e\u0000\u0000\u0246"+ - "\u0247\u0007\b\u0000\u0000\u0247\u0248\u0007\f\u0000\u0000\u0248\u0249"+ - "\u0007\t\u0000\u0000\u0249\u024a\u0007\u0000\u0000\u0000\u024a\u024b\u0001"+ - "\u0000\u0000\u0000\u024b\u024c\u0006\t\u0005\u0000\u024c&\u0001\u0000"+ - "\u0000\u0000\u024d\u024e\u0007\u0006\u0000\u0000\u024e\u024f\u0007\u0003"+ - "\u0000\u0000\u024f\u0250\u0007\t\u0000\u0000\u0250\u0251\u0007\f\u0000"+ - "\u0000\u0251\u0252\u0007\u0010\u0000\u0000\u0252\u0253\u0007\u0003\u0000"+ - "\u0000\u0253\u0254\u0001\u0000\u0000\u0000\u0254\u0255\u0006\n\u0006\u0000"+ - "\u0255(\u0001\u0000\u0000\u0000\u0256\u0257\u0007\u0006\u0000\u0000\u0257"+ - "\u0258\u0007\u0007\u0000\u0000\u0258\u0259\u0007\u0013\u0000\u0000\u0259"+ - "\u025a\u0001\u0000\u0000\u0000\u025a\u025b\u0006\u000b\u0000\u0000\u025b"+ - "*\u0001\u0000\u0000\u0000\u025c\u025d\u0007\u0002\u0000\u0000\u025d\u025e"+ - "\u0007\n\u0000\u0000\u025e\u025f\u0007\u0007\u0000\u0000\u025f\u0260\u0007"+ - "\u0013\u0000\u0000\u0260\u0261\u0001\u0000\u0000\u0000\u0261\u0262\u0006"+ - "\f\u0007\u0000\u0262,\u0001\u0000\u0000\u0000\u0263\u0264\u0007\u0002"+ - "\u0000\u0000\u0264\u0265\u0007\u0007\u0000\u0000\u0265\u0266\u0007\u0006"+ - "\u0000\u0000\u0266\u0267\u0007\u0005\u0000\u0000\u0267\u0268\u0001\u0000"+ - "\u0000\u0000\u0268\u0269\u0006\r\u0000\u0000\u0269.\u0001\u0000\u0000"+ - "\u0000\u026a\u026b\u0007\u0002\u0000\u0000\u026b\u026c\u0007\u0005\u0000"+ - "\u0000\u026c\u026d\u0007\f\u0000\u0000\u026d\u026e\u0007\u0005\u0000\u0000"+ - "\u026e\u026f\u0007\u0002\u0000\u0000\u026f\u0270\u0001\u0000\u0000\u0000"+ - "\u0270\u0271\u0006\u000e\u0000\u0000\u02710\u0001\u0000\u0000\u0000\u0272"+ - "\u0273\u0007\u0013\u0000\u0000\u0273\u0274\u0007\n\u0000\u0000\u0274\u0275"+ - "\u0007\u0003\u0000\u0000\u0275\u0276\u0007\u0006\u0000\u0000\u0276\u0277"+ - "\u0007\u0003\u0000\u0000\u0277\u0278\u0001\u0000\u0000\u0000\u0278\u0279"+ - "\u0006\u000f\u0000\u0000\u02792\u0001\u0000\u0000\u0000\u027a\u027b\u0007"+ - "\r\u0000\u0000\u027b\u027c\u0007\u0007\u0000\u0000\u027c\u027d\u0007\u0007"+ - "\u0000\u0000\u027d\u027e\u0007\u0012\u0000\u0000\u027e\u027f\u0007\u0014"+ - "\u0000\u0000\u027f\u0280\u0007\b\u0000\u0000\u0280\u0281\u0001\u0000\u0000"+ - "\u0000\u0281\u0282\u0006\u0010\b\u0000\u02824\u0001\u0000\u0000\u0000"+ - "\u0283\u0284\u0004\u0011\u0000\u0000\u0284\u0285\u0007\u0004\u0000\u0000"+ - "\u0285\u0286\u0007\n\u0000\u0000\u0286\u0287\u0007\f\u0000\u0000\u0287"+ - "\u0288\u0007\t\u0000\u0000\u0288\u0289\u0007\u0011\u0000\u0000\u0289\u028a"+ - "\u0007\u0003\u0000\u0000\u028a\u028b\u0005_\u0000\u0000\u028b\u028c\u0007"+ - "\b\u0000\u0000\u028c\u028d\u0007\u0007\u0000\u0000\u028d\u028e\u0007\u0001"+ - "\u0000\u0000\u028e\u028f\u0007\t\u0000\u0000\u028f\u0290\u0007\u0005\u0000"+ - "\u0000\u0290\u0291\u0001\u0000\u0000\u0000\u0291\u0292\u0006\u0011\t\u0000"+ - "\u02926\u0001\u0000\u0000\u0000\u0293\u0294\u0004\u0012\u0001\u0000\u0294"+ - "\u0295\u0007\u0001\u0000\u0000\u0295\u0296\u0007\t\u0000\u0000\u0296\u0297"+ - "\u0007\r\u0000\u0000\u0297\u0298\u0007\u0001\u0000\u0000\u0298\u0299\u0007"+ - "\t\u0000\u0000\u0299\u029a\u0007\u0003\u0000\u0000\u029a\u029b\u0007\u0002"+ - "\u0000\u0000\u029b\u029c\u0007\u0005\u0000\u0000\u029c\u029d\u0007\f\u0000"+ - "\u0000\u029d\u029e\u0007\u0005\u0000\u0000\u029e\u029f\u0007\u0002\u0000"+ - "\u0000\u029f\u02a0\u0001\u0000\u0000\u0000\u02a0\u02a1\u0006\u0012\u0000"+ - "\u0000\u02a18\u0001\u0000\u0000\u0000\u02a2\u02a3\u0004\u0013\u0002\u0000"+ - "\u02a3\u02a4\u0007\u0001\u0000\u0000\u02a4\u02a5\u0007\t\u0000\u0000\u02a5"+ - "\u02a6\u0007\u0002\u0000\u0000\u02a6\u02a7\u0007\u0001\u0000\u0000\u02a7"+ - "\u02a8\u0007\u0002\u0000\u0000\u02a8\u02a9\u0007\u0005\u0000\u0000\u02a9"+ - "\u02aa\u0005_\u0000\u0000\u02aa\u02ab\u0005\u8001\uf414\u0000\u0000\u02ab"+ - "\u02ac\u0001\u0000\u0000\u0000\u02ac\u02ad\u0006\u0013\u0001\u0000\u02ad"+ - ":\u0001\u0000\u0000\u0000\u02ae\u02af\u0004\u0014\u0003\u0000\u02af\u02b0"+ - "\u0007\r\u0000\u0000\u02b0\u02b1\u0007\u0007\u0000\u0000\u02b1\u02b2\u0007"+ - "\u0007\u0000\u0000\u02b2\u02b3\u0007\u0012\u0000\u0000\u02b3\u02b4\u0007"+ - "\u0014\u0000\u0000\u02b4\u02b5\u0007\b\u0000\u0000\u02b5\u02b6\u0005_"+ - "\u0000\u0000\u02b6\u02b7\u0005\u8001\uf414\u0000\u0000\u02b7\u02b8\u0001"+ - "\u0000\u0000\u0000\u02b8\u02b9\u0006\u0014\n\u0000\u02b9<\u0001\u0000"+ - "\u0000\u0000\u02ba\u02bb\u0004\u0015\u0004\u0000\u02bb\u02bc\u0007\u0010"+ - "\u0000\u0000\u02bc\u02bd\u0007\u0003\u0000\u0000\u02bd\u02be\u0007\u0005"+ - "\u0000\u0000\u02be\u02bf\u0007\u0006\u0000\u0000\u02bf\u02c0\u0007\u0001"+ - "\u0000\u0000\u02c0\u02c1\u0007\u0004\u0000\u0000\u02c1\u02c2\u0007\u0002"+ - "\u0000\u0000\u02c2\u02c3\u0001\u0000\u0000\u0000\u02c3\u02c4\u0006\u0015"+ - "\u000b\u0000\u02c4>\u0001\u0000\u0000\u0000\u02c5\u02c6\u0004\u0016\u0005"+ - "\u0000\u02c6\u02c7\u0007\u000f\u0000\u0000\u02c7\u02c8\u0007\u0014\u0000"+ - "\u0000\u02c8\u02c9\u0007\r\u0000\u0000\u02c9\u02ca\u0007\r\u0000\u0000"+ - "\u02ca\u02cb\u0001\u0000\u0000\u0000\u02cb\u02cc\u0006\u0016\b\u0000\u02cc"+ - "@\u0001\u0000\u0000\u0000\u02cd\u02ce\u0004\u0017\u0006\u0000\u02ce\u02cf"+ - "\u0007\r\u0000\u0000\u02cf\u02d0\u0007\u0003\u0000\u0000\u02d0\u02d1\u0007"+ - "\u000f\u0000\u0000\u02d1\u02d2\u0007\u0005\u0000\u0000\u02d2\u02d3\u0001"+ - "\u0000\u0000\u0000\u02d3\u02d4\u0006\u0017\b\u0000\u02d4B\u0001\u0000"+ - "\u0000\u0000\u02d5\u02d6\u0004\u0018\u0007\u0000\u02d6\u02d7\u0007\u0006"+ - "\u0000\u0000\u02d7\u02d8\u0007\u0001\u0000\u0000\u02d8\u02d9\u0007\u0011"+ - "\u0000\u0000\u02d9\u02da\u0007\n\u0000\u0000\u02da\u02db\u0007\u0005\u0000"+ - "\u0000\u02db\u02dc\u0001\u0000\u0000\u0000\u02dc\u02dd\u0006\u0018\b\u0000"+ - "\u02ddD\u0001\u0000\u0000\u0000\u02de\u02df\u0004\u0019\b\u0000\u02df"+ - "\u02e0\u0007\u000f\u0000\u0000\u02e0\u02e1\u0007\u0007\u0000\u0000\u02e1"+ - "\u02e2\u0007\u0006\u0000\u0000\u02e2\u02e3\u0007\u0012\u0000\u0000\u02e3"+ - "\u02e4\u0001\u0000\u0000\u0000\u02e4\u02e5\u0006\u0019\f\u0000\u02e5F"+ - "\u0001\u0000\u0000\u0000\u02e6\u02e8\b\u0015\u0000\u0000\u02e7\u02e6\u0001"+ - "\u0000\u0000\u0000\u02e8\u02e9\u0001\u0000\u0000\u0000\u02e9\u02e7\u0001"+ - "\u0000\u0000\u0000\u02e9\u02ea\u0001\u0000\u0000\u0000\u02ea\u02eb\u0001"+ - "\u0000\u0000\u0000\u02eb\u02ec\u0006\u001a\u0000\u0000\u02ecH\u0001\u0000"+ - "\u0000\u0000\u02ed\u02ee\u0005/\u0000\u0000\u02ee\u02ef\u0005/\u0000\u0000"+ - "\u02ef\u02f3\u0001\u0000\u0000\u0000\u02f0\u02f2\b\u0016\u0000\u0000\u02f1"+ - "\u02f0\u0001\u0000\u0000\u0000\u02f2\u02f5\u0001\u0000\u0000\u0000\u02f3"+ - "\u02f1\u0001\u0000\u0000\u0000\u02f3\u02f4\u0001\u0000\u0000\u0000\u02f4"+ - "\u02f7\u0001\u0000\u0000\u0000\u02f5\u02f3\u0001\u0000\u0000\u0000\u02f6"+ - "\u02f8\u0005\r\u0000\u0000\u02f7\u02f6\u0001\u0000\u0000\u0000\u02f7\u02f8"+ - "\u0001\u0000\u0000\u0000\u02f8\u02fa\u0001\u0000\u0000\u0000\u02f9\u02fb"+ - "\u0005\n\u0000\u0000\u02fa\u02f9\u0001\u0000\u0000\u0000\u02fa\u02fb\u0001"+ - "\u0000\u0000\u0000\u02fb\u02fc\u0001\u0000\u0000\u0000\u02fc\u02fd\u0006"+ - "\u001b\r\u0000\u02fdJ\u0001\u0000\u0000\u0000\u02fe\u02ff\u0005/\u0000"+ - "\u0000\u02ff\u0300\u0005*\u0000\u0000\u0300\u0305\u0001\u0000\u0000\u0000"+ - "\u0301\u0304\u0003K\u001c\u0000\u0302\u0304\t\u0000\u0000\u0000\u0303"+ - "\u0301\u0001\u0000\u0000\u0000\u0303\u0302\u0001\u0000\u0000\u0000\u0304"+ - "\u0307\u0001\u0000\u0000\u0000\u0305\u0306\u0001\u0000\u0000\u0000\u0305"+ - "\u0303\u0001\u0000\u0000\u0000\u0306\u0308\u0001\u0000\u0000\u0000\u0307"+ - "\u0305\u0001\u0000\u0000\u0000\u0308\u0309\u0005*\u0000\u0000\u0309\u030a"+ - "\u0005/\u0000\u0000\u030a\u030b\u0001\u0000\u0000\u0000\u030b\u030c\u0006"+ - "\u001c\r\u0000\u030cL\u0001\u0000\u0000\u0000\u030d\u030f\u0007\u0017"+ - "\u0000\u0000\u030e\u030d\u0001\u0000\u0000\u0000\u030f\u0310\u0001\u0000"+ - "\u0000\u0000\u0310\u030e\u0001\u0000\u0000\u0000\u0310\u0311\u0001\u0000"+ - "\u0000\u0000\u0311\u0312\u0001\u0000\u0000\u0000\u0312\u0313\u0006\u001d"+ - "\r\u0000\u0313N\u0001\u0000\u0000\u0000\u0314\u0315\u0005|\u0000\u0000"+ - "\u0315\u0316\u0001\u0000\u0000\u0000\u0316\u0317\u0006\u001e\u000e\u0000"+ - "\u0317P\u0001\u0000\u0000\u0000\u0318\u0319\u0007\u0018\u0000\u0000\u0319"+ - "R\u0001\u0000\u0000\u0000\u031a\u031b\u0007\u0019\u0000\u0000\u031bT\u0001"+ - "\u0000\u0000\u0000\u031c\u031d\u0005\\\u0000\u0000\u031d\u031e\u0007\u001a"+ - "\u0000\u0000\u031eV\u0001\u0000\u0000\u0000\u031f\u0320\b\u001b\u0000"+ - "\u0000\u0320X\u0001\u0000\u0000\u0000\u0321\u0323\u0007\u0003\u0000\u0000"+ - "\u0322\u0324\u0007\u001c\u0000\u0000\u0323\u0322\u0001\u0000\u0000\u0000"+ - "\u0323\u0324\u0001\u0000\u0000\u0000\u0324\u0326\u0001\u0000\u0000\u0000"+ - "\u0325\u0327\u0003Q\u001f\u0000\u0326\u0325\u0001\u0000\u0000\u0000\u0327"+ - "\u0328\u0001\u0000\u0000\u0000\u0328\u0326\u0001\u0000\u0000\u0000\u0328"+ - "\u0329\u0001\u0000\u0000\u0000\u0329Z\u0001\u0000\u0000\u0000\u032a\u032b"+ - "\u0005@\u0000\u0000\u032b\\\u0001\u0000\u0000\u0000\u032c\u032d\u0005"+ - "`\u0000\u0000\u032d^\u0001\u0000\u0000\u0000\u032e\u0332\b\u001d\u0000"+ - "\u0000\u032f\u0330\u0005`\u0000\u0000\u0330\u0332\u0005`\u0000\u0000\u0331"+ - "\u032e\u0001\u0000\u0000\u0000\u0331\u032f\u0001\u0000\u0000\u0000\u0332"+ - "`\u0001\u0000\u0000\u0000\u0333\u0334\u0005_\u0000\u0000\u0334b\u0001"+ - "\u0000\u0000\u0000\u0335\u0339\u0003S \u0000\u0336\u0339\u0003Q\u001f"+ - "\u0000\u0337\u0339\u0003a\'\u0000\u0338\u0335\u0001\u0000\u0000\u0000"+ - "\u0338\u0336\u0001\u0000\u0000\u0000\u0338\u0337\u0001\u0000\u0000\u0000"+ - "\u0339d\u0001\u0000\u0000\u0000\u033a\u033f\u0005\"\u0000\u0000\u033b"+ - "\u033e\u0003U!\u0000\u033c\u033e\u0003W\"\u0000\u033d\u033b\u0001\u0000"+ - "\u0000\u0000\u033d\u033c\u0001\u0000\u0000\u0000\u033e\u0341\u0001\u0000"+ - "\u0000\u0000\u033f\u033d\u0001\u0000\u0000\u0000\u033f\u0340\u0001\u0000"+ - "\u0000\u0000\u0340\u0342\u0001\u0000\u0000\u0000\u0341\u033f\u0001\u0000"+ - "\u0000\u0000\u0342\u0358\u0005\"\u0000\u0000\u0343\u0344\u0005\"\u0000"+ - "\u0000\u0344\u0345\u0005\"\u0000\u0000\u0345\u0346\u0005\"\u0000\u0000"+ - "\u0346\u034a\u0001\u0000\u0000\u0000\u0347\u0349\b\u0016\u0000\u0000\u0348"+ - "\u0347\u0001\u0000\u0000\u0000\u0349\u034c\u0001\u0000\u0000\u0000\u034a"+ - "\u034b\u0001\u0000\u0000\u0000\u034a\u0348\u0001\u0000\u0000\u0000\u034b"+ - "\u034d\u0001\u0000\u0000\u0000\u034c\u034a\u0001\u0000\u0000\u0000\u034d"+ - "\u034e\u0005\"\u0000\u0000\u034e\u034f\u0005\"\u0000\u0000\u034f\u0350"+ - "\u0005\"\u0000\u0000\u0350\u0352\u0001\u0000\u0000\u0000\u0351\u0353\u0005"+ - "\"\u0000\u0000\u0352\u0351\u0001\u0000\u0000\u0000\u0352\u0353\u0001\u0000"+ - "\u0000\u0000\u0353\u0355\u0001\u0000\u0000\u0000\u0354\u0356\u0005\"\u0000"+ - "\u0000\u0355\u0354\u0001\u0000\u0000\u0000\u0355\u0356\u0001\u0000\u0000"+ - "\u0000\u0356\u0358\u0001\u0000\u0000\u0000\u0357\u033a\u0001\u0000\u0000"+ - "\u0000\u0357\u0343\u0001\u0000\u0000\u0000\u0358f\u0001\u0000\u0000\u0000"+ - "\u0359\u035b\u0003Q\u001f\u0000\u035a\u0359\u0001\u0000\u0000\u0000\u035b"+ - "\u035c\u0001\u0000\u0000\u0000\u035c\u035a\u0001\u0000\u0000\u0000\u035c"+ - "\u035d\u0001\u0000\u0000\u0000\u035dh\u0001\u0000\u0000\u0000\u035e\u0360"+ - "\u0003Q\u001f\u0000\u035f\u035e\u0001\u0000\u0000\u0000\u0360\u0361\u0001"+ - "\u0000\u0000\u0000\u0361\u035f\u0001\u0000\u0000\u0000\u0361\u0362\u0001"+ - "\u0000\u0000\u0000\u0362\u0363\u0001\u0000\u0000\u0000\u0363\u0367\u0003"+ - "{4\u0000\u0364\u0366\u0003Q\u001f\u0000\u0365\u0364\u0001\u0000\u0000"+ - "\u0000\u0366\u0369\u0001\u0000\u0000\u0000\u0367\u0365\u0001\u0000\u0000"+ - "\u0000\u0367\u0368\u0001\u0000\u0000\u0000\u0368\u0389\u0001\u0000\u0000"+ - "\u0000\u0369\u0367\u0001\u0000\u0000\u0000\u036a\u036c\u0003{4\u0000\u036b"+ - "\u036d\u0003Q\u001f\u0000\u036c\u036b\u0001\u0000\u0000\u0000\u036d\u036e"+ - "\u0001\u0000\u0000\u0000\u036e\u036c\u0001\u0000\u0000\u0000\u036e\u036f"+ - "\u0001\u0000\u0000\u0000\u036f\u0389\u0001\u0000\u0000\u0000\u0370\u0372"+ - "\u0003Q\u001f\u0000\u0371\u0370\u0001\u0000\u0000\u0000\u0372\u0373\u0001"+ - "\u0000\u0000\u0000\u0373\u0371\u0001\u0000\u0000\u0000\u0373\u0374\u0001"+ - "\u0000\u0000\u0000\u0374\u037c\u0001\u0000\u0000\u0000\u0375\u0379\u0003"+ - "{4\u0000\u0376\u0378\u0003Q\u001f\u0000\u0377\u0376\u0001\u0000\u0000"+ - "\u0000\u0378\u037b\u0001\u0000\u0000\u0000\u0379\u0377\u0001\u0000\u0000"+ - "\u0000\u0379\u037a\u0001\u0000\u0000\u0000\u037a\u037d\u0001\u0000\u0000"+ - "\u0000\u037b\u0379\u0001\u0000\u0000\u0000\u037c\u0375\u0001\u0000\u0000"+ - "\u0000\u037c\u037d\u0001\u0000\u0000\u0000\u037d\u037e\u0001\u0000\u0000"+ - "\u0000\u037e\u037f\u0003Y#\u0000\u037f\u0389\u0001\u0000\u0000\u0000\u0380"+ - "\u0382\u0003{4\u0000\u0381\u0383\u0003Q\u001f\u0000\u0382\u0381\u0001"+ - "\u0000\u0000\u0000\u0383\u0384\u0001\u0000\u0000\u0000\u0384\u0382\u0001"+ - "\u0000\u0000\u0000\u0384\u0385\u0001\u0000\u0000\u0000\u0385\u0386\u0001"+ - "\u0000\u0000\u0000\u0386\u0387\u0003Y#\u0000\u0387\u0389\u0001\u0000\u0000"+ - "\u0000\u0388\u035f\u0001\u0000\u0000\u0000\u0388\u036a\u0001\u0000\u0000"+ - "\u0000\u0388\u0371\u0001\u0000\u0000\u0000\u0388\u0380\u0001\u0000\u0000"+ - "\u0000\u0389j\u0001\u0000\u0000\u0000\u038a\u038b\u0007\u001e\u0000\u0000"+ - "\u038b\u038c\u0007\u001f\u0000\u0000\u038cl\u0001\u0000\u0000\u0000\u038d"+ - "\u038e\u0007\f\u0000\u0000\u038e\u038f\u0007\t\u0000\u0000\u038f\u0390"+ - "\u0007\u0000\u0000\u0000\u0390n\u0001\u0000\u0000\u0000\u0391\u0392\u0007"+ - "\f\u0000\u0000\u0392\u0393\u0007\u0002\u0000\u0000\u0393\u0394\u0007\u0004"+ - "\u0000\u0000\u0394p\u0001\u0000\u0000\u0000\u0395\u0396\u0005=\u0000\u0000"+ - "\u0396r\u0001\u0000\u0000\u0000\u0397\u0398\u0005:\u0000\u0000\u0398\u0399"+ - "\u0005:\u0000\u0000\u0399t\u0001\u0000\u0000\u0000\u039a\u039b\u0005:"+ - "\u0000\u0000\u039bv\u0001\u0000\u0000\u0000\u039c\u039d\u0005,\u0000\u0000"+ - "\u039dx\u0001\u0000\u0000\u0000\u039e\u039f\u0007\u0000\u0000\u0000\u039f"+ - "\u03a0\u0007\u0003\u0000\u0000\u03a0\u03a1\u0007\u0002\u0000\u0000\u03a1"+ - "\u03a2\u0007\u0004\u0000\u0000\u03a2z\u0001\u0000\u0000\u0000\u03a3\u03a4"+ - "\u0005.\u0000\u0000\u03a4|\u0001\u0000\u0000\u0000\u03a5\u03a6\u0007\u000f"+ - "\u0000\u0000\u03a6\u03a7\u0007\f\u0000\u0000\u03a7\u03a8\u0007\r\u0000"+ - "\u0000\u03a8\u03a9\u0007\u0002\u0000\u0000\u03a9\u03aa\u0007\u0003\u0000"+ - "\u0000\u03aa~\u0001\u0000\u0000\u0000\u03ab\u03ac\u0007\u000f\u0000\u0000"+ - "\u03ac\u03ad\u0007\u0001\u0000\u0000\u03ad\u03ae\u0007\u0006\u0000\u0000"+ - "\u03ae\u03af\u0007\u0002\u0000\u0000\u03af\u03b0\u0007\u0005\u0000\u0000"+ - "\u03b0\u0080\u0001\u0000\u0000\u0000\u03b1\u03b2\u0007\u0001\u0000\u0000"+ - "\u03b2\u03b3\u0007\t\u0000\u0000\u03b3\u0082\u0001\u0000\u0000\u0000\u03b4"+ - "\u03b5\u0007\u0001\u0000\u0000\u03b5\u03b6\u0007\u0002\u0000\u0000\u03b6"+ - "\u0084\u0001\u0000\u0000\u0000\u03b7\u03b8\u0007\r\u0000\u0000\u03b8\u03b9"+ - "\u0007\f\u0000\u0000\u03b9\u03ba\u0007\u0002\u0000\u0000\u03ba\u03bb\u0007"+ - "\u0005\u0000\u0000\u03bb\u0086\u0001\u0000\u0000\u0000\u03bc\u03bd\u0007"+ - "\r\u0000\u0000\u03bd\u03be\u0007\u0001\u0000\u0000\u03be\u03bf\u0007\u0012"+ - "\u0000\u0000\u03bf\u03c0\u0007\u0003\u0000\u0000\u03c0\u0088\u0001\u0000"+ - "\u0000\u0000\u03c1\u03c2\u0005(\u0000\u0000\u03c2\u008a\u0001\u0000\u0000"+ - "\u0000\u03c3\u03c4\u0007\t\u0000\u0000\u03c4\u03c5\u0007\u0007\u0000\u0000"+ - "\u03c5\u03c6\u0007\u0005\u0000\u0000\u03c6\u008c\u0001\u0000\u0000\u0000"+ - "\u03c7\u03c8\u0007\t\u0000\u0000\u03c8\u03c9\u0007\u0014\u0000\u0000\u03c9"+ - "\u03ca\u0007\r\u0000\u0000\u03ca\u03cb\u0007\r\u0000\u0000\u03cb\u008e"+ - "\u0001\u0000\u0000\u0000\u03cc\u03cd\u0007\t\u0000\u0000\u03cd\u03ce\u0007"+ - "\u0014\u0000\u0000\u03ce\u03cf\u0007\r\u0000\u0000\u03cf\u03d0\u0007\r"+ - "\u0000\u0000\u03d0\u03d1\u0007\u0002\u0000\u0000\u03d1\u0090\u0001\u0000"+ - "\u0000\u0000\u03d2\u03d3\u0007\u0007\u0000\u0000\u03d3\u03d4\u0007\u0006"+ - "\u0000\u0000\u03d4\u0092\u0001\u0000\u0000\u0000\u03d5\u03d6\u0005?\u0000"+ - "\u0000\u03d6\u0094\u0001\u0000\u0000\u0000\u03d7\u03d8\u0007\u0006\u0000"+ - "\u0000\u03d8\u03d9\u0007\r\u0000\u0000\u03d9\u03da\u0007\u0001\u0000\u0000"+ - "\u03da\u03db\u0007\u0012\u0000\u0000\u03db\u03dc\u0007\u0003\u0000\u0000"+ - "\u03dc\u0096\u0001\u0000\u0000\u0000\u03dd\u03de\u0005)\u0000\u0000\u03de"+ - "\u0098\u0001\u0000\u0000\u0000\u03df\u03e0\u0007\u0005\u0000\u0000\u03e0"+ - "\u03e1\u0007\u0006\u0000\u0000\u03e1\u03e2\u0007\u0014\u0000\u0000\u03e2"+ - "\u03e3\u0007\u0003\u0000\u0000\u03e3\u009a\u0001\u0000\u0000\u0000\u03e4"+ - "\u03e5\u0005=\u0000\u0000\u03e5\u03e6\u0005=\u0000\u0000\u03e6\u009c\u0001"+ - "\u0000\u0000\u0000\u03e7\u03e8\u0005=\u0000\u0000\u03e8\u03e9\u0005~\u0000"+ - "\u0000\u03e9\u009e\u0001\u0000\u0000\u0000\u03ea\u03eb\u0005!\u0000\u0000"+ - "\u03eb\u03ec\u0005=\u0000\u0000\u03ec\u00a0\u0001\u0000\u0000\u0000\u03ed"+ - "\u03ee\u0005<\u0000\u0000\u03ee\u00a2\u0001\u0000\u0000\u0000\u03ef\u03f0"+ - "\u0005<\u0000\u0000\u03f0\u03f1\u0005=\u0000\u0000\u03f1\u00a4\u0001\u0000"+ - "\u0000\u0000\u03f2\u03f3\u0005>\u0000\u0000\u03f3\u00a6\u0001\u0000\u0000"+ - "\u0000\u03f4\u03f5\u0005>\u0000\u0000\u03f5\u03f6\u0005=\u0000\u0000\u03f6"+ - "\u00a8\u0001\u0000\u0000\u0000\u03f7\u03f8\u0005+\u0000\u0000\u03f8\u00aa"+ - "\u0001\u0000\u0000\u0000\u03f9\u03fa\u0005-\u0000\u0000\u03fa\u00ac\u0001"+ - "\u0000\u0000\u0000\u03fb\u03fc\u0005*\u0000\u0000\u03fc\u00ae\u0001\u0000"+ - "\u0000\u0000\u03fd\u03fe\u0005/\u0000\u0000\u03fe\u00b0\u0001\u0000\u0000"+ - "\u0000\u03ff\u0400\u0005%\u0000\u0000\u0400\u00b2\u0001\u0000\u0000\u0000"+ - "\u0401\u0402\u0005{\u0000\u0000\u0402\u00b4\u0001\u0000\u0000\u0000\u0403"+ - "\u0404\u0005}\u0000\u0000\u0404\u00b6\u0001\u0000\u0000\u0000\u0405\u0406"+ - "\u00031\u000f\u0000\u0406\u0407\u0001\u0000\u0000\u0000\u0407\u0408\u0006"+ - "R\u000f\u0000\u0408\u00b8\u0001\u0000\u0000\u0000\u0409\u040a\u0004S\t"+ - "\u0000\u040a\u040b\u0003-\r\u0000\u040b\u040c\u0001\u0000\u0000\u0000"+ - "\u040c\u040d\u0006S\u0010\u0000\u040d\u00ba\u0001\u0000\u0000\u0000\u040e"+ - "\u040f\u0004T\n\u0000\u040f\u0410\u0003#\b\u0000\u0410\u0411\u0001\u0000"+ - "\u0000\u0000\u0411\u0412\u0006T\u0011\u0000\u0412\u00bc\u0001\u0000\u0000"+ - "\u0000\u0413\u0416\u0003\u0093@\u0000\u0414\u0417\u0003S \u0000\u0415"+ - "\u0417\u0003a\'\u0000\u0416\u0414\u0001\u0000\u0000\u0000\u0416\u0415"+ - "\u0001\u0000\u0000\u0000\u0417\u041b\u0001\u0000\u0000\u0000\u0418\u041a"+ - "\u0003c(\u0000\u0419\u0418\u0001\u0000\u0000\u0000\u041a\u041d\u0001\u0000"+ - "\u0000\u0000\u041b\u0419\u0001\u0000\u0000\u0000\u041b\u041c\u0001\u0000"+ - "\u0000\u0000\u041c\u0425\u0001\u0000\u0000\u0000\u041d\u041b\u0001\u0000"+ - "\u0000\u0000\u041e\u0420\u0003\u0093@\u0000\u041f\u0421\u0003Q\u001f\u0000"+ - "\u0420\u041f\u0001\u0000\u0000\u0000\u0421\u0422\u0001\u0000\u0000\u0000"+ - "\u0422\u0420\u0001\u0000\u0000\u0000\u0422\u0423\u0001\u0000\u0000\u0000"+ - "\u0423\u0425\u0001\u0000\u0000\u0000\u0424\u0413\u0001\u0000\u0000\u0000"+ - "\u0424\u041e\u0001\u0000\u0000\u0000\u0425\u00be\u0001\u0000\u0000\u0000"+ - "\u0426\u0427\u0005[\u0000\u0000\u0427\u0428\u0001\u0000\u0000\u0000\u0428"+ - "\u0429\u0006V\u0000\u0000\u0429\u042a\u0006V\u0000\u0000\u042a\u00c0\u0001"+ - "\u0000\u0000\u0000\u042b\u042c\u0005]\u0000\u0000\u042c\u042d\u0001\u0000"+ - "\u0000\u0000\u042d\u042e\u0006W\u000e\u0000\u042e\u042f\u0006W\u000e\u0000"+ - "\u042f\u00c2\u0001\u0000\u0000\u0000\u0430\u0434\u0003S \u0000\u0431\u0433"+ - "\u0003c(\u0000\u0432\u0431\u0001\u0000\u0000\u0000\u0433\u0436\u0001\u0000"+ - "\u0000\u0000\u0434\u0432\u0001\u0000\u0000\u0000\u0434\u0435\u0001\u0000"+ - "\u0000\u0000\u0435\u0441\u0001\u0000\u0000\u0000\u0436\u0434\u0001\u0000"+ - "\u0000\u0000\u0437\u043a\u0003a\'\u0000\u0438\u043a\u0003[$\u0000\u0439"+ - "\u0437\u0001\u0000\u0000\u0000\u0439\u0438\u0001\u0000\u0000\u0000\u043a"+ - "\u043c\u0001\u0000\u0000\u0000\u043b\u043d\u0003c(\u0000\u043c\u043b\u0001"+ - "\u0000\u0000\u0000\u043d\u043e\u0001\u0000\u0000\u0000\u043e\u043c\u0001"+ - "\u0000\u0000\u0000\u043e\u043f\u0001\u0000\u0000\u0000\u043f\u0441\u0001"+ - "\u0000\u0000\u0000\u0440\u0430\u0001\u0000\u0000\u0000\u0440\u0439\u0001"+ - "\u0000\u0000\u0000\u0441\u00c4\u0001\u0000\u0000\u0000\u0442\u0444\u0003"+ - "]%\u0000\u0443\u0445\u0003_&\u0000\u0444\u0443\u0001\u0000\u0000\u0000"+ - "\u0445\u0446\u0001\u0000\u0000\u0000\u0446\u0444\u0001\u0000\u0000\u0000"+ - "\u0446\u0447\u0001\u0000\u0000\u0000\u0447\u0448\u0001\u0000\u0000\u0000"+ - "\u0448\u0449\u0003]%\u0000\u0449\u00c6\u0001\u0000\u0000\u0000\u044a\u044b"+ - "\u0003\u00c5Y\u0000\u044b\u00c8\u0001\u0000\u0000\u0000\u044c\u044d\u0003"+ - "I\u001b\u0000\u044d\u044e\u0001\u0000\u0000\u0000\u044e\u044f\u0006[\r"+ - "\u0000\u044f\u00ca\u0001\u0000\u0000\u0000\u0450\u0451\u0003K\u001c\u0000"+ - "\u0451\u0452\u0001\u0000\u0000\u0000\u0452\u0453\u0006\\\r\u0000\u0453"+ - "\u00cc\u0001\u0000\u0000\u0000\u0454\u0455\u0003M\u001d\u0000\u0455\u0456"+ - "\u0001\u0000\u0000\u0000\u0456\u0457\u0006]\r\u0000\u0457\u00ce\u0001"+ - "\u0000\u0000\u0000\u0458\u0459\u0003\u00bfV\u0000\u0459\u045a\u0001\u0000"+ - "\u0000\u0000\u045a\u045b\u0006^\u0012\u0000\u045b\u045c\u0006^\u0013\u0000"+ - "\u045c\u00d0\u0001\u0000\u0000\u0000\u045d\u045e\u0003O\u001e\u0000\u045e"+ - "\u045f\u0001\u0000\u0000\u0000\u045f\u0460\u0006_\u0014\u0000\u0460\u0461"+ - "\u0006_\u000e\u0000\u0461\u00d2\u0001\u0000\u0000\u0000\u0462\u0463\u0003"+ - "M\u001d\u0000\u0463\u0464\u0001\u0000\u0000\u0000\u0464\u0465\u0006`\r"+ - "\u0000\u0465\u00d4\u0001\u0000\u0000\u0000\u0466\u0467\u0003I\u001b\u0000"+ - "\u0467\u0468\u0001\u0000\u0000\u0000\u0468\u0469\u0006a\r\u0000\u0469"+ - "\u00d6\u0001\u0000\u0000\u0000\u046a\u046b\u0003K\u001c\u0000\u046b\u046c"+ - "\u0001\u0000\u0000\u0000\u046c\u046d\u0006b\r\u0000\u046d\u00d8\u0001"+ - "\u0000\u0000\u0000\u046e\u046f\u0003O\u001e\u0000\u046f\u0470\u0001\u0000"+ - "\u0000\u0000\u0470\u0471\u0006c\u0014\u0000\u0471\u0472\u0006c\u000e\u0000"+ - "\u0472\u00da\u0001\u0000\u0000\u0000\u0473\u0474\u0003\u00bfV\u0000\u0474"+ - "\u0475\u0001\u0000\u0000\u0000\u0475\u0476\u0006d\u0012\u0000\u0476\u00dc"+ - "\u0001\u0000\u0000\u0000\u0477\u0478\u0003\u00c1W\u0000\u0478\u0479\u0001"+ - "\u0000\u0000\u0000\u0479\u047a\u0006e\u0015\u0000\u047a\u00de\u0001\u0000"+ - "\u0000\u0000\u047b\u047c\u0003u1\u0000\u047c\u047d\u0001\u0000\u0000\u0000"+ - "\u047d\u047e\u0006f\u0016\u0000\u047e\u00e0\u0001\u0000\u0000\u0000\u047f"+ - "\u0480\u0003w2\u0000\u0480\u0481\u0001\u0000\u0000\u0000\u0481\u0482\u0006"+ - "g\u0017\u0000\u0482\u00e2\u0001\u0000\u0000\u0000\u0483\u0484\u0003q/"+ - "\u0000\u0484\u0485\u0001\u0000\u0000\u0000\u0485\u0486\u0006h\u0018\u0000"+ - "\u0486\u00e4\u0001\u0000\u0000\u0000\u0487\u0488\u0007\u0010\u0000\u0000"+ - "\u0488\u0489\u0007\u0003\u0000\u0000\u0489\u048a\u0007\u0005\u0000\u0000"+ - "\u048a\u048b\u0007\f\u0000\u0000\u048b\u048c\u0007\u0000\u0000\u0000\u048c"+ - "\u048d\u0007\f\u0000\u0000\u048d\u048e\u0007\u0005\u0000\u0000\u048e\u048f"+ - "\u0007\f\u0000\u0000\u048f\u00e6\u0001\u0000\u0000\u0000\u0490\u0494\b"+ - " \u0000\u0000\u0491\u0492\u0005/\u0000\u0000\u0492\u0494\b!\u0000\u0000"+ - "\u0493\u0490\u0001\u0000\u0000\u0000\u0493\u0491\u0001\u0000\u0000\u0000"+ - "\u0494\u00e8\u0001\u0000\u0000\u0000\u0495\u0497\u0003\u00e7j\u0000\u0496"+ - "\u0495\u0001\u0000\u0000\u0000\u0497\u0498\u0001\u0000\u0000\u0000\u0498"+ - "\u0496\u0001\u0000\u0000\u0000\u0498\u0499\u0001\u0000\u0000\u0000\u0499"+ - "\u00ea\u0001\u0000\u0000\u0000\u049a\u049b\u0003\u00e9k\u0000\u049b\u049c"+ - "\u0001\u0000\u0000\u0000\u049c\u049d\u0006l\u0019\u0000\u049d\u00ec\u0001"+ - "\u0000\u0000\u0000\u049e\u049f\u0003e)\u0000\u049f\u04a0\u0001\u0000\u0000"+ - "\u0000\u04a0\u04a1\u0006m\u001a\u0000\u04a1\u00ee\u0001\u0000\u0000\u0000"+ - "\u04a2\u04a3\u0003I\u001b\u0000\u04a3\u04a4\u0001\u0000\u0000\u0000\u04a4"+ - "\u04a5\u0006n\r\u0000\u04a5\u00f0\u0001\u0000\u0000\u0000\u04a6\u04a7"+ - "\u0003K\u001c\u0000\u04a7\u04a8\u0001\u0000\u0000\u0000\u04a8\u04a9\u0006"+ - "o\r\u0000\u04a9\u00f2\u0001\u0000\u0000\u0000\u04aa\u04ab\u0003M\u001d"+ - "\u0000\u04ab\u04ac\u0001\u0000\u0000\u0000\u04ac\u04ad\u0006p\r\u0000"+ - "\u04ad\u00f4\u0001\u0000\u0000\u0000\u04ae\u04af\u0003O\u001e\u0000\u04af"+ - "\u04b0\u0001\u0000\u0000\u0000\u04b0\u04b1\u0006q\u0014\u0000\u04b1\u04b2"+ - "\u0006q\u000e\u0000\u04b2\u00f6\u0001\u0000\u0000\u0000\u04b3\u04b4\u0003"+ - "{4\u0000\u04b4\u04b5\u0001\u0000\u0000\u0000\u04b5\u04b6\u0006r\u001b"+ - "\u0000\u04b6\u00f8\u0001\u0000\u0000\u0000\u04b7\u04b8\u0003w2\u0000\u04b8"+ - "\u04b9\u0001\u0000\u0000\u0000\u04b9\u04ba\u0006s\u0017\u0000\u04ba\u00fa"+ - "\u0001\u0000\u0000\u0000\u04bb\u04bc\u0003\u0093@\u0000\u04bc\u04bd\u0001"+ - "\u0000\u0000\u0000\u04bd\u04be\u0006t\u001c\u0000\u04be\u00fc\u0001\u0000"+ - "\u0000\u0000\u04bf\u04c0\u0003\u00bdU\u0000\u04c0\u04c1\u0001\u0000\u0000"+ - "\u0000\u04c1\u04c2\u0006u\u001d\u0000\u04c2\u00fe\u0001\u0000\u0000\u0000"+ - "\u04c3\u04c8\u0003S \u0000\u04c4\u04c8\u0003Q\u001f\u0000\u04c5\u04c8"+ - "\u0003a\'\u0000\u04c6\u04c8\u0003\u00adM\u0000\u04c7\u04c3\u0001\u0000"+ - "\u0000\u0000\u04c7\u04c4\u0001\u0000\u0000\u0000\u04c7\u04c5\u0001\u0000"+ - "\u0000\u0000\u04c7\u04c6\u0001\u0000\u0000\u0000\u04c8\u0100\u0001\u0000"+ - "\u0000\u0000\u04c9\u04cc\u0003S \u0000\u04ca\u04cc\u0003\u00adM\u0000"+ - "\u04cb\u04c9\u0001\u0000\u0000\u0000\u04cb\u04ca\u0001\u0000\u0000\u0000"+ - "\u04cc\u04d0\u0001\u0000\u0000\u0000\u04cd\u04cf\u0003\u00ffv\u0000\u04ce"+ - "\u04cd\u0001\u0000\u0000\u0000\u04cf\u04d2\u0001\u0000\u0000\u0000\u04d0"+ - "\u04ce\u0001\u0000\u0000\u0000\u04d0\u04d1\u0001\u0000\u0000\u0000\u04d1"+ - "\u04dd\u0001\u0000\u0000\u0000\u04d2\u04d0\u0001\u0000\u0000\u0000\u04d3"+ - "\u04d6\u0003a\'\u0000\u04d4\u04d6\u0003[$\u0000\u04d5\u04d3\u0001\u0000"+ - "\u0000\u0000\u04d5\u04d4\u0001\u0000\u0000\u0000\u04d6\u04d8\u0001\u0000"+ - "\u0000\u0000\u04d7\u04d9\u0003\u00ffv\u0000\u04d8\u04d7\u0001\u0000\u0000"+ - "\u0000\u04d9\u04da\u0001\u0000\u0000\u0000\u04da\u04d8\u0001\u0000\u0000"+ - "\u0000\u04da\u04db\u0001\u0000\u0000\u0000\u04db\u04dd\u0001\u0000\u0000"+ - "\u0000\u04dc\u04cb\u0001\u0000\u0000\u0000\u04dc\u04d5\u0001\u0000\u0000"+ - "\u0000\u04dd\u0102\u0001\u0000\u0000\u0000\u04de\u04e1\u0003\u0101w\u0000"+ - "\u04df\u04e1\u0003\u00c5Y\u0000\u04e0\u04de\u0001\u0000\u0000\u0000\u04e0"+ - "\u04df\u0001\u0000\u0000\u0000\u04e1\u04e2\u0001\u0000\u0000\u0000\u04e2"+ - "\u04e0\u0001\u0000\u0000\u0000\u04e2\u04e3\u0001\u0000\u0000\u0000\u04e3"+ - "\u0104\u0001\u0000\u0000\u0000\u04e4\u04e5\u0003I\u001b\u0000\u04e5\u04e6"+ - "\u0001\u0000\u0000\u0000\u04e6\u04e7\u0006y\r\u0000\u04e7\u0106\u0001"+ - "\u0000\u0000\u0000\u04e8\u04e9\u0003K\u001c\u0000\u04e9\u04ea\u0001\u0000"+ - "\u0000\u0000\u04ea\u04eb\u0006z\r\u0000\u04eb\u0108\u0001\u0000\u0000"+ - "\u0000\u04ec\u04ed\u0003M\u001d\u0000\u04ed\u04ee\u0001\u0000\u0000\u0000"+ - "\u04ee\u04ef\u0006{\r\u0000\u04ef\u010a\u0001\u0000\u0000\u0000\u04f0"+ - "\u04f1\u0003O\u001e\u0000\u04f1\u04f2\u0001\u0000\u0000\u0000\u04f2\u04f3"+ - "\u0006|\u0014\u0000\u04f3\u04f4\u0006|\u000e\u0000\u04f4\u010c\u0001\u0000"+ - "\u0000\u0000\u04f5\u04f6\u0003q/\u0000\u04f6\u04f7\u0001\u0000\u0000\u0000"+ - "\u04f7\u04f8\u0006}\u0018\u0000\u04f8\u010e\u0001\u0000\u0000\u0000\u04f9"+ - "\u04fa\u0003w2\u0000\u04fa\u04fb\u0001\u0000\u0000\u0000\u04fb\u04fc\u0006"+ - "~\u0017\u0000\u04fc\u0110\u0001\u0000\u0000\u0000\u04fd\u04fe\u0003{4"+ - "\u0000\u04fe\u04ff\u0001\u0000\u0000\u0000\u04ff\u0500\u0006\u007f\u001b"+ - "\u0000\u0500\u0112\u0001\u0000\u0000\u0000\u0501\u0502\u0003\u0093@\u0000"+ - "\u0502\u0503\u0001\u0000\u0000\u0000\u0503\u0504\u0006\u0080\u001c\u0000"+ - "\u0504\u0114\u0001\u0000\u0000\u0000\u0505\u0506\u0003\u00bdU\u0000\u0506"+ - "\u0507\u0001\u0000\u0000\u0000\u0507\u0508\u0006\u0081\u001d\u0000\u0508"+ - "\u0116\u0001\u0000\u0000\u0000\u0509\u050a\u0007\f\u0000\u0000\u050a\u050b"+ - "\u0007\u0002\u0000\u0000\u050b\u0118\u0001\u0000\u0000\u0000\u050c\u050d"+ - "\u0003\u0103x\u0000\u050d\u050e\u0001\u0000\u0000\u0000\u050e\u050f\u0006"+ - "\u0083\u001e\u0000\u050f\u011a\u0001\u0000\u0000\u0000\u0510\u0511\u0003"+ - "I\u001b\u0000\u0511\u0512\u0001\u0000\u0000\u0000\u0512\u0513\u0006\u0084"+ - "\r\u0000\u0513\u011c\u0001\u0000\u0000\u0000\u0514\u0515\u0003K\u001c"+ - "\u0000\u0515\u0516\u0001\u0000\u0000\u0000\u0516\u0517\u0006\u0085\r\u0000"+ - "\u0517\u011e\u0001\u0000\u0000\u0000\u0518\u0519\u0003M\u001d\u0000\u0519"+ - "\u051a\u0001\u0000\u0000\u0000\u051a\u051b\u0006\u0086\r\u0000\u051b\u0120"+ - "\u0001\u0000\u0000\u0000\u051c\u051d\u0003O\u001e\u0000\u051d\u051e\u0001"+ - "\u0000\u0000\u0000\u051e\u051f\u0006\u0087\u0014\u0000\u051f\u0520\u0006"+ - "\u0087\u000e\u0000\u0520\u0122\u0001\u0000\u0000\u0000\u0521\u0522\u0003"+ - "\u00bfV\u0000\u0522\u0523\u0001\u0000\u0000\u0000\u0523\u0524\u0006\u0088"+ - "\u0012\u0000\u0524\u0525\u0006\u0088\u001f\u0000\u0525\u0124\u0001\u0000"+ - "\u0000\u0000\u0526\u0527\u0007\u0007\u0000\u0000\u0527\u0528\u0007\t\u0000"+ - "\u0000\u0528\u0529\u0001\u0000\u0000\u0000\u0529\u052a\u0006\u0089 \u0000"+ - "\u052a\u0126\u0001\u0000\u0000\u0000\u052b\u052c\u0007\u0013\u0000\u0000"+ - "\u052c\u052d\u0007\u0001\u0000\u0000\u052d\u052e\u0007\u0005\u0000\u0000"+ - "\u052e\u052f\u0007\n\u0000\u0000\u052f\u0530\u0001\u0000\u0000\u0000\u0530"+ - "\u0531\u0006\u008a \u0000\u0531\u0128\u0001\u0000\u0000\u0000\u0532\u0533"+ - "\b\"\u0000\u0000\u0533\u012a\u0001\u0000\u0000\u0000\u0534\u0536\u0003"+ - "\u0129\u008b\u0000\u0535\u0534\u0001\u0000\u0000\u0000\u0536\u0537\u0001"+ - "\u0000\u0000\u0000\u0537\u0535\u0001\u0000\u0000\u0000\u0537\u0538\u0001"+ - "\u0000\u0000\u0000\u0538\u0539\u0001\u0000\u0000\u0000\u0539\u053a\u0003"+ - "u1\u0000\u053a\u053c\u0001\u0000\u0000\u0000\u053b\u0535\u0001\u0000\u0000"+ - "\u0000\u053b\u053c\u0001\u0000\u0000\u0000\u053c\u053e\u0001\u0000\u0000"+ - "\u0000\u053d\u053f\u0003\u0129\u008b\u0000\u053e\u053d\u0001\u0000\u0000"+ - "\u0000\u053f\u0540\u0001\u0000\u0000\u0000\u0540\u053e\u0001\u0000\u0000"+ - "\u0000\u0540\u0541\u0001\u0000\u0000\u0000\u0541\u012c\u0001\u0000\u0000"+ - "\u0000\u0542\u0543\u0003\u012b\u008c\u0000\u0543\u0544\u0001\u0000\u0000"+ - "\u0000\u0544\u0545\u0006\u008d!\u0000\u0545\u012e\u0001\u0000\u0000\u0000"+ - "\u0546\u0547\u0003I\u001b\u0000\u0547\u0548\u0001\u0000\u0000\u0000\u0548"+ - "\u0549\u0006\u008e\r\u0000\u0549\u0130\u0001\u0000\u0000\u0000\u054a\u054b"+ - "\u0003K\u001c\u0000\u054b\u054c\u0001\u0000\u0000\u0000\u054c\u054d\u0006"+ - "\u008f\r\u0000\u054d\u0132\u0001\u0000\u0000\u0000\u054e\u054f\u0003M"+ - "\u001d\u0000\u054f\u0550\u0001\u0000\u0000\u0000\u0550\u0551\u0006\u0090"+ - "\r\u0000\u0551\u0134\u0001\u0000\u0000\u0000\u0552\u0553\u0003O\u001e"+ - "\u0000\u0553\u0554\u0001\u0000\u0000\u0000\u0554\u0555\u0006\u0091\u0014"+ - "\u0000\u0555\u0556\u0006\u0091\u000e\u0000\u0556\u0557\u0006\u0091\u000e"+ - "\u0000\u0557\u0136\u0001\u0000\u0000\u0000\u0558\u0559\u0003q/\u0000\u0559"+ - "\u055a\u0001\u0000\u0000\u0000\u055a\u055b\u0006\u0092\u0018\u0000\u055b"+ - "\u0138\u0001\u0000\u0000\u0000\u055c\u055d\u0003w2\u0000\u055d\u055e\u0001"+ - "\u0000\u0000\u0000\u055e\u055f\u0006\u0093\u0017\u0000\u055f\u013a\u0001"+ - "\u0000\u0000\u0000\u0560\u0561\u0003{4\u0000\u0561\u0562\u0001\u0000\u0000"+ - "\u0000\u0562\u0563\u0006\u0094\u001b\u0000\u0563\u013c\u0001\u0000\u0000"+ - "\u0000\u0564\u0565\u0003\u0127\u008a\u0000\u0565\u0566\u0001\u0000\u0000"+ - "\u0000\u0566\u0567\u0006\u0095\"\u0000\u0567\u013e\u0001\u0000\u0000\u0000"+ - "\u0568\u0569\u0003\u0103x\u0000\u0569\u056a\u0001\u0000\u0000\u0000\u056a"+ - "\u056b\u0006\u0096\u001e\u0000\u056b\u0140\u0001\u0000\u0000\u0000\u056c"+ - "\u056d\u0003\u00c7Z\u0000\u056d\u056e\u0001\u0000\u0000\u0000\u056e\u056f"+ - "\u0006\u0097#\u0000\u056f\u0142\u0001\u0000\u0000\u0000\u0570\u0571\u0003"+ - "\u0093@\u0000\u0571\u0572\u0001\u0000\u0000\u0000\u0572\u0573\u0006\u0098"+ - "\u001c\u0000\u0573\u0144\u0001\u0000\u0000\u0000\u0574\u0575\u0003\u00bd"+ - "U\u0000\u0575\u0576\u0001\u0000\u0000\u0000\u0576\u0577\u0006\u0099\u001d"+ - "\u0000\u0577\u0146\u0001\u0000\u0000\u0000\u0578\u0579\u0003I\u001b\u0000"+ - "\u0579\u057a\u0001\u0000\u0000\u0000\u057a\u057b\u0006\u009a\r\u0000\u057b"+ - "\u0148\u0001\u0000\u0000\u0000\u057c\u057d\u0003K\u001c\u0000\u057d\u057e"+ - "\u0001\u0000\u0000\u0000\u057e\u057f\u0006\u009b\r\u0000\u057f\u014a\u0001"+ - "\u0000\u0000\u0000\u0580\u0581\u0003M\u001d\u0000\u0581\u0582\u0001\u0000"+ - "\u0000\u0000\u0582\u0583\u0006\u009c\r\u0000\u0583\u014c\u0001\u0000\u0000"+ - "\u0000\u0584\u0585\u0003O\u001e\u0000\u0585\u0586\u0001\u0000\u0000\u0000"+ - "\u0586\u0587\u0006\u009d\u0014\u0000\u0587\u0588\u0006\u009d\u000e\u0000"+ - "\u0588\u014e\u0001\u0000\u0000\u0000\u0589\u058a\u0003{4\u0000\u058a\u058b"+ - "\u0001\u0000\u0000\u0000\u058b\u058c\u0006\u009e\u001b\u0000\u058c\u0150"+ - "\u0001\u0000\u0000\u0000\u058d\u058e\u0003\u0093@\u0000\u058e\u058f\u0001"+ - "\u0000\u0000\u0000\u058f\u0590\u0006\u009f\u001c\u0000\u0590\u0152\u0001"+ - "\u0000\u0000\u0000\u0591\u0592\u0003\u00bdU\u0000\u0592\u0593\u0001\u0000"+ - "\u0000\u0000\u0593\u0594\u0006\u00a0\u001d\u0000\u0594\u0154\u0001\u0000"+ - "\u0000\u0000\u0595\u0596\u0003\u00c7Z\u0000\u0596\u0597\u0001\u0000\u0000"+ - "\u0000\u0597\u0598\u0006\u00a1#\u0000\u0598\u0156\u0001\u0000\u0000\u0000"+ - "\u0599\u059a\u0003\u00c3X\u0000\u059a\u059b\u0001\u0000\u0000\u0000\u059b"+ - "\u059c\u0006\u00a2$\u0000\u059c\u0158\u0001\u0000\u0000\u0000\u059d\u059e"+ - "\u0003I\u001b\u0000\u059e\u059f\u0001\u0000\u0000\u0000\u059f\u05a0\u0006"+ - "\u00a3\r\u0000\u05a0\u015a\u0001\u0000\u0000\u0000\u05a1\u05a2\u0003K"+ - "\u001c\u0000\u05a2\u05a3\u0001\u0000\u0000\u0000\u05a3\u05a4\u0006\u00a4"+ - "\r\u0000\u05a4\u015c\u0001\u0000\u0000\u0000\u05a5\u05a6\u0003M\u001d"+ - "\u0000\u05a6\u05a7\u0001\u0000\u0000\u0000\u05a7\u05a8\u0006\u00a5\r\u0000"+ - "\u05a8\u015e\u0001\u0000\u0000\u0000\u05a9\u05aa\u0003O\u001e\u0000\u05aa"+ - "\u05ab\u0001\u0000\u0000\u0000\u05ab\u05ac\u0006\u00a6\u0014\u0000\u05ac"+ - "\u05ad\u0006\u00a6\u000e\u0000\u05ad\u0160\u0001\u0000\u0000\u0000\u05ae"+ - "\u05af\u0007\u0001\u0000\u0000\u05af\u05b0\u0007\t\u0000\u0000\u05b0\u05b1"+ - "\u0007\u000f\u0000\u0000\u05b1\u05b2\u0007\u0007\u0000\u0000\u05b2\u0162"+ - "\u0001\u0000\u0000\u0000\u05b3\u05b4\u0003I\u001b\u0000\u05b4\u05b5\u0001"+ - "\u0000\u0000\u0000\u05b5\u05b6\u0006\u00a8\r\u0000\u05b6\u0164\u0001\u0000"+ - "\u0000\u0000\u05b7\u05b8\u0003K\u001c\u0000\u05b8\u05b9\u0001\u0000\u0000"+ - "\u0000\u05b9\u05ba\u0006\u00a9\r\u0000\u05ba\u0166\u0001\u0000\u0000\u0000"+ - "\u05bb\u05bc\u0003M\u001d\u0000\u05bc\u05bd\u0001\u0000\u0000\u0000\u05bd"+ - "\u05be\u0006\u00aa\r\u0000\u05be\u0168\u0001\u0000\u0000\u0000\u05bf\u05c0"+ - "\u0003\u00c1W\u0000\u05c0\u05c1\u0001\u0000\u0000\u0000\u05c1\u05c2\u0006"+ - "\u00ab\u0015\u0000\u05c2\u05c3\u0006\u00ab\u000e\u0000\u05c3\u016a\u0001"+ - "\u0000\u0000\u0000\u05c4\u05c5\u0003u1\u0000\u05c5\u05c6\u0001\u0000\u0000"+ - "\u0000\u05c6\u05c7\u0006\u00ac\u0016\u0000\u05c7\u016c\u0001\u0000\u0000"+ - "\u0000\u05c8\u05ce\u0003[$\u0000\u05c9\u05ce\u0003Q\u001f\u0000\u05ca"+ - "\u05ce\u0003{4\u0000\u05cb\u05ce\u0003S \u0000\u05cc\u05ce\u0003a\'\u0000"+ - "\u05cd\u05c8\u0001\u0000\u0000\u0000\u05cd\u05c9\u0001\u0000\u0000\u0000"+ - "\u05cd\u05ca\u0001\u0000\u0000\u0000\u05cd\u05cb\u0001\u0000\u0000\u0000"+ - "\u05cd\u05cc\u0001\u0000\u0000\u0000\u05ce\u05cf\u0001\u0000\u0000\u0000"+ - "\u05cf\u05cd\u0001\u0000\u0000\u0000\u05cf\u05d0\u0001\u0000\u0000\u0000"+ - "\u05d0\u016e\u0001\u0000\u0000\u0000\u05d1\u05d2\u0003I\u001b\u0000\u05d2"+ - "\u05d3\u0001\u0000\u0000\u0000\u05d3\u05d4\u0006\u00ae\r\u0000\u05d4\u0170"+ - "\u0001\u0000\u0000\u0000\u05d5\u05d6\u0003K\u001c\u0000\u05d6\u05d7\u0001"+ - "\u0000\u0000\u0000\u05d7\u05d8\u0006\u00af\r\u0000\u05d8\u0172\u0001\u0000"+ - "\u0000\u0000\u05d9\u05da\u0003M\u001d\u0000\u05da\u05db\u0001\u0000\u0000"+ - "\u0000\u05db\u05dc\u0006\u00b0\r\u0000\u05dc\u0174\u0001\u0000\u0000\u0000"+ - "\u05dd\u05de\u0003O\u001e\u0000\u05de\u05df\u0001\u0000\u0000\u0000\u05df"+ - "\u05e0\u0006\u00b1\u0014\u0000\u05e0\u05e1\u0006\u00b1\u000e\u0000\u05e1"+ - "\u0176\u0001\u0000\u0000\u0000\u05e2\u05e3\u0003u1\u0000\u05e3\u05e4\u0001"+ - "\u0000\u0000\u0000\u05e4\u05e5\u0006\u00b2\u0016\u0000\u05e5\u0178\u0001"+ - "\u0000\u0000\u0000\u05e6\u05e7\u0003w2\u0000\u05e7\u05e8\u0001\u0000\u0000"+ - "\u0000\u05e8\u05e9\u0006\u00b3\u0017\u0000\u05e9\u017a\u0001\u0000\u0000"+ - "\u0000\u05ea\u05eb\u0003{4\u0000\u05eb\u05ec\u0001\u0000\u0000\u0000\u05ec"+ - "\u05ed\u0006\u00b4\u001b\u0000\u05ed\u017c\u0001\u0000\u0000\u0000\u05ee"+ - "\u05ef\u0003\u0125\u0089\u0000\u05ef\u05f0\u0001\u0000\u0000\u0000\u05f0"+ - "\u05f1\u0006\u00b5%\u0000\u05f1\u05f2\u0006\u00b5&\u0000\u05f2\u017e\u0001"+ - "\u0000\u0000\u0000\u05f3\u05f4\u0003\u00e9k\u0000\u05f4\u05f5\u0001\u0000"+ - "\u0000\u0000\u05f5\u05f6\u0006\u00b6\u0019\u0000\u05f6\u0180\u0001\u0000"+ - "\u0000\u0000\u05f7\u05f8\u0003e)\u0000\u05f8\u05f9\u0001\u0000\u0000\u0000"+ - "\u05f9\u05fa\u0006\u00b7\u001a\u0000\u05fa\u0182\u0001\u0000\u0000\u0000"+ - "\u05fb\u05fc\u0003I\u001b\u0000\u05fc\u05fd\u0001\u0000\u0000\u0000\u05fd"+ - "\u05fe\u0006\u00b8\r\u0000\u05fe\u0184\u0001\u0000\u0000\u0000\u05ff\u0600"+ - "\u0003K\u001c\u0000\u0600\u0601\u0001\u0000\u0000\u0000\u0601\u0602\u0006"+ - "\u00b9\r\u0000\u0602\u0186\u0001\u0000\u0000\u0000\u0603\u0604\u0003M"+ - "\u001d\u0000\u0604\u0605\u0001\u0000\u0000\u0000\u0605\u0606\u0006\u00ba"+ - "\r\u0000\u0606\u0188\u0001\u0000\u0000\u0000\u0607\u0608\u0003O\u001e"+ - "\u0000\u0608\u0609\u0001\u0000\u0000\u0000\u0609\u060a\u0006\u00bb\u0014"+ - "\u0000\u060a\u060b\u0006\u00bb\u000e\u0000\u060b\u060c\u0006\u00bb\u000e"+ - "\u0000\u060c\u018a\u0001\u0000\u0000\u0000\u060d\u060e\u0003w2\u0000\u060e"+ - "\u060f\u0001\u0000\u0000\u0000\u060f\u0610\u0006\u00bc\u0017\u0000\u0610"+ - "\u018c\u0001\u0000\u0000\u0000\u0611\u0612\u0003{4\u0000\u0612\u0613\u0001"+ - "\u0000\u0000\u0000\u0613\u0614\u0006\u00bd\u001b\u0000\u0614\u018e\u0001"+ - "\u0000\u0000\u0000\u0615\u0616\u0003\u0103x\u0000\u0616\u0617\u0001\u0000"+ - "\u0000\u0000\u0617\u0618\u0006\u00be\u001e\u0000\u0618\u0190\u0001\u0000"+ - "\u0000\u0000\u0619\u061a\u0003I\u001b\u0000\u061a\u061b\u0001\u0000\u0000"+ - "\u0000\u061b\u061c\u0006\u00bf\r\u0000\u061c\u0192\u0001\u0000\u0000\u0000"+ - "\u061d\u061e\u0003K\u001c\u0000\u061e\u061f\u0001\u0000\u0000\u0000\u061f"+ - "\u0620\u0006\u00c0\r\u0000\u0620\u0194\u0001\u0000\u0000\u0000\u0621\u0622"+ - "\u0003M\u001d\u0000\u0622\u0623\u0001\u0000\u0000\u0000\u0623\u0624\u0006"+ - "\u00c1\r\u0000\u0624\u0196\u0001\u0000\u0000\u0000\u0625\u0626\u0003O"+ - "\u001e\u0000\u0626\u0627\u0001\u0000\u0000\u0000\u0627\u0628\u0006\u00c2"+ - "\u0014\u0000\u0628\u0629\u0006\u00c2\u000e\u0000\u0629\u0198\u0001\u0000"+ - "\u0000\u0000\u062a\u062b\u0007#\u0000\u0000\u062b\u062c\u0007\u0007\u0000"+ - "\u0000\u062c\u062d\u0007\u0001\u0000\u0000\u062d\u062e\u0007\t\u0000\u0000"+ - "\u062e\u019a\u0001\u0000\u0000\u0000\u062f\u0630\u0003\u0117\u0082\u0000"+ - "\u0630\u0631\u0001\u0000\u0000\u0000\u0631\u0632\u0006\u00c4\'\u0000\u0632"+ - "\u019c\u0001\u0000\u0000\u0000\u0633\u0634\u0003\u0125\u0089\u0000\u0634"+ - "\u0635\u0001\u0000\u0000\u0000\u0635\u0636\u0006\u00c5%\u0000\u0636\u0637"+ - "\u0006\u00c5\u000e\u0000\u0637\u0638\u0006\u00c5\u0000\u0000\u0638\u019e"+ - "\u0001\u0000\u0000\u0000\u0639\u063a\u0007\u0014\u0000\u0000\u063a\u063b"+ - "\u0007\u0002\u0000\u0000\u063b\u063c\u0007\u0001\u0000\u0000\u063c\u063d"+ - "\u0007\t\u0000\u0000\u063d\u063e\u0007\u0011\u0000\u0000\u063e\u063f\u0001"+ - "\u0000\u0000\u0000\u063f\u0640\u0006\u00c6\u000e\u0000\u0640\u0641\u0006"+ - "\u00c6\u0000\u0000\u0641\u01a0\u0001\u0000\u0000\u0000\u0642\u0643\u0003"+ - "\u00e9k\u0000\u0643\u0644\u0001\u0000\u0000\u0000\u0644\u0645\u0006\u00c7"+ - "\u0019\u0000\u0645\u01a2\u0001\u0000\u0000\u0000\u0646\u0647\u0003e)\u0000"+ - "\u0647\u0648\u0001\u0000\u0000\u0000\u0648\u0649\u0006\u00c8\u001a\u0000"+ - "\u0649\u01a4\u0001\u0000\u0000\u0000\u064a\u064b\u0003u1\u0000\u064b\u064c"+ - "\u0001\u0000\u0000\u0000\u064c\u064d\u0006\u00c9\u0016\u0000\u064d\u01a6"+ - "\u0001\u0000\u0000\u0000\u064e\u064f\u0003\u00c3X\u0000\u064f\u0650\u0001"+ - "\u0000\u0000\u0000\u0650\u0651\u0006\u00ca$\u0000\u0651\u01a8\u0001\u0000"+ - "\u0000\u0000\u0652\u0653\u0003\u00c7Z\u0000\u0653\u0654\u0001\u0000\u0000"+ - "\u0000\u0654\u0655\u0006\u00cb#\u0000\u0655\u01aa\u0001\u0000\u0000\u0000"+ - "\u0656\u0657\u0003I\u001b\u0000\u0657\u0658\u0001\u0000\u0000\u0000\u0658"+ - "\u0659\u0006\u00cc\r\u0000\u0659\u01ac\u0001\u0000\u0000\u0000\u065a\u065b"+ - "\u0003K\u001c\u0000\u065b\u065c\u0001\u0000\u0000\u0000\u065c\u065d\u0006"+ - "\u00cd\r\u0000\u065d\u01ae\u0001\u0000\u0000\u0000\u065e\u065f\u0003M"+ - "\u001d\u0000\u065f\u0660\u0001\u0000\u0000\u0000\u0660\u0661\u0006\u00ce"+ - "\r\u0000\u0661\u01b0\u0001\u0000\u0000\u0000\u0662\u0663\u0003O\u001e"+ - "\u0000\u0663\u0664\u0001\u0000\u0000\u0000\u0664\u0665\u0006\u00cf\u0014"+ - "\u0000\u0665\u0666\u0006\u00cf\u000e\u0000\u0666\u01b2\u0001\u0000\u0000"+ - "\u0000\u0667\u0668\u0003\u00e9k\u0000\u0668\u0669\u0001\u0000\u0000\u0000"+ - "\u0669\u066a\u0006\u00d0\u0019\u0000\u066a\u066b\u0006\u00d0\u000e\u0000"+ - "\u066b\u066c\u0006\u00d0(\u0000\u066c\u01b4\u0001\u0000\u0000\u0000\u066d"+ - "\u066e\u0003e)\u0000\u066e\u066f\u0001\u0000\u0000\u0000\u066f\u0670\u0006"+ - "\u00d1\u001a\u0000\u0670\u0671\u0006\u00d1\u000e\u0000\u0671\u0672\u0006"+ - "\u00d1(\u0000\u0672\u01b6\u0001\u0000\u0000\u0000\u0673\u0674\u0003I\u001b"+ - "\u0000\u0674\u0675\u0001\u0000\u0000\u0000\u0675\u0676\u0006\u00d2\r\u0000"+ - "\u0676\u01b8\u0001\u0000\u0000\u0000\u0677\u0678\u0003K\u001c\u0000\u0678"+ - "\u0679\u0001\u0000\u0000\u0000\u0679\u067a\u0006\u00d3\r\u0000\u067a\u01ba"+ - "\u0001\u0000\u0000\u0000\u067b\u067c\u0003M\u001d\u0000\u067c\u067d\u0001"+ - "\u0000\u0000\u0000\u067d\u067e\u0006\u00d4\r\u0000\u067e\u01bc\u0001\u0000"+ - "\u0000\u0000\u067f\u0680\u0003u1\u0000\u0680\u0681\u0001\u0000\u0000\u0000"+ - "\u0681\u0682\u0006\u00d5\u0016\u0000\u0682\u0683\u0006\u00d5\u000e\u0000"+ - "\u0683\u0684\u0006\u00d5\u000b\u0000\u0684\u01be\u0001\u0000\u0000\u0000"+ - "\u0685\u0686\u0003w2\u0000\u0686\u0687\u0001\u0000\u0000\u0000\u0687\u0688"+ - "\u0006\u00d6\u0017\u0000\u0688\u0689\u0006\u00d6\u000e\u0000\u0689\u068a"+ - "\u0006\u00d6\u000b\u0000\u068a\u01c0\u0001\u0000\u0000\u0000\u068b\u068c"+ - "\u0003I\u001b\u0000\u068c\u068d\u0001\u0000\u0000\u0000\u068d\u068e\u0006"+ - "\u00d7\r\u0000\u068e\u01c2\u0001\u0000\u0000\u0000\u068f\u0690\u0003K"+ - "\u001c\u0000\u0690\u0691\u0001\u0000\u0000\u0000\u0691\u0692\u0006\u00d8"+ - "\r\u0000\u0692\u01c4\u0001\u0000\u0000\u0000\u0693\u0694\u0003M\u001d"+ - "\u0000\u0694\u0695\u0001\u0000\u0000\u0000\u0695\u0696\u0006\u00d9\r\u0000"+ - "\u0696\u01c6\u0001\u0000\u0000\u0000\u0697\u0698\u0003\u00c7Z\u0000\u0698"+ - "\u0699\u0001\u0000\u0000\u0000\u0699\u069a\u0006\u00da\u000e\u0000\u069a"+ - "\u069b\u0006\u00da\u0000\u0000\u069b\u069c\u0006\u00da#\u0000\u069c\u01c8"+ - "\u0001\u0000\u0000\u0000\u069d\u069e\u0003\u00c3X\u0000\u069e\u069f\u0001"+ - "\u0000\u0000\u0000\u069f\u06a0\u0006\u00db\u000e\u0000\u06a0\u06a1\u0006"+ - "\u00db\u0000\u0000\u06a1\u06a2\u0006\u00db$\u0000\u06a2\u01ca\u0001\u0000"+ - "\u0000\u0000\u06a3\u06a4\u0003k,\u0000\u06a4\u06a5\u0001\u0000\u0000\u0000"+ - "\u06a5\u06a6\u0006\u00dc\u000e\u0000\u06a6\u06a7\u0006\u00dc\u0000\u0000"+ - "\u06a7\u06a8\u0006\u00dc)\u0000\u06a8\u01cc\u0001\u0000\u0000\u0000\u06a9"+ - "\u06aa\u0003O\u001e\u0000\u06aa\u06ab\u0001\u0000\u0000\u0000\u06ab\u06ac"+ - "\u0006\u00dd\u0014\u0000\u06ac\u06ad\u0006\u00dd\u000e\u0000\u06ad\u01ce"+ - "\u0001\u0000\u0000\u0000\u06ae\u06af\u0003O\u001e\u0000\u06af\u06b0\u0001"+ - "\u0000\u0000\u0000\u06b0\u06b1\u0006\u00de\u0014\u0000\u06b1\u06b2\u0006"+ - "\u00de\u000e\u0000\u06b2\u01d0\u0001\u0000\u0000\u0000\u06b3\u06b4\u0003"+ - "\u0125\u0089\u0000\u06b4\u06b5\u0001\u0000\u0000\u0000\u06b5\u06b6\u0006"+ - "\u00df%\u0000\u06b6\u01d2\u0001\u0000\u0000\u0000\u06b7\u06b8\u0003\u0117"+ - "\u0082\u0000\u06b8\u06b9\u0001\u0000\u0000\u0000\u06b9\u06ba\u0006\u00e0"+ - "\'\u0000\u06ba\u01d4\u0001\u0000\u0000\u0000\u06bb\u06bc\u0003{4\u0000"+ - "\u06bc\u06bd\u0001\u0000\u0000\u0000\u06bd\u06be\u0006\u00e1\u001b\u0000"+ - "\u06be\u01d6\u0001\u0000\u0000\u0000\u06bf\u06c0\u0003w2\u0000\u06c0\u06c1"+ - "\u0001\u0000\u0000\u0000\u06c1\u06c2\u0006\u00e2\u0017\u0000\u06c2\u01d8"+ - "\u0001\u0000\u0000\u0000\u06c3\u06c4\u0003\u00c7Z\u0000\u06c4\u06c5\u0001"+ - "\u0000\u0000\u0000\u06c5\u06c6\u0006\u00e3#\u0000\u06c6\u01da\u0001\u0000"+ - "\u0000\u0000\u06c7\u06c8\u0003\u00c3X\u0000\u06c8\u06c9\u0001\u0000\u0000"+ - "\u0000\u06c9\u06ca\u0006\u00e4$\u0000\u06ca\u01dc\u0001\u0000\u0000\u0000"+ - "\u06cb\u06cc\u0003I\u001b\u0000\u06cc\u06cd\u0001\u0000\u0000\u0000\u06cd"+ - "\u06ce\u0006\u00e5\r\u0000\u06ce\u01de\u0001\u0000\u0000\u0000\u06cf\u06d0"+ - "\u0003K\u001c\u0000\u06d0\u06d1\u0001\u0000\u0000\u0000\u06d1\u06d2\u0006"+ - "\u00e6\r\u0000\u06d2\u01e0\u0001\u0000\u0000\u0000\u06d3\u06d4\u0003M"+ - "\u001d\u0000\u06d4\u06d5\u0001\u0000\u0000\u0000\u06d5\u06d6\u0006\u00e7"+ - "\r\u0000\u06d6\u01e2\u0001\u0000\u0000\u0000\u06d7\u06d8\u0003O\u001e"+ - "\u0000\u06d8\u06d9\u0001\u0000\u0000\u0000\u06d9\u06da\u0006\u00e8\u0014"+ - "\u0000\u06da\u06db\u0006\u00e8\u000e\u0000\u06db\u01e4\u0001\u0000\u0000"+ - "\u0000\u06dc\u06dd\u0003\u00c3X\u0000\u06dd\u06de\u0001\u0000\u0000\u0000"+ - "\u06de\u06df\u0006\u00e9$\u0000\u06df\u01e6\u0001\u0000\u0000\u0000\u06e0"+ - "\u06e1\u0003M\u001d\u0000\u06e1\u06e2\u0001\u0000\u0000\u0000\u06e2\u06e3"+ - "\u0006\u00ea\r\u0000\u06e3\u01e8\u0001\u0000\u0000\u0000\u06e4\u06e5\u0003"+ - "I\u001b\u0000\u06e5\u06e6\u0001\u0000\u0000\u0000\u06e6\u06e7\u0006\u00eb"+ - "\r\u0000\u06e7\u01ea\u0001\u0000\u0000\u0000\u06e8\u06e9\u0003K\u001c"+ - "\u0000\u06e9\u06ea\u0001\u0000\u0000\u0000\u06ea\u06eb\u0006\u00ec\r\u0000"+ - "\u06eb\u01ec\u0001\u0000\u0000\u0000\u06ec\u06ed\u0003\u0089;\u0000\u06ed"+ - "\u06ee\u0001\u0000\u0000\u0000\u06ee\u06ef\u0006\u00ed*\u0000\u06ef\u06f0"+ - "\u0006\u00ed\u0013\u0000\u06f0\u01ee\u0001\u0000\u0000\u0000\u06f1\u06f2"+ - "\u0003\u0097B\u0000\u06f2\u06f3\u0001\u0000\u0000\u0000\u06f3\u06f4\u0006"+ - "\u00ee+\u0000\u06f4\u06f5\u0006\u00ee\u000e\u0000\u06f5\u01f0\u0001\u0000"+ - "\u0000\u0000\u06f6\u06f7\u0003O\u001e\u0000\u06f7\u06f8\u0001\u0000\u0000"+ - "\u0000\u06f8\u06f9\u0006\u00ef\u0014\u0000\u06f9\u06fa\u0006\u00ef\u000e"+ - "\u0000\u06fa\u01f2\u0001\u0000\u0000\u0000\u06fb\u06fc\u0003M\u001d\u0000"+ - "\u06fc\u06fd\u0001\u0000\u0000\u0000\u06fd\u06fe\u0006\u00f0\r\u0000\u06fe"+ - "\u01f4\u0001\u0000\u0000\u0000\u06ff\u0700\u0003I\u001b\u0000\u0700\u0701"+ - "\u0001\u0000\u0000\u0000\u0701\u0702\u0006\u00f1\r\u0000\u0702\u01f6\u0001"+ - "\u0000\u0000\u0000\u0703\u0704\u0003K\u001c\u0000\u0704\u0705\u0001\u0000"+ - "\u0000\u0000\u0705\u0706\u0006\u00f2\r\u0000\u0706\u01f8\u0001\u0000\u0000"+ - "\u0000E\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f"+ - "\r\u000e\u000f\u0010\u0011\u0012\u02e9\u02f3\u02f7\u02fa\u0303\u0305\u0310"+ - "\u0323\u0328\u0331\u0338\u033d\u033f\u034a\u0352\u0355\u0357\u035c\u0361"+ - "\u0367\u036e\u0373\u0379\u037c\u0384\u0388\u0416\u041b\u0422\u0424\u0434"+ - "\u0439\u043e\u0440\u0446\u0493\u0498\u04c7\u04cb\u04d0\u04d5\u04da\u04dc"+ - "\u04e0\u04e2\u0537\u053b\u0540\u05cd\u05cf,\u0005\u0001\u0000\u0005\u0004"+ - "\u0000\u0005\u0006\u0000\u0005\u0002\u0000\u0005\u0003\u0000\u0005\b\u0000"+ - "\u0005\u0005\u0000\u0005\t\u0000\u0005\r\u0000\u0005\u0010\u0000\u0005"+ - "\u000b\u0000\u0005\u000e\u0000\u0005\u0012\u0000\u0000\u0001\u0000\u0004"+ - "\u0000\u0000\u0007\u0010\u0000\u0007\u000e\u0000\u0007\t\u0000\u0007J"+ - "\u0000\u0005\u0000\u0000\u0007\u001f\u0000\u0007K\u0000\u0007(\u0000\u0007"+ - ")\u0000\u0007&\u0000\u0007U\u0000\u0007 \u0000\u0007+\u0000\u00077\u0000"+ - "\u0007I\u0000\u0007Y\u0000\u0005\n\u0000\u0005\u0007\u0000\u0007c\u0000"+ - "\u0007b\u0000\u0007M\u0000\u0007L\u0000\u0007a\u0000\u0005\f\u0000\u0007"+ - "]\u0000\u0005\u000f\u0000\u0007#\u0000\u00072\u0000\u00079\u0000"; + "\r\u01ab\u0001\u0000\u0000\u0000\u000e\u01ad\u0001\u0000\u0000\u0000\u000e"+ + "\u01af\u0001\u0000\u0000\u0000\u000e\u01b1\u0001\u0000\u0000\u0000\u000e"+ + "\u01b3\u0001\u0000\u0000\u0000\u000e\u01b5\u0001\u0000\u0000\u0000\u000e"+ + "\u01b7\u0001\u0000\u0000\u0000\u000f\u01b9\u0001\u0000\u0000\u0000\u000f"+ + "\u01bb\u0001\u0000\u0000\u0000\u000f\u01bd\u0001\u0000\u0000\u0000\u000f"+ + "\u01bf\u0001\u0000\u0000\u0000\u000f\u01c1\u0001\u0000\u0000\u0000\u000f"+ + "\u01c3\u0001\u0000\u0000\u0000\u000f\u01c5\u0001\u0000\u0000\u0000\u000f"+ + "\u01c7\u0001\u0000\u0000\u0000\u000f\u01c9\u0001\u0000\u0000\u0000\u0010"+ + "\u01cb\u0001\u0000\u0000\u0000\u0010\u01cd\u0001\u0000\u0000\u0000\u0010"+ + "\u01cf\u0001\u0000\u0000\u0000\u0010\u01d1\u0001\u0000\u0000\u0000\u0010"+ + "\u01d3\u0001\u0000\u0000\u0000\u0010\u01d5\u0001\u0000\u0000\u0000\u0010"+ + "\u01d7\u0001\u0000\u0000\u0000\u0010\u01d9\u0001\u0000\u0000\u0000\u0010"+ + "\u01db\u0001\u0000\u0000\u0000\u0010\u01dd\u0001\u0000\u0000\u0000\u0011"+ + "\u01df\u0001\u0000\u0000\u0000\u0011\u01e1\u0001\u0000\u0000\u0000\u0011"+ + "\u01e3\u0001\u0000\u0000\u0000\u0011\u01e5\u0001\u0000\u0000\u0000\u0011"+ + "\u01e7\u0001\u0000\u0000\u0000\u0012\u01e9\u0001\u0000\u0000\u0000\u0012"+ + "\u01eb\u0001\u0000\u0000\u0000\u0012\u01ed\u0001\u0000\u0000\u0000\u0012"+ + "\u01ef\u0001\u0000\u0000\u0000\u0012\u01f1\u0001\u0000\u0000\u0000\u0013"+ + "\u01f3\u0001\u0000\u0000\u0000\u0015\u01fd\u0001\u0000\u0000\u0000\u0017"+ + "\u0204\u0001\u0000\u0000\u0000\u0019\u020d\u0001\u0000\u0000\u0000\u001b"+ + "\u0214\u0001\u0000\u0000\u0000\u001d\u021e\u0001\u0000\u0000\u0000\u001f"+ + "\u0225\u0001\u0000\u0000\u0000!\u022c\u0001\u0000\u0000\u0000#\u0233\u0001"+ + "\u0000\u0000\u0000%\u023b\u0001\u0000\u0000\u0000\'\u0247\u0001\u0000"+ + "\u0000\u0000)\u0250\u0001\u0000\u0000\u0000+\u0256\u0001\u0000\u0000\u0000"+ + "-\u025d\u0001\u0000\u0000\u0000/\u0264\u0001\u0000\u0000\u00001\u026c"+ + "\u0001\u0000\u0000\u00003\u0274\u0001\u0000\u0000\u00005\u027d\u0001\u0000"+ + "\u0000\u00007\u028d\u0001\u0000\u0000\u00009\u029c\u0001\u0000\u0000\u0000"+ + ";\u02a8\u0001\u0000\u0000\u0000=\u02b4\u0001\u0000\u0000\u0000?\u02bf"+ + "\u0001\u0000\u0000\u0000A\u02c7\u0001\u0000\u0000\u0000C\u02cf\u0001\u0000"+ + "\u0000\u0000E\u02d8\u0001\u0000\u0000\u0000G\u02e1\u0001\u0000\u0000\u0000"+ + "I\u02e7\u0001\u0000\u0000\u0000K\u02f8\u0001\u0000\u0000\u0000M\u0308"+ + "\u0001\u0000\u0000\u0000O\u030e\u0001\u0000\u0000\u0000Q\u0312\u0001\u0000"+ + "\u0000\u0000S\u0314\u0001\u0000\u0000\u0000U\u0316\u0001\u0000\u0000\u0000"+ + "W\u0319\u0001\u0000\u0000\u0000Y\u031b\u0001\u0000\u0000\u0000[\u0324"+ + "\u0001\u0000\u0000\u0000]\u0326\u0001\u0000\u0000\u0000_\u032b\u0001\u0000"+ + "\u0000\u0000a\u032d\u0001\u0000\u0000\u0000c\u0332\u0001\u0000\u0000\u0000"+ + "e\u0351\u0001\u0000\u0000\u0000g\u0354\u0001\u0000\u0000\u0000i\u0382"+ + "\u0001\u0000\u0000\u0000k\u0384\u0001\u0000\u0000\u0000m\u0387\u0001\u0000"+ + "\u0000\u0000o\u038b\u0001\u0000\u0000\u0000q\u038f\u0001\u0000\u0000\u0000"+ + "s\u0391\u0001\u0000\u0000\u0000u\u0394\u0001\u0000\u0000\u0000w\u0396"+ + "\u0001\u0000\u0000\u0000y\u0398\u0001\u0000\u0000\u0000{\u039d\u0001\u0000"+ + "\u0000\u0000}\u039f\u0001\u0000\u0000\u0000\u007f\u03a5\u0001\u0000\u0000"+ + "\u0000\u0081\u03ab\u0001\u0000\u0000\u0000\u0083\u03ae\u0001\u0000\u0000"+ + "\u0000\u0085\u03b1\u0001\u0000\u0000\u0000\u0087\u03b6\u0001\u0000\u0000"+ + "\u0000\u0089\u03bb\u0001\u0000\u0000\u0000\u008b\u03bf\u0001\u0000\u0000"+ + "\u0000\u008d\u03c4\u0001\u0000\u0000\u0000\u008f\u03ca\u0001\u0000\u0000"+ + "\u0000\u0091\u03cd\u0001\u0000\u0000\u0000\u0093\u03cf\u0001\u0000\u0000"+ + "\u0000\u0095\u03d5\u0001\u0000\u0000\u0000\u0097\u03da\u0001\u0000\u0000"+ + "\u0000\u0099\u03dd\u0001\u0000\u0000\u0000\u009b\u03e0\u0001\u0000\u0000"+ + "\u0000\u009d\u03e3\u0001\u0000\u0000\u0000\u009f\u03e5\u0001\u0000\u0000"+ + "\u0000\u00a1\u03e8\u0001\u0000\u0000\u0000\u00a3\u03ea\u0001\u0000\u0000"+ + "\u0000\u00a5\u03ed\u0001\u0000\u0000\u0000\u00a7\u03ef\u0001\u0000\u0000"+ + "\u0000\u00a9\u03f1\u0001\u0000\u0000\u0000\u00ab\u03f3\u0001\u0000\u0000"+ + "\u0000\u00ad\u03f5\u0001\u0000\u0000\u0000\u00af\u03f7\u0001\u0000\u0000"+ + "\u0000\u00b1\u03f9\u0001\u0000\u0000\u0000\u00b3\u03fb\u0001\u0000\u0000"+ + "\u0000\u00b5\u0410\u0001\u0000\u0000\u0000\u00b7\u0412\u0001\u0000\u0000"+ + "\u0000\u00b9\u0417\u0001\u0000\u0000\u0000\u00bb\u041c\u0001\u0000\u0000"+ + "\u0000\u00bd\u0421\u0001\u0000\u0000\u0000\u00bf\u0436\u0001\u0000\u0000"+ + "\u0000\u00c1\u0438\u0001\u0000\u0000\u0000\u00c3\u0440\u0001\u0000\u0000"+ + "\u0000\u00c5\u0442\u0001\u0000\u0000\u0000\u00c7\u0446\u0001\u0000\u0000"+ + "\u0000\u00c9\u044a\u0001\u0000\u0000\u0000\u00cb\u044e\u0001\u0000\u0000"+ + "\u0000\u00cd\u0453\u0001\u0000\u0000\u0000\u00cf\u0458\u0001\u0000\u0000"+ + "\u0000\u00d1\u045c\u0001\u0000\u0000\u0000\u00d3\u0460\u0001\u0000\u0000"+ + "\u0000\u00d5\u0464\u0001\u0000\u0000\u0000\u00d7\u0469\u0001\u0000\u0000"+ + "\u0000\u00d9\u046d\u0001\u0000\u0000\u0000\u00db\u0471\u0001\u0000\u0000"+ + "\u0000\u00dd\u0475\u0001\u0000\u0000\u0000\u00df\u0479\u0001\u0000\u0000"+ + "\u0000\u00e1\u047d\u0001\u0000\u0000\u0000\u00e3\u0489\u0001\u0000\u0000"+ + "\u0000\u00e5\u048c\u0001\u0000\u0000\u0000\u00e7\u0490\u0001\u0000\u0000"+ + "\u0000\u00e9\u0494\u0001\u0000\u0000\u0000\u00eb\u0498\u0001\u0000\u0000"+ + "\u0000\u00ed\u049c\u0001\u0000\u0000\u0000\u00ef\u04a0\u0001\u0000\u0000"+ + "\u0000\u00f1\u04a4\u0001\u0000\u0000\u0000\u00f3\u04a9\u0001\u0000\u0000"+ + "\u0000\u00f5\u04ad\u0001\u0000\u0000\u0000\u00f7\u04b1\u0001\u0000\u0000"+ + "\u0000\u00f9\u04b5\u0001\u0000\u0000\u0000\u00fb\u04bd\u0001\u0000\u0000"+ + "\u0000\u00fd\u04d2\u0001\u0000\u0000\u0000\u00ff\u04d6\u0001\u0000\u0000"+ + "\u0000\u0101\u04da\u0001\u0000\u0000\u0000\u0103\u04de\u0001\u0000\u0000"+ + "\u0000\u0105\u04e2\u0001\u0000\u0000\u0000\u0107\u04e6\u0001\u0000\u0000"+ + "\u0000\u0109\u04eb\u0001\u0000\u0000\u0000\u010b\u04ef\u0001\u0000\u0000"+ + "\u0000\u010d\u04f3\u0001\u0000\u0000\u0000\u010f\u04f7\u0001\u0000\u0000"+ + "\u0000\u0111\u04fb\u0001\u0000\u0000\u0000\u0113\u04ff\u0001\u0000\u0000"+ + "\u0000\u0115\u0502\u0001\u0000\u0000\u0000\u0117\u0506\u0001\u0000\u0000"+ + "\u0000\u0119\u050a\u0001\u0000\u0000\u0000\u011b\u050e\u0001\u0000\u0000"+ + "\u0000\u011d\u0512\u0001\u0000\u0000\u0000\u011f\u0517\u0001\u0000\u0000"+ + "\u0000\u0121\u051c\u0001\u0000\u0000\u0000\u0123\u0521\u0001\u0000\u0000"+ + "\u0000\u0125\u0528\u0001\u0000\u0000\u0000\u0127\u0531\u0001\u0000\u0000"+ + "\u0000\u0129\u0538\u0001\u0000\u0000\u0000\u012b\u053c\u0001\u0000\u0000"+ + "\u0000\u012d\u0540\u0001\u0000\u0000\u0000\u012f\u0544\u0001\u0000\u0000"+ + "\u0000\u0131\u0548\u0001\u0000\u0000\u0000\u0133\u054e\u0001\u0000\u0000"+ + "\u0000\u0135\u0552\u0001\u0000\u0000\u0000\u0137\u0556\u0001\u0000\u0000"+ + "\u0000\u0139\u055a\u0001\u0000\u0000\u0000\u013b\u055e\u0001\u0000\u0000"+ + "\u0000\u013d\u0562\u0001\u0000\u0000\u0000\u013f\u0566\u0001\u0000\u0000"+ + "\u0000\u0141\u056a\u0001\u0000\u0000\u0000\u0143\u056e\u0001\u0000\u0000"+ + "\u0000\u0145\u0572\u0001\u0000\u0000\u0000\u0147\u0576\u0001\u0000\u0000"+ + "\u0000\u0149\u057a\u0001\u0000\u0000\u0000\u014b\u057f\u0001\u0000\u0000"+ + "\u0000\u014d\u0583\u0001\u0000\u0000\u0000\u014f\u0587\u0001\u0000\u0000"+ + "\u0000\u0151\u058b\u0001\u0000\u0000\u0000\u0153\u058f\u0001\u0000\u0000"+ + "\u0000\u0155\u0593\u0001\u0000\u0000\u0000\u0157\u0597\u0001\u0000\u0000"+ + "\u0000\u0159\u059b\u0001\u0000\u0000\u0000\u015b\u059f\u0001\u0000\u0000"+ + "\u0000\u015d\u05a4\u0001\u0000\u0000\u0000\u015f\u05a9\u0001\u0000\u0000"+ + "\u0000\u0161\u05ad\u0001\u0000\u0000\u0000\u0163\u05b1\u0001\u0000\u0000"+ + "\u0000\u0165\u05b5\u0001\u0000\u0000\u0000\u0167\u05ba\u0001\u0000\u0000"+ + "\u0000\u0169\u05c3\u0001\u0000\u0000\u0000\u016b\u05c7\u0001\u0000\u0000"+ + "\u0000\u016d\u05cb\u0001\u0000\u0000\u0000\u016f\u05cf\u0001\u0000\u0000"+ + "\u0000\u0171\u05d3\u0001\u0000\u0000\u0000\u0173\u05d8\u0001\u0000\u0000"+ + "\u0000\u0175\u05dc\u0001\u0000\u0000\u0000\u0177\u05e0\u0001\u0000\u0000"+ + "\u0000\u0179\u05e4\u0001\u0000\u0000\u0000\u017b\u05e9\u0001\u0000\u0000"+ + "\u0000\u017d\u05ed\u0001\u0000\u0000\u0000\u017f\u05f1\u0001\u0000\u0000"+ + "\u0000\u0181\u05f5\u0001\u0000\u0000\u0000\u0183\u05f9\u0001\u0000\u0000"+ + "\u0000\u0185\u05fd\u0001\u0000\u0000\u0000\u0187\u0603\u0001\u0000\u0000"+ + "\u0000\u0189\u0607\u0001\u0000\u0000\u0000\u018b\u060b\u0001\u0000\u0000"+ + "\u0000\u018d\u060f\u0001\u0000\u0000\u0000\u018f\u0613\u0001\u0000\u0000"+ + "\u0000\u0191\u0617\u0001\u0000\u0000\u0000\u0193\u061b\u0001\u0000\u0000"+ + "\u0000\u0195\u0620\u0001\u0000\u0000\u0000\u0197\u0625\u0001\u0000\u0000"+ + "\u0000\u0199\u0629\u0001\u0000\u0000\u0000\u019b\u062f\u0001\u0000\u0000"+ + "\u0000\u019d\u0638\u0001\u0000\u0000\u0000\u019f\u063c\u0001\u0000\u0000"+ + "\u0000\u01a1\u0640\u0001\u0000\u0000\u0000\u01a3\u0644\u0001\u0000\u0000"+ + "\u0000\u01a5\u0648\u0001\u0000\u0000\u0000\u01a7\u064c\u0001\u0000\u0000"+ + "\u0000\u01a9\u0650\u0001\u0000\u0000\u0000\u01ab\u0654\u0001\u0000\u0000"+ + "\u0000\u01ad\u0658\u0001\u0000\u0000\u0000\u01af\u065d\u0001\u0000\u0000"+ + "\u0000\u01b1\u0663\u0001\u0000\u0000\u0000\u01b3\u0669\u0001\u0000\u0000"+ + "\u0000\u01b5\u066d\u0001\u0000\u0000\u0000\u01b7\u0671\u0001\u0000\u0000"+ + "\u0000\u01b9\u0675\u0001\u0000\u0000\u0000\u01bb\u067b\u0001\u0000\u0000"+ + "\u0000\u01bd\u0681\u0001\u0000\u0000\u0000\u01bf\u0685\u0001\u0000\u0000"+ + "\u0000\u01c1\u0689\u0001\u0000\u0000\u0000\u01c3\u068d\u0001\u0000\u0000"+ + "\u0000\u01c5\u0693\u0001\u0000\u0000\u0000\u01c7\u0699\u0001\u0000\u0000"+ + "\u0000\u01c9\u069f\u0001\u0000\u0000\u0000\u01cb\u06a4\u0001\u0000\u0000"+ + "\u0000\u01cd\u06a9\u0001\u0000\u0000\u0000\u01cf\u06ad\u0001\u0000\u0000"+ + "\u0000\u01d1\u06b1\u0001\u0000\u0000\u0000\u01d3\u06b5\u0001\u0000\u0000"+ + "\u0000\u01d5\u06b9\u0001\u0000\u0000\u0000\u01d7\u06bd\u0001\u0000\u0000"+ + "\u0000\u01d9\u06c1\u0001\u0000\u0000\u0000\u01db\u06c5\u0001\u0000\u0000"+ + "\u0000\u01dd\u06c9\u0001\u0000\u0000\u0000\u01df\u06cd\u0001\u0000\u0000"+ + "\u0000\u01e1\u06d2\u0001\u0000\u0000\u0000\u01e3\u06d6\u0001\u0000\u0000"+ + "\u0000\u01e5\u06da\u0001\u0000\u0000\u0000\u01e7\u06de\u0001\u0000\u0000"+ + "\u0000\u01e9\u06e2\u0001\u0000\u0000\u0000\u01eb\u06e7\u0001\u0000\u0000"+ + "\u0000\u01ed\u06ec\u0001\u0000\u0000\u0000\u01ef\u06f0\u0001\u0000\u0000"+ + "\u0000\u01f1\u06f4\u0001\u0000\u0000\u0000\u01f3\u01f4\u0007\u0000\u0000"+ + "\u0000\u01f4\u01f5\u0007\u0001\u0000\u0000\u01f5\u01f6\u0007\u0002\u0000"+ + "\u0000\u01f6\u01f7\u0007\u0002\u0000\u0000\u01f7\u01f8\u0007\u0003\u0000"+ + "\u0000\u01f8\u01f9\u0007\u0004\u0000\u0000\u01f9\u01fa\u0007\u0005\u0000"+ + "\u0000\u01fa\u01fb\u0001\u0000\u0000\u0000\u01fb\u01fc\u0006\u0000\u0000"+ + "\u0000\u01fc\u0014\u0001\u0000\u0000\u0000\u01fd\u01fe\u0007\u0000\u0000"+ + "\u0000\u01fe\u01ff\u0007\u0006\u0000\u0000\u01ff\u0200\u0007\u0007\u0000"+ + "\u0000\u0200\u0201\u0007\b\u0000\u0000\u0201\u0202\u0001\u0000\u0000\u0000"+ + "\u0202\u0203\u0006\u0001\u0001\u0000\u0203\u0016\u0001\u0000\u0000\u0000"+ + "\u0204\u0205\u0007\u0003\u0000\u0000\u0205\u0206\u0007\t\u0000\u0000\u0206"+ + "\u0207\u0007\u0006\u0000\u0000\u0207\u0208\u0007\u0001\u0000\u0000\u0208"+ + "\u0209\u0007\u0004\u0000\u0000\u0209\u020a\u0007\n\u0000\u0000\u020a\u020b"+ + "\u0001\u0000\u0000\u0000\u020b\u020c\u0006\u0002\u0002\u0000\u020c\u0018"+ + "\u0001\u0000\u0000\u0000\u020d\u020e\u0007\u0003\u0000\u0000\u020e\u020f"+ + "\u0007\u000b\u0000\u0000\u020f\u0210\u0007\f\u0000\u0000\u0210\u0211\u0007"+ + "\r\u0000\u0000\u0211\u0212\u0001\u0000\u0000\u0000\u0212\u0213\u0006\u0003"+ + "\u0000\u0000\u0213\u001a\u0001\u0000\u0000\u0000\u0214\u0215\u0007\u0003"+ + "\u0000\u0000\u0215\u0216\u0007\u000e\u0000\u0000\u0216\u0217\u0007\b\u0000"+ + "\u0000\u0217\u0218\u0007\r\u0000\u0000\u0218\u0219\u0007\f\u0000\u0000"+ + "\u0219\u021a\u0007\u0001\u0000\u0000\u021a\u021b\u0007\t\u0000\u0000\u021b"+ + "\u021c\u0001\u0000\u0000\u0000\u021c\u021d\u0006\u0004\u0003\u0000\u021d"+ + "\u001c\u0001\u0000\u0000\u0000\u021e\u021f\u0007\u000f\u0000\u0000\u021f"+ + "\u0220\u0007\u0006\u0000\u0000\u0220\u0221\u0007\u0007\u0000\u0000\u0221"+ + "\u0222\u0007\u0010\u0000\u0000\u0222\u0223\u0001\u0000\u0000\u0000\u0223"+ + "\u0224\u0006\u0005\u0004\u0000\u0224\u001e\u0001\u0000\u0000\u0000\u0225"+ + "\u0226\u0007\u0011\u0000\u0000\u0226\u0227\u0007\u0006\u0000\u0000\u0227"+ + "\u0228\u0007\u0007\u0000\u0000\u0228\u0229\u0007\u0012\u0000\u0000\u0229"+ + "\u022a\u0001\u0000\u0000\u0000\u022a\u022b\u0006\u0006\u0000\u0000\u022b"+ + " \u0001\u0000\u0000\u0000\u022c\u022d\u0007\u0012\u0000\u0000\u022d\u022e"+ + "\u0007\u0003\u0000\u0000\u022e\u022f\u0007\u0003\u0000\u0000\u022f\u0230"+ + "\u0007\b\u0000\u0000\u0230\u0231\u0001\u0000\u0000\u0000\u0231\u0232\u0006"+ + "\u0007\u0001\u0000\u0232\"\u0001\u0000\u0000\u0000\u0233\u0234\u0007\r"+ + "\u0000\u0000\u0234\u0235\u0007\u0001\u0000\u0000\u0235\u0236\u0007\u0010"+ + "\u0000\u0000\u0236\u0237\u0007\u0001\u0000\u0000\u0237\u0238\u0007\u0005"+ + "\u0000\u0000\u0238\u0239\u0001\u0000\u0000\u0000\u0239\u023a\u0006\b\u0000"+ + "\u0000\u023a$\u0001\u0000\u0000\u0000\u023b\u023c\u0007\u0010\u0000\u0000"+ + "\u023c\u023d\u0007\u000b\u0000\u0000\u023d\u023e\u0005_\u0000\u0000\u023e"+ + "\u023f\u0007\u0003\u0000\u0000\u023f\u0240\u0007\u000e\u0000\u0000\u0240"+ + "\u0241\u0007\b\u0000\u0000\u0241\u0242\u0007\f\u0000\u0000\u0242\u0243"+ + "\u0007\t\u0000\u0000\u0243\u0244\u0007\u0000\u0000\u0000\u0244\u0245\u0001"+ + "\u0000\u0000\u0000\u0245\u0246\u0006\t\u0005\u0000\u0246&\u0001\u0000"+ + "\u0000\u0000\u0247\u0248\u0007\u0006\u0000\u0000\u0248\u0249\u0007\u0003"+ + "\u0000\u0000\u0249\u024a\u0007\t\u0000\u0000\u024a\u024b\u0007\f\u0000"+ + "\u0000\u024b\u024c\u0007\u0010\u0000\u0000\u024c\u024d\u0007\u0003\u0000"+ + "\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e\u024f\u0006\n\u0006\u0000"+ + "\u024f(\u0001\u0000\u0000\u0000\u0250\u0251\u0007\u0006\u0000\u0000\u0251"+ + "\u0252\u0007\u0007\u0000\u0000\u0252\u0253\u0007\u0013\u0000\u0000\u0253"+ + "\u0254\u0001\u0000\u0000\u0000\u0254\u0255\u0006\u000b\u0000\u0000\u0255"+ + "*\u0001\u0000\u0000\u0000\u0256\u0257\u0007\u0002\u0000\u0000\u0257\u0258"+ + "\u0007\n\u0000\u0000\u0258\u0259\u0007\u0007\u0000\u0000\u0259\u025a\u0007"+ + "\u0013\u0000\u0000\u025a\u025b\u0001\u0000\u0000\u0000\u025b\u025c\u0006"+ + "\f\u0007\u0000\u025c,\u0001\u0000\u0000\u0000\u025d\u025e\u0007\u0002"+ + "\u0000\u0000\u025e\u025f\u0007\u0007\u0000\u0000\u025f\u0260\u0007\u0006"+ + "\u0000\u0000\u0260\u0261\u0007\u0005\u0000\u0000\u0261\u0262\u0001\u0000"+ + "\u0000\u0000\u0262\u0263\u0006\r\u0000\u0000\u0263.\u0001\u0000\u0000"+ + "\u0000\u0264\u0265\u0007\u0002\u0000\u0000\u0265\u0266\u0007\u0005\u0000"+ + "\u0000\u0266\u0267\u0007\f\u0000\u0000\u0267\u0268\u0007\u0005\u0000\u0000"+ + "\u0268\u0269\u0007\u0002\u0000\u0000\u0269\u026a\u0001\u0000\u0000\u0000"+ + "\u026a\u026b\u0006\u000e\u0000\u0000\u026b0\u0001\u0000\u0000\u0000\u026c"+ + "\u026d\u0007\u0013\u0000\u0000\u026d\u026e\u0007\n\u0000\u0000\u026e\u026f"+ + "\u0007\u0003\u0000\u0000\u026f\u0270\u0007\u0006\u0000\u0000\u0270\u0271"+ + "\u0007\u0003\u0000\u0000\u0271\u0272\u0001\u0000\u0000\u0000\u0272\u0273"+ + "\u0006\u000f\u0000\u0000\u02732\u0001\u0000\u0000\u0000\u0274\u0275\u0007"+ + "\r\u0000\u0000\u0275\u0276\u0007\u0007\u0000\u0000\u0276\u0277\u0007\u0007"+ + "\u0000\u0000\u0277\u0278\u0007\u0012\u0000\u0000\u0278\u0279\u0007\u0014"+ + "\u0000\u0000\u0279\u027a\u0007\b\u0000\u0000\u027a\u027b\u0001\u0000\u0000"+ + "\u0000\u027b\u027c\u0006\u0010\b\u0000\u027c4\u0001\u0000\u0000\u0000"+ + "\u027d\u027e\u0004\u0011\u0000\u0000\u027e\u027f\u0007\u0004\u0000\u0000"+ + "\u027f\u0280\u0007\n\u0000\u0000\u0280\u0281\u0007\f\u0000\u0000\u0281"+ + "\u0282\u0007\t\u0000\u0000\u0282\u0283\u0007\u0011\u0000\u0000\u0283\u0284"+ + "\u0007\u0003\u0000\u0000\u0284\u0285\u0005_\u0000\u0000\u0285\u0286\u0007"+ + "\b\u0000\u0000\u0286\u0287\u0007\u0007\u0000\u0000\u0287\u0288\u0007\u0001"+ + "\u0000\u0000\u0288\u0289\u0007\t\u0000\u0000\u0289\u028a\u0007\u0005\u0000"+ + "\u0000\u028a\u028b\u0001\u0000\u0000\u0000\u028b\u028c\u0006\u0011\t\u0000"+ + "\u028c6\u0001\u0000\u0000\u0000\u028d\u028e\u0004\u0012\u0001\u0000\u028e"+ + "\u028f\u0007\u0001\u0000\u0000\u028f\u0290\u0007\t\u0000\u0000\u0290\u0291"+ + "\u0007\r\u0000\u0000\u0291\u0292\u0007\u0001\u0000\u0000\u0292\u0293\u0007"+ + "\t\u0000\u0000\u0293\u0294\u0007\u0003\u0000\u0000\u0294\u0295\u0007\u0002"+ + "\u0000\u0000\u0295\u0296\u0007\u0005\u0000\u0000\u0296\u0297\u0007\f\u0000"+ + "\u0000\u0297\u0298\u0007\u0005\u0000\u0000\u0298\u0299\u0007\u0002\u0000"+ + "\u0000\u0299\u029a\u0001\u0000\u0000\u0000\u029a\u029b\u0006\u0012\u0000"+ + "\u0000\u029b8\u0001\u0000\u0000\u0000\u029c\u029d\u0004\u0013\u0002\u0000"+ + "\u029d\u029e\u0007\u0001\u0000\u0000\u029e\u029f\u0007\t\u0000\u0000\u029f"+ + "\u02a0\u0007\u0002\u0000\u0000\u02a0\u02a1\u0007\u0001\u0000\u0000\u02a1"+ + "\u02a2\u0007\u0002\u0000\u0000\u02a2\u02a3\u0007\u0005\u0000\u0000\u02a3"+ + "\u02a4\u0005_\u0000\u0000\u02a4\u02a5\u0005\u8001\uf414\u0000\u0000\u02a5"+ + "\u02a6\u0001\u0000\u0000\u0000\u02a6\u02a7\u0006\u0013\u0001\u0000\u02a7"+ + ":\u0001\u0000\u0000\u0000\u02a8\u02a9\u0004\u0014\u0003\u0000\u02a9\u02aa"+ + "\u0007\r\u0000\u0000\u02aa\u02ab\u0007\u0007\u0000\u0000\u02ab\u02ac\u0007"+ + "\u0007\u0000\u0000\u02ac\u02ad\u0007\u0012\u0000\u0000\u02ad\u02ae\u0007"+ + "\u0014\u0000\u0000\u02ae\u02af\u0007\b\u0000\u0000\u02af\u02b0\u0005_"+ + "\u0000\u0000\u02b0\u02b1\u0005\u8001\uf414\u0000\u0000\u02b1\u02b2\u0001"+ + "\u0000\u0000\u0000\u02b2\u02b3\u0006\u0014\n\u0000\u02b3<\u0001\u0000"+ + "\u0000\u0000\u02b4\u02b5\u0004\u0015\u0004\u0000\u02b5\u02b6\u0007\u0010"+ + "\u0000\u0000\u02b6\u02b7\u0007\u0003\u0000\u0000\u02b7\u02b8\u0007\u0005"+ + "\u0000\u0000\u02b8\u02b9\u0007\u0006\u0000\u0000\u02b9\u02ba\u0007\u0001"+ + "\u0000\u0000\u02ba\u02bb\u0007\u0004\u0000\u0000\u02bb\u02bc\u0007\u0002"+ + "\u0000\u0000\u02bc\u02bd\u0001\u0000\u0000\u0000\u02bd\u02be\u0006\u0015"+ + "\u000b\u0000\u02be>\u0001\u0000\u0000\u0000\u02bf\u02c0\u0004\u0016\u0005"+ + "\u0000\u02c0\u02c1\u0007\u000f\u0000\u0000\u02c1\u02c2\u0007\u0014\u0000"+ + "\u0000\u02c2\u02c3\u0007\r\u0000\u0000\u02c3\u02c4\u0007\r\u0000\u0000"+ + "\u02c4\u02c5\u0001\u0000\u0000\u0000\u02c5\u02c6\u0006\u0016\b\u0000\u02c6"+ + "@\u0001\u0000\u0000\u0000\u02c7\u02c8\u0004\u0017\u0006\u0000\u02c8\u02c9"+ + "\u0007\r\u0000\u0000\u02c9\u02ca\u0007\u0003\u0000\u0000\u02ca\u02cb\u0007"+ + "\u000f\u0000\u0000\u02cb\u02cc\u0007\u0005\u0000\u0000\u02cc\u02cd\u0001"+ + "\u0000\u0000\u0000\u02cd\u02ce\u0006\u0017\b\u0000\u02ceB\u0001\u0000"+ + "\u0000\u0000\u02cf\u02d0\u0004\u0018\u0007\u0000\u02d0\u02d1\u0007\u0006"+ + "\u0000\u0000\u02d1\u02d2\u0007\u0001\u0000\u0000\u02d2\u02d3\u0007\u0011"+ + "\u0000\u0000\u02d3\u02d4\u0007\n\u0000\u0000\u02d4\u02d5\u0007\u0005\u0000"+ + "\u0000\u02d5\u02d6\u0001\u0000\u0000\u0000\u02d6\u02d7\u0006\u0018\b\u0000"+ + "\u02d7D\u0001\u0000\u0000\u0000\u02d8\u02d9\u0004\u0019\b\u0000\u02d9"+ + "\u02da\u0007\u000f\u0000\u0000\u02da\u02db\u0007\u0007\u0000\u0000\u02db"+ + "\u02dc\u0007\u0006\u0000\u0000\u02dc\u02dd\u0007\u0012\u0000\u0000\u02dd"+ + "\u02de\u0001\u0000\u0000\u0000\u02de\u02df\u0006\u0019\f\u0000\u02dfF"+ + "\u0001\u0000\u0000\u0000\u02e0\u02e2\b\u0015\u0000\u0000\u02e1\u02e0\u0001"+ + "\u0000\u0000\u0000\u02e2\u02e3\u0001\u0000\u0000\u0000\u02e3\u02e1\u0001"+ + "\u0000\u0000\u0000\u02e3\u02e4\u0001\u0000\u0000\u0000\u02e4\u02e5\u0001"+ + "\u0000\u0000\u0000\u02e5\u02e6\u0006\u001a\u0000\u0000\u02e6H\u0001\u0000"+ + "\u0000\u0000\u02e7\u02e8\u0005/\u0000\u0000\u02e8\u02e9\u0005/\u0000\u0000"+ + "\u02e9\u02ed\u0001\u0000\u0000\u0000\u02ea\u02ec\b\u0016\u0000\u0000\u02eb"+ + "\u02ea\u0001\u0000\u0000\u0000\u02ec\u02ef\u0001\u0000\u0000\u0000\u02ed"+ + "\u02eb\u0001\u0000\u0000\u0000\u02ed\u02ee\u0001\u0000\u0000\u0000\u02ee"+ + "\u02f1\u0001\u0000\u0000\u0000\u02ef\u02ed\u0001\u0000\u0000\u0000\u02f0"+ + "\u02f2\u0005\r\u0000\u0000\u02f1\u02f0\u0001\u0000\u0000\u0000\u02f1\u02f2"+ + "\u0001\u0000\u0000\u0000\u02f2\u02f4\u0001\u0000\u0000\u0000\u02f3\u02f5"+ + "\u0005\n\u0000\u0000\u02f4\u02f3\u0001\u0000\u0000\u0000\u02f4\u02f5\u0001"+ + "\u0000\u0000\u0000\u02f5\u02f6\u0001\u0000\u0000\u0000\u02f6\u02f7\u0006"+ + "\u001b\r\u0000\u02f7J\u0001\u0000\u0000\u0000\u02f8\u02f9\u0005/\u0000"+ + "\u0000\u02f9\u02fa\u0005*\u0000\u0000\u02fa\u02ff\u0001\u0000\u0000\u0000"+ + "\u02fb\u02fe\u0003K\u001c\u0000\u02fc\u02fe\t\u0000\u0000\u0000\u02fd"+ + "\u02fb\u0001\u0000\u0000\u0000\u02fd\u02fc\u0001\u0000\u0000\u0000\u02fe"+ + "\u0301\u0001\u0000\u0000\u0000\u02ff\u0300\u0001\u0000\u0000\u0000\u02ff"+ + "\u02fd\u0001\u0000\u0000\u0000\u0300\u0302\u0001\u0000\u0000\u0000\u0301"+ + "\u02ff\u0001\u0000\u0000\u0000\u0302\u0303\u0005*\u0000\u0000\u0303\u0304"+ + "\u0005/\u0000\u0000\u0304\u0305\u0001\u0000\u0000\u0000\u0305\u0306\u0006"+ + "\u001c\r\u0000\u0306L\u0001\u0000\u0000\u0000\u0307\u0309\u0007\u0017"+ + "\u0000\u0000\u0308\u0307\u0001\u0000\u0000\u0000\u0309\u030a\u0001\u0000"+ + "\u0000\u0000\u030a\u0308\u0001\u0000\u0000\u0000\u030a\u030b\u0001\u0000"+ + "\u0000\u0000\u030b\u030c\u0001\u0000\u0000\u0000\u030c\u030d\u0006\u001d"+ + "\r\u0000\u030dN\u0001\u0000\u0000\u0000\u030e\u030f\u0005|\u0000\u0000"+ + "\u030f\u0310\u0001\u0000\u0000\u0000\u0310\u0311\u0006\u001e\u000e\u0000"+ + "\u0311P\u0001\u0000\u0000\u0000\u0312\u0313\u0007\u0018\u0000\u0000\u0313"+ + "R\u0001\u0000\u0000\u0000\u0314\u0315\u0007\u0019\u0000\u0000\u0315T\u0001"+ + "\u0000\u0000\u0000\u0316\u0317\u0005\\\u0000\u0000\u0317\u0318\u0007\u001a"+ + "\u0000\u0000\u0318V\u0001\u0000\u0000\u0000\u0319\u031a\b\u001b\u0000"+ + "\u0000\u031aX\u0001\u0000\u0000\u0000\u031b\u031d\u0007\u0003\u0000\u0000"+ + "\u031c\u031e\u0007\u001c\u0000\u0000\u031d\u031c\u0001\u0000\u0000\u0000"+ + "\u031d\u031e\u0001\u0000\u0000\u0000\u031e\u0320\u0001\u0000\u0000\u0000"+ + "\u031f\u0321\u0003Q\u001f\u0000\u0320\u031f\u0001\u0000\u0000\u0000\u0321"+ + "\u0322\u0001\u0000\u0000\u0000\u0322\u0320\u0001\u0000\u0000\u0000\u0322"+ + "\u0323\u0001\u0000\u0000\u0000\u0323Z\u0001\u0000\u0000\u0000\u0324\u0325"+ + "\u0005@\u0000\u0000\u0325\\\u0001\u0000\u0000\u0000\u0326\u0327\u0005"+ + "`\u0000\u0000\u0327^\u0001\u0000\u0000\u0000\u0328\u032c\b\u001d\u0000"+ + "\u0000\u0329\u032a\u0005`\u0000\u0000\u032a\u032c\u0005`\u0000\u0000\u032b"+ + "\u0328\u0001\u0000\u0000\u0000\u032b\u0329\u0001\u0000\u0000\u0000\u032c"+ + "`\u0001\u0000\u0000\u0000\u032d\u032e\u0005_\u0000\u0000\u032eb\u0001"+ + "\u0000\u0000\u0000\u032f\u0333\u0003S \u0000\u0330\u0333\u0003Q\u001f"+ + "\u0000\u0331\u0333\u0003a\'\u0000\u0332\u032f\u0001\u0000\u0000\u0000"+ + "\u0332\u0330\u0001\u0000\u0000\u0000\u0332\u0331\u0001\u0000\u0000\u0000"+ + "\u0333d\u0001\u0000\u0000\u0000\u0334\u0339\u0005\"\u0000\u0000\u0335"+ + "\u0338\u0003U!\u0000\u0336\u0338\u0003W\"\u0000\u0337\u0335\u0001\u0000"+ + "\u0000\u0000\u0337\u0336\u0001\u0000\u0000\u0000\u0338\u033b\u0001\u0000"+ + "\u0000\u0000\u0339\u0337\u0001\u0000\u0000\u0000\u0339\u033a\u0001\u0000"+ + "\u0000\u0000\u033a\u033c\u0001\u0000\u0000\u0000\u033b\u0339\u0001\u0000"+ + "\u0000\u0000\u033c\u0352\u0005\"\u0000\u0000\u033d\u033e\u0005\"\u0000"+ + "\u0000\u033e\u033f\u0005\"\u0000\u0000\u033f\u0340\u0005\"\u0000\u0000"+ + "\u0340\u0344\u0001\u0000\u0000\u0000\u0341\u0343\b\u0016\u0000\u0000\u0342"+ + "\u0341\u0001\u0000\u0000\u0000\u0343\u0346\u0001\u0000\u0000\u0000\u0344"+ + "\u0345\u0001\u0000\u0000\u0000\u0344\u0342\u0001\u0000\u0000\u0000\u0345"+ + "\u0347\u0001\u0000\u0000\u0000\u0346\u0344\u0001\u0000\u0000\u0000\u0347"+ + "\u0348\u0005\"\u0000\u0000\u0348\u0349\u0005\"\u0000\u0000\u0349\u034a"+ + "\u0005\"\u0000\u0000\u034a\u034c\u0001\u0000\u0000\u0000\u034b\u034d\u0005"+ + "\"\u0000\u0000\u034c\u034b\u0001\u0000\u0000\u0000\u034c\u034d\u0001\u0000"+ + "\u0000\u0000\u034d\u034f\u0001\u0000\u0000\u0000\u034e\u0350\u0005\"\u0000"+ + "\u0000\u034f\u034e\u0001\u0000\u0000\u0000\u034f\u0350\u0001\u0000\u0000"+ + "\u0000\u0350\u0352\u0001\u0000\u0000\u0000\u0351\u0334\u0001\u0000\u0000"+ + "\u0000\u0351\u033d\u0001\u0000\u0000\u0000\u0352f\u0001\u0000\u0000\u0000"+ + "\u0353\u0355\u0003Q\u001f\u0000\u0354\u0353\u0001\u0000\u0000\u0000\u0355"+ + "\u0356\u0001\u0000\u0000\u0000\u0356\u0354\u0001\u0000\u0000\u0000\u0356"+ + "\u0357\u0001\u0000\u0000\u0000\u0357h\u0001\u0000\u0000\u0000\u0358\u035a"+ + "\u0003Q\u001f\u0000\u0359\u0358\u0001\u0000\u0000\u0000\u035a\u035b\u0001"+ + "\u0000\u0000\u0000\u035b\u0359\u0001\u0000\u0000\u0000\u035b\u035c\u0001"+ + "\u0000\u0000\u0000\u035c\u035d\u0001\u0000\u0000\u0000\u035d\u0361\u0003"+ + "{4\u0000\u035e\u0360\u0003Q\u001f\u0000\u035f\u035e\u0001\u0000\u0000"+ + "\u0000\u0360\u0363\u0001\u0000\u0000\u0000\u0361\u035f\u0001\u0000\u0000"+ + "\u0000\u0361\u0362\u0001\u0000\u0000\u0000\u0362\u0383\u0001\u0000\u0000"+ + "\u0000\u0363\u0361\u0001\u0000\u0000\u0000\u0364\u0366\u0003{4\u0000\u0365"+ + "\u0367\u0003Q\u001f\u0000\u0366\u0365\u0001\u0000\u0000\u0000\u0367\u0368"+ + "\u0001\u0000\u0000\u0000\u0368\u0366\u0001\u0000\u0000\u0000\u0368\u0369"+ + "\u0001\u0000\u0000\u0000\u0369\u0383\u0001\u0000\u0000\u0000\u036a\u036c"+ + "\u0003Q\u001f\u0000\u036b\u036a\u0001\u0000\u0000\u0000\u036c\u036d\u0001"+ + "\u0000\u0000\u0000\u036d\u036b\u0001\u0000\u0000\u0000\u036d\u036e\u0001"+ + "\u0000\u0000\u0000\u036e\u0376\u0001\u0000\u0000\u0000\u036f\u0373\u0003"+ + "{4\u0000\u0370\u0372\u0003Q\u001f\u0000\u0371\u0370\u0001\u0000\u0000"+ + "\u0000\u0372\u0375\u0001\u0000\u0000\u0000\u0373\u0371\u0001\u0000\u0000"+ + "\u0000\u0373\u0374\u0001\u0000\u0000\u0000\u0374\u0377\u0001\u0000\u0000"+ + "\u0000\u0375\u0373\u0001\u0000\u0000\u0000\u0376\u036f\u0001\u0000\u0000"+ + "\u0000\u0376\u0377\u0001\u0000\u0000\u0000\u0377\u0378\u0001\u0000\u0000"+ + "\u0000\u0378\u0379\u0003Y#\u0000\u0379\u0383\u0001\u0000\u0000\u0000\u037a"+ + "\u037c\u0003{4\u0000\u037b\u037d\u0003Q\u001f\u0000\u037c\u037b\u0001"+ + "\u0000\u0000\u0000\u037d\u037e\u0001\u0000\u0000\u0000\u037e\u037c\u0001"+ + "\u0000\u0000\u0000\u037e\u037f\u0001\u0000\u0000\u0000\u037f\u0380\u0001"+ + "\u0000\u0000\u0000\u0380\u0381\u0003Y#\u0000\u0381\u0383\u0001\u0000\u0000"+ + "\u0000\u0382\u0359\u0001\u0000\u0000\u0000\u0382\u0364\u0001\u0000\u0000"+ + "\u0000\u0382\u036b\u0001\u0000\u0000\u0000\u0382\u037a\u0001\u0000\u0000"+ + "\u0000\u0383j\u0001\u0000\u0000\u0000\u0384\u0385\u0007\u001e\u0000\u0000"+ + "\u0385\u0386\u0007\u001f\u0000\u0000\u0386l\u0001\u0000\u0000\u0000\u0387"+ + "\u0388\u0007\f\u0000\u0000\u0388\u0389\u0007\t\u0000\u0000\u0389\u038a"+ + "\u0007\u0000\u0000\u0000\u038an\u0001\u0000\u0000\u0000\u038b\u038c\u0007"+ + "\f\u0000\u0000\u038c\u038d\u0007\u0002\u0000\u0000\u038d\u038e\u0007\u0004"+ + "\u0000\u0000\u038ep\u0001\u0000\u0000\u0000\u038f\u0390\u0005=\u0000\u0000"+ + "\u0390r\u0001\u0000\u0000\u0000\u0391\u0392\u0005:\u0000\u0000\u0392\u0393"+ + "\u0005:\u0000\u0000\u0393t\u0001\u0000\u0000\u0000\u0394\u0395\u0005:"+ + "\u0000\u0000\u0395v\u0001\u0000\u0000\u0000\u0396\u0397\u0005,\u0000\u0000"+ + "\u0397x\u0001\u0000\u0000\u0000\u0398\u0399\u0007\u0000\u0000\u0000\u0399"+ + "\u039a\u0007\u0003\u0000\u0000\u039a\u039b\u0007\u0002\u0000\u0000\u039b"+ + "\u039c\u0007\u0004\u0000\u0000\u039cz\u0001\u0000\u0000\u0000\u039d\u039e"+ + "\u0005.\u0000\u0000\u039e|\u0001\u0000\u0000\u0000\u039f\u03a0\u0007\u000f"+ + "\u0000\u0000\u03a0\u03a1\u0007\f\u0000\u0000\u03a1\u03a2\u0007\r\u0000"+ + "\u0000\u03a2\u03a3\u0007\u0002\u0000\u0000\u03a3\u03a4\u0007\u0003\u0000"+ + "\u0000\u03a4~\u0001\u0000\u0000\u0000\u03a5\u03a6\u0007\u000f\u0000\u0000"+ + "\u03a6\u03a7\u0007\u0001\u0000\u0000\u03a7\u03a8\u0007\u0006\u0000\u0000"+ + "\u03a8\u03a9\u0007\u0002\u0000\u0000\u03a9\u03aa\u0007\u0005\u0000\u0000"+ + "\u03aa\u0080\u0001\u0000\u0000\u0000\u03ab\u03ac\u0007\u0001\u0000\u0000"+ + "\u03ac\u03ad\u0007\t\u0000\u0000\u03ad\u0082\u0001\u0000\u0000\u0000\u03ae"+ + "\u03af\u0007\u0001\u0000\u0000\u03af\u03b0\u0007\u0002\u0000\u0000\u03b0"+ + "\u0084\u0001\u0000\u0000\u0000\u03b1\u03b2\u0007\r\u0000\u0000\u03b2\u03b3"+ + "\u0007\f\u0000\u0000\u03b3\u03b4\u0007\u0002\u0000\u0000\u03b4\u03b5\u0007"+ + "\u0005\u0000\u0000\u03b5\u0086\u0001\u0000\u0000\u0000\u03b6\u03b7\u0007"+ + "\r\u0000\u0000\u03b7\u03b8\u0007\u0001\u0000\u0000\u03b8\u03b9\u0007\u0012"+ + "\u0000\u0000\u03b9\u03ba\u0007\u0003\u0000\u0000\u03ba\u0088\u0001\u0000"+ + "\u0000\u0000\u03bb\u03bc\u0007\t\u0000\u0000\u03bc\u03bd\u0007\u0007\u0000"+ + "\u0000\u03bd\u03be\u0007\u0005\u0000\u0000\u03be\u008a\u0001\u0000\u0000"+ + "\u0000\u03bf\u03c0\u0007\t\u0000\u0000\u03c0\u03c1\u0007\u0014\u0000\u0000"+ + "\u03c1\u03c2\u0007\r\u0000\u0000\u03c2\u03c3\u0007\r\u0000\u0000\u03c3"+ + "\u008c\u0001\u0000\u0000\u0000\u03c4\u03c5\u0007\t\u0000\u0000\u03c5\u03c6"+ + "\u0007\u0014\u0000\u0000\u03c6\u03c7\u0007\r\u0000\u0000\u03c7\u03c8\u0007"+ + "\r\u0000\u0000\u03c8\u03c9\u0007\u0002\u0000\u0000\u03c9\u008e\u0001\u0000"+ + "\u0000\u0000\u03ca\u03cb\u0007\u0007\u0000\u0000\u03cb\u03cc\u0007\u0006"+ + "\u0000\u0000\u03cc\u0090\u0001\u0000\u0000\u0000\u03cd\u03ce\u0005?\u0000"+ + "\u0000\u03ce\u0092\u0001\u0000\u0000\u0000\u03cf\u03d0\u0007\u0006\u0000"+ + "\u0000\u03d0\u03d1\u0007\r\u0000\u0000\u03d1\u03d2\u0007\u0001\u0000\u0000"+ + "\u03d2\u03d3\u0007\u0012\u0000\u0000\u03d3\u03d4\u0007\u0003\u0000\u0000"+ + "\u03d4\u0094\u0001\u0000\u0000\u0000\u03d5\u03d6\u0007\u0005\u0000\u0000"+ + "\u03d6\u03d7\u0007\u0006\u0000\u0000\u03d7\u03d8\u0007\u0014\u0000\u0000"+ + "\u03d8\u03d9\u0007\u0003\u0000\u0000\u03d9\u0096\u0001\u0000\u0000\u0000"+ + "\u03da\u03db\u0005=\u0000\u0000\u03db\u03dc\u0005=\u0000\u0000\u03dc\u0098"+ + "\u0001\u0000\u0000\u0000\u03dd\u03de\u0005=\u0000\u0000\u03de\u03df\u0005"+ + "~\u0000\u0000\u03df\u009a\u0001\u0000\u0000\u0000\u03e0\u03e1\u0005!\u0000"+ + "\u0000\u03e1\u03e2\u0005=\u0000\u0000\u03e2\u009c\u0001\u0000\u0000\u0000"+ + "\u03e3\u03e4\u0005<\u0000\u0000\u03e4\u009e\u0001\u0000\u0000\u0000\u03e5"+ + "\u03e6\u0005<\u0000\u0000\u03e6\u03e7\u0005=\u0000\u0000\u03e7\u00a0\u0001"+ + "\u0000\u0000\u0000\u03e8\u03e9\u0005>\u0000\u0000\u03e9\u00a2\u0001\u0000"+ + "\u0000\u0000\u03ea\u03eb\u0005>\u0000\u0000\u03eb\u03ec\u0005=\u0000\u0000"+ + "\u03ec\u00a4\u0001\u0000\u0000\u0000\u03ed\u03ee\u0005+\u0000\u0000\u03ee"+ + "\u00a6\u0001\u0000\u0000\u0000\u03ef\u03f0\u0005-\u0000\u0000\u03f0\u00a8"+ + "\u0001\u0000\u0000\u0000\u03f1\u03f2\u0005*\u0000\u0000\u03f2\u00aa\u0001"+ + "\u0000\u0000\u0000\u03f3\u03f4\u0005/\u0000\u0000\u03f4\u00ac\u0001\u0000"+ + "\u0000\u0000\u03f5\u03f6\u0005%\u0000\u0000\u03f6\u00ae\u0001\u0000\u0000"+ + "\u0000\u03f7\u03f8\u0005{\u0000\u0000\u03f8\u00b0\u0001\u0000\u0000\u0000"+ + "\u03f9\u03fa\u0005}\u0000\u0000\u03fa\u00b2\u0001\u0000\u0000\u0000\u03fb"+ + "\u03fc\u00031\u000f\u0000\u03fc\u03fd\u0001\u0000\u0000\u0000\u03fd\u03fe"+ + "\u0006P\u000f\u0000\u03fe\u00b4\u0001\u0000\u0000\u0000\u03ff\u0402\u0003"+ + "\u0091?\u0000\u0400\u0403\u0003S \u0000\u0401\u0403\u0003a\'\u0000\u0402"+ + "\u0400\u0001\u0000\u0000\u0000\u0402\u0401\u0001\u0000\u0000\u0000\u0403"+ + "\u0407\u0001\u0000\u0000\u0000\u0404\u0406\u0003c(\u0000\u0405\u0404\u0001"+ + "\u0000\u0000\u0000\u0406\u0409\u0001\u0000\u0000\u0000\u0407\u0405\u0001"+ + "\u0000\u0000\u0000\u0407\u0408\u0001\u0000\u0000\u0000\u0408\u0411\u0001"+ + "\u0000\u0000\u0000\u0409\u0407\u0001\u0000\u0000\u0000\u040a\u040c\u0003"+ + "\u0091?\u0000\u040b\u040d\u0003Q\u001f\u0000\u040c\u040b\u0001\u0000\u0000"+ + "\u0000\u040d\u040e\u0001\u0000\u0000\u0000\u040e\u040c\u0001\u0000\u0000"+ + "\u0000\u040e\u040f\u0001\u0000\u0000\u0000\u040f\u0411\u0001\u0000\u0000"+ + "\u0000\u0410\u03ff\u0001\u0000\u0000\u0000\u0410\u040a\u0001\u0000\u0000"+ + "\u0000\u0411\u00b6\u0001\u0000\u0000\u0000\u0412\u0413\u0005[\u0000\u0000"+ + "\u0413\u0414\u0001\u0000\u0000\u0000\u0414\u0415\u0006R\u0000\u0000\u0415"+ + "\u0416\u0006R\u0000\u0000\u0416\u00b8\u0001\u0000\u0000\u0000\u0417\u0418"+ + "\u0005]\u0000\u0000\u0418\u0419\u0001\u0000\u0000\u0000\u0419\u041a\u0006"+ + "S\u000e\u0000\u041a\u041b\u0006S\u000e\u0000\u041b\u00ba\u0001\u0000\u0000"+ + "\u0000\u041c\u041d\u0005(\u0000\u0000\u041d\u041e\u0001\u0000\u0000\u0000"+ + "\u041e\u041f\u0006T\u0000\u0000\u041f\u0420\u0006T\u0000\u0000\u0420\u00bc"+ + "\u0001\u0000\u0000\u0000\u0421\u0422\u0005)\u0000\u0000\u0422\u0423\u0001"+ + "\u0000\u0000\u0000\u0423\u0424\u0006U\u000e\u0000\u0424\u0425\u0006U\u000e"+ + "\u0000\u0425\u00be\u0001\u0000\u0000\u0000\u0426\u042a\u0003S \u0000\u0427"+ + "\u0429\u0003c(\u0000\u0428\u0427\u0001\u0000\u0000\u0000\u0429\u042c\u0001"+ + "\u0000\u0000\u0000\u042a\u0428\u0001\u0000\u0000\u0000\u042a\u042b\u0001"+ + "\u0000\u0000\u0000\u042b\u0437\u0001\u0000\u0000\u0000\u042c\u042a\u0001"+ + "\u0000\u0000\u0000\u042d\u0430\u0003a\'\u0000\u042e\u0430\u0003[$\u0000"+ + "\u042f\u042d\u0001\u0000\u0000\u0000\u042f\u042e\u0001\u0000\u0000\u0000"+ + "\u0430\u0432\u0001\u0000\u0000\u0000\u0431\u0433\u0003c(\u0000\u0432\u0431"+ + "\u0001\u0000\u0000\u0000\u0433\u0434\u0001\u0000\u0000\u0000\u0434\u0432"+ + "\u0001\u0000\u0000\u0000\u0434\u0435\u0001\u0000\u0000\u0000\u0435\u0437"+ + "\u0001\u0000\u0000\u0000\u0436\u0426\u0001\u0000\u0000\u0000\u0436\u042f"+ + "\u0001\u0000\u0000\u0000\u0437\u00c0\u0001\u0000\u0000\u0000\u0438\u043a"+ + "\u0003]%\u0000\u0439\u043b\u0003_&\u0000\u043a\u0439\u0001\u0000\u0000"+ + "\u0000\u043b\u043c\u0001\u0000\u0000\u0000\u043c\u043a\u0001\u0000\u0000"+ + "\u0000\u043c\u043d\u0001\u0000\u0000\u0000\u043d\u043e\u0001\u0000\u0000"+ + "\u0000\u043e\u043f\u0003]%\u0000\u043f\u00c2\u0001\u0000\u0000\u0000\u0440"+ + "\u0441\u0003\u00c1W\u0000\u0441\u00c4\u0001\u0000\u0000\u0000\u0442\u0443"+ + "\u0003I\u001b\u0000\u0443\u0444\u0001\u0000\u0000\u0000\u0444\u0445\u0006"+ + "Y\r\u0000\u0445\u00c6\u0001\u0000\u0000\u0000\u0446\u0447\u0003K\u001c"+ + "\u0000\u0447\u0448\u0001\u0000\u0000\u0000\u0448\u0449\u0006Z\r\u0000"+ + "\u0449\u00c8\u0001\u0000\u0000\u0000\u044a\u044b\u0003M\u001d\u0000\u044b"+ + "\u044c\u0001\u0000\u0000\u0000\u044c\u044d\u0006[\r\u0000\u044d\u00ca"+ + "\u0001\u0000\u0000\u0000\u044e\u044f\u0003\u00b7R\u0000\u044f\u0450\u0001"+ + "\u0000\u0000\u0000\u0450\u0451\u0006\\\u0010\u0000\u0451\u0452\u0006\\"+ + "\u0011\u0000\u0452\u00cc\u0001\u0000\u0000\u0000\u0453\u0454\u0003O\u001e"+ + "\u0000\u0454\u0455\u0001\u0000\u0000\u0000\u0455\u0456\u0006]\u0012\u0000"+ + "\u0456\u0457\u0006]\u000e\u0000\u0457\u00ce\u0001\u0000\u0000\u0000\u0458"+ + "\u0459\u0003M\u001d\u0000\u0459\u045a\u0001\u0000\u0000\u0000\u045a\u045b"+ + "\u0006^\r\u0000\u045b\u00d0\u0001\u0000\u0000\u0000\u045c\u045d\u0003"+ + "I\u001b\u0000\u045d\u045e\u0001\u0000\u0000\u0000\u045e\u045f\u0006_\r"+ + "\u0000\u045f\u00d2\u0001\u0000\u0000\u0000\u0460\u0461\u0003K\u001c\u0000"+ + "\u0461\u0462\u0001\u0000\u0000\u0000\u0462\u0463\u0006`\r\u0000\u0463"+ + "\u00d4\u0001\u0000\u0000\u0000\u0464\u0465\u0003O\u001e\u0000\u0465\u0466"+ + "\u0001\u0000\u0000\u0000\u0466\u0467\u0006a\u0012\u0000\u0467\u0468\u0006"+ + "a\u000e\u0000\u0468\u00d6\u0001\u0000\u0000\u0000\u0469\u046a\u0003\u00b7"+ + "R\u0000\u046a\u046b\u0001\u0000\u0000\u0000\u046b\u046c\u0006b\u0010\u0000"+ + "\u046c\u00d8\u0001\u0000\u0000\u0000\u046d\u046e\u0003\u00b9S\u0000\u046e"+ + "\u046f\u0001\u0000\u0000\u0000\u046f\u0470\u0006c\u0013\u0000\u0470\u00da"+ + "\u0001\u0000\u0000\u0000\u0471\u0472\u0003u1\u0000\u0472\u0473\u0001\u0000"+ + "\u0000\u0000\u0473\u0474\u0006d\u0014\u0000\u0474\u00dc\u0001\u0000\u0000"+ + "\u0000\u0475\u0476\u0003w2\u0000\u0476\u0477\u0001\u0000\u0000\u0000\u0477"+ + "\u0478\u0006e\u0015\u0000\u0478\u00de\u0001\u0000\u0000\u0000\u0479\u047a"+ + "\u0003q/\u0000\u047a\u047b\u0001\u0000\u0000\u0000\u047b\u047c\u0006f"+ + "\u0016\u0000\u047c\u00e0\u0001\u0000\u0000\u0000\u047d\u047e\u0007\u0010"+ + "\u0000\u0000\u047e\u047f\u0007\u0003\u0000\u0000\u047f\u0480\u0007\u0005"+ + "\u0000\u0000\u0480\u0481\u0007\f\u0000\u0000\u0481\u0482\u0007\u0000\u0000"+ + "\u0000\u0482\u0483\u0007\f\u0000\u0000\u0483\u0484\u0007\u0005\u0000\u0000"+ + "\u0484\u0485\u0007\f\u0000\u0000\u0485\u00e2\u0001\u0000\u0000\u0000\u0486"+ + "\u048a\b \u0000\u0000\u0487\u0488\u0005/\u0000\u0000\u0488\u048a\b!\u0000"+ + "\u0000\u0489\u0486\u0001\u0000\u0000\u0000\u0489\u0487\u0001\u0000\u0000"+ + "\u0000\u048a\u00e4\u0001\u0000\u0000\u0000\u048b\u048d\u0003\u00e3h\u0000"+ + "\u048c\u048b\u0001\u0000\u0000\u0000\u048d\u048e\u0001\u0000\u0000\u0000"+ + "\u048e\u048c\u0001\u0000\u0000\u0000\u048e\u048f\u0001\u0000\u0000\u0000"+ + "\u048f\u00e6\u0001\u0000\u0000\u0000\u0490\u0491\u0003\u00e5i\u0000\u0491"+ + "\u0492\u0001\u0000\u0000\u0000\u0492\u0493\u0006j\u0017\u0000\u0493\u00e8"+ + "\u0001\u0000\u0000\u0000\u0494\u0495\u0003e)\u0000\u0495\u0496\u0001\u0000"+ + "\u0000\u0000\u0496\u0497\u0006k\u0018\u0000\u0497\u00ea\u0001\u0000\u0000"+ + "\u0000\u0498\u0499\u0003I\u001b\u0000\u0499\u049a\u0001\u0000\u0000\u0000"+ + "\u049a\u049b\u0006l\r\u0000\u049b\u00ec\u0001\u0000\u0000\u0000\u049c"+ + "\u049d\u0003K\u001c\u0000\u049d\u049e\u0001\u0000\u0000\u0000\u049e\u049f"+ + "\u0006m\r\u0000\u049f\u00ee\u0001\u0000\u0000\u0000\u04a0\u04a1\u0003"+ + "M\u001d\u0000\u04a1\u04a2\u0001\u0000\u0000\u0000\u04a2\u04a3\u0006n\r"+ + "\u0000\u04a3\u00f0\u0001\u0000\u0000\u0000\u04a4\u04a5\u0003O\u001e\u0000"+ + "\u04a5\u04a6\u0001\u0000\u0000\u0000\u04a6\u04a7\u0006o\u0012\u0000\u04a7"+ + "\u04a8\u0006o\u000e\u0000\u04a8\u00f2\u0001\u0000\u0000\u0000\u04a9\u04aa"+ + "\u0003{4\u0000\u04aa\u04ab\u0001\u0000\u0000\u0000\u04ab\u04ac\u0006p"+ + "\u0019\u0000\u04ac\u00f4\u0001\u0000\u0000\u0000\u04ad\u04ae\u0003w2\u0000"+ + "\u04ae\u04af\u0001\u0000\u0000\u0000\u04af\u04b0\u0006q\u0015\u0000\u04b0"+ + "\u00f6\u0001\u0000\u0000\u0000\u04b1\u04b2\u0003\u0091?\u0000\u04b2\u04b3"+ + "\u0001\u0000\u0000\u0000\u04b3\u04b4\u0006r\u001a\u0000\u04b4\u00f8\u0001"+ + "\u0000\u0000\u0000\u04b5\u04b6\u0003\u00b5Q\u0000\u04b6\u04b7\u0001\u0000"+ + "\u0000\u0000\u04b7\u04b8\u0006s\u001b\u0000\u04b8\u00fa\u0001\u0000\u0000"+ + "\u0000\u04b9\u04be\u0003S \u0000\u04ba\u04be\u0003Q\u001f\u0000\u04bb"+ + "\u04be\u0003a\'\u0000\u04bc\u04be\u0003\u00a9K\u0000\u04bd\u04b9\u0001"+ + "\u0000\u0000\u0000\u04bd\u04ba\u0001\u0000\u0000\u0000\u04bd\u04bb\u0001"+ + "\u0000\u0000\u0000\u04bd\u04bc\u0001\u0000\u0000\u0000\u04be\u00fc\u0001"+ + "\u0000\u0000\u0000\u04bf\u04c2\u0003S \u0000\u04c0\u04c2\u0003\u00a9K"+ + "\u0000\u04c1\u04bf\u0001\u0000\u0000\u0000\u04c1\u04c0\u0001\u0000\u0000"+ + "\u0000\u04c2\u04c6\u0001\u0000\u0000\u0000\u04c3\u04c5\u0003\u00fbt\u0000"+ + "\u04c4\u04c3\u0001\u0000\u0000\u0000\u04c5\u04c8\u0001\u0000\u0000\u0000"+ + "\u04c6\u04c4\u0001\u0000\u0000\u0000\u04c6\u04c7\u0001\u0000\u0000\u0000"+ + "\u04c7\u04d3\u0001\u0000\u0000\u0000\u04c8\u04c6\u0001\u0000\u0000\u0000"+ + "\u04c9\u04cc\u0003a\'\u0000\u04ca\u04cc\u0003[$\u0000\u04cb\u04c9\u0001"+ + "\u0000\u0000\u0000\u04cb\u04ca\u0001\u0000\u0000\u0000\u04cc\u04ce\u0001"+ + "\u0000\u0000\u0000\u04cd\u04cf\u0003\u00fbt\u0000\u04ce\u04cd\u0001\u0000"+ + "\u0000\u0000\u04cf\u04d0\u0001\u0000\u0000\u0000\u04d0\u04ce\u0001\u0000"+ + "\u0000\u0000\u04d0\u04d1\u0001\u0000\u0000\u0000\u04d1\u04d3\u0001\u0000"+ + "\u0000\u0000\u04d2\u04c1\u0001\u0000\u0000\u0000\u04d2\u04cb\u0001\u0000"+ + "\u0000\u0000\u04d3\u00fe\u0001\u0000\u0000\u0000\u04d4\u04d7\u0003\u00fd"+ + "u\u0000\u04d5\u04d7\u0003\u00c1W\u0000\u04d6\u04d4\u0001\u0000\u0000\u0000"+ + "\u04d6\u04d5\u0001\u0000\u0000\u0000\u04d7\u04d8\u0001\u0000\u0000\u0000"+ + "\u04d8\u04d6\u0001\u0000\u0000\u0000\u04d8\u04d9\u0001\u0000\u0000\u0000"+ + "\u04d9\u0100\u0001\u0000\u0000\u0000\u04da\u04db\u0003I\u001b\u0000\u04db"+ + "\u04dc\u0001\u0000\u0000\u0000\u04dc\u04dd\u0006w\r\u0000\u04dd\u0102"+ + "\u0001\u0000\u0000\u0000\u04de\u04df\u0003K\u001c\u0000\u04df\u04e0\u0001"+ + "\u0000\u0000\u0000\u04e0\u04e1\u0006x\r\u0000\u04e1\u0104\u0001\u0000"+ + "\u0000\u0000\u04e2\u04e3\u0003M\u001d\u0000\u04e3\u04e4\u0001\u0000\u0000"+ + "\u0000\u04e4\u04e5\u0006y\r\u0000\u04e5\u0106\u0001\u0000\u0000\u0000"+ + "\u04e6\u04e7\u0003O\u001e\u0000\u04e7\u04e8\u0001\u0000\u0000\u0000\u04e8"+ + "\u04e9\u0006z\u0012\u0000\u04e9\u04ea\u0006z\u000e\u0000\u04ea\u0108\u0001"+ + "\u0000\u0000\u0000\u04eb\u04ec\u0003q/\u0000\u04ec\u04ed\u0001\u0000\u0000"+ + "\u0000\u04ed\u04ee\u0006{\u0016\u0000\u04ee\u010a\u0001\u0000\u0000\u0000"+ + "\u04ef\u04f0\u0003w2\u0000\u04f0\u04f1\u0001\u0000\u0000\u0000\u04f1\u04f2"+ + "\u0006|\u0015\u0000\u04f2\u010c\u0001\u0000\u0000\u0000\u04f3\u04f4\u0003"+ + "{4\u0000\u04f4\u04f5\u0001\u0000\u0000\u0000\u04f5\u04f6\u0006}\u0019"+ + "\u0000\u04f6\u010e\u0001\u0000\u0000\u0000\u04f7\u04f8\u0003\u0091?\u0000"+ + "\u04f8\u04f9\u0001\u0000\u0000\u0000\u04f9\u04fa\u0006~\u001a\u0000\u04fa"+ + "\u0110\u0001\u0000\u0000\u0000\u04fb\u04fc\u0003\u00b5Q\u0000\u04fc\u04fd"+ + "\u0001\u0000\u0000\u0000\u04fd\u04fe\u0006\u007f\u001b\u0000\u04fe\u0112"+ + "\u0001\u0000\u0000\u0000\u04ff\u0500\u0007\f\u0000\u0000\u0500\u0501\u0007"+ + "\u0002\u0000\u0000\u0501\u0114\u0001\u0000\u0000\u0000\u0502\u0503\u0003"+ + "\u00ffv\u0000\u0503\u0504\u0001\u0000\u0000\u0000\u0504\u0505\u0006\u0081"+ + "\u001c\u0000\u0505\u0116\u0001\u0000\u0000\u0000\u0506\u0507\u0003I\u001b"+ + "\u0000\u0507\u0508\u0001\u0000\u0000\u0000\u0508\u0509\u0006\u0082\r\u0000"+ + "\u0509\u0118\u0001\u0000\u0000\u0000\u050a\u050b\u0003K\u001c\u0000\u050b"+ + "\u050c\u0001\u0000\u0000\u0000\u050c\u050d\u0006\u0083\r\u0000\u050d\u011a"+ + "\u0001\u0000\u0000\u0000\u050e\u050f\u0003M\u001d\u0000\u050f\u0510\u0001"+ + "\u0000\u0000\u0000\u0510\u0511\u0006\u0084\r\u0000\u0511\u011c\u0001\u0000"+ + "\u0000\u0000\u0512\u0513\u0003O\u001e\u0000\u0513\u0514\u0001\u0000\u0000"+ + "\u0000\u0514\u0515\u0006\u0085\u0012\u0000\u0515\u0516\u0006\u0085\u000e"+ + "\u0000\u0516\u011e\u0001\u0000\u0000\u0000\u0517\u0518\u0003\u00b7R\u0000"+ + "\u0518\u0519\u0001\u0000\u0000\u0000\u0519\u051a\u0006\u0086\u0010\u0000"+ + "\u051a\u051b\u0006\u0086\u001d\u0000\u051b\u0120\u0001\u0000\u0000\u0000"+ + "\u051c\u051d\u0007\u0007\u0000\u0000\u051d\u051e\u0007\t\u0000\u0000\u051e"+ + "\u051f\u0001\u0000\u0000\u0000\u051f\u0520\u0006\u0087\u001e\u0000\u0520"+ + "\u0122\u0001\u0000\u0000\u0000\u0521\u0522\u0007\u0013\u0000\u0000\u0522"+ + "\u0523\u0007\u0001\u0000\u0000\u0523\u0524\u0007\u0005\u0000\u0000\u0524"+ + "\u0525\u0007\n\u0000\u0000\u0525\u0526\u0001\u0000\u0000\u0000\u0526\u0527"+ + "\u0006\u0088\u001e\u0000\u0527\u0124\u0001\u0000\u0000\u0000\u0528\u0529"+ + "\b\"\u0000\u0000\u0529\u0126\u0001\u0000\u0000\u0000\u052a\u052c\u0003"+ + "\u0125\u0089\u0000\u052b\u052a\u0001\u0000\u0000\u0000\u052c\u052d\u0001"+ + "\u0000\u0000\u0000\u052d\u052b\u0001\u0000\u0000\u0000\u052d\u052e\u0001"+ + "\u0000\u0000\u0000\u052e\u052f\u0001\u0000\u0000\u0000\u052f\u0530\u0003"+ + "u1\u0000\u0530\u0532\u0001\u0000\u0000\u0000\u0531\u052b\u0001\u0000\u0000"+ + "\u0000\u0531\u0532\u0001\u0000\u0000\u0000\u0532\u0534\u0001\u0000\u0000"+ + "\u0000\u0533\u0535\u0003\u0125\u0089\u0000\u0534\u0533\u0001\u0000\u0000"+ + "\u0000\u0535\u0536\u0001\u0000\u0000\u0000\u0536\u0534\u0001\u0000\u0000"+ + "\u0000\u0536\u0537\u0001\u0000\u0000\u0000\u0537\u0128\u0001\u0000\u0000"+ + "\u0000\u0538\u0539\u0003\u0127\u008a\u0000\u0539\u053a\u0001\u0000\u0000"+ + "\u0000\u053a\u053b\u0006\u008b\u001f\u0000\u053b\u012a\u0001\u0000\u0000"+ + "\u0000\u053c\u053d\u0003I\u001b\u0000\u053d\u053e\u0001\u0000\u0000\u0000"+ + "\u053e\u053f\u0006\u008c\r\u0000\u053f\u012c\u0001\u0000\u0000\u0000\u0540"+ + "\u0541\u0003K\u001c\u0000\u0541\u0542\u0001\u0000\u0000\u0000\u0542\u0543"+ + "\u0006\u008d\r\u0000\u0543\u012e\u0001\u0000\u0000\u0000\u0544\u0545\u0003"+ + "M\u001d\u0000\u0545\u0546\u0001\u0000\u0000\u0000\u0546\u0547\u0006\u008e"+ + "\r\u0000\u0547\u0130\u0001\u0000\u0000\u0000\u0548\u0549\u0003O\u001e"+ + "\u0000\u0549\u054a\u0001\u0000\u0000\u0000\u054a\u054b\u0006\u008f\u0012"+ + "\u0000\u054b\u054c\u0006\u008f\u000e\u0000\u054c\u054d\u0006\u008f\u000e"+ + "\u0000\u054d\u0132\u0001\u0000\u0000\u0000\u054e\u054f\u0003q/\u0000\u054f"+ + "\u0550\u0001\u0000\u0000\u0000\u0550\u0551\u0006\u0090\u0016\u0000\u0551"+ + "\u0134\u0001\u0000\u0000\u0000\u0552\u0553\u0003w2\u0000\u0553\u0554\u0001"+ + "\u0000\u0000\u0000\u0554\u0555\u0006\u0091\u0015\u0000\u0555\u0136\u0001"+ + "\u0000\u0000\u0000\u0556\u0557\u0003{4\u0000\u0557\u0558\u0001\u0000\u0000"+ + "\u0000\u0558\u0559\u0006\u0092\u0019\u0000\u0559\u0138\u0001\u0000\u0000"+ + "\u0000\u055a\u055b\u0003\u0123\u0088\u0000\u055b\u055c\u0001\u0000\u0000"+ + "\u0000\u055c\u055d\u0006\u0093 \u0000\u055d\u013a\u0001\u0000\u0000\u0000"+ + "\u055e\u055f\u0003\u00ffv\u0000\u055f\u0560\u0001\u0000\u0000\u0000\u0560"+ + "\u0561\u0006\u0094\u001c\u0000\u0561\u013c\u0001\u0000\u0000\u0000\u0562"+ + "\u0563\u0003\u00c3X\u0000\u0563\u0564\u0001\u0000\u0000\u0000\u0564\u0565"+ + "\u0006\u0095!\u0000\u0565\u013e\u0001\u0000\u0000\u0000\u0566\u0567\u0003"+ + "\u0091?\u0000\u0567\u0568\u0001\u0000\u0000\u0000\u0568\u0569\u0006\u0096"+ + "\u001a\u0000\u0569\u0140\u0001\u0000\u0000\u0000\u056a\u056b\u0003\u00b5"+ + "Q\u0000\u056b\u056c\u0001\u0000\u0000\u0000\u056c\u056d\u0006\u0097\u001b"+ + "\u0000\u056d\u0142\u0001\u0000\u0000\u0000\u056e\u056f\u0003I\u001b\u0000"+ + "\u056f\u0570\u0001\u0000\u0000\u0000\u0570\u0571\u0006\u0098\r\u0000\u0571"+ + "\u0144\u0001\u0000\u0000\u0000\u0572\u0573\u0003K\u001c\u0000\u0573\u0574"+ + "\u0001\u0000\u0000\u0000\u0574\u0575\u0006\u0099\r\u0000\u0575\u0146\u0001"+ + "\u0000\u0000\u0000\u0576\u0577\u0003M\u001d\u0000\u0577\u0578\u0001\u0000"+ + "\u0000\u0000\u0578\u0579\u0006\u009a\r\u0000\u0579\u0148\u0001\u0000\u0000"+ + "\u0000\u057a\u057b\u0003O\u001e\u0000\u057b\u057c\u0001\u0000\u0000\u0000"+ + "\u057c\u057d\u0006\u009b\u0012\u0000\u057d\u057e\u0006\u009b\u000e\u0000"+ + "\u057e\u014a\u0001\u0000\u0000\u0000\u057f\u0580\u0003{4\u0000\u0580\u0581"+ + "\u0001\u0000\u0000\u0000\u0581\u0582\u0006\u009c\u0019\u0000\u0582\u014c"+ + "\u0001\u0000\u0000\u0000\u0583\u0584\u0003\u0091?\u0000\u0584\u0585\u0001"+ + "\u0000\u0000\u0000\u0585\u0586\u0006\u009d\u001a\u0000\u0586\u014e\u0001"+ + "\u0000\u0000\u0000\u0587\u0588\u0003\u00b5Q\u0000\u0588\u0589\u0001\u0000"+ + "\u0000\u0000\u0589\u058a\u0006\u009e\u001b\u0000\u058a\u0150\u0001\u0000"+ + "\u0000\u0000\u058b\u058c\u0003\u00c3X\u0000\u058c\u058d\u0001\u0000\u0000"+ + "\u0000\u058d\u058e\u0006\u009f!\u0000\u058e\u0152\u0001\u0000\u0000\u0000"+ + "\u058f\u0590\u0003\u00bfV\u0000\u0590\u0591\u0001\u0000\u0000\u0000\u0591"+ + "\u0592\u0006\u00a0\"\u0000\u0592\u0154\u0001\u0000\u0000\u0000\u0593\u0594"+ + "\u0003I\u001b\u0000\u0594\u0595\u0001\u0000\u0000\u0000\u0595\u0596\u0006"+ + "\u00a1\r\u0000\u0596\u0156\u0001\u0000\u0000\u0000\u0597\u0598\u0003K"+ + "\u001c\u0000\u0598\u0599\u0001\u0000\u0000\u0000\u0599\u059a\u0006\u00a2"+ + "\r\u0000\u059a\u0158\u0001\u0000\u0000\u0000\u059b\u059c\u0003M\u001d"+ + "\u0000\u059c\u059d\u0001\u0000\u0000\u0000\u059d\u059e\u0006\u00a3\r\u0000"+ + "\u059e\u015a\u0001\u0000\u0000\u0000\u059f\u05a0\u0003O\u001e\u0000\u05a0"+ + "\u05a1\u0001\u0000\u0000\u0000\u05a1\u05a2\u0006\u00a4\u0012\u0000\u05a2"+ + "\u05a3\u0006\u00a4\u000e\u0000\u05a3\u015c\u0001\u0000\u0000\u0000\u05a4"+ + "\u05a5\u0007\u0001\u0000\u0000\u05a5\u05a6\u0007\t\u0000\u0000\u05a6\u05a7"+ + "\u0007\u000f\u0000\u0000\u05a7\u05a8\u0007\u0007\u0000\u0000\u05a8\u015e"+ + "\u0001\u0000\u0000\u0000\u05a9\u05aa\u0003I\u001b\u0000\u05aa\u05ab\u0001"+ + "\u0000\u0000\u0000\u05ab\u05ac\u0006\u00a6\r\u0000\u05ac\u0160\u0001\u0000"+ + "\u0000\u0000\u05ad\u05ae\u0003K\u001c\u0000\u05ae\u05af\u0001\u0000\u0000"+ + "\u0000\u05af\u05b0\u0006\u00a7\r\u0000\u05b0\u0162\u0001\u0000\u0000\u0000"+ + "\u05b1\u05b2\u0003M\u001d\u0000\u05b2\u05b3\u0001\u0000\u0000\u0000\u05b3"+ + "\u05b4\u0006\u00a8\r\u0000\u05b4\u0164\u0001\u0000\u0000\u0000\u05b5\u05b6"+ + "\u0003\u00b9S\u0000\u05b6\u05b7\u0001\u0000\u0000\u0000\u05b7\u05b8\u0006"+ + "\u00a9\u0013\u0000\u05b8\u05b9\u0006\u00a9\u000e\u0000\u05b9\u0166\u0001"+ + "\u0000\u0000\u0000\u05ba\u05bb\u0003u1\u0000\u05bb\u05bc\u0001\u0000\u0000"+ + "\u0000\u05bc\u05bd\u0006\u00aa\u0014\u0000\u05bd\u0168\u0001\u0000\u0000"+ + "\u0000\u05be\u05c4\u0003[$\u0000\u05bf\u05c4\u0003Q\u001f\u0000\u05c0"+ + "\u05c4\u0003{4\u0000\u05c1\u05c4\u0003S \u0000\u05c2\u05c4\u0003a\'\u0000"+ + "\u05c3\u05be\u0001\u0000\u0000\u0000\u05c3\u05bf\u0001\u0000\u0000\u0000"+ + "\u05c3\u05c0\u0001\u0000\u0000\u0000\u05c3\u05c1\u0001\u0000\u0000\u0000"+ + "\u05c3\u05c2\u0001\u0000\u0000\u0000\u05c4\u05c5\u0001\u0000\u0000\u0000"+ + "\u05c5\u05c3\u0001\u0000\u0000\u0000\u05c5\u05c6\u0001\u0000\u0000\u0000"+ + "\u05c6\u016a\u0001\u0000\u0000\u0000\u05c7\u05c8\u0003I\u001b\u0000\u05c8"+ + "\u05c9\u0001\u0000\u0000\u0000\u05c9\u05ca\u0006\u00ac\r\u0000\u05ca\u016c"+ + "\u0001\u0000\u0000\u0000\u05cb\u05cc\u0003K\u001c\u0000\u05cc\u05cd\u0001"+ + "\u0000\u0000\u0000\u05cd\u05ce\u0006\u00ad\r\u0000\u05ce\u016e\u0001\u0000"+ + "\u0000\u0000\u05cf\u05d0\u0003M\u001d\u0000\u05d0\u05d1\u0001\u0000\u0000"+ + "\u0000\u05d1\u05d2\u0006\u00ae\r\u0000\u05d2\u0170\u0001\u0000\u0000\u0000"+ + "\u05d3\u05d4\u0003O\u001e\u0000\u05d4\u05d5\u0001\u0000\u0000\u0000\u05d5"+ + "\u05d6\u0006\u00af\u0012\u0000\u05d6\u05d7\u0006\u00af\u000e\u0000\u05d7"+ + "\u0172\u0001\u0000\u0000\u0000\u05d8\u05d9\u0003u1\u0000\u05d9\u05da\u0001"+ + "\u0000\u0000\u0000\u05da\u05db\u0006\u00b0\u0014\u0000\u05db\u0174\u0001"+ + "\u0000\u0000\u0000\u05dc\u05dd\u0003w2\u0000\u05dd\u05de\u0001\u0000\u0000"+ + "\u0000\u05de\u05df\u0006\u00b1\u0015\u0000\u05df\u0176\u0001\u0000\u0000"+ + "\u0000\u05e0\u05e1\u0003{4\u0000\u05e1\u05e2\u0001\u0000\u0000\u0000\u05e2"+ + "\u05e3\u0006\u00b2\u0019\u0000\u05e3\u0178\u0001\u0000\u0000\u0000\u05e4"+ + "\u05e5\u0003\u0121\u0087\u0000\u05e5\u05e6\u0001\u0000\u0000\u0000\u05e6"+ + "\u05e7\u0006\u00b3#\u0000\u05e7\u05e8\u0006\u00b3$\u0000\u05e8\u017a\u0001"+ + "\u0000\u0000\u0000\u05e9\u05ea\u0003\u00e5i\u0000\u05ea\u05eb\u0001\u0000"+ + "\u0000\u0000\u05eb\u05ec\u0006\u00b4\u0017\u0000\u05ec\u017c\u0001\u0000"+ + "\u0000\u0000\u05ed\u05ee\u0003e)\u0000\u05ee\u05ef\u0001\u0000\u0000\u0000"+ + "\u05ef\u05f0\u0006\u00b5\u0018\u0000\u05f0\u017e\u0001\u0000\u0000\u0000"+ + "\u05f1\u05f2\u0003I\u001b\u0000\u05f2\u05f3\u0001\u0000\u0000\u0000\u05f3"+ + "\u05f4\u0006\u00b6\r\u0000\u05f4\u0180\u0001\u0000\u0000\u0000\u05f5\u05f6"+ + "\u0003K\u001c\u0000\u05f6\u05f7\u0001\u0000\u0000\u0000\u05f7\u05f8\u0006"+ + "\u00b7\r\u0000\u05f8\u0182\u0001\u0000\u0000\u0000\u05f9\u05fa\u0003M"+ + "\u001d\u0000\u05fa\u05fb\u0001\u0000\u0000\u0000\u05fb\u05fc\u0006\u00b8"+ + "\r\u0000\u05fc\u0184\u0001\u0000\u0000\u0000\u05fd\u05fe\u0003O\u001e"+ + "\u0000\u05fe\u05ff\u0001\u0000\u0000\u0000\u05ff\u0600\u0006\u00b9\u0012"+ + "\u0000\u0600\u0601\u0006\u00b9\u000e\u0000\u0601\u0602\u0006\u00b9\u000e"+ + "\u0000\u0602\u0186\u0001\u0000\u0000\u0000\u0603\u0604\u0003w2\u0000\u0604"+ + "\u0605\u0001\u0000\u0000\u0000\u0605\u0606\u0006\u00ba\u0015\u0000\u0606"+ + "\u0188\u0001\u0000\u0000\u0000\u0607\u0608\u0003{4\u0000\u0608\u0609\u0001"+ + "\u0000\u0000\u0000\u0609\u060a\u0006\u00bb\u0019\u0000\u060a\u018a\u0001"+ + "\u0000\u0000\u0000\u060b\u060c\u0003\u00ffv\u0000\u060c\u060d\u0001\u0000"+ + "\u0000\u0000\u060d\u060e\u0006\u00bc\u001c\u0000\u060e\u018c\u0001\u0000"+ + "\u0000\u0000\u060f\u0610\u0003I\u001b\u0000\u0610\u0611\u0001\u0000\u0000"+ + "\u0000\u0611\u0612\u0006\u00bd\r\u0000\u0612\u018e\u0001\u0000\u0000\u0000"+ + "\u0613\u0614\u0003K\u001c\u0000\u0614\u0615\u0001\u0000\u0000\u0000\u0615"+ + "\u0616\u0006\u00be\r\u0000\u0616\u0190\u0001\u0000\u0000\u0000\u0617\u0618"+ + "\u0003M\u001d\u0000\u0618\u0619\u0001\u0000\u0000\u0000\u0619\u061a\u0006"+ + "\u00bf\r\u0000\u061a\u0192\u0001\u0000\u0000\u0000\u061b\u061c\u0003O"+ + "\u001e\u0000\u061c\u061d\u0001\u0000\u0000\u0000\u061d\u061e\u0006\u00c0"+ + "\u0012\u0000\u061e\u061f\u0006\u00c0\u000e\u0000\u061f\u0194\u0001\u0000"+ + "\u0000\u0000\u0620\u0621\u0007#\u0000\u0000\u0621\u0622\u0007\u0007\u0000"+ + "\u0000\u0622\u0623\u0007\u0001\u0000\u0000\u0623\u0624\u0007\t\u0000\u0000"+ + "\u0624\u0196\u0001\u0000\u0000\u0000\u0625\u0626\u0003\u0113\u0080\u0000"+ + "\u0626\u0627\u0001\u0000\u0000\u0000\u0627\u0628\u0006\u00c2%\u0000\u0628"+ + "\u0198\u0001\u0000\u0000\u0000\u0629\u062a\u0003\u0121\u0087\u0000\u062a"+ + "\u062b\u0001\u0000\u0000\u0000\u062b\u062c\u0006\u00c3#\u0000\u062c\u062d"+ + "\u0006\u00c3\u000e\u0000\u062d\u062e\u0006\u00c3\u0000\u0000\u062e\u019a"+ + "\u0001\u0000\u0000\u0000\u062f\u0630\u0007\u0014\u0000\u0000\u0630\u0631"+ + "\u0007\u0002\u0000\u0000\u0631\u0632\u0007\u0001\u0000\u0000\u0632\u0633"+ + "\u0007\t\u0000\u0000\u0633\u0634\u0007\u0011\u0000\u0000\u0634\u0635\u0001"+ + "\u0000\u0000\u0000\u0635\u0636\u0006\u00c4\u000e\u0000\u0636\u0637\u0006"+ + "\u00c4\u0000\u0000\u0637\u019c\u0001\u0000\u0000\u0000\u0638\u0639\u0003"+ + "\u00e5i\u0000\u0639\u063a\u0001\u0000\u0000\u0000\u063a\u063b\u0006\u00c5"+ + "\u0017\u0000\u063b\u019e\u0001\u0000\u0000\u0000\u063c\u063d\u0003e)\u0000"+ + "\u063d\u063e\u0001\u0000\u0000\u0000\u063e\u063f\u0006\u00c6\u0018\u0000"+ + "\u063f\u01a0\u0001\u0000\u0000\u0000\u0640\u0641\u0003u1\u0000\u0641\u0642"+ + "\u0001\u0000\u0000\u0000\u0642\u0643\u0006\u00c7\u0014\u0000\u0643\u01a2"+ + "\u0001\u0000\u0000\u0000\u0644\u0645\u0003\u00bfV\u0000\u0645\u0646\u0001"+ + "\u0000\u0000\u0000\u0646\u0647\u0006\u00c8\"\u0000\u0647\u01a4\u0001\u0000"+ + "\u0000\u0000\u0648\u0649\u0003\u00c3X\u0000\u0649\u064a\u0001\u0000\u0000"+ + "\u0000\u064a\u064b\u0006\u00c9!\u0000\u064b\u01a6\u0001\u0000\u0000\u0000"+ + "\u064c\u064d\u0003I\u001b\u0000\u064d\u064e\u0001\u0000\u0000\u0000\u064e"+ + "\u064f\u0006\u00ca\r\u0000\u064f\u01a8\u0001\u0000\u0000\u0000\u0650\u0651"+ + "\u0003K\u001c\u0000\u0651\u0652\u0001\u0000\u0000\u0000\u0652\u0653\u0006"+ + "\u00cb\r\u0000\u0653\u01aa\u0001\u0000\u0000\u0000\u0654\u0655\u0003M"+ + "\u001d\u0000\u0655\u0656\u0001\u0000\u0000\u0000\u0656\u0657\u0006\u00cc"+ + "\r\u0000\u0657\u01ac\u0001\u0000\u0000\u0000\u0658\u0659\u0003O\u001e"+ + "\u0000\u0659\u065a\u0001\u0000\u0000\u0000\u065a\u065b\u0006\u00cd\u0012"+ + "\u0000\u065b\u065c\u0006\u00cd\u000e\u0000\u065c\u01ae\u0001\u0000\u0000"+ + "\u0000\u065d\u065e\u0003\u00e5i\u0000\u065e\u065f\u0001\u0000\u0000\u0000"+ + "\u065f\u0660\u0006\u00ce\u0017\u0000\u0660\u0661\u0006\u00ce\u000e\u0000"+ + "\u0661\u0662\u0006\u00ce&\u0000\u0662\u01b0\u0001\u0000\u0000\u0000\u0663"+ + "\u0664\u0003e)\u0000\u0664\u0665\u0001\u0000\u0000\u0000\u0665\u0666\u0006"+ + "\u00cf\u0018\u0000\u0666\u0667\u0006\u00cf\u000e\u0000\u0667\u0668\u0006"+ + "\u00cf&\u0000\u0668\u01b2\u0001\u0000\u0000\u0000\u0669\u066a\u0003I\u001b"+ + "\u0000\u066a\u066b\u0001\u0000\u0000\u0000\u066b\u066c\u0006\u00d0\r\u0000"+ + "\u066c\u01b4\u0001\u0000\u0000\u0000\u066d\u066e\u0003K\u001c\u0000\u066e"+ + "\u066f\u0001\u0000\u0000\u0000\u066f\u0670\u0006\u00d1\r\u0000\u0670\u01b6"+ + "\u0001\u0000\u0000\u0000\u0671\u0672\u0003M\u001d\u0000\u0672\u0673\u0001"+ + "\u0000\u0000\u0000\u0673\u0674\u0006\u00d2\r\u0000\u0674\u01b8\u0001\u0000"+ + "\u0000\u0000\u0675\u0676\u0003u1\u0000\u0676\u0677\u0001\u0000\u0000\u0000"+ + "\u0677\u0678\u0006\u00d3\u0014\u0000\u0678\u0679\u0006\u00d3\u000e\u0000"+ + "\u0679\u067a\u0006\u00d3\u000b\u0000\u067a\u01ba\u0001\u0000\u0000\u0000"+ + "\u067b\u067c\u0003w2\u0000\u067c\u067d\u0001\u0000\u0000\u0000\u067d\u067e"+ + "\u0006\u00d4\u0015\u0000\u067e\u067f\u0006\u00d4\u000e\u0000\u067f\u0680"+ + "\u0006\u00d4\u000b\u0000\u0680\u01bc\u0001\u0000\u0000\u0000\u0681\u0682"+ + "\u0003I\u001b\u0000\u0682\u0683\u0001\u0000\u0000\u0000\u0683\u0684\u0006"+ + "\u00d5\r\u0000\u0684\u01be\u0001\u0000\u0000\u0000\u0685\u0686\u0003K"+ + "\u001c\u0000\u0686\u0687\u0001\u0000\u0000\u0000\u0687\u0688\u0006\u00d6"+ + "\r\u0000\u0688\u01c0\u0001\u0000\u0000\u0000\u0689\u068a\u0003M\u001d"+ + "\u0000\u068a\u068b\u0001\u0000\u0000\u0000\u068b\u068c\u0006\u00d7\r\u0000"+ + "\u068c\u01c2\u0001\u0000\u0000\u0000\u068d\u068e\u0003\u00c3X\u0000\u068e"+ + "\u068f\u0001\u0000\u0000\u0000\u068f\u0690\u0006\u00d8\u000e\u0000\u0690"+ + "\u0691\u0006\u00d8\u0000\u0000\u0691\u0692\u0006\u00d8!\u0000\u0692\u01c4"+ + "\u0001\u0000\u0000\u0000\u0693\u0694\u0003\u00bfV\u0000\u0694\u0695\u0001"+ + "\u0000\u0000\u0000\u0695\u0696\u0006\u00d9\u000e\u0000\u0696\u0697\u0006"+ + "\u00d9\u0000\u0000\u0697\u0698\u0006\u00d9\"\u0000\u0698\u01c6\u0001\u0000"+ + "\u0000\u0000\u0699\u069a\u0003k,\u0000\u069a\u069b\u0001\u0000\u0000\u0000"+ + "\u069b\u069c\u0006\u00da\u000e\u0000\u069c\u069d\u0006\u00da\u0000\u0000"+ + "\u069d\u069e\u0006\u00da\'\u0000\u069e\u01c8\u0001\u0000\u0000\u0000\u069f"+ + "\u06a0\u0003O\u001e\u0000\u06a0\u06a1\u0001\u0000\u0000\u0000\u06a1\u06a2"+ + "\u0006\u00db\u0012\u0000\u06a2\u06a3\u0006\u00db\u000e\u0000\u06a3\u01ca"+ + "\u0001\u0000\u0000\u0000\u06a4\u06a5\u0003O\u001e\u0000\u06a5\u06a6\u0001"+ + "\u0000\u0000\u0000\u06a6\u06a7\u0006\u00dc\u0012\u0000\u06a7\u06a8\u0006"+ + "\u00dc\u000e\u0000\u06a8\u01cc\u0001\u0000\u0000\u0000\u06a9\u06aa\u0003"+ + "\u0121\u0087\u0000\u06aa\u06ab\u0001\u0000\u0000\u0000\u06ab\u06ac\u0006"+ + "\u00dd#\u0000\u06ac\u01ce\u0001\u0000\u0000\u0000\u06ad\u06ae\u0003\u0113"+ + "\u0080\u0000\u06ae\u06af\u0001\u0000\u0000\u0000\u06af\u06b0\u0006\u00de"+ + "%\u0000\u06b0\u01d0\u0001\u0000\u0000\u0000\u06b1\u06b2\u0003{4\u0000"+ + "\u06b2\u06b3\u0001\u0000\u0000\u0000\u06b3\u06b4\u0006\u00df\u0019\u0000"+ + "\u06b4\u01d2\u0001\u0000\u0000\u0000\u06b5\u06b6\u0003w2\u0000\u06b6\u06b7"+ + "\u0001\u0000\u0000\u0000\u06b7\u06b8\u0006\u00e0\u0015\u0000\u06b8\u01d4"+ + "\u0001\u0000\u0000\u0000\u06b9\u06ba\u0003\u00c3X\u0000\u06ba\u06bb\u0001"+ + "\u0000\u0000\u0000\u06bb\u06bc\u0006\u00e1!\u0000\u06bc\u01d6\u0001\u0000"+ + "\u0000\u0000\u06bd\u06be\u0003\u00bfV\u0000\u06be\u06bf\u0001\u0000\u0000"+ + "\u0000\u06bf\u06c0\u0006\u00e2\"\u0000\u06c0\u01d8\u0001\u0000\u0000\u0000"+ + "\u06c1\u06c2\u0003I\u001b\u0000\u06c2\u06c3\u0001\u0000\u0000\u0000\u06c3"+ + "\u06c4\u0006\u00e3\r\u0000\u06c4\u01da\u0001\u0000\u0000\u0000\u06c5\u06c6"+ + "\u0003K\u001c\u0000\u06c6\u06c7\u0001\u0000\u0000\u0000\u06c7\u06c8\u0006"+ + "\u00e4\r\u0000\u06c8\u01dc\u0001\u0000\u0000\u0000\u06c9\u06ca\u0003M"+ + "\u001d\u0000\u06ca\u06cb\u0001\u0000\u0000\u0000\u06cb\u06cc\u0006\u00e5"+ + "\r\u0000\u06cc\u01de\u0001\u0000\u0000\u0000\u06cd\u06ce\u0003O\u001e"+ + "\u0000\u06ce\u06cf\u0001\u0000\u0000\u0000\u06cf\u06d0\u0006\u00e6\u0012"+ + "\u0000\u06d0\u06d1\u0006\u00e6\u000e\u0000\u06d1\u01e0\u0001\u0000\u0000"+ + "\u0000\u06d2\u06d3\u0003\u00bfV\u0000\u06d3\u06d4\u0001\u0000\u0000\u0000"+ + "\u06d4\u06d5\u0006\u00e7\"\u0000\u06d5\u01e2\u0001\u0000\u0000\u0000\u06d6"+ + "\u06d7\u0003M\u001d\u0000\u06d7\u06d8\u0001\u0000\u0000\u0000\u06d8\u06d9"+ + "\u0006\u00e8\r\u0000\u06d9\u01e4\u0001\u0000\u0000\u0000\u06da\u06db\u0003"+ + "I\u001b\u0000\u06db\u06dc\u0001\u0000\u0000\u0000\u06dc\u06dd\u0006\u00e9"+ + "\r\u0000\u06dd\u01e6\u0001\u0000\u0000\u0000\u06de\u06df\u0003K\u001c"+ + "\u0000\u06df\u06e0\u0001\u0000\u0000\u0000\u06e0\u06e1\u0006\u00ea\r\u0000"+ + "\u06e1\u01e8\u0001\u0000\u0000\u0000\u06e2\u06e3\u0003\u00bbT\u0000\u06e3"+ + "\u06e4\u0001\u0000\u0000\u0000\u06e4\u06e5\u0006\u00eb(\u0000\u06e5\u06e6"+ + "\u0006\u00eb\u0011\u0000\u06e6\u01ea\u0001\u0000\u0000\u0000\u06e7\u06e8"+ + "\u0003O\u001e\u0000\u06e8\u06e9\u0001\u0000\u0000\u0000\u06e9\u06ea\u0006"+ + "\u00ec\u0012\u0000\u06ea\u06eb\u0006\u00ec\u000e\u0000\u06eb\u01ec\u0001"+ + "\u0000\u0000\u0000\u06ec\u06ed\u0003M\u001d\u0000\u06ed\u06ee\u0001\u0000"+ + "\u0000\u0000\u06ee\u06ef\u0006\u00ed\r\u0000\u06ef\u01ee\u0001\u0000\u0000"+ + "\u0000\u06f0\u06f1\u0003I\u001b\u0000\u06f1\u06f2\u0001\u0000\u0000\u0000"+ + "\u06f2\u06f3\u0006\u00ee\r\u0000\u06f3\u01f0\u0001\u0000\u0000\u0000\u06f4"+ + "\u06f5\u0003K\u001c\u0000\u06f5\u06f6\u0001\u0000\u0000\u0000\u06f6\u06f7"+ + "\u0006\u00ef\r\u0000\u06f7\u01f2\u0001\u0000\u0000\u0000E\u0000\u0001"+ + "\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010"+ + "\u0011\u0012\u02e3\u02ed\u02f1\u02f4\u02fd\u02ff\u030a\u031d\u0322\u032b"+ + "\u0332\u0337\u0339\u0344\u034c\u034f\u0351\u0356\u035b\u0361\u0368\u036d"+ + "\u0373\u0376\u037e\u0382\u0402\u0407\u040e\u0410\u042a\u042f\u0434\u0436"+ + "\u043c\u0489\u048e\u04bd\u04c1\u04c6\u04cb\u04d0\u04d2\u04d6\u04d8\u052d"+ + "\u0531\u0536\u05c3\u05c5)\u0005\u0001\u0000\u0005\u0004\u0000\u0005\u0006"+ + "\u0000\u0005\u0002\u0000\u0005\u0003\u0000\u0005\b\u0000\u0005\u0005\u0000"+ + "\u0005\t\u0000\u0005\r\u0000\u0005\u0010\u0000\u0005\u000b\u0000\u0005"+ + "\u000e\u0000\u0005\u0012\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007"+ + "\u0010\u0000\u0007H\u0000\u0005\u0000\u0000\u0007\u001f\u0000\u0007I\u0000"+ + "\u0007(\u0000\u0007)\u0000\u0007&\u0000\u0007U\u0000\u0007 \u0000\u0007"+ + "+\u0000\u00076\u0000\u0007G\u0000\u0007Y\u0000\u0005\n\u0000\u0005\u0007"+ + "\u0000\u0007c\u0000\u0007b\u0000\u0007M\u0000\u0007L\u0000\u0007a\u0000"+ + "\u0005\f\u0000\u0007]\u0000\u0005\u000f\u0000\u0007#\u0000\u0007J\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index cfee76afb26b1..c0b1e8c308d64 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -49,14 +49,12 @@ null 'is' 'last' 'like' -'(' 'not' 'null' 'nulls' 'or' '?' 'rlike' -')' 'true' '==' '=~' @@ -76,6 +74,8 @@ null null ']' null +')' +null null null null @@ -194,14 +194,12 @@ IN IS LAST LIKE -LP NOT NULL NULLS OR PARAM RLIKE -RP TRUE EQ CIEQ @@ -220,6 +218,8 @@ RIGHT_BRACES NAMED_OR_POSITIONAL_PARAM OPENING_BRACKET CLOSING_BRACKET +LP +RP UNQUOTED_IDENTIFIER QUOTED_IDENTIFIER EXPR_LINE_COMMENT @@ -365,4 +365,4 @@ forkSubQueryProcessingCommand atn: -[4, 1, 142, 706, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 156, 8, 1, 10, 1, 12, 1, 159, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 167, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 192, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 204, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 211, 8, 5, 10, 5, 12, 5, 214, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 221, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 226, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 234, 8, 5, 10, 5, 12, 5, 237, 9, 5, 1, 6, 1, 6, 3, 6, 241, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 248, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 253, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 258, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 268, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 274, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 282, 8, 9, 10, 9, 12, 9, 285, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 295, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 300, 8, 10, 10, 10, 12, 10, 303, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 311, 8, 11, 10, 11, 12, 11, 314, 9, 11, 1, 11, 1, 11, 3, 11, 318, 8, 11, 3, 11, 320, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 5, 13, 330, 8, 13, 10, 13, 12, 13, 333, 9, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 5, 17, 349, 8, 17, 10, 17, 12, 17, 352, 9, 17, 1, 18, 1, 18, 1, 18, 3, 18, 357, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 365, 8, 19, 10, 19, 12, 19, 368, 9, 19, 1, 19, 3, 19, 371, 8, 19, 1, 20, 1, 20, 1, 20, 3, 20, 376, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 388, 8, 23, 10, 23, 12, 23, 391, 9, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 397, 8, 24, 10, 24, 12, 24, 400, 9, 24, 1, 24, 3, 24, 403, 8, 24, 1, 24, 1, 24, 3, 24, 407, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 414, 8, 26, 1, 26, 1, 26, 3, 26, 418, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 423, 8, 27, 10, 27, 12, 27, 426, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 431, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 436, 8, 29, 10, 29, 12, 29, 439, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 444, 8, 30, 10, 30, 12, 30, 447, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 452, 8, 31, 10, 31, 12, 31, 455, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 3, 33, 461, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 476, 8, 34, 10, 34, 12, 34, 479, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 487, 8, 34, 10, 34, 12, 34, 490, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 498, 8, 34, 10, 34, 12, 34, 501, 9, 34, 1, 34, 1, 34, 3, 34, 505, 8, 34, 1, 35, 1, 35, 3, 35, 509, 8, 35, 1, 36, 1, 36, 3, 36, 513, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 522, 8, 38, 10, 38, 12, 38, 525, 9, 38, 1, 39, 1, 39, 3, 39, 529, 8, 39, 1, 39, 1, 39, 3, 39, 533, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 545, 8, 42, 10, 42, 12, 42, 548, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 558, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 570, 8, 47, 10, 47, 12, 47, 573, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 583, 8, 50, 1, 51, 3, 51, 586, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 591, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 613, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 619, 8, 58, 10, 58, 12, 58, 622, 9, 58, 3, 58, 624, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 629, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 642, 8, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 64, 5, 64, 655, 8, 64, 10, 64, 12, 64, 658, 9, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 66, 3, 66, 666, 8, 66, 1, 66, 1, 66, 1, 66, 1, 66, 1, 66, 3, 66, 673, 8, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 4, 69, 682, 8, 69, 11, 69, 12, 69, 683, 1, 70, 1, 70, 1, 70, 1, 70, 1, 71, 1, 71, 1, 71, 1, 71, 1, 71, 1, 71, 5, 71, 696, 8, 71, 10, 71, 12, 71, 699, 9, 71, 1, 72, 1, 72, 1, 72, 3, 72, 704, 8, 72, 1, 72, 0, 5, 2, 10, 18, 20, 142, 73, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, 136, 138, 140, 142, 144, 0, 9, 1, 0, 66, 67, 1, 0, 68, 70, 2, 0, 32, 32, 85, 85, 1, 0, 76, 77, 2, 0, 37, 37, 42, 42, 2, 0, 45, 45, 48, 48, 2, 0, 44, 44, 58, 58, 2, 0, 59, 59, 61, 65, 2, 0, 17, 17, 24, 25, 733, 0, 146, 1, 0, 0, 0, 2, 149, 1, 0, 0, 0, 4, 166, 1, 0, 0, 0, 6, 191, 1, 0, 0, 0, 8, 193, 1, 0, 0, 0, 10, 225, 1, 0, 0, 0, 12, 252, 1, 0, 0, 0, 14, 254, 1, 0, 0, 0, 16, 267, 1, 0, 0, 0, 18, 273, 1, 0, 0, 0, 20, 294, 1, 0, 0, 0, 22, 304, 1, 0, 0, 0, 24, 323, 1, 0, 0, 0, 26, 325, 1, 0, 0, 0, 28, 336, 1, 0, 0, 0, 30, 340, 1, 0, 0, 0, 32, 342, 1, 0, 0, 0, 34, 345, 1, 0, 0, 0, 36, 356, 1, 0, 0, 0, 38, 360, 1, 0, 0, 0, 40, 375, 1, 0, 0, 0, 42, 379, 1, 0, 0, 0, 44, 381, 1, 0, 0, 0, 46, 383, 1, 0, 0, 0, 48, 392, 1, 0, 0, 0, 50, 408, 1, 0, 0, 0, 52, 411, 1, 0, 0, 0, 54, 419, 1, 0, 0, 0, 56, 427, 1, 0, 0, 0, 58, 432, 1, 0, 0, 0, 60, 440, 1, 0, 0, 0, 62, 448, 1, 0, 0, 0, 64, 456, 1, 0, 0, 0, 66, 460, 1, 0, 0, 0, 68, 504, 1, 0, 0, 0, 70, 508, 1, 0, 0, 0, 72, 512, 1, 0, 0, 0, 74, 514, 1, 0, 0, 0, 76, 517, 1, 0, 0, 0, 78, 526, 1, 0, 0, 0, 80, 534, 1, 0, 0, 0, 82, 537, 1, 0, 0, 0, 84, 540, 1, 0, 0, 0, 86, 549, 1, 0, 0, 0, 88, 553, 1, 0, 0, 0, 90, 559, 1, 0, 0, 0, 92, 563, 1, 0, 0, 0, 94, 566, 1, 0, 0, 0, 96, 574, 1, 0, 0, 0, 98, 578, 1, 0, 0, 0, 100, 582, 1, 0, 0, 0, 102, 585, 1, 0, 0, 0, 104, 590, 1, 0, 0, 0, 106, 594, 1, 0, 0, 0, 108, 596, 1, 0, 0, 0, 110, 598, 1, 0, 0, 0, 112, 601, 1, 0, 0, 0, 114, 605, 1, 0, 0, 0, 116, 608, 1, 0, 0, 0, 118, 628, 1, 0, 0, 0, 120, 632, 1, 0, 0, 0, 122, 637, 1, 0, 0, 0, 124, 643, 1, 0, 0, 0, 126, 648, 1, 0, 0, 0, 128, 650, 1, 0, 0, 0, 130, 659, 1, 0, 0, 0, 132, 661, 1, 0, 0, 0, 134, 674, 1, 0, 0, 0, 136, 677, 1, 0, 0, 0, 138, 681, 1, 0, 0, 0, 140, 685, 1, 0, 0, 0, 142, 689, 1, 0, 0, 0, 144, 703, 1, 0, 0, 0, 146, 147, 3, 2, 1, 0, 147, 148, 5, 0, 0, 1, 148, 1, 1, 0, 0, 0, 149, 150, 6, 1, -1, 0, 150, 151, 3, 4, 2, 0, 151, 157, 1, 0, 0, 0, 152, 153, 10, 1, 0, 0, 153, 154, 5, 31, 0, 0, 154, 156, 3, 6, 3, 0, 155, 152, 1, 0, 0, 0, 156, 159, 1, 0, 0, 0, 157, 155, 1, 0, 0, 0, 157, 158, 1, 0, 0, 0, 158, 3, 1, 0, 0, 0, 159, 157, 1, 0, 0, 0, 160, 167, 3, 110, 55, 0, 161, 167, 3, 38, 19, 0, 162, 167, 3, 32, 16, 0, 163, 167, 3, 114, 57, 0, 164, 165, 4, 2, 1, 0, 165, 167, 3, 48, 24, 0, 166, 160, 1, 0, 0, 0, 166, 161, 1, 0, 0, 0, 166, 162, 1, 0, 0, 0, 166, 163, 1, 0, 0, 0, 166, 164, 1, 0, 0, 0, 167, 5, 1, 0, 0, 0, 168, 192, 3, 50, 25, 0, 169, 192, 3, 8, 4, 0, 170, 192, 3, 80, 40, 0, 171, 192, 3, 74, 37, 0, 172, 192, 3, 52, 26, 0, 173, 192, 3, 76, 38, 0, 174, 192, 3, 82, 41, 0, 175, 192, 3, 84, 42, 0, 176, 192, 3, 88, 44, 0, 177, 192, 3, 90, 45, 0, 178, 192, 3, 116, 58, 0, 179, 192, 3, 92, 46, 0, 180, 192, 3, 124, 62, 0, 181, 182, 4, 3, 2, 0, 182, 192, 3, 122, 61, 0, 183, 184, 4, 3, 3, 0, 184, 192, 3, 120, 60, 0, 185, 186, 4, 3, 4, 0, 186, 192, 3, 132, 66, 0, 187, 188, 4, 3, 5, 0, 188, 192, 3, 134, 67, 0, 189, 190, 4, 3, 6, 0, 190, 192, 3, 136, 68, 0, 191, 168, 1, 0, 0, 0, 191, 169, 1, 0, 0, 0, 191, 170, 1, 0, 0, 0, 191, 171, 1, 0, 0, 0, 191, 172, 1, 0, 0, 0, 191, 173, 1, 0, 0, 0, 191, 174, 1, 0, 0, 0, 191, 175, 1, 0, 0, 0, 191, 176, 1, 0, 0, 0, 191, 177, 1, 0, 0, 0, 191, 178, 1, 0, 0, 0, 191, 179, 1, 0, 0, 0, 191, 180, 1, 0, 0, 0, 191, 181, 1, 0, 0, 0, 191, 183, 1, 0, 0, 0, 191, 185, 1, 0, 0, 0, 191, 187, 1, 0, 0, 0, 191, 189, 1, 0, 0, 0, 192, 7, 1, 0, 0, 0, 193, 194, 5, 16, 0, 0, 194, 195, 3, 10, 5, 0, 195, 9, 1, 0, 0, 0, 196, 197, 6, 5, -1, 0, 197, 198, 5, 51, 0, 0, 198, 226, 3, 10, 5, 8, 199, 226, 3, 16, 8, 0, 200, 226, 3, 12, 6, 0, 201, 203, 3, 16, 8, 0, 202, 204, 5, 51, 0, 0, 203, 202, 1, 0, 0, 0, 203, 204, 1, 0, 0, 0, 204, 205, 1, 0, 0, 0, 205, 206, 5, 46, 0, 0, 206, 207, 5, 50, 0, 0, 207, 212, 3, 16, 8, 0, 208, 209, 5, 41, 0, 0, 209, 211, 3, 16, 8, 0, 210, 208, 1, 0, 0, 0, 211, 214, 1, 0, 0, 0, 212, 210, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 215, 1, 0, 0, 0, 214, 212, 1, 0, 0, 0, 215, 216, 5, 57, 0, 0, 216, 226, 1, 0, 0, 0, 217, 218, 3, 16, 8, 0, 218, 220, 5, 47, 0, 0, 219, 221, 5, 51, 0, 0, 220, 219, 1, 0, 0, 0, 220, 221, 1, 0, 0, 0, 221, 222, 1, 0, 0, 0, 222, 223, 5, 52, 0, 0, 223, 226, 1, 0, 0, 0, 224, 226, 3, 14, 7, 0, 225, 196, 1, 0, 0, 0, 225, 199, 1, 0, 0, 0, 225, 200, 1, 0, 0, 0, 225, 201, 1, 0, 0, 0, 225, 217, 1, 0, 0, 0, 225, 224, 1, 0, 0, 0, 226, 235, 1, 0, 0, 0, 227, 228, 10, 5, 0, 0, 228, 229, 5, 36, 0, 0, 229, 234, 3, 10, 5, 6, 230, 231, 10, 4, 0, 0, 231, 232, 5, 54, 0, 0, 232, 234, 3, 10, 5, 5, 233, 227, 1, 0, 0, 0, 233, 230, 1, 0, 0, 0, 234, 237, 1, 0, 0, 0, 235, 233, 1, 0, 0, 0, 235, 236, 1, 0, 0, 0, 236, 11, 1, 0, 0, 0, 237, 235, 1, 0, 0, 0, 238, 240, 3, 16, 8, 0, 239, 241, 5, 51, 0, 0, 240, 239, 1, 0, 0, 0, 240, 241, 1, 0, 0, 0, 241, 242, 1, 0, 0, 0, 242, 243, 5, 49, 0, 0, 243, 244, 3, 106, 53, 0, 244, 253, 1, 0, 0, 0, 245, 247, 3, 16, 8, 0, 246, 248, 5, 51, 0, 0, 247, 246, 1, 0, 0, 0, 247, 248, 1, 0, 0, 0, 248, 249, 1, 0, 0, 0, 249, 250, 5, 56, 0, 0, 250, 251, 3, 106, 53, 0, 251, 253, 1, 0, 0, 0, 252, 238, 1, 0, 0, 0, 252, 245, 1, 0, 0, 0, 253, 13, 1, 0, 0, 0, 254, 257, 3, 58, 29, 0, 255, 256, 5, 39, 0, 0, 256, 258, 3, 30, 15, 0, 257, 255, 1, 0, 0, 0, 257, 258, 1, 0, 0, 0, 258, 259, 1, 0, 0, 0, 259, 260, 5, 40, 0, 0, 260, 261, 3, 68, 34, 0, 261, 15, 1, 0, 0, 0, 262, 268, 3, 18, 9, 0, 263, 264, 3, 18, 9, 0, 264, 265, 3, 108, 54, 0, 265, 266, 3, 18, 9, 0, 266, 268, 1, 0, 0, 0, 267, 262, 1, 0, 0, 0, 267, 263, 1, 0, 0, 0, 268, 17, 1, 0, 0, 0, 269, 270, 6, 9, -1, 0, 270, 274, 3, 20, 10, 0, 271, 272, 7, 0, 0, 0, 272, 274, 3, 18, 9, 3, 273, 269, 1, 0, 0, 0, 273, 271, 1, 0, 0, 0, 274, 283, 1, 0, 0, 0, 275, 276, 10, 2, 0, 0, 276, 277, 7, 1, 0, 0, 277, 282, 3, 18, 9, 3, 278, 279, 10, 1, 0, 0, 279, 280, 7, 0, 0, 0, 280, 282, 3, 18, 9, 2, 281, 275, 1, 0, 0, 0, 281, 278, 1, 0, 0, 0, 282, 285, 1, 0, 0, 0, 283, 281, 1, 0, 0, 0, 283, 284, 1, 0, 0, 0, 284, 19, 1, 0, 0, 0, 285, 283, 1, 0, 0, 0, 286, 287, 6, 10, -1, 0, 287, 295, 3, 68, 34, 0, 288, 295, 3, 58, 29, 0, 289, 295, 3, 22, 11, 0, 290, 291, 5, 50, 0, 0, 291, 292, 3, 10, 5, 0, 292, 293, 5, 57, 0, 0, 293, 295, 1, 0, 0, 0, 294, 286, 1, 0, 0, 0, 294, 288, 1, 0, 0, 0, 294, 289, 1, 0, 0, 0, 294, 290, 1, 0, 0, 0, 295, 301, 1, 0, 0, 0, 296, 297, 10, 1, 0, 0, 297, 298, 5, 39, 0, 0, 298, 300, 3, 30, 15, 0, 299, 296, 1, 0, 0, 0, 300, 303, 1, 0, 0, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 21, 1, 0, 0, 0, 303, 301, 1, 0, 0, 0, 304, 305, 3, 24, 12, 0, 305, 319, 5, 50, 0, 0, 306, 320, 5, 68, 0, 0, 307, 312, 3, 10, 5, 0, 308, 309, 5, 41, 0, 0, 309, 311, 3, 10, 5, 0, 310, 308, 1, 0, 0, 0, 311, 314, 1, 0, 0, 0, 312, 310, 1, 0, 0, 0, 312, 313, 1, 0, 0, 0, 313, 317, 1, 0, 0, 0, 314, 312, 1, 0, 0, 0, 315, 316, 5, 41, 0, 0, 316, 318, 3, 26, 13, 0, 317, 315, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 320, 1, 0, 0, 0, 319, 306, 1, 0, 0, 0, 319, 307, 1, 0, 0, 0, 319, 320, 1, 0, 0, 0, 320, 321, 1, 0, 0, 0, 321, 322, 5, 57, 0, 0, 322, 23, 1, 0, 0, 0, 323, 324, 3, 72, 36, 0, 324, 25, 1, 0, 0, 0, 325, 326, 5, 71, 0, 0, 326, 331, 3, 28, 14, 0, 327, 328, 5, 41, 0, 0, 328, 330, 3, 28, 14, 0, 329, 327, 1, 0, 0, 0, 330, 333, 1, 0, 0, 0, 331, 329, 1, 0, 0, 0, 331, 332, 1, 0, 0, 0, 332, 334, 1, 0, 0, 0, 333, 331, 1, 0, 0, 0, 334, 335, 5, 72, 0, 0, 335, 27, 1, 0, 0, 0, 336, 337, 3, 106, 53, 0, 337, 338, 5, 40, 0, 0, 338, 339, 3, 68, 34, 0, 339, 29, 1, 0, 0, 0, 340, 341, 3, 64, 32, 0, 341, 31, 1, 0, 0, 0, 342, 343, 5, 12, 0, 0, 343, 344, 3, 34, 17, 0, 344, 33, 1, 0, 0, 0, 345, 350, 3, 36, 18, 0, 346, 347, 5, 41, 0, 0, 347, 349, 3, 36, 18, 0, 348, 346, 1, 0, 0, 0, 349, 352, 1, 0, 0, 0, 350, 348, 1, 0, 0, 0, 350, 351, 1, 0, 0, 0, 351, 35, 1, 0, 0, 0, 352, 350, 1, 0, 0, 0, 353, 354, 3, 58, 29, 0, 354, 355, 5, 38, 0, 0, 355, 357, 1, 0, 0, 0, 356, 353, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 358, 1, 0, 0, 0, 358, 359, 3, 10, 5, 0, 359, 37, 1, 0, 0, 0, 360, 361, 5, 6, 0, 0, 361, 366, 3, 40, 20, 0, 362, 363, 5, 41, 0, 0, 363, 365, 3, 40, 20, 0, 364, 362, 1, 0, 0, 0, 365, 368, 1, 0, 0, 0, 366, 364, 1, 0, 0, 0, 366, 367, 1, 0, 0, 0, 367, 370, 1, 0, 0, 0, 368, 366, 1, 0, 0, 0, 369, 371, 3, 46, 23, 0, 370, 369, 1, 0, 0, 0, 370, 371, 1, 0, 0, 0, 371, 39, 1, 0, 0, 0, 372, 373, 3, 42, 21, 0, 373, 374, 5, 40, 0, 0, 374, 376, 1, 0, 0, 0, 375, 372, 1, 0, 0, 0, 375, 376, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 378, 3, 44, 22, 0, 378, 41, 1, 0, 0, 0, 379, 380, 7, 2, 0, 0, 380, 43, 1, 0, 0, 0, 381, 382, 7, 2, 0, 0, 382, 45, 1, 0, 0, 0, 383, 384, 5, 84, 0, 0, 384, 389, 5, 85, 0, 0, 385, 386, 5, 41, 0, 0, 386, 388, 5, 85, 0, 0, 387, 385, 1, 0, 0, 0, 388, 391, 1, 0, 0, 0, 389, 387, 1, 0, 0, 0, 389, 390, 1, 0, 0, 0, 390, 47, 1, 0, 0, 0, 391, 389, 1, 0, 0, 0, 392, 393, 5, 22, 0, 0, 393, 398, 3, 40, 20, 0, 394, 395, 5, 41, 0, 0, 395, 397, 3, 40, 20, 0, 396, 394, 1, 0, 0, 0, 397, 400, 1, 0, 0, 0, 398, 396, 1, 0, 0, 0, 398, 399, 1, 0, 0, 0, 399, 402, 1, 0, 0, 0, 400, 398, 1, 0, 0, 0, 401, 403, 3, 54, 27, 0, 402, 401, 1, 0, 0, 0, 402, 403, 1, 0, 0, 0, 403, 406, 1, 0, 0, 0, 404, 405, 5, 35, 0, 0, 405, 407, 3, 34, 17, 0, 406, 404, 1, 0, 0, 0, 406, 407, 1, 0, 0, 0, 407, 49, 1, 0, 0, 0, 408, 409, 5, 4, 0, 0, 409, 410, 3, 34, 17, 0, 410, 51, 1, 0, 0, 0, 411, 413, 5, 15, 0, 0, 412, 414, 3, 54, 27, 0, 413, 412, 1, 0, 0, 0, 413, 414, 1, 0, 0, 0, 414, 417, 1, 0, 0, 0, 415, 416, 5, 35, 0, 0, 416, 418, 3, 34, 17, 0, 417, 415, 1, 0, 0, 0, 417, 418, 1, 0, 0, 0, 418, 53, 1, 0, 0, 0, 419, 424, 3, 56, 28, 0, 420, 421, 5, 41, 0, 0, 421, 423, 3, 56, 28, 0, 422, 420, 1, 0, 0, 0, 423, 426, 1, 0, 0, 0, 424, 422, 1, 0, 0, 0, 424, 425, 1, 0, 0, 0, 425, 55, 1, 0, 0, 0, 426, 424, 1, 0, 0, 0, 427, 430, 3, 36, 18, 0, 428, 429, 5, 16, 0, 0, 429, 431, 3, 10, 5, 0, 430, 428, 1, 0, 0, 0, 430, 431, 1, 0, 0, 0, 431, 57, 1, 0, 0, 0, 432, 437, 3, 72, 36, 0, 433, 434, 5, 43, 0, 0, 434, 436, 3, 72, 36, 0, 435, 433, 1, 0, 0, 0, 436, 439, 1, 0, 0, 0, 437, 435, 1, 0, 0, 0, 437, 438, 1, 0, 0, 0, 438, 59, 1, 0, 0, 0, 439, 437, 1, 0, 0, 0, 440, 445, 3, 66, 33, 0, 441, 442, 5, 43, 0, 0, 442, 444, 3, 66, 33, 0, 443, 441, 1, 0, 0, 0, 444, 447, 1, 0, 0, 0, 445, 443, 1, 0, 0, 0, 445, 446, 1, 0, 0, 0, 446, 61, 1, 0, 0, 0, 447, 445, 1, 0, 0, 0, 448, 453, 3, 60, 30, 0, 449, 450, 5, 41, 0, 0, 450, 452, 3, 60, 30, 0, 451, 449, 1, 0, 0, 0, 452, 455, 1, 0, 0, 0, 453, 451, 1, 0, 0, 0, 453, 454, 1, 0, 0, 0, 454, 63, 1, 0, 0, 0, 455, 453, 1, 0, 0, 0, 456, 457, 7, 3, 0, 0, 457, 65, 1, 0, 0, 0, 458, 461, 5, 89, 0, 0, 459, 461, 3, 70, 35, 0, 460, 458, 1, 0, 0, 0, 460, 459, 1, 0, 0, 0, 461, 67, 1, 0, 0, 0, 462, 505, 5, 52, 0, 0, 463, 464, 3, 104, 52, 0, 464, 465, 5, 76, 0, 0, 465, 505, 1, 0, 0, 0, 466, 505, 3, 102, 51, 0, 467, 505, 3, 104, 52, 0, 468, 505, 3, 98, 49, 0, 469, 505, 3, 70, 35, 0, 470, 505, 3, 106, 53, 0, 471, 472, 5, 74, 0, 0, 472, 477, 3, 100, 50, 0, 473, 474, 5, 41, 0, 0, 474, 476, 3, 100, 50, 0, 475, 473, 1, 0, 0, 0, 476, 479, 1, 0, 0, 0, 477, 475, 1, 0, 0, 0, 477, 478, 1, 0, 0, 0, 478, 480, 1, 0, 0, 0, 479, 477, 1, 0, 0, 0, 480, 481, 5, 75, 0, 0, 481, 505, 1, 0, 0, 0, 482, 483, 5, 74, 0, 0, 483, 488, 3, 98, 49, 0, 484, 485, 5, 41, 0, 0, 485, 487, 3, 98, 49, 0, 486, 484, 1, 0, 0, 0, 487, 490, 1, 0, 0, 0, 488, 486, 1, 0, 0, 0, 488, 489, 1, 0, 0, 0, 489, 491, 1, 0, 0, 0, 490, 488, 1, 0, 0, 0, 491, 492, 5, 75, 0, 0, 492, 505, 1, 0, 0, 0, 493, 494, 5, 74, 0, 0, 494, 499, 3, 106, 53, 0, 495, 496, 5, 41, 0, 0, 496, 498, 3, 106, 53, 0, 497, 495, 1, 0, 0, 0, 498, 501, 1, 0, 0, 0, 499, 497, 1, 0, 0, 0, 499, 500, 1, 0, 0, 0, 500, 502, 1, 0, 0, 0, 501, 499, 1, 0, 0, 0, 502, 503, 5, 75, 0, 0, 503, 505, 1, 0, 0, 0, 504, 462, 1, 0, 0, 0, 504, 463, 1, 0, 0, 0, 504, 466, 1, 0, 0, 0, 504, 467, 1, 0, 0, 0, 504, 468, 1, 0, 0, 0, 504, 469, 1, 0, 0, 0, 504, 470, 1, 0, 0, 0, 504, 471, 1, 0, 0, 0, 504, 482, 1, 0, 0, 0, 504, 493, 1, 0, 0, 0, 505, 69, 1, 0, 0, 0, 506, 509, 5, 55, 0, 0, 507, 509, 5, 73, 0, 0, 508, 506, 1, 0, 0, 0, 508, 507, 1, 0, 0, 0, 509, 71, 1, 0, 0, 0, 510, 513, 3, 64, 32, 0, 511, 513, 3, 70, 35, 0, 512, 510, 1, 0, 0, 0, 512, 511, 1, 0, 0, 0, 513, 73, 1, 0, 0, 0, 514, 515, 5, 9, 0, 0, 515, 516, 5, 33, 0, 0, 516, 75, 1, 0, 0, 0, 517, 518, 5, 14, 0, 0, 518, 523, 3, 78, 39, 0, 519, 520, 5, 41, 0, 0, 520, 522, 3, 78, 39, 0, 521, 519, 1, 0, 0, 0, 522, 525, 1, 0, 0, 0, 523, 521, 1, 0, 0, 0, 523, 524, 1, 0, 0, 0, 524, 77, 1, 0, 0, 0, 525, 523, 1, 0, 0, 0, 526, 528, 3, 10, 5, 0, 527, 529, 7, 4, 0, 0, 528, 527, 1, 0, 0, 0, 528, 529, 1, 0, 0, 0, 529, 532, 1, 0, 0, 0, 530, 531, 5, 53, 0, 0, 531, 533, 7, 5, 0, 0, 532, 530, 1, 0, 0, 0, 532, 533, 1, 0, 0, 0, 533, 79, 1, 0, 0, 0, 534, 535, 5, 8, 0, 0, 535, 536, 3, 62, 31, 0, 536, 81, 1, 0, 0, 0, 537, 538, 5, 2, 0, 0, 538, 539, 3, 62, 31, 0, 539, 83, 1, 0, 0, 0, 540, 541, 5, 11, 0, 0, 541, 546, 3, 86, 43, 0, 542, 543, 5, 41, 0, 0, 543, 545, 3, 86, 43, 0, 544, 542, 1, 0, 0, 0, 545, 548, 1, 0, 0, 0, 546, 544, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 85, 1, 0, 0, 0, 548, 546, 1, 0, 0, 0, 549, 550, 3, 60, 30, 0, 550, 551, 5, 93, 0, 0, 551, 552, 3, 60, 30, 0, 552, 87, 1, 0, 0, 0, 553, 554, 5, 1, 0, 0, 554, 555, 3, 20, 10, 0, 555, 557, 3, 106, 53, 0, 556, 558, 3, 94, 47, 0, 557, 556, 1, 0, 0, 0, 557, 558, 1, 0, 0, 0, 558, 89, 1, 0, 0, 0, 559, 560, 5, 7, 0, 0, 560, 561, 3, 20, 10, 0, 561, 562, 3, 106, 53, 0, 562, 91, 1, 0, 0, 0, 563, 564, 5, 10, 0, 0, 564, 565, 3, 58, 29, 0, 565, 93, 1, 0, 0, 0, 566, 571, 3, 96, 48, 0, 567, 568, 5, 41, 0, 0, 568, 570, 3, 96, 48, 0, 569, 567, 1, 0, 0, 0, 570, 573, 1, 0, 0, 0, 571, 569, 1, 0, 0, 0, 571, 572, 1, 0, 0, 0, 572, 95, 1, 0, 0, 0, 573, 571, 1, 0, 0, 0, 574, 575, 3, 64, 32, 0, 575, 576, 5, 38, 0, 0, 576, 577, 3, 68, 34, 0, 577, 97, 1, 0, 0, 0, 578, 579, 7, 6, 0, 0, 579, 99, 1, 0, 0, 0, 580, 583, 3, 102, 51, 0, 581, 583, 3, 104, 52, 0, 582, 580, 1, 0, 0, 0, 582, 581, 1, 0, 0, 0, 583, 101, 1, 0, 0, 0, 584, 586, 7, 0, 0, 0, 585, 584, 1, 0, 0, 0, 585, 586, 1, 0, 0, 0, 586, 587, 1, 0, 0, 0, 587, 588, 5, 34, 0, 0, 588, 103, 1, 0, 0, 0, 589, 591, 7, 0, 0, 0, 590, 589, 1, 0, 0, 0, 590, 591, 1, 0, 0, 0, 591, 592, 1, 0, 0, 0, 592, 593, 5, 33, 0, 0, 593, 105, 1, 0, 0, 0, 594, 595, 5, 32, 0, 0, 595, 107, 1, 0, 0, 0, 596, 597, 7, 7, 0, 0, 597, 109, 1, 0, 0, 0, 598, 599, 5, 5, 0, 0, 599, 600, 3, 112, 56, 0, 600, 111, 1, 0, 0, 0, 601, 602, 5, 74, 0, 0, 602, 603, 3, 2, 1, 0, 603, 604, 5, 75, 0, 0, 604, 113, 1, 0, 0, 0, 605, 606, 5, 13, 0, 0, 606, 607, 5, 109, 0, 0, 607, 115, 1, 0, 0, 0, 608, 609, 5, 3, 0, 0, 609, 612, 5, 99, 0, 0, 610, 611, 5, 97, 0, 0, 611, 613, 3, 60, 30, 0, 612, 610, 1, 0, 0, 0, 612, 613, 1, 0, 0, 0, 613, 623, 1, 0, 0, 0, 614, 615, 5, 98, 0, 0, 615, 620, 3, 118, 59, 0, 616, 617, 5, 41, 0, 0, 617, 619, 3, 118, 59, 0, 618, 616, 1, 0, 0, 0, 619, 622, 1, 0, 0, 0, 620, 618, 1, 0, 0, 0, 620, 621, 1, 0, 0, 0, 621, 624, 1, 0, 0, 0, 622, 620, 1, 0, 0, 0, 623, 614, 1, 0, 0, 0, 623, 624, 1, 0, 0, 0, 624, 117, 1, 0, 0, 0, 625, 626, 3, 60, 30, 0, 626, 627, 5, 38, 0, 0, 627, 629, 1, 0, 0, 0, 628, 625, 1, 0, 0, 0, 628, 629, 1, 0, 0, 0, 629, 630, 1, 0, 0, 0, 630, 631, 3, 60, 30, 0, 631, 119, 1, 0, 0, 0, 632, 633, 5, 21, 0, 0, 633, 634, 3, 40, 20, 0, 634, 635, 5, 97, 0, 0, 635, 636, 3, 62, 31, 0, 636, 121, 1, 0, 0, 0, 637, 638, 5, 19, 0, 0, 638, 641, 3, 54, 27, 0, 639, 640, 5, 35, 0, 0, 640, 642, 3, 34, 17, 0, 641, 639, 1, 0, 0, 0, 641, 642, 1, 0, 0, 0, 642, 123, 1, 0, 0, 0, 643, 644, 7, 8, 0, 0, 644, 645, 5, 123, 0, 0, 645, 646, 3, 126, 63, 0, 646, 647, 3, 128, 64, 0, 647, 125, 1, 0, 0, 0, 648, 649, 3, 40, 20, 0, 649, 127, 1, 0, 0, 0, 650, 651, 5, 97, 0, 0, 651, 656, 3, 130, 65, 0, 652, 653, 5, 41, 0, 0, 653, 655, 3, 130, 65, 0, 654, 652, 1, 0, 0, 0, 655, 658, 1, 0, 0, 0, 656, 654, 1, 0, 0, 0, 656, 657, 1, 0, 0, 0, 657, 129, 1, 0, 0, 0, 658, 656, 1, 0, 0, 0, 659, 660, 3, 16, 8, 0, 660, 131, 1, 0, 0, 0, 661, 662, 5, 18, 0, 0, 662, 665, 3, 58, 29, 0, 663, 664, 5, 97, 0, 0, 664, 666, 3, 58, 29, 0, 665, 663, 1, 0, 0, 0, 665, 666, 1, 0, 0, 0, 666, 672, 1, 0, 0, 0, 667, 668, 5, 93, 0, 0, 668, 669, 3, 58, 29, 0, 669, 670, 5, 41, 0, 0, 670, 671, 3, 58, 29, 0, 671, 673, 1, 0, 0, 0, 672, 667, 1, 0, 0, 0, 672, 673, 1, 0, 0, 0, 673, 133, 1, 0, 0, 0, 674, 675, 5, 20, 0, 0, 675, 676, 3, 62, 31, 0, 676, 135, 1, 0, 0, 0, 677, 678, 5, 26, 0, 0, 678, 679, 3, 138, 69, 0, 679, 137, 1, 0, 0, 0, 680, 682, 3, 140, 70, 0, 681, 680, 1, 0, 0, 0, 682, 683, 1, 0, 0, 0, 683, 681, 1, 0, 0, 0, 683, 684, 1, 0, 0, 0, 684, 139, 1, 0, 0, 0, 685, 686, 5, 50, 0, 0, 686, 687, 3, 142, 71, 0, 687, 688, 5, 57, 0, 0, 688, 141, 1, 0, 0, 0, 689, 690, 6, 71, -1, 0, 690, 691, 3, 144, 72, 0, 691, 697, 1, 0, 0, 0, 692, 693, 10, 1, 0, 0, 693, 694, 5, 31, 0, 0, 694, 696, 3, 144, 72, 0, 695, 692, 1, 0, 0, 0, 696, 699, 1, 0, 0, 0, 697, 695, 1, 0, 0, 0, 697, 698, 1, 0, 0, 0, 698, 143, 1, 0, 0, 0, 699, 697, 1, 0, 0, 0, 700, 704, 3, 8, 4, 0, 701, 704, 3, 76, 38, 0, 702, 704, 3, 74, 37, 0, 703, 700, 1, 0, 0, 0, 703, 701, 1, 0, 0, 0, 703, 702, 1, 0, 0, 0, 704, 145, 1, 0, 0, 0, 66, 157, 166, 191, 203, 212, 220, 225, 233, 235, 240, 247, 252, 257, 267, 273, 281, 283, 294, 301, 312, 317, 319, 331, 350, 356, 366, 370, 375, 389, 398, 402, 406, 413, 417, 424, 430, 437, 445, 453, 460, 477, 488, 499, 504, 508, 512, 523, 528, 532, 546, 557, 571, 582, 585, 590, 612, 620, 623, 628, 641, 656, 665, 672, 683, 697, 703] \ No newline at end of file +[4, 1, 142, 706, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 156, 8, 1, 10, 1, 12, 1, 159, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 167, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 192, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 204, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 211, 8, 5, 10, 5, 12, 5, 214, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 221, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 226, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 234, 8, 5, 10, 5, 12, 5, 237, 9, 5, 1, 6, 1, 6, 3, 6, 241, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 248, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 253, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 258, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 268, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 274, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 282, 8, 9, 10, 9, 12, 9, 285, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 295, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 300, 8, 10, 10, 10, 12, 10, 303, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 311, 8, 11, 10, 11, 12, 11, 314, 9, 11, 1, 11, 1, 11, 3, 11, 318, 8, 11, 3, 11, 320, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 5, 13, 330, 8, 13, 10, 13, 12, 13, 333, 9, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 5, 17, 349, 8, 17, 10, 17, 12, 17, 352, 9, 17, 1, 18, 1, 18, 1, 18, 3, 18, 357, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 5, 19, 365, 8, 19, 10, 19, 12, 19, 368, 9, 19, 1, 19, 3, 19, 371, 8, 19, 1, 20, 1, 20, 1, 20, 3, 20, 376, 8, 20, 1, 20, 1, 20, 1, 21, 1, 21, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 388, 8, 23, 10, 23, 12, 23, 391, 9, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 397, 8, 24, 10, 24, 12, 24, 400, 9, 24, 1, 24, 3, 24, 403, 8, 24, 1, 24, 1, 24, 3, 24, 407, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 414, 8, 26, 1, 26, 1, 26, 3, 26, 418, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 423, 8, 27, 10, 27, 12, 27, 426, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 431, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 436, 8, 29, 10, 29, 12, 29, 439, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 444, 8, 30, 10, 30, 12, 30, 447, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 452, 8, 31, 10, 31, 12, 31, 455, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 3, 33, 461, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 476, 8, 34, 10, 34, 12, 34, 479, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 487, 8, 34, 10, 34, 12, 34, 490, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 498, 8, 34, 10, 34, 12, 34, 501, 9, 34, 1, 34, 1, 34, 3, 34, 505, 8, 34, 1, 35, 1, 35, 3, 35, 509, 8, 35, 1, 36, 1, 36, 3, 36, 513, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 522, 8, 38, 10, 38, 12, 38, 525, 9, 38, 1, 39, 1, 39, 3, 39, 529, 8, 39, 1, 39, 1, 39, 3, 39, 533, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 545, 8, 42, 10, 42, 12, 42, 548, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 558, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 570, 8, 47, 10, 47, 12, 47, 573, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 583, 8, 50, 1, 51, 3, 51, 586, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 591, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 613, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 619, 8, 58, 10, 58, 12, 58, 622, 9, 58, 3, 58, 624, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 629, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 642, 8, 61, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 64, 5, 64, 655, 8, 64, 10, 64, 12, 64, 658, 9, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 66, 3, 66, 666, 8, 66, 1, 66, 1, 66, 1, 66, 1, 66, 1, 66, 3, 66, 673, 8, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 4, 69, 682, 8, 69, 11, 69, 12, 69, 683, 1, 70, 1, 70, 1, 70, 1, 70, 1, 71, 1, 71, 1, 71, 1, 71, 1, 71, 1, 71, 5, 71, 696, 8, 71, 10, 71, 12, 71, 699, 9, 71, 1, 72, 1, 72, 1, 72, 3, 72, 704, 8, 72, 1, 72, 0, 5, 2, 10, 18, 20, 142, 73, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, 136, 138, 140, 142, 144, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 32, 32, 85, 85, 1, 0, 76, 77, 2, 0, 37, 37, 42, 42, 2, 0, 45, 45, 48, 48, 2, 0, 44, 44, 56, 56, 2, 0, 57, 57, 59, 63, 2, 0, 17, 17, 24, 25, 733, 0, 146, 1, 0, 0, 0, 2, 149, 1, 0, 0, 0, 4, 166, 1, 0, 0, 0, 6, 191, 1, 0, 0, 0, 8, 193, 1, 0, 0, 0, 10, 225, 1, 0, 0, 0, 12, 252, 1, 0, 0, 0, 14, 254, 1, 0, 0, 0, 16, 267, 1, 0, 0, 0, 18, 273, 1, 0, 0, 0, 20, 294, 1, 0, 0, 0, 22, 304, 1, 0, 0, 0, 24, 323, 1, 0, 0, 0, 26, 325, 1, 0, 0, 0, 28, 336, 1, 0, 0, 0, 30, 340, 1, 0, 0, 0, 32, 342, 1, 0, 0, 0, 34, 345, 1, 0, 0, 0, 36, 356, 1, 0, 0, 0, 38, 360, 1, 0, 0, 0, 40, 375, 1, 0, 0, 0, 42, 379, 1, 0, 0, 0, 44, 381, 1, 0, 0, 0, 46, 383, 1, 0, 0, 0, 48, 392, 1, 0, 0, 0, 50, 408, 1, 0, 0, 0, 52, 411, 1, 0, 0, 0, 54, 419, 1, 0, 0, 0, 56, 427, 1, 0, 0, 0, 58, 432, 1, 0, 0, 0, 60, 440, 1, 0, 0, 0, 62, 448, 1, 0, 0, 0, 64, 456, 1, 0, 0, 0, 66, 460, 1, 0, 0, 0, 68, 504, 1, 0, 0, 0, 70, 508, 1, 0, 0, 0, 72, 512, 1, 0, 0, 0, 74, 514, 1, 0, 0, 0, 76, 517, 1, 0, 0, 0, 78, 526, 1, 0, 0, 0, 80, 534, 1, 0, 0, 0, 82, 537, 1, 0, 0, 0, 84, 540, 1, 0, 0, 0, 86, 549, 1, 0, 0, 0, 88, 553, 1, 0, 0, 0, 90, 559, 1, 0, 0, 0, 92, 563, 1, 0, 0, 0, 94, 566, 1, 0, 0, 0, 96, 574, 1, 0, 0, 0, 98, 578, 1, 0, 0, 0, 100, 582, 1, 0, 0, 0, 102, 585, 1, 0, 0, 0, 104, 590, 1, 0, 0, 0, 106, 594, 1, 0, 0, 0, 108, 596, 1, 0, 0, 0, 110, 598, 1, 0, 0, 0, 112, 601, 1, 0, 0, 0, 114, 605, 1, 0, 0, 0, 116, 608, 1, 0, 0, 0, 118, 628, 1, 0, 0, 0, 120, 632, 1, 0, 0, 0, 122, 637, 1, 0, 0, 0, 124, 643, 1, 0, 0, 0, 126, 648, 1, 0, 0, 0, 128, 650, 1, 0, 0, 0, 130, 659, 1, 0, 0, 0, 132, 661, 1, 0, 0, 0, 134, 674, 1, 0, 0, 0, 136, 677, 1, 0, 0, 0, 138, 681, 1, 0, 0, 0, 140, 685, 1, 0, 0, 0, 142, 689, 1, 0, 0, 0, 144, 703, 1, 0, 0, 0, 146, 147, 3, 2, 1, 0, 147, 148, 5, 0, 0, 1, 148, 1, 1, 0, 0, 0, 149, 150, 6, 1, -1, 0, 150, 151, 3, 4, 2, 0, 151, 157, 1, 0, 0, 0, 152, 153, 10, 1, 0, 0, 153, 154, 5, 31, 0, 0, 154, 156, 3, 6, 3, 0, 155, 152, 1, 0, 0, 0, 156, 159, 1, 0, 0, 0, 157, 155, 1, 0, 0, 0, 157, 158, 1, 0, 0, 0, 158, 3, 1, 0, 0, 0, 159, 157, 1, 0, 0, 0, 160, 167, 3, 110, 55, 0, 161, 167, 3, 38, 19, 0, 162, 167, 3, 32, 16, 0, 163, 167, 3, 114, 57, 0, 164, 165, 4, 2, 1, 0, 165, 167, 3, 48, 24, 0, 166, 160, 1, 0, 0, 0, 166, 161, 1, 0, 0, 0, 166, 162, 1, 0, 0, 0, 166, 163, 1, 0, 0, 0, 166, 164, 1, 0, 0, 0, 167, 5, 1, 0, 0, 0, 168, 192, 3, 50, 25, 0, 169, 192, 3, 8, 4, 0, 170, 192, 3, 80, 40, 0, 171, 192, 3, 74, 37, 0, 172, 192, 3, 52, 26, 0, 173, 192, 3, 76, 38, 0, 174, 192, 3, 82, 41, 0, 175, 192, 3, 84, 42, 0, 176, 192, 3, 88, 44, 0, 177, 192, 3, 90, 45, 0, 178, 192, 3, 116, 58, 0, 179, 192, 3, 92, 46, 0, 180, 192, 3, 124, 62, 0, 181, 182, 4, 3, 2, 0, 182, 192, 3, 122, 61, 0, 183, 184, 4, 3, 3, 0, 184, 192, 3, 120, 60, 0, 185, 186, 4, 3, 4, 0, 186, 192, 3, 132, 66, 0, 187, 188, 4, 3, 5, 0, 188, 192, 3, 134, 67, 0, 189, 190, 4, 3, 6, 0, 190, 192, 3, 136, 68, 0, 191, 168, 1, 0, 0, 0, 191, 169, 1, 0, 0, 0, 191, 170, 1, 0, 0, 0, 191, 171, 1, 0, 0, 0, 191, 172, 1, 0, 0, 0, 191, 173, 1, 0, 0, 0, 191, 174, 1, 0, 0, 0, 191, 175, 1, 0, 0, 0, 191, 176, 1, 0, 0, 0, 191, 177, 1, 0, 0, 0, 191, 178, 1, 0, 0, 0, 191, 179, 1, 0, 0, 0, 191, 180, 1, 0, 0, 0, 191, 181, 1, 0, 0, 0, 191, 183, 1, 0, 0, 0, 191, 185, 1, 0, 0, 0, 191, 187, 1, 0, 0, 0, 191, 189, 1, 0, 0, 0, 192, 7, 1, 0, 0, 0, 193, 194, 5, 16, 0, 0, 194, 195, 3, 10, 5, 0, 195, 9, 1, 0, 0, 0, 196, 197, 6, 5, -1, 0, 197, 198, 5, 50, 0, 0, 198, 226, 3, 10, 5, 8, 199, 226, 3, 16, 8, 0, 200, 226, 3, 12, 6, 0, 201, 203, 3, 16, 8, 0, 202, 204, 5, 50, 0, 0, 203, 202, 1, 0, 0, 0, 203, 204, 1, 0, 0, 0, 204, 205, 1, 0, 0, 0, 205, 206, 5, 46, 0, 0, 206, 207, 5, 74, 0, 0, 207, 212, 3, 16, 8, 0, 208, 209, 5, 41, 0, 0, 209, 211, 3, 16, 8, 0, 210, 208, 1, 0, 0, 0, 211, 214, 1, 0, 0, 0, 212, 210, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 215, 1, 0, 0, 0, 214, 212, 1, 0, 0, 0, 215, 216, 5, 75, 0, 0, 216, 226, 1, 0, 0, 0, 217, 218, 3, 16, 8, 0, 218, 220, 5, 47, 0, 0, 219, 221, 5, 50, 0, 0, 220, 219, 1, 0, 0, 0, 220, 221, 1, 0, 0, 0, 221, 222, 1, 0, 0, 0, 222, 223, 5, 51, 0, 0, 223, 226, 1, 0, 0, 0, 224, 226, 3, 14, 7, 0, 225, 196, 1, 0, 0, 0, 225, 199, 1, 0, 0, 0, 225, 200, 1, 0, 0, 0, 225, 201, 1, 0, 0, 0, 225, 217, 1, 0, 0, 0, 225, 224, 1, 0, 0, 0, 226, 235, 1, 0, 0, 0, 227, 228, 10, 5, 0, 0, 228, 229, 5, 36, 0, 0, 229, 234, 3, 10, 5, 6, 230, 231, 10, 4, 0, 0, 231, 232, 5, 53, 0, 0, 232, 234, 3, 10, 5, 5, 233, 227, 1, 0, 0, 0, 233, 230, 1, 0, 0, 0, 234, 237, 1, 0, 0, 0, 235, 233, 1, 0, 0, 0, 235, 236, 1, 0, 0, 0, 236, 11, 1, 0, 0, 0, 237, 235, 1, 0, 0, 0, 238, 240, 3, 16, 8, 0, 239, 241, 5, 50, 0, 0, 240, 239, 1, 0, 0, 0, 240, 241, 1, 0, 0, 0, 241, 242, 1, 0, 0, 0, 242, 243, 5, 49, 0, 0, 243, 244, 3, 106, 53, 0, 244, 253, 1, 0, 0, 0, 245, 247, 3, 16, 8, 0, 246, 248, 5, 50, 0, 0, 247, 246, 1, 0, 0, 0, 247, 248, 1, 0, 0, 0, 248, 249, 1, 0, 0, 0, 249, 250, 5, 55, 0, 0, 250, 251, 3, 106, 53, 0, 251, 253, 1, 0, 0, 0, 252, 238, 1, 0, 0, 0, 252, 245, 1, 0, 0, 0, 253, 13, 1, 0, 0, 0, 254, 257, 3, 58, 29, 0, 255, 256, 5, 39, 0, 0, 256, 258, 3, 30, 15, 0, 257, 255, 1, 0, 0, 0, 257, 258, 1, 0, 0, 0, 258, 259, 1, 0, 0, 0, 259, 260, 5, 40, 0, 0, 260, 261, 3, 68, 34, 0, 261, 15, 1, 0, 0, 0, 262, 268, 3, 18, 9, 0, 263, 264, 3, 18, 9, 0, 264, 265, 3, 108, 54, 0, 265, 266, 3, 18, 9, 0, 266, 268, 1, 0, 0, 0, 267, 262, 1, 0, 0, 0, 267, 263, 1, 0, 0, 0, 268, 17, 1, 0, 0, 0, 269, 270, 6, 9, -1, 0, 270, 274, 3, 20, 10, 0, 271, 272, 7, 0, 0, 0, 272, 274, 3, 18, 9, 3, 273, 269, 1, 0, 0, 0, 273, 271, 1, 0, 0, 0, 274, 283, 1, 0, 0, 0, 275, 276, 10, 2, 0, 0, 276, 277, 7, 1, 0, 0, 277, 282, 3, 18, 9, 3, 278, 279, 10, 1, 0, 0, 279, 280, 7, 0, 0, 0, 280, 282, 3, 18, 9, 2, 281, 275, 1, 0, 0, 0, 281, 278, 1, 0, 0, 0, 282, 285, 1, 0, 0, 0, 283, 281, 1, 0, 0, 0, 283, 284, 1, 0, 0, 0, 284, 19, 1, 0, 0, 0, 285, 283, 1, 0, 0, 0, 286, 287, 6, 10, -1, 0, 287, 295, 3, 68, 34, 0, 288, 295, 3, 58, 29, 0, 289, 295, 3, 22, 11, 0, 290, 291, 5, 74, 0, 0, 291, 292, 3, 10, 5, 0, 292, 293, 5, 75, 0, 0, 293, 295, 1, 0, 0, 0, 294, 286, 1, 0, 0, 0, 294, 288, 1, 0, 0, 0, 294, 289, 1, 0, 0, 0, 294, 290, 1, 0, 0, 0, 295, 301, 1, 0, 0, 0, 296, 297, 10, 1, 0, 0, 297, 298, 5, 39, 0, 0, 298, 300, 3, 30, 15, 0, 299, 296, 1, 0, 0, 0, 300, 303, 1, 0, 0, 0, 301, 299, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 21, 1, 0, 0, 0, 303, 301, 1, 0, 0, 0, 304, 305, 3, 24, 12, 0, 305, 319, 5, 74, 0, 0, 306, 320, 5, 66, 0, 0, 307, 312, 3, 10, 5, 0, 308, 309, 5, 41, 0, 0, 309, 311, 3, 10, 5, 0, 310, 308, 1, 0, 0, 0, 311, 314, 1, 0, 0, 0, 312, 310, 1, 0, 0, 0, 312, 313, 1, 0, 0, 0, 313, 317, 1, 0, 0, 0, 314, 312, 1, 0, 0, 0, 315, 316, 5, 41, 0, 0, 316, 318, 3, 26, 13, 0, 317, 315, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 320, 1, 0, 0, 0, 319, 306, 1, 0, 0, 0, 319, 307, 1, 0, 0, 0, 319, 320, 1, 0, 0, 0, 320, 321, 1, 0, 0, 0, 321, 322, 5, 75, 0, 0, 322, 23, 1, 0, 0, 0, 323, 324, 3, 72, 36, 0, 324, 25, 1, 0, 0, 0, 325, 326, 5, 69, 0, 0, 326, 331, 3, 28, 14, 0, 327, 328, 5, 41, 0, 0, 328, 330, 3, 28, 14, 0, 329, 327, 1, 0, 0, 0, 330, 333, 1, 0, 0, 0, 331, 329, 1, 0, 0, 0, 331, 332, 1, 0, 0, 0, 332, 334, 1, 0, 0, 0, 333, 331, 1, 0, 0, 0, 334, 335, 5, 70, 0, 0, 335, 27, 1, 0, 0, 0, 336, 337, 3, 106, 53, 0, 337, 338, 5, 40, 0, 0, 338, 339, 3, 68, 34, 0, 339, 29, 1, 0, 0, 0, 340, 341, 3, 64, 32, 0, 341, 31, 1, 0, 0, 0, 342, 343, 5, 12, 0, 0, 343, 344, 3, 34, 17, 0, 344, 33, 1, 0, 0, 0, 345, 350, 3, 36, 18, 0, 346, 347, 5, 41, 0, 0, 347, 349, 3, 36, 18, 0, 348, 346, 1, 0, 0, 0, 349, 352, 1, 0, 0, 0, 350, 348, 1, 0, 0, 0, 350, 351, 1, 0, 0, 0, 351, 35, 1, 0, 0, 0, 352, 350, 1, 0, 0, 0, 353, 354, 3, 58, 29, 0, 354, 355, 5, 38, 0, 0, 355, 357, 1, 0, 0, 0, 356, 353, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 358, 1, 0, 0, 0, 358, 359, 3, 10, 5, 0, 359, 37, 1, 0, 0, 0, 360, 361, 5, 6, 0, 0, 361, 366, 3, 40, 20, 0, 362, 363, 5, 41, 0, 0, 363, 365, 3, 40, 20, 0, 364, 362, 1, 0, 0, 0, 365, 368, 1, 0, 0, 0, 366, 364, 1, 0, 0, 0, 366, 367, 1, 0, 0, 0, 367, 370, 1, 0, 0, 0, 368, 366, 1, 0, 0, 0, 369, 371, 3, 46, 23, 0, 370, 369, 1, 0, 0, 0, 370, 371, 1, 0, 0, 0, 371, 39, 1, 0, 0, 0, 372, 373, 3, 42, 21, 0, 373, 374, 5, 40, 0, 0, 374, 376, 1, 0, 0, 0, 375, 372, 1, 0, 0, 0, 375, 376, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 378, 3, 44, 22, 0, 378, 41, 1, 0, 0, 0, 379, 380, 7, 2, 0, 0, 380, 43, 1, 0, 0, 0, 381, 382, 7, 2, 0, 0, 382, 45, 1, 0, 0, 0, 383, 384, 5, 84, 0, 0, 384, 389, 5, 85, 0, 0, 385, 386, 5, 41, 0, 0, 386, 388, 5, 85, 0, 0, 387, 385, 1, 0, 0, 0, 388, 391, 1, 0, 0, 0, 389, 387, 1, 0, 0, 0, 389, 390, 1, 0, 0, 0, 390, 47, 1, 0, 0, 0, 391, 389, 1, 0, 0, 0, 392, 393, 5, 22, 0, 0, 393, 398, 3, 40, 20, 0, 394, 395, 5, 41, 0, 0, 395, 397, 3, 40, 20, 0, 396, 394, 1, 0, 0, 0, 397, 400, 1, 0, 0, 0, 398, 396, 1, 0, 0, 0, 398, 399, 1, 0, 0, 0, 399, 402, 1, 0, 0, 0, 400, 398, 1, 0, 0, 0, 401, 403, 3, 54, 27, 0, 402, 401, 1, 0, 0, 0, 402, 403, 1, 0, 0, 0, 403, 406, 1, 0, 0, 0, 404, 405, 5, 35, 0, 0, 405, 407, 3, 34, 17, 0, 406, 404, 1, 0, 0, 0, 406, 407, 1, 0, 0, 0, 407, 49, 1, 0, 0, 0, 408, 409, 5, 4, 0, 0, 409, 410, 3, 34, 17, 0, 410, 51, 1, 0, 0, 0, 411, 413, 5, 15, 0, 0, 412, 414, 3, 54, 27, 0, 413, 412, 1, 0, 0, 0, 413, 414, 1, 0, 0, 0, 414, 417, 1, 0, 0, 0, 415, 416, 5, 35, 0, 0, 416, 418, 3, 34, 17, 0, 417, 415, 1, 0, 0, 0, 417, 418, 1, 0, 0, 0, 418, 53, 1, 0, 0, 0, 419, 424, 3, 56, 28, 0, 420, 421, 5, 41, 0, 0, 421, 423, 3, 56, 28, 0, 422, 420, 1, 0, 0, 0, 423, 426, 1, 0, 0, 0, 424, 422, 1, 0, 0, 0, 424, 425, 1, 0, 0, 0, 425, 55, 1, 0, 0, 0, 426, 424, 1, 0, 0, 0, 427, 430, 3, 36, 18, 0, 428, 429, 5, 16, 0, 0, 429, 431, 3, 10, 5, 0, 430, 428, 1, 0, 0, 0, 430, 431, 1, 0, 0, 0, 431, 57, 1, 0, 0, 0, 432, 437, 3, 72, 36, 0, 433, 434, 5, 43, 0, 0, 434, 436, 3, 72, 36, 0, 435, 433, 1, 0, 0, 0, 436, 439, 1, 0, 0, 0, 437, 435, 1, 0, 0, 0, 437, 438, 1, 0, 0, 0, 438, 59, 1, 0, 0, 0, 439, 437, 1, 0, 0, 0, 440, 445, 3, 66, 33, 0, 441, 442, 5, 43, 0, 0, 442, 444, 3, 66, 33, 0, 443, 441, 1, 0, 0, 0, 444, 447, 1, 0, 0, 0, 445, 443, 1, 0, 0, 0, 445, 446, 1, 0, 0, 0, 446, 61, 1, 0, 0, 0, 447, 445, 1, 0, 0, 0, 448, 453, 3, 60, 30, 0, 449, 450, 5, 41, 0, 0, 450, 452, 3, 60, 30, 0, 451, 449, 1, 0, 0, 0, 452, 455, 1, 0, 0, 0, 453, 451, 1, 0, 0, 0, 453, 454, 1, 0, 0, 0, 454, 63, 1, 0, 0, 0, 455, 453, 1, 0, 0, 0, 456, 457, 7, 3, 0, 0, 457, 65, 1, 0, 0, 0, 458, 461, 5, 89, 0, 0, 459, 461, 3, 70, 35, 0, 460, 458, 1, 0, 0, 0, 460, 459, 1, 0, 0, 0, 461, 67, 1, 0, 0, 0, 462, 505, 5, 51, 0, 0, 463, 464, 3, 104, 52, 0, 464, 465, 5, 76, 0, 0, 465, 505, 1, 0, 0, 0, 466, 505, 3, 102, 51, 0, 467, 505, 3, 104, 52, 0, 468, 505, 3, 98, 49, 0, 469, 505, 3, 70, 35, 0, 470, 505, 3, 106, 53, 0, 471, 472, 5, 72, 0, 0, 472, 477, 3, 100, 50, 0, 473, 474, 5, 41, 0, 0, 474, 476, 3, 100, 50, 0, 475, 473, 1, 0, 0, 0, 476, 479, 1, 0, 0, 0, 477, 475, 1, 0, 0, 0, 477, 478, 1, 0, 0, 0, 478, 480, 1, 0, 0, 0, 479, 477, 1, 0, 0, 0, 480, 481, 5, 73, 0, 0, 481, 505, 1, 0, 0, 0, 482, 483, 5, 72, 0, 0, 483, 488, 3, 98, 49, 0, 484, 485, 5, 41, 0, 0, 485, 487, 3, 98, 49, 0, 486, 484, 1, 0, 0, 0, 487, 490, 1, 0, 0, 0, 488, 486, 1, 0, 0, 0, 488, 489, 1, 0, 0, 0, 489, 491, 1, 0, 0, 0, 490, 488, 1, 0, 0, 0, 491, 492, 5, 73, 0, 0, 492, 505, 1, 0, 0, 0, 493, 494, 5, 72, 0, 0, 494, 499, 3, 106, 53, 0, 495, 496, 5, 41, 0, 0, 496, 498, 3, 106, 53, 0, 497, 495, 1, 0, 0, 0, 498, 501, 1, 0, 0, 0, 499, 497, 1, 0, 0, 0, 499, 500, 1, 0, 0, 0, 500, 502, 1, 0, 0, 0, 501, 499, 1, 0, 0, 0, 502, 503, 5, 73, 0, 0, 503, 505, 1, 0, 0, 0, 504, 462, 1, 0, 0, 0, 504, 463, 1, 0, 0, 0, 504, 466, 1, 0, 0, 0, 504, 467, 1, 0, 0, 0, 504, 468, 1, 0, 0, 0, 504, 469, 1, 0, 0, 0, 504, 470, 1, 0, 0, 0, 504, 471, 1, 0, 0, 0, 504, 482, 1, 0, 0, 0, 504, 493, 1, 0, 0, 0, 505, 69, 1, 0, 0, 0, 506, 509, 5, 54, 0, 0, 507, 509, 5, 71, 0, 0, 508, 506, 1, 0, 0, 0, 508, 507, 1, 0, 0, 0, 509, 71, 1, 0, 0, 0, 510, 513, 3, 64, 32, 0, 511, 513, 3, 70, 35, 0, 512, 510, 1, 0, 0, 0, 512, 511, 1, 0, 0, 0, 513, 73, 1, 0, 0, 0, 514, 515, 5, 9, 0, 0, 515, 516, 5, 33, 0, 0, 516, 75, 1, 0, 0, 0, 517, 518, 5, 14, 0, 0, 518, 523, 3, 78, 39, 0, 519, 520, 5, 41, 0, 0, 520, 522, 3, 78, 39, 0, 521, 519, 1, 0, 0, 0, 522, 525, 1, 0, 0, 0, 523, 521, 1, 0, 0, 0, 523, 524, 1, 0, 0, 0, 524, 77, 1, 0, 0, 0, 525, 523, 1, 0, 0, 0, 526, 528, 3, 10, 5, 0, 527, 529, 7, 4, 0, 0, 528, 527, 1, 0, 0, 0, 528, 529, 1, 0, 0, 0, 529, 532, 1, 0, 0, 0, 530, 531, 5, 52, 0, 0, 531, 533, 7, 5, 0, 0, 532, 530, 1, 0, 0, 0, 532, 533, 1, 0, 0, 0, 533, 79, 1, 0, 0, 0, 534, 535, 5, 8, 0, 0, 535, 536, 3, 62, 31, 0, 536, 81, 1, 0, 0, 0, 537, 538, 5, 2, 0, 0, 538, 539, 3, 62, 31, 0, 539, 83, 1, 0, 0, 0, 540, 541, 5, 11, 0, 0, 541, 546, 3, 86, 43, 0, 542, 543, 5, 41, 0, 0, 543, 545, 3, 86, 43, 0, 544, 542, 1, 0, 0, 0, 545, 548, 1, 0, 0, 0, 546, 544, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 85, 1, 0, 0, 0, 548, 546, 1, 0, 0, 0, 549, 550, 3, 60, 30, 0, 550, 551, 5, 93, 0, 0, 551, 552, 3, 60, 30, 0, 552, 87, 1, 0, 0, 0, 553, 554, 5, 1, 0, 0, 554, 555, 3, 20, 10, 0, 555, 557, 3, 106, 53, 0, 556, 558, 3, 94, 47, 0, 557, 556, 1, 0, 0, 0, 557, 558, 1, 0, 0, 0, 558, 89, 1, 0, 0, 0, 559, 560, 5, 7, 0, 0, 560, 561, 3, 20, 10, 0, 561, 562, 3, 106, 53, 0, 562, 91, 1, 0, 0, 0, 563, 564, 5, 10, 0, 0, 564, 565, 3, 58, 29, 0, 565, 93, 1, 0, 0, 0, 566, 571, 3, 96, 48, 0, 567, 568, 5, 41, 0, 0, 568, 570, 3, 96, 48, 0, 569, 567, 1, 0, 0, 0, 570, 573, 1, 0, 0, 0, 571, 569, 1, 0, 0, 0, 571, 572, 1, 0, 0, 0, 572, 95, 1, 0, 0, 0, 573, 571, 1, 0, 0, 0, 574, 575, 3, 64, 32, 0, 575, 576, 5, 38, 0, 0, 576, 577, 3, 68, 34, 0, 577, 97, 1, 0, 0, 0, 578, 579, 7, 6, 0, 0, 579, 99, 1, 0, 0, 0, 580, 583, 3, 102, 51, 0, 581, 583, 3, 104, 52, 0, 582, 580, 1, 0, 0, 0, 582, 581, 1, 0, 0, 0, 583, 101, 1, 0, 0, 0, 584, 586, 7, 0, 0, 0, 585, 584, 1, 0, 0, 0, 585, 586, 1, 0, 0, 0, 586, 587, 1, 0, 0, 0, 587, 588, 5, 34, 0, 0, 588, 103, 1, 0, 0, 0, 589, 591, 7, 0, 0, 0, 590, 589, 1, 0, 0, 0, 590, 591, 1, 0, 0, 0, 591, 592, 1, 0, 0, 0, 592, 593, 5, 33, 0, 0, 593, 105, 1, 0, 0, 0, 594, 595, 5, 32, 0, 0, 595, 107, 1, 0, 0, 0, 596, 597, 7, 7, 0, 0, 597, 109, 1, 0, 0, 0, 598, 599, 5, 5, 0, 0, 599, 600, 3, 112, 56, 0, 600, 111, 1, 0, 0, 0, 601, 602, 5, 72, 0, 0, 602, 603, 3, 2, 1, 0, 603, 604, 5, 73, 0, 0, 604, 113, 1, 0, 0, 0, 605, 606, 5, 13, 0, 0, 606, 607, 5, 109, 0, 0, 607, 115, 1, 0, 0, 0, 608, 609, 5, 3, 0, 0, 609, 612, 5, 99, 0, 0, 610, 611, 5, 97, 0, 0, 611, 613, 3, 60, 30, 0, 612, 610, 1, 0, 0, 0, 612, 613, 1, 0, 0, 0, 613, 623, 1, 0, 0, 0, 614, 615, 5, 98, 0, 0, 615, 620, 3, 118, 59, 0, 616, 617, 5, 41, 0, 0, 617, 619, 3, 118, 59, 0, 618, 616, 1, 0, 0, 0, 619, 622, 1, 0, 0, 0, 620, 618, 1, 0, 0, 0, 620, 621, 1, 0, 0, 0, 621, 624, 1, 0, 0, 0, 622, 620, 1, 0, 0, 0, 623, 614, 1, 0, 0, 0, 623, 624, 1, 0, 0, 0, 624, 117, 1, 0, 0, 0, 625, 626, 3, 60, 30, 0, 626, 627, 5, 38, 0, 0, 627, 629, 1, 0, 0, 0, 628, 625, 1, 0, 0, 0, 628, 629, 1, 0, 0, 0, 629, 630, 1, 0, 0, 0, 630, 631, 3, 60, 30, 0, 631, 119, 1, 0, 0, 0, 632, 633, 5, 21, 0, 0, 633, 634, 3, 40, 20, 0, 634, 635, 5, 97, 0, 0, 635, 636, 3, 62, 31, 0, 636, 121, 1, 0, 0, 0, 637, 638, 5, 19, 0, 0, 638, 641, 3, 54, 27, 0, 639, 640, 5, 35, 0, 0, 640, 642, 3, 34, 17, 0, 641, 639, 1, 0, 0, 0, 641, 642, 1, 0, 0, 0, 642, 123, 1, 0, 0, 0, 643, 644, 7, 8, 0, 0, 644, 645, 5, 123, 0, 0, 645, 646, 3, 126, 63, 0, 646, 647, 3, 128, 64, 0, 647, 125, 1, 0, 0, 0, 648, 649, 3, 40, 20, 0, 649, 127, 1, 0, 0, 0, 650, 651, 5, 97, 0, 0, 651, 656, 3, 130, 65, 0, 652, 653, 5, 41, 0, 0, 653, 655, 3, 130, 65, 0, 654, 652, 1, 0, 0, 0, 655, 658, 1, 0, 0, 0, 656, 654, 1, 0, 0, 0, 656, 657, 1, 0, 0, 0, 657, 129, 1, 0, 0, 0, 658, 656, 1, 0, 0, 0, 659, 660, 3, 16, 8, 0, 660, 131, 1, 0, 0, 0, 661, 662, 5, 18, 0, 0, 662, 665, 3, 58, 29, 0, 663, 664, 5, 97, 0, 0, 664, 666, 3, 58, 29, 0, 665, 663, 1, 0, 0, 0, 665, 666, 1, 0, 0, 0, 666, 672, 1, 0, 0, 0, 667, 668, 5, 93, 0, 0, 668, 669, 3, 58, 29, 0, 669, 670, 5, 41, 0, 0, 670, 671, 3, 58, 29, 0, 671, 673, 1, 0, 0, 0, 672, 667, 1, 0, 0, 0, 672, 673, 1, 0, 0, 0, 673, 133, 1, 0, 0, 0, 674, 675, 5, 20, 0, 0, 675, 676, 3, 62, 31, 0, 676, 135, 1, 0, 0, 0, 677, 678, 5, 26, 0, 0, 678, 679, 3, 138, 69, 0, 679, 137, 1, 0, 0, 0, 680, 682, 3, 140, 70, 0, 681, 680, 1, 0, 0, 0, 682, 683, 1, 0, 0, 0, 683, 681, 1, 0, 0, 0, 683, 684, 1, 0, 0, 0, 684, 139, 1, 0, 0, 0, 685, 686, 5, 74, 0, 0, 686, 687, 3, 142, 71, 0, 687, 688, 5, 75, 0, 0, 688, 141, 1, 0, 0, 0, 689, 690, 6, 71, -1, 0, 690, 691, 3, 144, 72, 0, 691, 697, 1, 0, 0, 0, 692, 693, 10, 1, 0, 0, 693, 694, 5, 31, 0, 0, 694, 696, 3, 144, 72, 0, 695, 692, 1, 0, 0, 0, 696, 699, 1, 0, 0, 0, 697, 695, 1, 0, 0, 0, 697, 698, 1, 0, 0, 0, 698, 143, 1, 0, 0, 0, 699, 697, 1, 0, 0, 0, 700, 704, 3, 8, 4, 0, 701, 704, 3, 76, 38, 0, 702, 704, 3, 74, 37, 0, 703, 700, 1, 0, 0, 0, 703, 701, 1, 0, 0, 0, 703, 702, 1, 0, 0, 0, 704, 145, 1, 0, 0, 0, 66, 157, 166, 191, 203, 212, 220, 225, 233, 235, 240, 247, 252, 257, 267, 273, 281, 283, 294, 301, 312, 317, 319, 331, 350, 356, 366, 370, 375, 389, 398, 402, 406, 413, 417, 424, 430, 437, 445, 453, 460, 477, 488, 499, 504, 508, 512, 523, 528, 532, 546, 557, 571, 582, 585, 590, 612, 620, 623, 628, 641, 656, 665, 672, 683, 697, 703] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index 533b2793bb159..2c03f767c58ad 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -32,21 +32,21 @@ public class EsqlBaseParser extends ParserConfig { DEV_FORK=26, UNKNOWN_CMD=27, LINE_COMMENT=28, MULTILINE_COMMENT=29, WS=30, PIPE=31, QUOTED_STRING=32, INTEGER_LITERAL=33, DECIMAL_LITERAL=34, BY=35, AND=36, ASC=37, ASSIGN=38, CAST_OP=39, COLON=40, COMMA=41, DESC=42, DOT=43, - FALSE=44, FIRST=45, IN=46, IS=47, LAST=48, LIKE=49, LP=50, NOT=51, NULL=52, - NULLS=53, OR=54, PARAM=55, RLIKE=56, RP=57, TRUE=58, EQ=59, CIEQ=60, NEQ=61, - LT=62, LTE=63, GT=64, GTE=65, PLUS=66, MINUS=67, ASTERISK=68, SLASH=69, - PERCENT=70, LEFT_BRACES=71, RIGHT_BRACES=72, NAMED_OR_POSITIONAL_PARAM=73, - OPENING_BRACKET=74, CLOSING_BRACKET=75, UNQUOTED_IDENTIFIER=76, QUOTED_IDENTIFIER=77, - EXPR_LINE_COMMENT=78, EXPR_MULTILINE_COMMENT=79, EXPR_WS=80, EXPLAIN_WS=81, - EXPLAIN_LINE_COMMENT=82, EXPLAIN_MULTILINE_COMMENT=83, METADATA=84, UNQUOTED_SOURCE=85, - FROM_LINE_COMMENT=86, FROM_MULTILINE_COMMENT=87, FROM_WS=88, ID_PATTERN=89, - PROJECT_LINE_COMMENT=90, PROJECT_MULTILINE_COMMENT=91, PROJECT_WS=92, - AS=93, RENAME_LINE_COMMENT=94, RENAME_MULTILINE_COMMENT=95, RENAME_WS=96, - ON=97, WITH=98, ENRICH_POLICY_NAME=99, ENRICH_LINE_COMMENT=100, ENRICH_MULTILINE_COMMENT=101, - ENRICH_WS=102, ENRICH_FIELD_LINE_COMMENT=103, ENRICH_FIELD_MULTILINE_COMMENT=104, - ENRICH_FIELD_WS=105, MVEXPAND_LINE_COMMENT=106, MVEXPAND_MULTILINE_COMMENT=107, - MVEXPAND_WS=108, INFO=109, SHOW_LINE_COMMENT=110, SHOW_MULTILINE_COMMENT=111, - SHOW_WS=112, SETTING=113, SETTING_LINE_COMMENT=114, SETTTING_MULTILINE_COMMENT=115, + FALSE=44, FIRST=45, IN=46, IS=47, LAST=48, LIKE=49, NOT=50, NULL=51, NULLS=52, + OR=53, PARAM=54, RLIKE=55, TRUE=56, EQ=57, CIEQ=58, NEQ=59, LT=60, LTE=61, + GT=62, GTE=63, PLUS=64, MINUS=65, ASTERISK=66, SLASH=67, PERCENT=68, LEFT_BRACES=69, + RIGHT_BRACES=70, NAMED_OR_POSITIONAL_PARAM=71, OPENING_BRACKET=72, CLOSING_BRACKET=73, + LP=74, RP=75, UNQUOTED_IDENTIFIER=76, QUOTED_IDENTIFIER=77, EXPR_LINE_COMMENT=78, + EXPR_MULTILINE_COMMENT=79, EXPR_WS=80, EXPLAIN_WS=81, EXPLAIN_LINE_COMMENT=82, + EXPLAIN_MULTILINE_COMMENT=83, METADATA=84, UNQUOTED_SOURCE=85, FROM_LINE_COMMENT=86, + FROM_MULTILINE_COMMENT=87, FROM_WS=88, ID_PATTERN=89, PROJECT_LINE_COMMENT=90, + PROJECT_MULTILINE_COMMENT=91, PROJECT_WS=92, AS=93, RENAME_LINE_COMMENT=94, + RENAME_MULTILINE_COMMENT=95, RENAME_WS=96, ON=97, WITH=98, ENRICH_POLICY_NAME=99, + ENRICH_LINE_COMMENT=100, ENRICH_MULTILINE_COMMENT=101, ENRICH_WS=102, + ENRICH_FIELD_LINE_COMMENT=103, ENRICH_FIELD_MULTILINE_COMMENT=104, ENRICH_FIELD_WS=105, + MVEXPAND_LINE_COMMENT=106, MVEXPAND_MULTILINE_COMMENT=107, MVEXPAND_WS=108, + INFO=109, SHOW_LINE_COMMENT=110, SHOW_MULTILINE_COMMENT=111, SHOW_WS=112, + SETTING=113, SETTING_LINE_COMMENT=114, SETTTING_MULTILINE_COMMENT=115, SETTING_WS=116, LOOKUP_LINE_COMMENT=117, LOOKUP_MULTILINE_COMMENT=118, LOOKUP_WS=119, LOOKUP_FIELD_LINE_COMMENT=120, LOOKUP_FIELD_MULTILINE_COMMENT=121, LOOKUP_FIELD_WS=122, JOIN=123, USING=124, JOIN_LINE_COMMENT=125, JOIN_MULTILINE_COMMENT=126, @@ -108,10 +108,10 @@ private static String[] makeLiteralNames() { "'sort'", "'stats'", "'where'", "'lookup'", null, null, null, null, null, null, null, null, null, null, null, null, null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", "':'", "','", "'desc'", "'.'", - "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", "'('", "'not'", - "'null'", "'nulls'", "'or'", "'?'", "'rlike'", "')'", "'true'", "'=='", - "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", - "'%'", "'{'", "'}'", null, null, "']'", null, null, null, null, null, + "'false'", "'first'", "'in'", "'is'", "'last'", "'like'", "'not'", "'null'", + "'nulls'", "'or'", "'?'", "'rlike'", "'true'", "'=='", "'=~'", "'!='", + "'<'", "'<='", "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", "'{'", + "'}'", null, null, "']'", null, "')'", null, null, null, null, null, null, null, null, "'metadata'", null, null, null, null, null, null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, null, null, null, null, null, null, null, null, "'info'", null, null, null, null, @@ -128,17 +128,17 @@ private static String[] makeSymbolicNames() { "DEV_FORK", "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COLON", "COMMA", "DESC", "DOT", "FALSE", - "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", "NULL", "NULLS", "OR", - "PARAM", "RLIKE", "RP", "TRUE", "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", - "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", "LEFT_BRACES", - "RIGHT_BRACES", "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", - "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", "EXPR_MULTILINE_COMMENT", - "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", "EXPLAIN_MULTILINE_COMMENT", - "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", "FROM_MULTILINE_COMMENT", - "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", "PROJECT_MULTILINE_COMMENT", - "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", "RENAME_MULTILINE_COMMENT", - "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", "ENRICH_LINE_COMMENT", - "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", + "FIRST", "IN", "IS", "LAST", "LIKE", "NOT", "NULL", "NULLS", "OR", "PARAM", + "RLIKE", "TRUE", "EQ", "CIEQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", + "MINUS", "ASTERISK", "SLASH", "PERCENT", "LEFT_BRACES", "RIGHT_BRACES", + "NAMED_OR_POSITIONAL_PARAM", "OPENING_BRACKET", "CLOSING_BRACKET", "LP", + "RP", "UNQUOTED_IDENTIFIER", "QUOTED_IDENTIFIER", "EXPR_LINE_COMMENT", + "EXPR_MULTILINE_COMMENT", "EXPR_WS", "EXPLAIN_WS", "EXPLAIN_LINE_COMMENT", + "EXPLAIN_MULTILINE_COMMENT", "METADATA", "UNQUOTED_SOURCE", "FROM_LINE_COMMENT", + "FROM_MULTILINE_COMMENT", "FROM_WS", "ID_PATTERN", "PROJECT_LINE_COMMENT", + "PROJECT_MULTILINE_COMMENT", "PROJECT_WS", "AS", "RENAME_LINE_COMMENT", + "RENAME_MULTILINE_COMMENT", "RENAME_WS", "ON", "WITH", "ENRICH_POLICY_NAME", + "ENRICH_LINE_COMMENT", "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", "SHOW_WS", "SETTING", "SETTING_LINE_COMMENT", @@ -1557,7 +1557,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE setState(276); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); - if ( !(((((_la - 68)) & ~0x3f) == 0 && ((1L << (_la - 68)) & 7L) != 0)) ) { + if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & 7L) != 0)) ) { ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this); } else { @@ -1901,7 +1901,6 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx case INTEGER_LITERAL: case DECIMAL_LITERAL: case FALSE: - case LP: case NOT: case NULL: case PARAM: @@ -1910,6 +1909,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx case MINUS: case NAMED_OR_POSITIONAL_PARAM: case OPENING_BRACKET: + case LP: case UNQUOTED_IDENTIFIER: case QUOTED_IDENTIFIER: { @@ -5055,7 +5055,7 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx { setState(596); _la = _input.LA(1); - if ( !(((((_la - 59)) & ~0x3f) == 0 && ((1L << (_la - 59)) & 125L) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -432345564227567616L) != 0)) ) { _errHandler.recoverInline(this); } else { @@ -6449,8 +6449,8 @@ private boolean forkSubQueryCommand_sempred(ForkSubQueryCommandContext _localctx "\u0002\n\u0012\u0014\u008eI\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ "TVXZ\\^`bdfhjlnprtvxz|~\u0080\u0082\u0084\u0086\u0088\u008a\u008c\u008e"+ - "\u0090\u0000\t\u0001\u0000BC\u0001\u0000DF\u0002\u0000 UU\u0001\u0000"+ - "LM\u0002\u0000%%**\u0002\u0000--00\u0002\u0000,,::\u0002\u0000;;=A\u0002"+ + "\u0090\u0000\t\u0001\u0000@A\u0001\u0000BD\u0002\u0000 UU\u0001\u0000"+ + "LM\u0002\u0000%%**\u0002\u0000--00\u0002\u0000,,88\u0002\u000099;?\u0002"+ "\u0000\u0011\u0011\u0018\u0019\u02dd\u0000\u0092\u0001\u0000\u0000\u0000"+ "\u0002\u0095\u0001\u0000\u0000\u0000\u0004\u00a6\u0001\u0000\u0000\u0000"+ "\u0006\u00bf\u0001\u0000\u0000\u0000\b\u00c1\u0001\u0000\u0000\u0000\n"+ @@ -6518,38 +6518,38 @@ private boolean forkSubQueryCommand_sempred(ForkSubQueryCommandContext _localctx "\u00bf\u00bb\u0001\u0000\u0000\u0000\u00bf\u00bd\u0001\u0000\u0000\u0000"+ "\u00c0\u0007\u0001\u0000\u0000\u0000\u00c1\u00c2\u0005\u0010\u0000\u0000"+ "\u00c2\u00c3\u0003\n\u0005\u0000\u00c3\t\u0001\u0000\u0000\u0000\u00c4"+ - "\u00c5\u0006\u0005\uffff\uffff\u0000\u00c5\u00c6\u00053\u0000\u0000\u00c6"+ + "\u00c5\u0006\u0005\uffff\uffff\u0000\u00c5\u00c6\u00052\u0000\u0000\u00c6"+ "\u00e2\u0003\n\u0005\b\u00c7\u00e2\u0003\u0010\b\u0000\u00c8\u00e2\u0003"+ - "\f\u0006\u0000\u00c9\u00cb\u0003\u0010\b\u0000\u00ca\u00cc\u00053\u0000"+ + "\f\u0006\u0000\u00c9\u00cb\u0003\u0010\b\u0000\u00ca\u00cc\u00052\u0000"+ "\u0000\u00cb\u00ca\u0001\u0000\u0000\u0000\u00cb\u00cc\u0001\u0000\u0000"+ "\u0000\u00cc\u00cd\u0001\u0000\u0000\u0000\u00cd\u00ce\u0005.\u0000\u0000"+ - "\u00ce\u00cf\u00052\u0000\u0000\u00cf\u00d4\u0003\u0010\b\u0000\u00d0"+ + "\u00ce\u00cf\u0005J\u0000\u0000\u00cf\u00d4\u0003\u0010\b\u0000\u00d0"+ "\u00d1\u0005)\u0000\u0000\u00d1\u00d3\u0003\u0010\b\u0000\u00d2\u00d0"+ "\u0001\u0000\u0000\u0000\u00d3\u00d6\u0001\u0000\u0000\u0000\u00d4\u00d2"+ "\u0001\u0000\u0000\u0000\u00d4\u00d5\u0001\u0000\u0000\u0000\u00d5\u00d7"+ "\u0001\u0000\u0000\u0000\u00d6\u00d4\u0001\u0000\u0000\u0000\u00d7\u00d8"+ - "\u00059\u0000\u0000\u00d8\u00e2\u0001\u0000\u0000\u0000\u00d9\u00da\u0003"+ - "\u0010\b\u0000\u00da\u00dc\u0005/\u0000\u0000\u00db\u00dd\u00053\u0000"+ + "\u0005K\u0000\u0000\u00d8\u00e2\u0001\u0000\u0000\u0000\u00d9\u00da\u0003"+ + "\u0010\b\u0000\u00da\u00dc\u0005/\u0000\u0000\u00db\u00dd\u00052\u0000"+ "\u0000\u00dc\u00db\u0001\u0000\u0000\u0000\u00dc\u00dd\u0001\u0000\u0000"+ - "\u0000\u00dd\u00de\u0001\u0000\u0000\u0000\u00de\u00df\u00054\u0000\u0000"+ + "\u0000\u00dd\u00de\u0001\u0000\u0000\u0000\u00de\u00df\u00053\u0000\u0000"+ "\u00df\u00e2\u0001\u0000\u0000\u0000\u00e0\u00e2\u0003\u000e\u0007\u0000"+ "\u00e1\u00c4\u0001\u0000\u0000\u0000\u00e1\u00c7\u0001\u0000\u0000\u0000"+ "\u00e1\u00c8\u0001\u0000\u0000\u0000\u00e1\u00c9\u0001\u0000\u0000\u0000"+ "\u00e1\u00d9\u0001\u0000\u0000\u0000\u00e1\u00e0\u0001\u0000\u0000\u0000"+ "\u00e2\u00eb\u0001\u0000\u0000\u0000\u00e3\u00e4\n\u0005\u0000\u0000\u00e4"+ "\u00e5\u0005$\u0000\u0000\u00e5\u00ea\u0003\n\u0005\u0006\u00e6\u00e7"+ - "\n\u0004\u0000\u0000\u00e7\u00e8\u00056\u0000\u0000\u00e8\u00ea\u0003"+ + "\n\u0004\u0000\u0000\u00e7\u00e8\u00055\u0000\u0000\u00e8\u00ea\u0003"+ "\n\u0005\u0005\u00e9\u00e3\u0001\u0000\u0000\u0000\u00e9\u00e6\u0001\u0000"+ "\u0000\u0000\u00ea\u00ed\u0001\u0000\u0000\u0000\u00eb\u00e9\u0001\u0000"+ "\u0000\u0000\u00eb\u00ec\u0001\u0000\u0000\u0000\u00ec\u000b\u0001\u0000"+ "\u0000\u0000\u00ed\u00eb\u0001\u0000\u0000\u0000\u00ee\u00f0\u0003\u0010"+ - "\b\u0000\u00ef\u00f1\u00053\u0000\u0000\u00f0\u00ef\u0001\u0000\u0000"+ + "\b\u0000\u00ef\u00f1\u00052\u0000\u0000\u00f0\u00ef\u0001\u0000\u0000"+ "\u0000\u00f0\u00f1\u0001\u0000\u0000\u0000\u00f1\u00f2\u0001\u0000\u0000"+ "\u0000\u00f2\u00f3\u00051\u0000\u0000\u00f3\u00f4\u0003j5\u0000\u00f4"+ "\u00fd\u0001\u0000\u0000\u0000\u00f5\u00f7\u0003\u0010\b\u0000\u00f6\u00f8"+ - "\u00053\u0000\u0000\u00f7\u00f6\u0001\u0000\u0000\u0000\u00f7\u00f8\u0001"+ + "\u00052\u0000\u0000\u00f7\u00f6\u0001\u0000\u0000\u0000\u00f7\u00f8\u0001"+ "\u0000\u0000\u0000\u00f8\u00f9\u0001\u0000\u0000\u0000\u00f9\u00fa\u0005"+ - "8\u0000\u0000\u00fa\u00fb\u0003j5\u0000\u00fb\u00fd\u0001\u0000\u0000"+ + "7\u0000\u0000\u00fa\u00fb\u0003j5\u0000\u00fb\u00fd\u0001\u0000\u0000"+ "\u0000\u00fc\u00ee\u0001\u0000\u0000\u0000\u00fc\u00f5\u0001\u0000\u0000"+ "\u0000\u00fd\r\u0001\u0000\u0000\u0000\u00fe\u0101\u0003:\u001d\u0000"+ "\u00ff\u0100\u0005\'\u0000\u0000\u0100\u0102\u0003\u001e\u000f\u0000\u0101"+ @@ -6571,8 +6571,8 @@ private boolean forkSubQueryCommand_sempred(ForkSubQueryCommandContext _localctx "\u011c\u0013\u0001\u0000\u0000\u0000\u011d\u011b\u0001\u0000\u0000\u0000"+ "\u011e\u011f\u0006\n\uffff\uffff\u0000\u011f\u0127\u0003D\"\u0000\u0120"+ "\u0127\u0003:\u001d\u0000\u0121\u0127\u0003\u0016\u000b\u0000\u0122\u0123"+ - "\u00052\u0000\u0000\u0123\u0124\u0003\n\u0005\u0000\u0124\u0125\u0005"+ - "9\u0000\u0000\u0125\u0127\u0001\u0000\u0000\u0000\u0126\u011e\u0001\u0000"+ + "\u0005J\u0000\u0000\u0123\u0124\u0003\n\u0005\u0000\u0124\u0125\u0005"+ + "K\u0000\u0000\u0125\u0127\u0001\u0000\u0000\u0000\u0126\u011e\u0001\u0000"+ "\u0000\u0000\u0126\u0120\u0001\u0000\u0000\u0000\u0126\u0121\u0001\u0000"+ "\u0000\u0000\u0126\u0122\u0001\u0000\u0000\u0000\u0127\u012d\u0001\u0000"+ "\u0000\u0000\u0128\u0129\n\u0001\u0000\u0000\u0129\u012a\u0005\'\u0000"+ @@ -6580,7 +6580,7 @@ private boolean forkSubQueryCommand_sempred(ForkSubQueryCommandContext _localctx "\u0000\u012c\u012f\u0001\u0000\u0000\u0000\u012d\u012b\u0001\u0000\u0000"+ "\u0000\u012d\u012e\u0001\u0000\u0000\u0000\u012e\u0015\u0001\u0000\u0000"+ "\u0000\u012f\u012d\u0001\u0000\u0000\u0000\u0130\u0131\u0003\u0018\f\u0000"+ - "\u0131\u013f\u00052\u0000\u0000\u0132\u0140\u0005D\u0000\u0000\u0133\u0138"+ + "\u0131\u013f\u0005J\u0000\u0000\u0132\u0140\u0005B\u0000\u0000\u0133\u0138"+ "\u0003\n\u0005\u0000\u0134\u0135\u0005)\u0000\u0000\u0135\u0137\u0003"+ "\n\u0005\u0000\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u013a\u0001\u0000"+ "\u0000\u0000\u0138\u0136\u0001\u0000\u0000\u0000\u0138\u0139\u0001\u0000"+ @@ -6589,14 +6589,14 @@ private boolean forkSubQueryCommand_sempred(ForkSubQueryCommandContext _localctx "\u0000\u013d\u013b\u0001\u0000\u0000\u0000\u013d\u013e\u0001\u0000\u0000"+ "\u0000\u013e\u0140\u0001\u0000\u0000\u0000\u013f\u0132\u0001\u0000\u0000"+ "\u0000\u013f\u0133\u0001\u0000\u0000\u0000\u013f\u0140\u0001\u0000\u0000"+ - "\u0000\u0140\u0141\u0001\u0000\u0000\u0000\u0141\u0142\u00059\u0000\u0000"+ + "\u0000\u0140\u0141\u0001\u0000\u0000\u0000\u0141\u0142\u0005K\u0000\u0000"+ "\u0142\u0017\u0001\u0000\u0000\u0000\u0143\u0144\u0003H$\u0000\u0144\u0019"+ - "\u0001\u0000\u0000\u0000\u0145\u0146\u0005G\u0000\u0000\u0146\u014b\u0003"+ + "\u0001\u0000\u0000\u0000\u0145\u0146\u0005E\u0000\u0000\u0146\u014b\u0003"+ "\u001c\u000e\u0000\u0147\u0148\u0005)\u0000\u0000\u0148\u014a\u0003\u001c"+ "\u000e\u0000\u0149\u0147\u0001\u0000\u0000\u0000\u014a\u014d\u0001\u0000"+ "\u0000\u0000\u014b\u0149\u0001\u0000\u0000\u0000\u014b\u014c\u0001\u0000"+ "\u0000\u0000\u014c\u014e\u0001\u0000\u0000\u0000\u014d\u014b\u0001\u0000"+ - "\u0000\u0000\u014e\u014f\u0005H\u0000\u0000\u014f\u001b\u0001\u0000\u0000"+ + "\u0000\u0000\u014e\u014f\u0005F\u0000\u0000\u014f\u001b\u0001\u0000\u0000"+ "\u0000\u0150\u0151\u0003j5\u0000\u0151\u0152\u0005(\u0000\u0000\u0152"+ "\u0153\u0003D\"\u0000\u0153\u001d\u0001\u0000\u0000\u0000\u0154\u0155"+ "\u0003@ \u0000\u0155\u001f\u0001\u0000\u0000\u0000\u0156\u0157\u0005\f"+ @@ -6663,33 +6663,33 @@ private boolean forkSubQueryCommand_sempred(ForkSubQueryCommandContext _localctx "\u0000\u01c8\u01c9\u0007\u0003\u0000\u0000\u01c9A\u0001\u0000\u0000\u0000"+ "\u01ca\u01cd\u0005Y\u0000\u0000\u01cb\u01cd\u0003F#\u0000\u01cc\u01ca"+ "\u0001\u0000\u0000\u0000\u01cc\u01cb\u0001\u0000\u0000\u0000\u01cdC\u0001"+ - "\u0000\u0000\u0000\u01ce\u01f9\u00054\u0000\u0000\u01cf\u01d0\u0003h4"+ + "\u0000\u0000\u0000\u01ce\u01f9\u00053\u0000\u0000\u01cf\u01d0\u0003h4"+ "\u0000\u01d0\u01d1\u0005L\u0000\u0000\u01d1\u01f9\u0001\u0000\u0000\u0000"+ "\u01d2\u01f9\u0003f3\u0000\u01d3\u01f9\u0003h4\u0000\u01d4\u01f9\u0003"+ "b1\u0000\u01d5\u01f9\u0003F#\u0000\u01d6\u01f9\u0003j5\u0000\u01d7\u01d8"+ - "\u0005J\u0000\u0000\u01d8\u01dd\u0003d2\u0000\u01d9\u01da\u0005)\u0000"+ + "\u0005H\u0000\u0000\u01d8\u01dd\u0003d2\u0000\u01d9\u01da\u0005)\u0000"+ "\u0000\u01da\u01dc\u0003d2\u0000\u01db\u01d9\u0001\u0000\u0000\u0000\u01dc"+ "\u01df\u0001\u0000\u0000\u0000\u01dd\u01db\u0001\u0000\u0000\u0000\u01dd"+ "\u01de\u0001\u0000\u0000\u0000\u01de\u01e0\u0001\u0000\u0000\u0000\u01df"+ - "\u01dd\u0001\u0000\u0000\u0000\u01e0\u01e1\u0005K\u0000\u0000\u01e1\u01f9"+ - "\u0001\u0000\u0000\u0000\u01e2\u01e3\u0005J\u0000\u0000\u01e3\u01e8\u0003"+ + "\u01dd\u0001\u0000\u0000\u0000\u01e0\u01e1\u0005I\u0000\u0000\u01e1\u01f9"+ + "\u0001\u0000\u0000\u0000\u01e2\u01e3\u0005H\u0000\u0000\u01e3\u01e8\u0003"+ "b1\u0000\u01e4\u01e5\u0005)\u0000\u0000\u01e5\u01e7\u0003b1\u0000\u01e6"+ "\u01e4\u0001\u0000\u0000\u0000\u01e7\u01ea\u0001\u0000\u0000\u0000\u01e8"+ "\u01e6\u0001\u0000\u0000\u0000\u01e8\u01e9\u0001\u0000\u0000\u0000\u01e9"+ "\u01eb\u0001\u0000\u0000\u0000\u01ea\u01e8\u0001\u0000\u0000\u0000\u01eb"+ - "\u01ec\u0005K\u0000\u0000\u01ec\u01f9\u0001\u0000\u0000\u0000\u01ed\u01ee"+ - "\u0005J\u0000\u0000\u01ee\u01f3\u0003j5\u0000\u01ef\u01f0\u0005)\u0000"+ + "\u01ec\u0005I\u0000\u0000\u01ec\u01f9\u0001\u0000\u0000\u0000\u01ed\u01ee"+ + "\u0005H\u0000\u0000\u01ee\u01f3\u0003j5\u0000\u01ef\u01f0\u0005)\u0000"+ "\u0000\u01f0\u01f2\u0003j5\u0000\u01f1\u01ef\u0001\u0000\u0000\u0000\u01f2"+ "\u01f5\u0001\u0000\u0000\u0000\u01f3\u01f1\u0001\u0000\u0000\u0000\u01f3"+ "\u01f4\u0001\u0000\u0000\u0000\u01f4\u01f6\u0001\u0000\u0000\u0000\u01f5"+ - "\u01f3\u0001\u0000\u0000\u0000\u01f6\u01f7\u0005K\u0000\u0000\u01f7\u01f9"+ + "\u01f3\u0001\u0000\u0000\u0000\u01f6\u01f7\u0005I\u0000\u0000\u01f7\u01f9"+ "\u0001\u0000\u0000\u0000\u01f8\u01ce\u0001\u0000\u0000\u0000\u01f8\u01cf"+ "\u0001\u0000\u0000\u0000\u01f8\u01d2\u0001\u0000\u0000\u0000\u01f8\u01d3"+ "\u0001\u0000\u0000\u0000\u01f8\u01d4\u0001\u0000\u0000\u0000\u01f8\u01d5"+ "\u0001\u0000\u0000\u0000\u01f8\u01d6\u0001\u0000\u0000\u0000\u01f8\u01d7"+ "\u0001\u0000\u0000\u0000\u01f8\u01e2\u0001\u0000\u0000\u0000\u01f8\u01ed"+ "\u0001\u0000\u0000\u0000\u01f9E\u0001\u0000\u0000\u0000\u01fa\u01fd\u0005"+ - "7\u0000\u0000\u01fb\u01fd\u0005I\u0000\u0000\u01fc\u01fa\u0001\u0000\u0000"+ + "6\u0000\u0000\u01fb\u01fd\u0005G\u0000\u0000\u01fc\u01fa\u0001\u0000\u0000"+ "\u0000\u01fc\u01fb\u0001\u0000\u0000\u0000\u01fdG\u0001\u0000\u0000\u0000"+ "\u01fe\u0201\u0003@ \u0000\u01ff\u0201\u0003F#\u0000\u0200\u01fe\u0001"+ "\u0000\u0000\u0000\u0200\u01ff\u0001\u0000\u0000\u0000\u0201I\u0001\u0000"+ @@ -6701,7 +6701,7 @@ private boolean forkSubQueryCommand_sempred(ForkSubQueryCommandContext _localctx "\u0000\u0000\u0000\u020cM\u0001\u0000\u0000\u0000\u020d\u020b\u0001\u0000"+ "\u0000\u0000\u020e\u0210\u0003\n\u0005\u0000\u020f\u0211\u0007\u0004\u0000"+ "\u0000\u0210\u020f\u0001\u0000\u0000\u0000\u0210\u0211\u0001\u0000\u0000"+ - "\u0000\u0211\u0214\u0001\u0000\u0000\u0000\u0212\u0213\u00055\u0000\u0000"+ + "\u0000\u0211\u0214\u0001\u0000\u0000\u0000\u0212\u0213\u00054\u0000\u0000"+ "\u0213\u0215\u0007\u0005\u0000\u0000\u0214\u0212\u0001\u0000\u0000\u0000"+ "\u0214\u0215\u0001\u0000\u0000\u0000\u0215O\u0001\u0000\u0000\u0000\u0216"+ "\u0217\u0005\b\u0000\u0000\u0217\u0218\u0003>\u001f\u0000\u0218Q\u0001"+ @@ -6736,8 +6736,8 @@ private boolean forkSubQueryCommand_sempred(ForkSubQueryCommandContext _localctx "\u0000\u0251i\u0001\u0000\u0000\u0000\u0252\u0253\u0005 \u0000\u0000\u0253"+ "k\u0001\u0000\u0000\u0000\u0254\u0255\u0007\u0007\u0000\u0000\u0255m\u0001"+ "\u0000\u0000\u0000\u0256\u0257\u0005\u0005\u0000\u0000\u0257\u0258\u0003"+ - "p8\u0000\u0258o\u0001\u0000\u0000\u0000\u0259\u025a\u0005J\u0000\u0000"+ - "\u025a\u025b\u0003\u0002\u0001\u0000\u025b\u025c\u0005K\u0000\u0000\u025c"+ + "p8\u0000\u0258o\u0001\u0000\u0000\u0000\u0259\u025a\u0005H\u0000\u0000"+ + "\u025a\u025b\u0003\u0002\u0001\u0000\u025b\u025c\u0005I\u0000\u0000\u025c"+ "q\u0001\u0000\u0000\u0000\u025d\u025e\u0005\r\u0000\u0000\u025e\u025f"+ "\u0005m\u0000\u0000\u025fs\u0001\u0000\u0000\u0000\u0260\u0261\u0005\u0003"+ "\u0000\u0000\u0261\u0264\u0005c\u0000\u0000\u0262\u0263\u0005a\u0000\u0000"+ @@ -6779,8 +6779,8 @@ private boolean forkSubQueryCommand_sempred(ForkSubQueryCommandContext _localctx "\u02a7\u0089\u0001\u0000\u0000\u0000\u02a8\u02aa\u0003\u008cF\u0000\u02a9"+ "\u02a8\u0001\u0000\u0000\u0000\u02aa\u02ab\u0001\u0000\u0000\u0000\u02ab"+ "\u02a9\u0001\u0000\u0000\u0000\u02ab\u02ac\u0001\u0000\u0000\u0000\u02ac"+ - "\u008b\u0001\u0000\u0000\u0000\u02ad\u02ae\u00052\u0000\u0000\u02ae\u02af"+ - "\u0003\u008eG\u0000\u02af\u02b0\u00059\u0000\u0000\u02b0\u008d\u0001\u0000"+ + "\u008b\u0001\u0000\u0000\u0000\u02ad\u02ae\u0005J\u0000\u0000\u02ae\u02af"+ + "\u0003\u008eG\u0000\u02af\u02b0\u0005K\u0000\u0000\u02b0\u008d\u0001\u0000"+ "\u0000\u0000\u02b1\u02b2\u0006G\uffff\uffff\u0000\u02b2\u02b3\u0003\u0090"+ "H\u0000\u02b3\u02b9\u0001\u0000\u0000\u0000\u02b4\u02b5\n\u0001\u0000"+ "\u0000\u02b5\u02b6\u0005\u001f\u0000\u0000\u02b6\u02b8\u0003\u0090H\u0000"+ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 2c76f76a67f77..b63a41faa9006 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2330,13 +2330,37 @@ public void testRateRequiresCounterTypes() { ); } - public void testCoalesceWithMixedNumericTypes() { + public void testConditionalFunctionsWithMixedNumericTypes() { LogicalPlan plan = analyze(""" from test | eval x = coalesce(salary_change, null, 0), y = coalesce(languages, null, 0), z = coalesce(languages.long, null, 0) , w = coalesce(salary_change, null, 0::long) | keep x, y, z, w """, "mapping-default.json"); + validateConditionalFunctions(plan); + + plan = analyze(""" + from test + | eval x = case(languages == 1, salary_change, languages == 2, salary, languages == 3, salary_change.long, 0) + , y = case(languages == 1, salary_change.int, languages == 2, salary, 0) + , z = case(languages == 1, salary_change.long, languages == 2, salary, 0::long) + , w = case(languages == 1, salary_change, languages == 2, salary, languages == 3, salary_change.long, null) + | keep x, y, z, w + """, "mapping-default.json"); + validateConditionalFunctions(plan); + + plan = analyze(""" + from test + | eval x = greatest(salary_change, salary, salary_change.long) + , y = least(salary_change.int, salary) + , z = greatest(salary_change.long, salary, null) + , w = least(null, salary_change, salary_change.long, salary, null) + | keep x, y, z, w + """, "mapping-default.json"); + validateConditionalFunctions(plan); + } + + private void validateConditionalFunctions(LogicalPlan plan) { var limit = as(plan, Limit.class); var esqlProject = as(limit.child(), EsqlProject.class); List projections = esqlProject.projections(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 71d36ce0ffcf8..76e63aa853e2b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1671,108 +1671,162 @@ public void testTermTargetsExistingField() throws Exception { assertEquals("1:38: Unknown column [first_name]", error("from test | keep emp_no | where term(first_name, \"Anna\")")); } - public void testCoalesceWithMixedNumericTypes() { - assertEquals( - "1:22: second argument of [coalesce(languages, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(languages, height)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages.long, height)] must be [long], found value [height] type [double]", - error("from test | eval x = coalesce(languages.long, height)") - ); - assertEquals( - "1:22: second argument of [coalesce(salary, languages.long)] must be [integer], found value [languages.long] type [long]", - error("from test | eval x = coalesce(salary, languages.long)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages.short, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(languages.short, height)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages.byte, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(languages.byte, height)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages, height.float)] must be [integer], found value [height.float] type [double]", - error("from test | eval x = coalesce(languages, height.float)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages, height.scaled_float)] must be [integer], " - + "found value [height.scaled_float] type [double]", - error("from test | eval x = coalesce(languages, height.scaled_float)") - ); - assertEquals( - "1:22: second argument of [coalesce(languages, height.half_float)] must be [integer], " - + "found value [height.half_float] type [double]", - error("from test | eval x = coalesce(languages, height.half_float)") - ); + public void testConditionalFunctionsWithMixedNumericTypes() { + for (String functionName : List.of("coalesce", "greatest", "least")) { + assertEquals( + "1:22: second argument of [" + functionName + "(languages, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(languages, height)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages.long, height)] must be [long], found value [height] type [double]", + error("from test | eval x = " + functionName + "(languages.long, height)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(salary, languages.long)] must be [integer], found value [languages.long] type [long]", + error("from test | eval x = " + functionName + "(salary, languages.long)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages.short, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(languages.short, height)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages.byte, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(languages.byte, height)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages, height.float)] must be [integer], found value [height.float] type [double]", + error("from test | eval x = " + functionName + "(languages, height.float)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages, height.scaled_float)] must be [integer], " + + "found value [height.scaled_float] type [double]", + error("from test | eval x = " + functionName + "(languages, height.scaled_float)") + ); + assertEquals( + "1:22: second argument of [" + + functionName + + "(languages, height.half_float)] must be [integer], " + + "found value [height.half_float] type [double]", + error("from test | eval x = " + functionName + "(languages, height.half_float)") + ); - assertEquals( - "1:22: third argument of [coalesce(null, languages, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(null, languages, height)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages.long, height)] must be [long], found value [height] type [double]", - error("from test | eval x = coalesce(null, languages.long, height)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, salary, languages.long)] must be [integer], " - + "found value [languages.long] type [long]", - error("from test | eval x = coalesce(null, salary, languages.long)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages.short, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(null, languages.short, height)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages.byte, height)] must be [integer], found value [height] type [double]", - error("from test | eval x = coalesce(null, languages.byte, height)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages, height.float)] must be [integer], " - + "found value [height.float] type [double]", - error("from test | eval x = coalesce(null, languages, height.float)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages, height.scaled_float)] must be [integer], " - + "found value [height.scaled_float] type [double]", - error("from test | eval x = coalesce(null, languages, height.scaled_float)") - ); - assertEquals( - "1:22: third argument of [coalesce(null, languages, height.half_float)] must be [integer], " - + "found value [height.half_float] type [double]", - error("from test | eval x = coalesce(null, languages, height.half_float)") - ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(null, languages, height)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages.long, height)] must be [long], found value [height] type [double]", + error("from test | eval x = " + functionName + "(null, languages.long, height)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, salary, languages.long)] must be [integer], " + + "found value [languages.long] type [long]", + error("from test | eval x = " + functionName + "(null, salary, languages.long)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages.short, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(null, languages.short, height)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages.byte, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = " + functionName + "(null, languages.byte, height)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages, height.float)] must be [integer], " + + "found value [height.float] type [double]", + error("from test | eval x = " + functionName + "(null, languages, height.float)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages, height.scaled_float)] must be [integer], " + + "found value [height.scaled_float] type [double]", + error("from test | eval x = " + functionName + "(null, languages, height.scaled_float)") + ); + assertEquals( + "1:22: third argument of [" + + functionName + + "(null, languages, height.half_float)] must be [integer], " + + "found value [height.half_float] type [double]", + error("from test | eval x = " + functionName + "(null, languages, height.half_float)") + ); - // counter - assertEquals( - "1:23: second argument of [coalesce(network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]", - error("FROM tests | eval x = coalesce(network.bytes_in, 0)", tsdb) - ); + // counter + assertEquals( + "1:23: second argument of [" + + functionName + + "(network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]", + error("FROM tests | eval x = " + functionName + "(network.bytes_in, 0)", tsdb) + ); - assertEquals( - "1:23: second argument of [coalesce(network.bytes_in, to_long(0))] must be [counter_long], " - + "found value [to_long(0)] type [long]", - error("FROM tests | eval x = coalesce(network.bytes_in, to_long(0))", tsdb) - ); - assertEquals( - "1:23: second argument of [coalesce(network.bytes_in, 0.0)] must be [counter_long], found value [0.0] type [double]", - error("FROM tests | eval x = coalesce(network.bytes_in, 0.0)", tsdb) - ); + assertEquals( + "1:23: second argument of [" + + functionName + + "(network.bytes_in, to_long(0))] must be [counter_long], " + + "found value [to_long(0)] type [long]", + error("FROM tests | eval x = " + functionName + "(network.bytes_in, to_long(0))", tsdb) + ); + assertEquals( + "1:23: second argument of [" + + functionName + + "(network.bytes_in, 0.0)] must be [counter_long], found value [0.0] type [double]", + error("FROM tests | eval x = " + functionName + "(network.bytes_in, 0.0)", tsdb) + ); - assertEquals( - "1:23: third argument of [coalesce(null, network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]", - error("FROM tests | eval x = coalesce(null, network.bytes_in, 0)", tsdb) - ); + assertEquals( + "1:23: third argument of [" + + functionName + + "(null, network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]", + error("FROM tests | eval x = " + functionName + "(null, network.bytes_in, 0)", tsdb) + ); + + assertEquals( + "1:23: third argument of [" + + functionName + + "(null, network.bytes_in, to_long(0))] must be [counter_long], " + + "found value [to_long(0)] type [long]", + error("FROM tests | eval x = " + functionName + "(null, network.bytes_in, to_long(0))", tsdb) + ); + assertEquals( + "1:23: third argument of [" + + functionName + + "(null, network.bytes_in, 0.0)] must be [counter_long], found value [0.0] type [double]", + error("FROM tests | eval x = " + functionName + "(null, network.bytes_in, 0.0)", tsdb) + ); + } + // case, a subset tests of coalesce/greatest/least assertEquals( - "1:23: third argument of [coalesce(null, network.bytes_in, to_long(0))] must be [counter_long], " - + "found value [to_long(0)] type [long]", - error("FROM tests | eval x = coalesce(null, network.bytes_in, to_long(0))", tsdb) + "1:22: third argument of [case(languages == 1, salary, height)] must be [integer], found value [height] type [double]", + error("from test | eval x = case(languages == 1, salary, height)") ); assertEquals( - "1:23: third argument of [coalesce(null, network.bytes_in, 0.0)] must be [counter_long], found value [0.0] type [double]", - error("FROM tests | eval x = coalesce(null, network.bytes_in, 0.0)", tsdb) + "1:23: third argument of [case(name == \"a\", network.bytes_in, 0)] must be [counter_long], found value [0] type [integer]", + error("FROM tests | eval x = case(name == \"a\", network.bytes_in, 0)", tsdb) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index a1e421b80d608..50ef33381a65d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -3085,7 +3085,28 @@ public void testInvalidFork() { "line 1:20: mismatched input 'FORK' expecting {'limit', 'sort', 'where'}" ); expectError("FROM foo* | FORK ( x+1 ) ( WHERE y>2 )", "line 1:20: mismatched input 'x+1' expecting {'limit', 'sort', 'where'}"); - expectError("FROM foo* | FORK ( LIMIT 10 ) ( y+2 )", "line 1:33: mismatched input 'y' expecting {'limit', 'sort', 'where'}"); + expectError("FROM foo* | FORK ( LIMIT 10 ) ( y+2 )", "line 1:33: mismatched input 'y+2' expecting {'limit', 'sort', 'where'}"); + } + + public void testFieldNamesAsCommands() throws Exception { + String[] keywords = new String[] { + "dissect", + "drop", + "enrich", + "eval", + "explain", + "from", + "grok", + "keep", + "limit", + "mv_expand", + "rename", + "sort", + "stats" }; + for (String keyword : keywords) { + var plan = statement("FROM test | STATS avg(" + keyword + ")"); + var aggregate = as(plan, Aggregate.class); + } } static Alias alias(String name, Expression value) { diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetServicesIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetServicesIT.java index 9d4cec798964a..859a065b6e1a0 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetServicesIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetServicesIT.java @@ -25,7 +25,7 @@ public class InferenceGetServicesIT extends BaseMockEISAuthServerTest { @SuppressWarnings("unchecked") public void testGetServicesWithoutTaskType() throws IOException { List services = getAllServices(); - assertThat(services.size(), equalTo(19)); + assertThat(services.size(), equalTo(20)); String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { @@ -53,6 +53,7 @@ public void testGetServicesWithoutTaskType() throws IOException { "test_reranking_service", "test_service", "text_embedding_test_service", + "voyageai", "watsonxai" ).toArray(), providers @@ -62,7 +63,7 @@ public void testGetServicesWithoutTaskType() throws IOException { @SuppressWarnings("unchecked") public void testGetServicesWithTextEmbeddingTaskType() throws IOException { List services = getServices(TaskType.TEXT_EMBEDDING); - assertThat(services.size(), equalTo(14)); + assertThat(services.size(), equalTo(15)); String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { @@ -85,6 +86,7 @@ public void testGetServicesWithTextEmbeddingTaskType() throws IOException { "mistral", "openai", "text_embedding_test_service", + "voyageai", "watsonxai" ).toArray(), providers @@ -94,7 +96,7 @@ public void testGetServicesWithTextEmbeddingTaskType() throws IOException { @SuppressWarnings("unchecked") public void testGetServicesWithRerankTaskType() throws IOException { List services = getServices(TaskType.RERANK); - assertThat(services.size(), equalTo(6)); + assertThat(services.size(), equalTo(7)); String[] providers = new String[services.size()]; for (int i = 0; i < services.size(); i++) { @@ -103,7 +105,8 @@ public void testGetServicesWithRerankTaskType() throws IOException { } assertArrayEquals( - List.of("alibabacloud-ai-search", "cohere", "elasticsearch", "googlevertexai", "jinaai", "test_reranking_service").toArray(), + List.of("alibabacloud-ai-search", "cohere", "elasticsearch", "googlevertexai", "jinaai", "test_reranking_service", "voyageai") + .toArray(), providers ); } diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java index 1f17e335462a7..da7acf122bb72 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java @@ -34,8 +34,8 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -164,24 +164,24 @@ public void chunkedInfer( } } - private InferenceTextEmbeddingFloatResults makeResults(List input, int dimensions) { - List embeddings = new ArrayList<>(); + private TextEmbeddingFloatResults makeResults(List input, int dimensions) { + List embeddings = new ArrayList<>(); for (String inputString : input) { List floatEmbeddings = generateEmbedding(inputString, dimensions); - embeddings.add(InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(floatEmbeddings)); + embeddings.add(TextEmbeddingFloatResults.Embedding.of(floatEmbeddings)); } - return new InferenceTextEmbeddingFloatResults(embeddings); + return new TextEmbeddingFloatResults(embeddings); } private List makeChunkedResults(List input, int dimensions) { - InferenceTextEmbeddingFloatResults nonChunkedResults = makeResults(input, dimensions); + TextEmbeddingFloatResults nonChunkedResults = makeResults(input, dimensions); var results = new ArrayList(); for (int i = 0; i < input.size(); i++) { results.add( - new ChunkedInferenceEmbeddingFloat( + new ChunkedInferenceEmbedding( List.of( - new ChunkedInferenceEmbeddingFloat.FloatEmbeddingChunk( + new TextEmbeddingFloatResults.Chunk( nonChunkedResults.embeddings().get(i).values(), input.get(i), new ChunkedInference.TextOffset(0, input.get(i).length()) diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java index f700f6672fd63..fcc175a051964 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java @@ -32,7 +32,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; @@ -171,9 +171,9 @@ private List makeChunkedResults(List input) { tokens.add(new WeightedToken("feature_" + j, generateEmbedding(input.get(i), j))); } results.add( - new ChunkedInferenceEmbeddingSparse( + new ChunkedInferenceEmbedding( List.of( - new ChunkedInferenceEmbeddingSparse.SparseEmbeddingChunk( + new SparseEmbeddingResults.Chunk( tokens, input.get(i), new ChunkedInference.TextOffset(0, input.get(i).length()) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index 563247de44f81..d57a6b86e4e71 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -18,13 +18,13 @@ import org.elasticsearch.inference.TaskSettings; import org.elasticsearch.inference.UnifiedCompletionRequest; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.LegacyTextEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.core.inference.results.StreamingUnifiedChatCompletionResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; import org.elasticsearch.xpack.inference.chunking.SentenceBoundaryChunkingSettings; import org.elasticsearch.xpack.inference.chunking.WordBoundaryChunkingSettings; @@ -59,6 +59,7 @@ import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankServiceSettings; import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankTaskSettings; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSparseEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.elastic.completion.ElasticInferenceServiceCompletionServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalTextEmbeddingServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticRerankerServiceSettings; @@ -90,6 +91,11 @@ import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsTaskSettings; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettings; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankTaskSettings; import java.util.ArrayList; import java.util.List; @@ -142,6 +148,7 @@ public static List getNamedWriteables() { addEisNamedWriteables(namedWriteables); addAlibabaCloudSearchNamedWriteables(namedWriteables); addJinaAINamedWriteables(namedWriteables); + addVoyageAINamedWriteables(namedWriteables); addUnifiedNamedWriteables(namedWriteables); @@ -467,18 +474,10 @@ private static void addInferenceResultsNamedWriteables(List ); } + private static void addVoyageAINamedWriteables(List namedWriteables) { + namedWriteables.add( + new NamedWriteableRegistry.Entry(ServiceSettings.class, VoyageAIServiceSettings.NAME, VoyageAIServiceSettings::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + ServiceSettings.class, + VoyageAIEmbeddingsServiceSettings.NAME, + VoyageAIEmbeddingsServiceSettings::new + ) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(TaskSettings.class, VoyageAIEmbeddingsTaskSettings.NAME, VoyageAIEmbeddingsTaskSettings::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(ServiceSettings.class, VoyageAIRerankServiceSettings.NAME, VoyageAIRerankServiceSettings::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(TaskSettings.class, VoyageAIRerankTaskSettings.NAME, VoyageAIRerankTaskSettings::new) + ); + } + private static void addEisNamedWriteables(List namedWriteables) { namedWriteables.add( new NamedWriteableRegistry.Entry( @@ -634,5 +655,12 @@ private static void addEisNamedWriteables(List nam ElasticInferenceServiceSparseEmbeddingsServiceSettings::new ) ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + ServiceSettings.class, + ElasticInferenceServiceCompletionServiceSettings.NAME, + ElasticInferenceServiceCompletionServiceSettings::new + ) + ); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index d865b241bb4e0..0b01ad5e3c66f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -128,6 +128,7 @@ import org.elasticsearch.xpack.inference.services.jinaai.JinaAIService; import org.elasticsearch.xpack.inference.services.mistral.MistralService; import org.elasticsearch.xpack.inference.services.openai.OpenAiService; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIService; import org.elasticsearch.xpack.inference.telemetry.InferenceStats; import java.util.ArrayList; @@ -359,6 +360,7 @@ public List getInferenceServiceFactories() { context -> new AlibabaCloudSearchService(httpFactory.get(), serviceComponents.get()), context -> new IbmWatsonxService(httpFactory.get(), serviceComponents.get()), context -> new JinaAIService(httpFactory.get(), serviceComponents.get()), + context -> new VoyageAIService(httpFactory.get(), serviceComponents.get()), ElasticsearchInternalService::new ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java index fb796c2afdfeb..d8751a542392d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java @@ -10,24 +10,19 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.util.concurrent.AtomicArray; -import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingByte; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; -import org.elasticsearch.xpack.core.inference.results.InferenceByteEmbedding; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.EmbeddingResults; +import org.elasticsearch.xpack.inference.chunking.Chunker.ChunkOffset; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; +import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.stream.Collectors; /** @@ -43,145 +38,72 @@ */ public class EmbeddingRequestChunker { - public enum EmbeddingType { - FLOAT, - BYTE, - SPARSE; + // Visible for testing + record Request(int inputIndex, int chunkIndex, ChunkOffset chunk, List inputs) { + public String chunkText() { + return inputs.get(inputIndex).substring(chunk.start(), chunk.end()); + } + } - public static EmbeddingType fromDenseVectorElementType(DenseVectorFieldMapper.ElementType elementType) { - return switch (elementType) { - case BYTE -> EmbeddingType.BYTE; - case FLOAT -> EmbeddingType.FLOAT; - case BIT -> throw new IllegalArgumentException("Bit vectors are not supported"); - }; + public record BatchRequest(List requests) { + public List inputs() { + return requests.stream().map(Request::chunkText).collect(Collectors.toList()); } - }; + } - public static final int DEFAULT_WORDS_PER_CHUNK = 250; - public static final int DEFAULT_CHUNK_OVERLAP = 100; + public record BatchRequestAndListener(BatchRequest batch, ActionListener listener) {} - private final List batchedRequests = new ArrayList<>(); + private static final int DEFAULT_WORDS_PER_CHUNK = 250; + private static final int DEFAULT_CHUNK_OVERLAP = 100; + + private final List inputs; + private final List> requests; + private final List batchRequests; private final AtomicInteger resultCount = new AtomicInteger(); - private final int maxNumberOfInputsPerBatch; - private final int wordsPerChunk; - private final int chunkOverlap; - private final EmbeddingType embeddingType; - private final ChunkingSettings chunkingSettings; - private List chunkedOffsets; - private List>> floatResults; - private List>> byteResults; - private List>> sparseResults; - private AtomicArray errors; + private final List>> results; + private final AtomicArray errors; private ActionListener> finalListener; - public EmbeddingRequestChunker(List inputs, int maxNumberOfInputsPerBatch, EmbeddingType embeddingType) { - this(inputs, maxNumberOfInputsPerBatch, DEFAULT_WORDS_PER_CHUNK, DEFAULT_CHUNK_OVERLAP, embeddingType); + public EmbeddingRequestChunker(List inputs, int maxNumberOfInputsPerBatch) { + this(inputs, maxNumberOfInputsPerBatch, null); } - public EmbeddingRequestChunker( - List inputs, - int maxNumberOfInputsPerBatch, - int wordsPerChunk, - int chunkOverlap, - EmbeddingType embeddingType - ) { - this.maxNumberOfInputsPerBatch = maxNumberOfInputsPerBatch; - this.wordsPerChunk = wordsPerChunk; - this.chunkOverlap = chunkOverlap; - this.embeddingType = embeddingType; - this.chunkingSettings = null; - splitIntoBatchedRequests(inputs); + public EmbeddingRequestChunker(List inputs, int maxNumberOfInputsPerBatch, int wordsPerChunk, int chunkOverlap) { + this(inputs, maxNumberOfInputsPerBatch, new WordBoundaryChunkingSettings(wordsPerChunk, chunkOverlap)); } - public EmbeddingRequestChunker( - List inputs, - int maxNumberOfInputsPerBatch, - EmbeddingType embeddingType, - ChunkingSettings chunkingSettings - ) { - this.maxNumberOfInputsPerBatch = maxNumberOfInputsPerBatch; - this.wordsPerChunk = DEFAULT_WORDS_PER_CHUNK; // Can be removed after ChunkingConfigurationFeatureFlag is enabled - this.chunkOverlap = DEFAULT_CHUNK_OVERLAP; // Can be removed after ChunkingConfigurationFeatureFlag is enabled - this.embeddingType = embeddingType; - this.chunkingSettings = chunkingSettings; - splitIntoBatchedRequests(inputs); - } + public EmbeddingRequestChunker(List inputs, int maxNumberOfInputsPerBatch, ChunkingSettings chunkingSettings) { + this.inputs = inputs; + this.results = new ArrayList<>(inputs.size()); + this.errors = new AtomicArray<>(inputs.size()); - private void splitIntoBatchedRequests(List inputs) { - Function> chunkFunction; - if (chunkingSettings != null) { - var chunker = ChunkerBuilder.fromChunkingStrategy(chunkingSettings.getChunkingStrategy()); - chunkFunction = input -> chunker.chunk(input, chunkingSettings); - } else { - var chunker = new WordBoundaryChunker(); - chunkFunction = input -> chunker.chunk(input, wordsPerChunk, chunkOverlap); + if (chunkingSettings == null) { + chunkingSettings = new WordBoundaryChunkingSettings(DEFAULT_WORDS_PER_CHUNK, DEFAULT_CHUNK_OVERLAP); } + Chunker chunker = ChunkerBuilder.fromChunkingStrategy(chunkingSettings.getChunkingStrategy()); - chunkedOffsets = new ArrayList<>(inputs.size()); - switch (embeddingType) { - case FLOAT -> floatResults = new ArrayList<>(inputs.size()); - case BYTE -> byteResults = new ArrayList<>(inputs.size()); - case SPARSE -> sparseResults = new ArrayList<>(inputs.size()); - } - errors = new AtomicArray<>(inputs.size()); + this.requests = new ArrayList<>(inputs.size()); - for (int i = 0; i < inputs.size(); i++) { - var chunks = chunkFunction.apply(inputs.get(i)); - var offSetsAndInput = new ChunkOffsetsAndInput(chunks, inputs.get(i)); - int numberOfSubBatches = addToBatches(offSetsAndInput, i); - // size the results array with the expected number of request/responses - switch (embeddingType) { - case FLOAT -> floatResults.add(new AtomicArray<>(numberOfSubBatches)); - case BYTE -> byteResults.add(new AtomicArray<>(numberOfSubBatches)); - case SPARSE -> sparseResults.add(new AtomicArray<>(numberOfSubBatches)); + for (int inputIndex = 0; inputIndex < inputs.size(); inputIndex++) { + List chunks = chunker.chunk(inputs.get(inputIndex), chunkingSettings); + List requestForInput = new ArrayList<>(chunks.size()); + for (int chunkIndex = 0; chunkIndex < chunks.size(); chunkIndex++) { + requestForInput.add(new Request(inputIndex, chunkIndex, chunks.get(chunkIndex), inputs)); } - chunkedOffsets.add(offSetsAndInput); - } - } - - private int addToBatches(ChunkOffsetsAndInput chunk, int inputIndex) { - BatchRequest lastBatch; - if (batchedRequests.isEmpty()) { - lastBatch = new BatchRequest(new ArrayList<>()); - batchedRequests.add(lastBatch); - } else { - lastBatch = batchedRequests.get(batchedRequests.size() - 1); - } - - int freeSpace = maxNumberOfInputsPerBatch - lastBatch.size(); - assert freeSpace >= 0; - - // chunks may span multiple batches, - // the chunkIndex keeps them ordered. - int chunkIndex = 0; - - if (freeSpace > 0) { - // use any free space in the previous batch before creating new batches - int toAdd = Math.min(freeSpace, chunk.offsets().size()); - lastBatch.addSubBatch( - new SubBatch( - new ChunkOffsetsAndInput(chunk.offsets().subList(0, toAdd), chunk.input()), - new SubBatchPositionsAndCount(inputIndex, chunkIndex++, toAdd) - ) - ); - } - - int start = freeSpace; - while (start < chunk.offsets().size()) { - int toAdd = Math.min(maxNumberOfInputsPerBatch, chunk.offsets().size() - start); - var batch = new BatchRequest(new ArrayList<>()); - batch.addSubBatch( - new SubBatch( - new ChunkOffsetsAndInput(chunk.offsets().subList(start, start + toAdd), chunk.input()), - new SubBatchPositionsAndCount(inputIndex, chunkIndex++, toAdd) - ) - ); - batchedRequests.add(batch); - start += toAdd; + requests.add(requestForInput); + // size the results array with the expected number of request/responses + results.add(new AtomicReferenceArray<>(chunks.size())); } - return chunkIndex; + AtomicInteger counter = new AtomicInteger(); + this.batchRequests = requests.stream() + .flatMap(List::stream) + .collect(Collectors.groupingBy(it -> counter.getAndIncrement() / maxNumberOfInputsPerBatch)) + .values() + .stream() + .map(BatchRequest::new) + .toList(); } /** @@ -191,23 +113,7 @@ private int addToBatches(ChunkOffsetsAndInput chunk, int inputIndex) { */ public List batchRequestsWithListeners(ActionListener> finalListener) { this.finalListener = finalListener; - - int numberOfRequests = batchedRequests.size(); - - var requests = new ArrayList(numberOfRequests); - for (var batch : batchedRequests) { - requests.add( - new BatchRequestAndListener( - batch, - new DebatchingListener( - batch.subBatches().stream().map(SubBatch::positions).collect(Collectors.toList()), - numberOfRequests - ) - ) - ); - } - - return requests; + return batchRequests.stream().map(req -> new BatchRequestAndListener(req, new DebatchingListener(req))).toList(); } /** @@ -220,266 +126,83 @@ public List batchRequestsWithListeners(ActionListener { - private final List positions; - private final int totalNumberOfRequests; + private final BatchRequest request; - DebatchingListener(List positions, int totalNumberOfRequests) { - this.positions = positions; - this.totalNumberOfRequests = totalNumberOfRequests; + DebatchingListener(BatchRequest request) { + this.request = request; } @Override public void onResponse(InferenceServiceResults inferenceServiceResults) { - switch (embeddingType) { - case FLOAT -> handleFloatResults(inferenceServiceResults); - case BYTE -> handleByteResults(inferenceServiceResults); - case SPARSE -> handleSparseResults(inferenceServiceResults); - } - } - - private void handleFloatResults(InferenceServiceResults inferenceServiceResults) { - if (inferenceServiceResults instanceof InferenceTextEmbeddingFloatResults floatEmbeddings) { - if (failIfNumRequestsDoNotMatch(floatEmbeddings.embeddings().size())) { - return; - } - - int start = 0; - for (var pos : positions) { - floatResults.get(pos.inputIndex()) - .setOnce(pos.chunkIndex(), floatEmbeddings.embeddings().subList(start, start + pos.embeddingCount())); - start += pos.embeddingCount(); - } - - if (resultCount.incrementAndGet() == totalNumberOfRequests) { - sendResponse(); - } - } else { - onFailure( - unexpectedResultTypeException(inferenceServiceResults.getWriteableName(), InferenceTextEmbeddingFloatResults.NAME) - ); - } - } - - private void handleByteResults(InferenceServiceResults inferenceServiceResults) { - if (inferenceServiceResults instanceof InferenceTextEmbeddingByteResults byteEmbeddings) { - if (failIfNumRequestsDoNotMatch(byteEmbeddings.embeddings().size())) { - return; - } - - int start = 0; - for (var pos : positions) { - byteResults.get(pos.inputIndex()) - .setOnce(pos.chunkIndex(), byteEmbeddings.embeddings().subList(start, start + pos.embeddingCount())); - start += pos.embeddingCount(); - } - - if (resultCount.incrementAndGet() == totalNumberOfRequests) { - sendResponse(); - } - } else { - onFailure( - unexpectedResultTypeException(inferenceServiceResults.getWriteableName(), InferenceTextEmbeddingByteResults.NAME) - ); - } - } - - private void handleSparseResults(InferenceServiceResults inferenceServiceResults) { - if (inferenceServiceResults instanceof SparseEmbeddingResults sparseEmbeddings) { - if (failIfNumRequestsDoNotMatch(sparseEmbeddings.embeddings().size())) { + if (inferenceServiceResults instanceof EmbeddingResults embeddingResults) { + if (embeddingResults.embeddings().size() != request.requests.size()) { + onFailure(numResultsDoesntMatchException(embeddingResults.embeddings().size(), request.requests.size())); return; } - - int start = 0; - for (var pos : positions) { - sparseResults.get(pos.inputIndex()) - .setOnce(pos.chunkIndex(), sparseEmbeddings.embeddings().subList(start, start + pos.embeddingCount())); - start += pos.embeddingCount(); + for (int i = 0; i < embeddingResults.embeddings().size(); i++) { + results.get(request.requests().get(i).inputIndex()) + .set(request.requests().get(i).chunkIndex(), embeddingResults.embeddings().get(i)); } - - if (resultCount.incrementAndGet() == totalNumberOfRequests) { - sendResponse(); + if (resultCount.incrementAndGet() == batchRequests.size()) { + sendFinalResponse(); } } else { - onFailure( - unexpectedResultTypeException(inferenceServiceResults.getWriteableName(), InferenceTextEmbeddingByteResults.NAME) - ); + onFailure(unexpectedResultTypeException(inferenceServiceResults.getWriteableName())); } } - private boolean failIfNumRequestsDoNotMatch(int numberOfResults) { - int numberOfRequests = positions.stream().mapToInt(SubBatchPositionsAndCount::embeddingCount).sum(); - if (numberOfRequests != numberOfResults) { - onFailure( - new ElasticsearchStatusException( - "Error the number of embedding responses [{}] does not equal the number of " + "requests [{}]", - RestStatus.INTERNAL_SERVER_ERROR, - numberOfResults, - numberOfRequests - ) - ); - return true; - } - return false; + private ElasticsearchStatusException numResultsDoesntMatchException(int numResults, int numRequests) { + return new ElasticsearchStatusException( + "Error the number of embedding responses [{}] does not equal the number of requests [{}]", + RestStatus.INTERNAL_SERVER_ERROR, + numResults, + numRequests + ); } - private ElasticsearchStatusException unexpectedResultTypeException(String got, String expected) { + private ElasticsearchStatusException unexpectedResultTypeException(String resultType) { return new ElasticsearchStatusException( - "Unexpected inference result type [" + got + "], expected a [" + expected + "]", - RestStatus.INTERNAL_SERVER_ERROR + "Unexpected inference result type [{}], expected [EmbeddingResults]", + RestStatus.INTERNAL_SERVER_ERROR, + resultType ); } @Override public void onFailure(Exception e) { - for (var pos : positions) { - errors.set(pos.inputIndex(), e); - } - - if (resultCount.incrementAndGet() == totalNumberOfRequests) { - sendResponse(); + for (Request request : request.requests) { + errors.set(request.inputIndex(), e); } - } - - private void sendResponse() { - var response = new ArrayList(chunkedOffsets.size()); - for (int i = 0; i < chunkedOffsets.size(); i++) { - if (errors.get(i) != null) { - response.add(new ChunkedInferenceError(errors.get(i))); - } else { - response.add(mergeResultsWithInputs(i)); - } + if (resultCount.incrementAndGet() == batchRequests.size()) { + sendFinalResponse(); } - - finalListener.onResponse(response); - } - } - - private ChunkedInference mergeResultsWithInputs(int resultIndex) { - return switch (embeddingType) { - case FLOAT -> mergeFloatResultsWithInputs(chunkedOffsets.get(resultIndex), floatResults.get(resultIndex)); - case BYTE -> mergeByteResultsWithInputs(chunkedOffsets.get(resultIndex), byteResults.get(resultIndex)); - case SPARSE -> mergeSparseResultsWithInputs(chunkedOffsets.get(resultIndex), sparseResults.get(resultIndex)); - }; - } - - private ChunkedInferenceEmbeddingFloat mergeFloatResultsWithInputs( - ChunkOffsetsAndInput chunks, - AtomicArray> debatchedResults - ) { - var all = new ArrayList(); - for (int i = 0; i < debatchedResults.length(); i++) { - var subBatch = debatchedResults.get(i); - all.addAll(subBatch); - } - - assert chunks.size() == all.size(); - - var embeddingChunks = new ArrayList(); - for (int i = 0; i < chunks.size(); i++) { - embeddingChunks.add( - new ChunkedInferenceEmbeddingFloat.FloatEmbeddingChunk( - all.get(i).values(), - chunks.chunkText(i), - new ChunkedInference.TextOffset(chunks.offsets().get(i).start(), chunks.offsets().get(i).end()) - ) - ); - } - - return new ChunkedInferenceEmbeddingFloat(embeddingChunks); - } - - private ChunkedInferenceEmbeddingByte mergeByteResultsWithInputs( - ChunkOffsetsAndInput chunks, - AtomicArray> debatchedResults - ) { - var all = new ArrayList(); - for (int i = 0; i < debatchedResults.length(); i++) { - var subBatch = debatchedResults.get(i); - all.addAll(subBatch); - } - - assert chunks.size() == all.size(); - - var embeddingChunks = new ArrayList(); - for (int i = 0; i < chunks.size(); i++) { - embeddingChunks.add( - new ChunkedInferenceEmbeddingByte.ByteEmbeddingChunk( - all.get(i).values(), - chunks.chunkText(i), - new ChunkedInference.TextOffset(chunks.offsets().get(i).start(), chunks.offsets().get(i).end()) - ) - ); - } - - return new ChunkedInferenceEmbeddingByte(embeddingChunks); - } - - private ChunkedInferenceEmbeddingSparse mergeSparseResultsWithInputs( - ChunkOffsetsAndInput chunks, - AtomicArray> debatchedResults - ) { - var all = new ArrayList(); - for (int i = 0; i < debatchedResults.length(); i++) { - var subBatch = debatchedResults.get(i); - all.addAll(subBatch); - } - - assert chunks.size() == all.size(); - - var embeddingChunks = new ArrayList(); - for (int i = 0; i < chunks.size(); i++) { - embeddingChunks.add( - new ChunkedInferenceEmbeddingSparse.SparseEmbeddingChunk( - all.get(i).tokens(), - chunks.chunkText(i), - new ChunkedInference.TextOffset(chunks.offsets().get(i).start(), chunks.offsets().get(i).end()) - ) - ); - } - - return new ChunkedInferenceEmbeddingSparse(embeddingChunks); - } - - public record BatchRequest(List subBatches) { - public int size() { - return subBatches.stream().mapToInt(SubBatch::size).sum(); - } - - public void addSubBatch(SubBatch sb) { - subBatches.add(sb); - } - - public List inputs() { - return subBatches.stream().flatMap(s -> s.requests().toChunkText().stream()).collect(Collectors.toList()); } } - public record BatchRequestAndListener(BatchRequest batch, ActionListener listener) { - - } - - /** - * Used for mapping batched requests back to the original input - */ - record SubBatchPositionsAndCount(int inputIndex, int chunkIndex, int embeddingCount) {} - - record SubBatch(ChunkOffsetsAndInput requests, SubBatchPositionsAndCount positions) { - int size() { - return requests.offsets().size(); + private void sendFinalResponse() { + var response = new ArrayList(inputs.size()); + for (int i = 0; i < inputs.size(); i++) { + if (errors.get(i) != null) { + response.add(new ChunkedInferenceError(errors.get(i))); + } else { + response.add(mergeResultsWithInputs(i)); + } } + finalListener.onResponse(response); } - record ChunkOffsetsAndInput(List offsets, String input) { - List toChunkText() { - return offsets.stream().map(o -> input.substring(o.start(), o.end())).collect(Collectors.toList()); - } - - int size() { - return offsets.size(); - } - - String chunkText(int index) { - return input.substring(offsets.get(index).start(), offsets.get(index).end()); + private ChunkedInference mergeResultsWithInputs(int index) { + List chunks = new ArrayList<>(); + List request = requests.get(index); + AtomicReferenceArray> result = results.get(index); + for (int i = 0; i < request.size(); i++) { + EmbeddingResults.Chunk chunk = result.get(i) + .toChunk( + request.get(i).chunkText(), + new ChunkedInference.TextOffset(request.get(i).chunk.start(), request.get(i).chunk.end()) + ); + chunks.add(chunk); } + return new ChunkedInferenceEmbedding(chunks); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIActionCreator.java new file mode 100644 index 0000000000000..6a4a9e5f93639 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIActionCreator.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action.voyageai; + +import org.elasticsearch.inference.InputType; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.external.http.sender.VoyageAIEmbeddingsRequestManager; +import org.elasticsearch.xpack.inference.external.http.sender.VoyageAIRerankRequestManager; +import org.elasticsearch.xpack.inference.services.ServiceComponents; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankModel; + +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; + +/** + * Provides a way to construct an {@link ExecutableAction} using the visitor pattern based on the voyageai model type. + */ +public class VoyageAIActionCreator implements VoyageAIActionVisitor { + private final Sender sender; + private final ServiceComponents serviceComponents; + + public VoyageAIActionCreator(Sender sender, ServiceComponents serviceComponents) { + this.sender = Objects.requireNonNull(sender); + this.serviceComponents = Objects.requireNonNull(serviceComponents); + } + + @Override + public ExecutableAction create(VoyageAIEmbeddingsModel model, Map taskSettings, InputType inputType) { + var overriddenModel = VoyageAIEmbeddingsModel.of(model, taskSettings, inputType); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(overriddenModel.uri(), "VoyageAI embeddings"); + var requestCreator = VoyageAIEmbeddingsRequestManager.of(overriddenModel, serviceComponents.threadPool()); + return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); + } + + @Override + public ExecutableAction create(VoyageAIRerankModel model, Map taskSettings) { + var overriddenModel = VoyageAIRerankModel.of(model, taskSettings); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(overriddenModel.uri(), "VoyageAI rerank"); + var requestCreator = VoyageAIRerankRequestManager.of(overriddenModel, serviceComponents.threadPool()); + return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIActionVisitor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIActionVisitor.java new file mode 100644 index 0000000000000..d6732dba95475 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIActionVisitor.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action.voyageai; + +import org.elasticsearch.inference.InputType; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankModel; + +import java.util.Map; + +public interface VoyageAIActionVisitor { + ExecutableAction create(VoyageAIEmbeddingsModel model, Map taskSettings, InputType inputType); + + ExecutableAction create(VoyageAIRerankModel model, Map taskSettings); +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/VoyageAIEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/VoyageAIEmbeddingsRequestManager.java new file mode 100644 index 0000000000000..be186dc54f8f2 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/VoyageAIEmbeddingsRequestManager.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; +import org.elasticsearch.xpack.inference.external.request.voyageai.VoyageAIEmbeddingsRequest; +import org.elasticsearch.xpack.inference.external.response.voyageai.VoyageAIEmbeddingsResponseEntity; +import org.elasticsearch.xpack.inference.external.voyageai.VoyageAIResponseHandler; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModel; + +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +public class VoyageAIEmbeddingsRequestManager extends VoyageAIRequestManager { + private static final Logger logger = LogManager.getLogger(VoyageAIEmbeddingsRequestManager.class); + private static final ResponseHandler HANDLER = createEmbeddingsHandler(); + + private static ResponseHandler createEmbeddingsHandler() { + return new VoyageAIResponseHandler("voyageai text embedding", VoyageAIEmbeddingsResponseEntity::fromResponse); + } + + public static VoyageAIEmbeddingsRequestManager of(VoyageAIEmbeddingsModel model, ThreadPool threadPool) { + return new VoyageAIEmbeddingsRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool)); + } + + private final VoyageAIEmbeddingsModel model; + + private VoyageAIEmbeddingsRequestManager(VoyageAIEmbeddingsModel model, ThreadPool threadPool) { + super(threadPool, model); + this.model = Objects.requireNonNull(model); + } + + @Override + public void execute( + InferenceInputs inferenceInputs, + RequestSender requestSender, + Supplier hasRequestCompletedFunction, + ActionListener listener + ) { + List docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs(); + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest(docsInput, model); + + execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/VoyageAIRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/VoyageAIRequestManager.java new file mode 100644 index 0000000000000..99a0617ff510a --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/VoyageAIRequestManager.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIModel; + +import java.util.Map; +import java.util.Objects; + +abstract class VoyageAIRequestManager extends BaseRequestManager { + private static final String DEFAULT_MODEL_FAMILY = "default_model_family"; + private static final Map MODEL_TO_MODEL_FAMILY = Map.of( + "voyage-multimodal-3", + "embed_multimodal", + "voyage-3-large", + "embed_large", + "voyage-code-3", + "embed_large", + "voyage-3", + "embed_medium", + "voyage-3-lite", + "embed_small", + "voyage-finance-2", + "embed_large", + "voyage-law-2", + "embed_large", + "voyage-code-2", + "embed_large", + "rerank-2", + "rerank_large", + "rerank-2-lite", + "rerank_small" + ); + + protected VoyageAIRequestManager(ThreadPool threadPool, VoyageAIModel model) { + super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model), model.rateLimitServiceSettings().rateLimitSettings()); + } + + record RateLimitGrouping(int apiKeyHash) { + public static RateLimitGrouping of(VoyageAIModel model) { + Objects.requireNonNull(model); + String modelId = model.getServiceSettings().modelId(); + String modelFamily = MODEL_TO_MODEL_FAMILY.getOrDefault(modelId, DEFAULT_MODEL_FAMILY); + + return new RateLimitGrouping(modelFamily.hashCode()); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/VoyageAIRerankRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/VoyageAIRerankRequestManager.java new file mode 100644 index 0000000000000..ca91ad4dda276 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/VoyageAIRerankRequestManager.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; +import org.elasticsearch.xpack.inference.external.request.voyageai.VoyageAIRerankRequest; +import org.elasticsearch.xpack.inference.external.response.voyageai.VoyageAIRerankResponseEntity; +import org.elasticsearch.xpack.inference.external.voyageai.VoyageAIResponseHandler; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankModel; + +import java.util.Objects; +import java.util.function.Supplier; + +public class VoyageAIRerankRequestManager extends VoyageAIRequestManager { + private static final Logger logger = LogManager.getLogger(VoyageAIRerankRequestManager.class); + private static final ResponseHandler HANDLER = createVoyageAIResponseHandler(); + + private static ResponseHandler createVoyageAIResponseHandler() { + return new VoyageAIResponseHandler("voyageai rerank", (request, response) -> VoyageAIRerankResponseEntity.fromResponse(response)); + } + + public static VoyageAIRerankRequestManager of(VoyageAIRerankModel model, ThreadPool threadPool) { + return new VoyageAIRerankRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool)); + } + + private final VoyageAIRerankModel model; + + private VoyageAIRerankRequestManager(VoyageAIRerankModel model, ThreadPool threadPool) { + super(threadPool, model); + this.model = model; + } + + @Override + public void execute( + InferenceInputs inferenceInputs, + RequestSender requestSender, + Supplier hasRequestCompletedFunction, + ActionListener listener + ) { + var rerankInput = QueryAndDocsInputs.of(inferenceInputs); + VoyageAIRerankRequest request = new VoyageAIRerankRequest(rerankInput.getQuery(), rerankInput.getChunks(), model); + + execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequest.java new file mode 100644 index 0000000000000..a24f5dc8e14ea --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequest.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.inference.external.request.HttpRequest; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.voyageai.VoyageAIAccount; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettings; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Objects; + +public class VoyageAIEmbeddingsRequest extends VoyageAIRequest { + + private final VoyageAIAccount account; + private final List input; + private final VoyageAIEmbeddingsServiceSettings serviceSettings; + private final VoyageAIEmbeddingsTaskSettings taskSettings; + private final String model; + private final String inferenceEntityId; + + public VoyageAIEmbeddingsRequest(List input, VoyageAIEmbeddingsModel embeddingsModel) { + Objects.requireNonNull(embeddingsModel); + + account = VoyageAIAccount.of(embeddingsModel); + this.input = Objects.requireNonNull(input); + serviceSettings = embeddingsModel.getServiceSettings(); + taskSettings = embeddingsModel.getTaskSettings(); + model = embeddingsModel.getServiceSettings().getCommonSettings().modelId(); + inferenceEntityId = embeddingsModel.getInferenceEntityId(); + } + + @Override + public HttpRequest createHttpRequest() { + HttpPost httpPost = new HttpPost(account.uri()); + + ByteArrayEntity byteEntity = new ByteArrayEntity( + Strings.toString(new VoyageAIEmbeddingsRequestEntity(input, serviceSettings, taskSettings, model)) + .getBytes(StandardCharsets.UTF_8) + ); + httpPost.setEntity(byteEntity); + + decorateWithHeaders(httpPost, account); + + return new HttpRequest(httpPost, getInferenceEntityId()); + } + + @Override + public String getInferenceEntityId() { + return inferenceEntityId; + } + + @Override + public URI getURI() { + return account.uri(); + } + + @Override + public Request truncate() { + return this; + } + + @Override + public boolean[] getTruncationInfo() { + return null; + } + + public VoyageAIEmbeddingsTaskSettings getTaskSettings() { + return taskSettings; + } + + public VoyageAIEmbeddingsServiceSettings getServiceSettings() { + return serviceSettings; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequestEntity.java new file mode 100644 index 0000000000000..8191443edf75c --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequestEntity.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.elasticsearch.inference.InputType; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettings; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettings.invalidInputTypeMessage; + +public record VoyageAIEmbeddingsRequestEntity( + List input, + VoyageAIEmbeddingsServiceSettings serviceSettings, + VoyageAIEmbeddingsTaskSettings taskSettings, + String model +) implements ToXContentObject { + + private static final String DOCUMENT = "document"; + private static final String QUERY = "query"; + private static final String INPUT_FIELD = "input"; + private static final String MODEL_FIELD = "model"; + public static final String INPUT_TYPE_FIELD = "input_type"; + public static final String TRUNCATION_FIELD = "truncation"; + public static final String OUTPUT_DIMENSION = "output_dimension"; + static final String OUTPUT_DTYPE_FIELD = "output_dtype"; + + public VoyageAIEmbeddingsRequestEntity { + Objects.requireNonNull(input); + Objects.requireNonNull(model); + Objects.requireNonNull(taskSettings); + Objects.requireNonNull(serviceSettings); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(INPUT_FIELD, input); + builder.field(MODEL_FIELD, model); + + var inputType = convertToString(taskSettings.getInputType()); + if (inputType != null) { + builder.field(INPUT_TYPE_FIELD, inputType); + } + + if (taskSettings.getTruncation() != null) { + builder.field(TRUNCATION_FIELD, taskSettings.getTruncation()); + } + + if (serviceSettings.dimensions() != null) { + builder.field(OUTPUT_DIMENSION, serviceSettings.dimensions()); + } + + if (serviceSettings.getEmbeddingType() != null) { + builder.field(OUTPUT_DTYPE_FIELD, serviceSettings.getEmbeddingType().toRequestString()); + } + + builder.endObject(); + return builder; + } + + static String convertToString(InputType inputType) { + return switch (inputType) { + case null -> null; + case INGEST -> DOCUMENT; + case SEARCH -> QUERY; + default -> { + assert false : invalidInputTypeMessage(inputType); + yield null; + } + }; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRequest.java new file mode 100644 index 0000000000000..5455a2f0301f2 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRequest.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.voyageai.VoyageAIAccount; + +import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader; + +public abstract class VoyageAIRequest implements Request { + + public static void decorateWithHeaders(HttpPost request, VoyageAIAccount account) { + request.setHeader(HttpHeaders.CONTENT_TYPE, XContentType.JSON.mediaType()); + request.setHeader(createAuthBearerHeader(account.apiKey())); + request.setHeader(VoyageAIUtils.createRequestSourceHeader()); + } + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequest.java new file mode 100644 index 0000000000000..37d15fe1fe2c5 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequest.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.inference.external.request.HttpRequest; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.voyageai.VoyageAIAccount; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankModel; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankTaskSettings; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Objects; + +public class VoyageAIRerankRequest extends VoyageAIRequest { + + private final VoyageAIAccount account; + private final String query; + private final List input; + private final VoyageAIRerankTaskSettings taskSettings; + private final String model; + private final String inferenceEntityId; + + public VoyageAIRerankRequest(String query, List input, VoyageAIRerankModel model) { + Objects.requireNonNull(model); + + this.account = VoyageAIAccount.of(model); + this.input = Objects.requireNonNull(input); + this.query = Objects.requireNonNull(query); + taskSettings = model.getTaskSettings(); + this.model = model.getServiceSettings().modelId(); + inferenceEntityId = model.getInferenceEntityId(); + } + + @Override + public HttpRequest createHttpRequest() { + HttpPost httpPost = new HttpPost(account.uri()); + + ByteArrayEntity byteEntity = new ByteArrayEntity( + Strings.toString(new VoyageAIRerankRequestEntity(query, input, taskSettings, model)).getBytes(StandardCharsets.UTF_8) + ); + httpPost.setEntity(byteEntity); + + decorateWithHeaders(httpPost, account); + + return new HttpRequest(httpPost, getInferenceEntityId()); + } + + @Override + public String getInferenceEntityId() { + return inferenceEntityId; + } + + @Override + public URI getURI() { + return account.uri(); + } + + @Override + public Request truncate() { + return this; + } + + @Override + public boolean[] getTruncationInfo() { + return null; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequestEntity.java new file mode 100644 index 0000000000000..0f7baaa35044e --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequestEntity.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankTaskSettings; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public record VoyageAIRerankRequestEntity(String model, String query, List documents, VoyageAIRerankTaskSettings taskSettings) + implements + ToXContentObject { + + private static final String DOCUMENTS_FIELD = "documents"; + private static final String QUERY_FIELD = "query"; + private static final String MODEL_FIELD = "model"; + public static final String TRUNCATION_FIELD = "truncation"; + public static final String RETURN_DOCUMENTS_FIELD = "return_documents"; + + public VoyageAIRerankRequestEntity { + Objects.requireNonNull(query); + Objects.requireNonNull(documents); + Objects.requireNonNull(model); + Objects.requireNonNull(taskSettings); + } + + public VoyageAIRerankRequestEntity(String query, List input, VoyageAIRerankTaskSettings taskSettings, String model) { + this(model, query, input, taskSettings != null ? taskSettings : VoyageAIRerankTaskSettings.EMPTY_SETTINGS); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder.field(MODEL_FIELD, model); + builder.field(QUERY_FIELD, query); + builder.field(DOCUMENTS_FIELD, documents); + + if (taskSettings.getDoesReturnDocuments() != null) { + builder.field(VoyageAIRerankTaskSettings.RETURN_DOCUMENTS, taskSettings.getDoesReturnDocuments()); + } + + if (taskSettings.getTopKDocumentsOnly() != null) { + builder.field(VoyageAIRerankTaskSettings.TOP_K_DOCS_ONLY, taskSettings.getTopKDocumentsOnly()); + } + + if (taskSettings.getTruncation() != null) { + builder.field(TRUNCATION_FIELD, taskSettings.getTruncation()); + } + + builder.endObject(); + return builder; + } + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIUtils.java new file mode 100644 index 0000000000000..130093826ee0f --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIUtils.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; + +public class VoyageAIUtils { + public static final String HOST = "api.voyageai.com"; + public static final String VERSION_1 = "v1"; + public static final String EMBEDDINGS_PATH = "embeddings"; + public static final String RERANK_PATH = "rerank"; + public static final String REQUEST_SOURCE_HEADER = "Request-Source"; + public static final String ELASTIC_REQUEST_SOURCE = "unspecified:elasticsearch"; + + public static Header createRequestSourceHeader() { + return new BasicHeader(REQUEST_SOURCE_HEADER, ELASTIC_REQUEST_SOURCE); + } + + private VoyageAIUtils() {} +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntity.java index 33fa645b107bc..554ab655def66 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntity.java @@ -9,7 +9,7 @@ import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; @@ -70,21 +70,20 @@ public class AlibabaCloudSearchEmbeddingsResponseEntity extends AlibabaCloudSear * * */ - public static InferenceTextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { + public static TextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { return fromResponse(request, response, parser -> { positionParserAtTokenAfterField(parser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE); - List embeddingList = XContentParserUtils.parseList( + List embeddingList = XContentParserUtils.parseList( parser, AlibabaCloudSearchEmbeddingsResponseEntity::parseEmbeddingObject ); - return new InferenceTextEmbeddingFloatResults(embeddingList); + return new TextEmbeddingFloatResults(embeddingList); }); } - private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseEmbeddingObject(XContentParser parser) - throws IOException { + private static TextEmbeddingFloatResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE); @@ -96,7 +95,7 @@ private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseE // if there are additional fields within this object, lets skip them, so we can begin parsing the next embedding array parser.skipChildren(); - return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValues); + return TextEmbeddingFloatResults.Embedding.of(embeddingValues); } private static float parseEmbeddingList(XContentParser parser) throws IOException { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/amazonbedrock/embeddings/AmazonBedrockEmbeddingsResponse.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/amazonbedrock/embeddings/AmazonBedrockEmbeddingsResponse.java index 1848e082dec46..017654ebb191d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/amazonbedrock/embeddings/AmazonBedrockEmbeddingsResponse.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/amazonbedrock/embeddings/AmazonBedrockEmbeddingsResponse.java @@ -16,7 +16,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.request.amazonbedrock.AmazonBedrockRequest; import org.elasticsearch.xpack.inference.external.request.amazonbedrock.embeddings.AmazonBedrockEmbeddingsRequest; import org.elasticsearch.xpack.inference.external.response.XContentUtils; @@ -48,7 +48,7 @@ public InferenceServiceResults accept(AmazonBedrockRequest request) { throw new ElasticsearchException("unexpected request type [" + request.getClass() + "]"); } - public static InferenceTextEmbeddingFloatResults fromResponse(InvokeModelResponse response, AmazonBedrockProvider provider) { + public static TextEmbeddingFloatResults fromResponse(InvokeModelResponse response, AmazonBedrockProvider provider) { var charset = StandardCharsets.UTF_8; var bodyText = String.valueOf(charset.decode(response.body().asByteBuffer())); @@ -63,16 +63,14 @@ public static InferenceTextEmbeddingFloatResults fromResponse(InvokeModelRespons var embeddingList = parseEmbeddings(jsonParser, provider); - return new InferenceTextEmbeddingFloatResults(embeddingList); + return new TextEmbeddingFloatResults(embeddingList); } catch (IOException e) { throw new ElasticsearchException(e); } } - private static List parseEmbeddings( - XContentParser jsonParser, - AmazonBedrockProvider provider - ) throws IOException { + private static List parseEmbeddings(XContentParser jsonParser, AmazonBedrockProvider provider) + throws IOException { switch (provider) { case AMAZONTITAN -> { return parseTitanEmbeddings(jsonParser); @@ -84,8 +82,7 @@ private static List } } - private static List parseTitanEmbeddings(XContentParser parser) - throws IOException { + private static List parseTitanEmbeddings(XContentParser parser) throws IOException { /* Titan response: { @@ -95,12 +92,11 @@ private static List */ positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE); List embeddingValuesList = parseList(parser, XContentUtils::parseFloat); - var embeddingValues = InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValuesList); + var embeddingValues = TextEmbeddingFloatResults.Embedding.of(embeddingValuesList); return List.of(embeddingValues); } - private static List parseCohereEmbeddings(XContentParser parser) - throws IOException { + private static List parseCohereEmbeddings(XContentParser parser) throws IOException { /* Cohere response: { @@ -115,7 +111,7 @@ private static List */ positionParserAtTokenAfterField(parser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE); - List embeddingList = parseList( + List embeddingList = parseList( parser, AmazonBedrockEmbeddingsResponse::parseCohereEmbeddingsListItem ); @@ -123,10 +119,9 @@ private static List return embeddingList; } - private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseCohereEmbeddingsListItem(XContentParser parser) - throws IOException { + private static TextEmbeddingFloatResults.Embedding parseCohereEmbeddingsListItem(XContentParser parser) throws IOException { List embeddingValuesList = parseList(parser, XContentUtils::parseFloat); - return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValuesList); + return TextEmbeddingFloatResults.Embedding.of(embeddingValuesList); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java index 2e574d477b057..cbd570e662b83 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntity.java @@ -17,10 +17,9 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.results.InferenceByteEmbedding; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingBitResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingBitResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.XContentUtils; @@ -192,20 +191,20 @@ private static InferenceServiceResults parseBitEmbeddingsArray(XContentParser pa // Cohere returns array of binary embeddings encoded as bytes with int8 precision so we can reuse the byte parser var embeddingList = parseList(parser, CohereEmbeddingsResponseEntity::parseByteArrayEntry); - return new InferenceTextEmbeddingBitResults(embeddingList); + return new TextEmbeddingBitResults(embeddingList); } private static InferenceServiceResults parseByteEmbeddingsArray(XContentParser parser) throws IOException { var embeddingList = parseList(parser, CohereEmbeddingsResponseEntity::parseByteArrayEntry); - return new InferenceTextEmbeddingByteResults(embeddingList); + return new TextEmbeddingByteResults(embeddingList); } - private static InferenceByteEmbedding parseByteArrayEntry(XContentParser parser) throws IOException { + private static TextEmbeddingByteResults.Embedding parseByteArrayEntry(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); List embeddingValuesList = parseList(parser, CohereEmbeddingsResponseEntity::parseEmbeddingInt8Entry); - return InferenceByteEmbedding.of(embeddingValuesList); + return TextEmbeddingByteResults.Embedding.of(embeddingValuesList); } private static Byte parseEmbeddingInt8Entry(XContentParser parser) throws IOException { @@ -226,14 +225,13 @@ private static void checkByteBounds(short value) { private static InferenceServiceResults parseFloatEmbeddingsArray(XContentParser parser) throws IOException { var embeddingList = parseList(parser, CohereEmbeddingsResponseEntity::parseFloatArrayEntry); - return new InferenceTextEmbeddingFloatResults(embeddingList); + return new TextEmbeddingFloatResults(embeddingList); } - private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseFloatArrayEntry(XContentParser parser) - throws IOException { + private static TextEmbeddingFloatResults.Embedding parseFloatArrayEntry(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); List embeddingValuesList = parseList(parser, XContentUtils::parseFloat); - return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValuesList); + return TextEmbeddingFloatResults.Embedding.of(embeddingValuesList); } private CohereEmbeddingsResponseEntity() {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java index 543b8e39d85f8..177241670dcc9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntity.java @@ -12,7 +12,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.XContentUtils; @@ -70,7 +70,7 @@ public class GoogleAiStudioEmbeddingsResponseEntity { * */ - public static InferenceTextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { + public static TextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { @@ -81,17 +81,16 @@ public static InferenceTextEmbeddingFloatResults fromResponse(Request request, H positionParserAtTokenAfterField(jsonParser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE); - List embeddingList = parseList( + List embeddingList = parseList( jsonParser, GoogleAiStudioEmbeddingsResponseEntity::parseEmbeddingObject ); - return new InferenceTextEmbeddingFloatResults(embeddingList); + return new TextEmbeddingFloatResults(embeddingList); } } - private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseEmbeddingObject(XContentParser parser) - throws IOException { + private static TextEmbeddingFloatResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); positionParserAtTokenAfterField(parser, "values", FAILED_TO_FIND_FIELD_TEMPLATE); @@ -100,7 +99,7 @@ private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseE // parse and discard the rest of the object consumeUntilObjectEnd(parser); - return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValuesList); + return TextEmbeddingFloatResults.Embedding.of(embeddingValuesList); } private GoogleAiStudioEmbeddingsResponseEntity() {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiEmbeddingsResponseEntity.java index 7205ea83d0a7a..631698eb121a9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiEmbeddingsResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiEmbeddingsResponseEntity.java @@ -13,7 +13,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; @@ -64,7 +64,7 @@ public class GoogleVertexAiEmbeddingsResponseEntity { * */ - public static InferenceTextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { + public static TextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { @@ -75,17 +75,16 @@ public static InferenceTextEmbeddingFloatResults fromResponse(Request request, H positionParserAtTokenAfterField(jsonParser, "predictions", FAILED_TO_FIND_FIELD_TEMPLATE); - List embeddingList = parseList( + List embeddingList = parseList( jsonParser, GoogleVertexAiEmbeddingsResponseEntity::parseEmbeddingObject ); - return new InferenceTextEmbeddingFloatResults(embeddingList); + return new TextEmbeddingFloatResults(embeddingList); } } - private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseEmbeddingObject(XContentParser parser) - throws IOException { + private static TextEmbeddingFloatResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); positionParserAtTokenAfterField(parser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE); @@ -100,7 +99,7 @@ private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseE consumeUntilObjectEnd(parser); consumeUntilObjectEnd(parser); - return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValueList); + return TextEmbeddingFloatResults.Embedding.of(embeddingValueList); } private static float parseEmbeddingList(XContentParser parser) throws IOException { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java index cdfe36447b88c..423fbf4aaa5d6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntity.java @@ -14,7 +14,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.XContentUtils; @@ -35,7 +35,7 @@ public class HuggingFaceEmbeddingsResponseEntity { * Parse the response from hugging face. The known formats are an array of arrays and object with an {@code embeddings} field containing * an array of arrays. */ - public static InferenceTextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { + public static TextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { @@ -93,13 +93,13 @@ public static InferenceTextEmbeddingFloatResults fromResponse(Request request, H * sentence-transformers/all-MiniLM-L6-v2 * sentence-transformers/all-MiniLM-L12-v2 */ - private static InferenceTextEmbeddingFloatResults parseArrayFormat(XContentParser parser) throws IOException { - List embeddingList = parseList( + private static TextEmbeddingFloatResults parseArrayFormat(XContentParser parser) throws IOException { + List embeddingList = parseList( parser, HuggingFaceEmbeddingsResponseEntity::parseEmbeddingEntry ); - return new InferenceTextEmbeddingFloatResults(embeddingList); + return new TextEmbeddingFloatResults(embeddingList); } /** @@ -138,23 +138,22 @@ private static InferenceTextEmbeddingFloatResults parseArrayFormat(XContentParse * intfloat/multilingual-e5-small * sentence-transformers/all-mpnet-base-v2 */ - private static InferenceTextEmbeddingFloatResults parseObjectFormat(XContentParser parser) throws IOException { + private static TextEmbeddingFloatResults parseObjectFormat(XContentParser parser) throws IOException { positionParserAtTokenAfterField(parser, "embeddings", FAILED_TO_FIND_FIELD_TEMPLATE); - List embeddingList = parseList( + List embeddingList = parseList( parser, HuggingFaceEmbeddingsResponseEntity::parseEmbeddingEntry ); - return new InferenceTextEmbeddingFloatResults(embeddingList); + return new TextEmbeddingFloatResults(embeddingList); } - private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseEmbeddingEntry(XContentParser parser) - throws IOException { + private static TextEmbeddingFloatResults.Embedding parseEmbeddingEntry(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); List embeddingValuesList = parseList(parser, XContentUtils::parseFloat); - return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValuesList); + return TextEmbeddingFloatResults.Embedding.of(embeddingValuesList); } private HuggingFaceEmbeddingsResponseEntity() {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxEmbeddingsResponseEntity.java index 2702556bb983d..81b1d9cd0f3fb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxEmbeddingsResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxEmbeddingsResponseEntity.java @@ -12,7 +12,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.XContentUtils; @@ -30,7 +30,7 @@ public class IbmWatsonxEmbeddingsResponseEntity { private static final String FAILED_TO_FIND_FIELD_TEMPLATE = "Failed to find required field [%s] in IBM Watsonx embeddings response"; - public static InferenceTextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { + public static TextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { @@ -41,17 +41,16 @@ public static InferenceTextEmbeddingFloatResults fromResponse(Request request, H positionParserAtTokenAfterField(jsonParser, "results", FAILED_TO_FIND_FIELD_TEMPLATE); - List embeddingList = parseList( + List embeddingList = parseList( jsonParser, IbmWatsonxEmbeddingsResponseEntity::parseEmbeddingObject ); - return new InferenceTextEmbeddingFloatResults(embeddingList); + return new TextEmbeddingFloatResults(embeddingList); } } - private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseEmbeddingObject(XContentParser parser) - throws IOException { + private static TextEmbeddingFloatResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE); @@ -60,7 +59,7 @@ private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseE // parse and discard the rest of the object consumeUntilObjectEnd(parser); - return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValuesList); + return TextEmbeddingFloatResults.Embedding.of(embeddingValuesList); } private IbmWatsonxEmbeddingsResponseEntity() {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java index 26bde5f5f48ad..b1782bb560ac0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntity.java @@ -14,7 +14,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.XContentUtils; @@ -73,7 +73,7 @@ public class JinaAIEmbeddingsResponseEntity { * * */ - public static InferenceTextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { + public static TextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { @@ -84,17 +84,16 @@ public static InferenceTextEmbeddingFloatResults fromResponse(Request request, H positionParserAtTokenAfterField(jsonParser, "data", FAILED_TO_FIND_FIELD_TEMPLATE); - List embeddingList = parseList( + List embeddingList = parseList( jsonParser, JinaAIEmbeddingsResponseEntity::parseEmbeddingObject ); - return new InferenceTextEmbeddingFloatResults(embeddingList); + return new TextEmbeddingFloatResults(embeddingList); } } - private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseEmbeddingObject(XContentParser parser) - throws IOException { + private static TextEmbeddingFloatResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE); @@ -103,7 +102,7 @@ private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseE // parse and discard the rest of the object consumeUntilObjectEnd(parser); - return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValuesList); + return TextEmbeddingFloatResults.Embedding.of(embeddingValuesList); } private JinaAIEmbeddingsResponseEntity() {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java index ad6df06247080..bda5741af2038 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntity.java @@ -14,7 +14,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.XContentUtils; @@ -74,7 +74,7 @@ public class OpenAiEmbeddingsResponseEntity { * * */ - public static InferenceTextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { + public static TextEmbeddingFloatResults fromResponse(Request request, HttpResult response) throws IOException { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { @@ -85,17 +85,16 @@ public static InferenceTextEmbeddingFloatResults fromResponse(Request request, H positionParserAtTokenAfterField(jsonParser, "data", FAILED_TO_FIND_FIELD_TEMPLATE); - List embeddingList = parseList( + List embeddingList = parseList( jsonParser, OpenAiEmbeddingsResponseEntity::parseEmbeddingObject ); - return new InferenceTextEmbeddingFloatResults(embeddingList); + return new TextEmbeddingFloatResults(embeddingList); } } - private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseEmbeddingObject(XContentParser parser) - throws IOException { + private static TextEmbeddingFloatResults.Embedding parseEmbeddingObject(XContentParser parser) throws IOException { ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); positionParserAtTokenAfterField(parser, "embedding", FAILED_TO_FIND_FIELD_TEMPLATE); @@ -104,7 +103,7 @@ private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding parseE // parse and discard the rest of the object consumeUntilObjectEnd(parser); - return InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(embeddingValuesList); + return TextEmbeddingFloatResults.Embedding.of(embeddingValuesList); } private OpenAiEmbeddingsResponseEntity() {} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIEmbeddingsResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIEmbeddingsResponseEntity.java new file mode 100644 index 0000000000000..ed30df1a8bb20 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIEmbeddingsResponseEntity.java @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * this file was contributed to by a generative AI + */ + +package org.elasticsearch.xpack.inference.external.response.voyageai; + +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingBitResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.request.voyageai.VoyageAIEmbeddingsRequest; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingType; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingType.toLowerCase; + +public class VoyageAIEmbeddingsResponseEntity { + private static final String VALID_EMBEDDING_TYPES_STRING = supportedEmbeddingTypes(); + + private static String supportedEmbeddingTypes() { + String[] validTypes = new String[] { + toLowerCase(VoyageAIEmbeddingType.FLOAT), + toLowerCase(VoyageAIEmbeddingType.INT8), + toLowerCase(VoyageAIEmbeddingType.BIT) }; + Arrays.sort(validTypes); + return String.join(", ", validTypes); + } + + record EmbeddingInt8Result(List entries) { + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + EmbeddingInt8Result.class.getSimpleName(), + true, + args -> new EmbeddingInt8Result((List) args[0]) + ); + + static { + PARSER.declareObjectArray(constructorArg(), EmbeddingInt8ResultEntry.PARSER::apply, new ParseField("data")); + } + } + + record EmbeddingInt8ResultEntry(Integer index, List embedding) { + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + EmbeddingInt8ResultEntry.class.getSimpleName(), + true, + args -> new EmbeddingInt8ResultEntry((Integer) args[0], (List) args[1]) + ); + + static { + PARSER.declareInt(constructorArg(), new ParseField("index")); + PARSER.declareIntArray(constructorArg(), new ParseField("embedding")); + } + + private static void checkByteBounds(Integer value) { + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { + throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte"); + } + } + + public TextEmbeddingByteResults.Embedding toInferenceByteEmbedding() { + embedding.forEach(EmbeddingInt8ResultEntry::checkByteBounds); + return TextEmbeddingByteResults.Embedding.of(embedding.stream().map(Integer::byteValue).toList()); + } + } + + record EmbeddingFloatResult(List entries) { + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + EmbeddingFloatResult.class.getSimpleName(), + true, + args -> new EmbeddingFloatResult((List) args[0]) + ); + + static { + PARSER.declareObjectArray(constructorArg(), EmbeddingFloatResultEntry.PARSER::apply, new ParseField("data")); + } + } + + record EmbeddingFloatResultEntry(Integer index, List embedding) { + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + EmbeddingFloatResultEntry.class.getSimpleName(), + true, + args -> new EmbeddingFloatResultEntry((Integer) args[0], (List) args[1]) + ); + + static { + PARSER.declareInt(constructorArg(), new ParseField("index")); + PARSER.declareFloatArray(constructorArg(), new ParseField("embedding")); + } + + public TextEmbeddingFloatResults.Embedding toInferenceFloatEmbedding() { + return TextEmbeddingFloatResults.Embedding.of(embedding); + } + } + + /** + * Parses the VoyageAI json response. + * For a request like: + * + * 
+     *     
+     *        {
+     *          "input": [
+     *            "Sample text 1",
+     *            "Sample text 2"
+     *          ],
+     *          "model": "voyage-3-large"
+     *        }
+     *     
+     *
+ * + * The response would look like: + * + * 
+     * 
+     * {
+     *  "object": "list",
+     *  "data": [
+     *      {
+     *          "object": "embedding",
+     *          "embedding": [
+     *              -0.009327292,
+     *              -0.0028842222,
+     *          ],
+     *          "index": 0
+     *      },
+     *      {
+     *          "object": "embedding",
+     *          "embedding": [ ... ],
+     *          "index": 1
+     *      }
+     *  ],
+     *  "model": "voyage-3-large",
+     *  "usage": {
+     *      "total_tokens": 10
+     *  }
+     * }
+     * 
+     *
+ */ + public static InferenceServiceResults fromResponse(Request request, HttpResult response) throws IOException { + var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); + VoyageAIEmbeddingType embeddingType = ((VoyageAIEmbeddingsRequest) request).getServiceSettings().getEmbeddingType(); + + try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { + if (embeddingType == null || embeddingType == VoyageAIEmbeddingType.FLOAT) { + var embeddingResult = EmbeddingFloatResult.PARSER.apply(jsonParser, null); + + List embeddingList = embeddingResult.entries.stream() + .map(EmbeddingFloatResultEntry::toInferenceFloatEmbedding) + .toList(); + return new TextEmbeddingFloatResults(embeddingList); + } else if (embeddingType == VoyageAIEmbeddingType.INT8) { + var embeddingResult = EmbeddingInt8Result.PARSER.apply(jsonParser, null); + List embeddingList = embeddingResult.entries.stream() + .map(EmbeddingInt8ResultEntry::toInferenceByteEmbedding) + .toList(); + return new TextEmbeddingByteResults(embeddingList); + } else if (embeddingType == VoyageAIEmbeddingType.BIT || embeddingType == VoyageAIEmbeddingType.BINARY) { + var embeddingResult = EmbeddingInt8Result.PARSER.apply(jsonParser, null); + List embeddingList = embeddingResult.entries.stream() + .map(EmbeddingInt8ResultEntry::toInferenceByteEmbedding) + .toList(); + return new TextEmbeddingBitResults(embeddingList); + } else { + throw new IllegalArgumentException( + "Illegal embedding_type value: " + embeddingType + ". Supported types are: " + VALID_EMBEDDING_TYPES_STRING + ); + } + } + } + + private VoyageAIEmbeddingsResponseEntity() {} +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIErrorResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIErrorResponseEntity.java new file mode 100644 index 0000000000000..41ffaa61fcd26 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIErrorResponseEntity.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.voyageai; + +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; + +public class VoyageAIErrorResponseEntity extends ErrorResponse { + + private VoyageAIErrorResponseEntity(String errorMessage) { + super(errorMessage); + } + + /** + * Parse an HTTP response into a VoyageAIErrorResponseEntity + * + * @param response The error response + * @return An error entity if the response is JSON with a `detail` field containing the error message + * or null if the response does not contain the message field + */ + public static ErrorResponse fromResponse(HttpResult response) { + try ( + XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON) + .createParser(XContentParserConfiguration.EMPTY, response.body()) + ) { + var responseMap = jsonParser.map(); + var message = (String) responseMap.get("detail"); + if (message != null) { + return new VoyageAIErrorResponseEntity(message); + } + } catch (Exception e) { + // swallow the error + } + + return ErrorResponse.UNDEFINED_ERROR; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIRerankResponseEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIRerankResponseEntity.java new file mode 100644 index 0000000000000..5438ba3644753 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIRerankResponseEntity.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * this file was contributed to by a generative AI + */ + +package org.elasticsearch.xpack.inference.external.response.voyageai; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class VoyageAIRerankResponseEntity { + + private static final Logger logger = LogManager.getLogger(VoyageAIRerankResponseEntity.class); + + record RerankResult(List entries) { + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + RerankResult.class.getSimpleName(), + true, + args -> new RerankResult((List) args[0]) + ); + + static { + PARSER.declareObjectArray(constructorArg(), RerankResultEntry.PARSER::apply, new ParseField("data")); + } + } + + record RerankResultEntry(Float relevanceScore, Integer index, @Nullable String document) { + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + RerankResultEntry.class.getSimpleName(), + args -> new RerankResultEntry((Float) args[0], (Integer) args[1], (String) args[2]) + ); + + static { + PARSER.declareFloat(constructorArg(), new ParseField("relevance_score")); + PARSER.declareInt(constructorArg(), new ParseField("index")); + PARSER.declareString(optionalConstructorArg(), new ParseField("document")); + } + + public RankedDocsResults.RankedDoc toRankedDoc() { + return new RankedDocsResults.RankedDoc(index, relevanceScore, document); + } + } + + /** + * Parses the VoyageAI ranked response. + * For a request like: + * "model": "rerank-2", + * "query": "What is the capital of the United States?", + * "top_k": 2, + * "documents": ["Carson City is the capital city of the American state of Nevada.", + * "The Commonwealth of the Northern Mariana ... Its capital is Saipan.", + * "Washington, D.C. (also known as simply Washington or D.C., ... It is a federal district.", + * "Capital punishment (the death penalty) ... As of 2017, capital punishment is legal in 30 of the 50 states."] + *

+ * The response will look like (without whitespace): + * { + * "object": "list", + * "data": [ + * { + * "relevance_score": 0.4375, + * "index": 0 + * }, + * { + * "relevance_score": 0.421875, + * "index": 1 + * } + * ], + * "model": "rerank-2", + * "usage": { + * "total_tokens": 26 + * } + * } + * @param response the http response from VoyageAI + * @return the parsed response + * @throws IOException if there is an error parsing the response + */ + public static InferenceServiceResults fromResponse(HttpResult response) throws IOException { + var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); + + try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, response.body())) { + var rerankResult = RerankResult.PARSER.apply(jsonParser, null); + + return new RankedDocsResults(rerankResult.entries.stream().map(RerankResultEntry::toRankedDoc).toList()); + } + } + + private VoyageAIRerankResponseEntity() {} +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/voyageai/VoyageAIAccount.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/voyageai/VoyageAIAccount.java new file mode 100644 index 0000000000000..bb7de2fc8ad0c --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/voyageai/VoyageAIAccount.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.voyageai; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIModel; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Objects; + +public record VoyageAIAccount(URI uri, SecureString apiKey) { + + public static VoyageAIAccount of(VoyageAIModel model) { + try { + var uri = model.buildUri(); + return new VoyageAIAccount(uri, model.apiKey()); + } catch (URISyntaxException e) { + // using bad request here so that potentially sensitive URL information does not get logged + throw new ElasticsearchStatusException("Failed to construct VoyageAI URL", RestStatus.BAD_REQUEST, e); + } + } + + public VoyageAIAccount { + Objects.requireNonNull(uri); + Objects.requireNonNull(apiKey); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/voyageai/VoyageAIResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/voyageai/VoyageAIResponseHandler.java new file mode 100644 index 0000000000000..1611426034a52 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/voyageai/VoyageAIResponseHandler.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.voyageai; + +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; +import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; +import org.elasticsearch.xpack.inference.external.http.retry.RetryException; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.elasticsearch.xpack.inference.external.response.voyageai.VoyageAIErrorResponseEntity; + +/** + * Defines how to handle various errors returned from the VoyageAI integration. + * + */ +public class VoyageAIResponseHandler extends BaseResponseHandler { + static final String VALIDATION_ERROR_MESSAGE = "Received an input validation error response"; + static final String PAYMENT_ERROR_MESSAGE = "Payment required"; + + public VoyageAIResponseHandler(String requestType, ResponseParser parseFunction) { + super(requestType, parseFunction, VoyageAIErrorResponseEntity::fromResponse); + } + + /** + * Validates the status code throws an RetryException if not in the range [200, 300). + * + * @param request The http request + * @param result The http response and body + * @throws RetryException Throws if status code is {@code >= 300 or < 200 } + */ + @Override + protected void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { + if (result.isSuccessfulResponse()) { + return; + } + + // handle error codes + int statusCode = result.response().getStatusLine().getStatusCode(); + if (statusCode == 500) { + throw new RetryException(true, buildError(SERVER_ERROR, request, result)); + } else if (statusCode > 500) { + throw new RetryException(false, buildError(SERVER_ERROR, request, result)); + } else if (statusCode == 429) { + throw new RetryException(true, buildError(RATE_LIMIT, request, result)); + } else if (statusCode == 400 || statusCode == 422) { + throw new RetryException(false, buildError(VALIDATION_ERROR_MESSAGE, request, result)); + } else if (statusCode == 401) { + throw new RetryException(false, buildError(AUTHENTICATION, request, result)); + } else if (statusCode == 402) { + throw new RetryException(false, buildError(PAYMENT_ERROR_MESSAGE, request, result)); + } else if (statusCode >= 300 && statusCode < 400) { + throw new RetryException(false, buildError(REDIRECTION, request, result)); + } else { + throw new RetryException(false, buildError(UNSUCCESSFUL, request, result)); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index 13d641101a1cf..7330d45b6f16c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -22,8 +22,8 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.inference.results.TextEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; import org.elasticsearch.xpack.inference.services.settings.ApiKeySecrets; @@ -741,7 +741,7 @@ public static void getEmbeddingSize(Model model, InferenceService service, Actio InputType.INGEST, InferenceAction.Request.DEFAULT_TIMEOUT, listener.delegateFailureAndWrap((delegate, r) -> { - if (r instanceof TextEmbedding embeddingResults) { + if (r instanceof TextEmbeddingResults embeddingResults) { try { delegate.onResponse(embeddingResults.getFirstEmbeddingSize()); } catch (Exception e) { @@ -754,7 +754,7 @@ public static void getEmbeddingSize(Model model, InferenceService service, Actio new ElasticsearchStatusException( "Could not determine embedding size. " + "Expected a result of type [" - + InferenceTextEmbeddingFloatResults.NAME + + TextEmbeddingFloatResults.NAME + "] got [" + r.getWriteableName() + "]", diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java index 589ca1e033f06..dd2b29ec3efe6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchService.java @@ -308,7 +308,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), EMBEDDING_MAX_BATCH_SIZE, - getEmbeddingTypeFromTaskType(alibabaCloudSearchModel.getTaskType()), alibabaCloudSearchModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); @@ -318,14 +317,6 @@ protected void doChunkedInfer( } } - private EmbeddingRequestChunker.EmbeddingType getEmbeddingTypeFromTaskType(TaskType taskType) { - return switch (taskType) { - case TEXT_EMBEDDING -> EmbeddingRequestChunker.EmbeddingType.FLOAT; - case SPARSE_EMBEDDING -> EmbeddingRequestChunker.EmbeddingType.SPARSE; - default -> throw new IllegalArgumentException("Unsupported task type for chunking: " + taskType); - }; - } - /** * For text embedding models get the embedding size and * update the service settings. diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java index 493acd3c0cd1a..b9361a2e66232 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java @@ -132,7 +132,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), maxBatchSize, - EmbeddingRequestChunker.EmbeddingType.FLOAT, baseAmazonBedrockModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java index 34a5c2b4cc1e9..c82d0753edee0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java @@ -124,7 +124,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.FLOAT, baseAzureAiStudioModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java index 9a77b63337978..0f3e84e7c13e9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java @@ -284,7 +284,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.FLOAT, azureOpenAiModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index 6c2d3bb96d74d..08ab5f1ba86d0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -286,7 +286,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(model.getServiceSettings().elementType()), cohereModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java index 849babe1a9ab7..9ea816757fe13 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java @@ -33,7 +33,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; @@ -578,7 +578,7 @@ public void checkModelConfig(Model model, ActionListener listener) { private static List translateToChunkedResults(InferenceInputs inputs, InferenceServiceResults inferenceResults) { if (inferenceResults instanceof SparseEmbeddingResults sparseEmbeddingResults) { var inputsAsList = DocumentsOnlyInput.of(inputs).getInputs(); - return ChunkedInferenceEmbeddingSparse.listOf(inputsAsList, sparseEmbeddingResults); + return ChunkedInferenceEmbedding.listOf(inputsAsList, sparseEmbeddingResults); } else if (inferenceResults instanceof ErrorInferenceResults error) { return List.of(new ChunkedInferenceError(error.getException())); } else { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index ddc5e3e1aa36c..1beb476832b2a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -35,9 +35,9 @@ import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.action.GetDeploymentStatsAction; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.InferModelAction; @@ -635,7 +635,7 @@ public void inferTextEmbedding( ); ActionListener mlResultsListener = listener.delegateFailureAndWrap( - (l, inferenceResult) -> l.onResponse(InferenceTextEmbeddingFloatResults.of(inferenceResult.getInferenceResults())) + (l, inferenceResult) -> l.onResponse(TextEmbeddingFloatResults.of(inferenceResult.getInferenceResults())) ); var maybeDeployListener = mlResultsListener.delegateResponse( @@ -728,7 +728,6 @@ public void chunkedInfer( List batchedRequests = new EmbeddingRequestChunker( input, EMBEDDING_MAX_BATCH_SIZE, - embeddingTypeFromTaskTypeAndSettings(model.getTaskType(), esModel.internalServiceSettings), esModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); @@ -751,13 +750,11 @@ private static void translateToChunkedResult( ActionListener chunkPartListener ) { if (taskType == TaskType.TEXT_EMBEDDING) { - var translated = new ArrayList(); + var translated = new ArrayList(); for (var inferenceResult : inferenceResults) { if (inferenceResult instanceof MlTextEmbeddingResults mlTextEmbeddingResult) { - translated.add( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(mlTextEmbeddingResult.getInferenceAsFloat()) - ); + translated.add(new TextEmbeddingFloatResults.Embedding(mlTextEmbeddingResult.getInferenceAsFloat())); } else if (inferenceResult instanceof ErrorInferenceResults error) { chunkPartListener.onFailure(error.getException()); return; @@ -768,7 +765,7 @@ private static void translateToChunkedResult( return; } } - chunkPartListener.onResponse(new InferenceTextEmbeddingFloatResults(translated)); + chunkPartListener.onResponse(new TextEmbeddingFloatResults(translated)); } else { // sparse var translated = new ArrayList(); @@ -946,23 +943,6 @@ boolean isDefaultId(String inferenceId) { return DEFAULT_ELSER_ID.equals(inferenceId) || DEFAULT_E5_ID.equals(inferenceId) || DEFAULT_RERANK_ID.equals(inferenceId); } - static EmbeddingRequestChunker.EmbeddingType embeddingTypeFromTaskTypeAndSettings( - TaskType taskType, - ElasticsearchInternalServiceSettings serviceSettings - ) { - return switch (taskType) { - case SPARSE_EMBEDDING -> EmbeddingRequestChunker.EmbeddingType.SPARSE; - case TEXT_EMBEDDING -> serviceSettings.elementType() == null - ? EmbeddingRequestChunker.EmbeddingType.FLOAT - : EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(serviceSettings.elementType()); - default -> throw new ElasticsearchStatusException( - "Chunking is not supported for task type [{}]", - RestStatus.BAD_REQUEST, - taskType - ); - }; - } - private void validateAgainstDeployment( String modelId, String deploymentId, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java index 205cc545a23f0..29b5fca47eabe 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioService.java @@ -331,7 +331,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.FLOAT, googleAiStudioModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java index 3e921f669e864..29f7cdee75704 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java @@ -231,7 +231,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.FLOAT, googleVertexAiModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java index 73c1446b9bb26..ce1a31c90ed79 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java @@ -130,7 +130,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.FLOAT, huggingFaceModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java index 79001f17a4e96..434b94e6f8ac4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java @@ -25,11 +25,10 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; @@ -111,16 +110,16 @@ protected void doChunkedInfer( } private static List translateToChunkedResults(DocumentsOnlyInput inputs, InferenceServiceResults inferenceResults) { - if (inferenceResults instanceof InferenceTextEmbeddingFloatResults textEmbeddingResults) { + if (inferenceResults instanceof TextEmbeddingFloatResults textEmbeddingResults) { validateInputSizeAgainstEmbeddings(inputs.getInputs(), textEmbeddingResults.embeddings().size()); var results = new ArrayList(inputs.getInputs().size()); for (int i = 0; i < inputs.getInputs().size(); i++) { results.add( - new ChunkedInferenceEmbeddingFloat( + new ChunkedInferenceEmbedding( List.of( - new ChunkedInferenceEmbeddingFloat.FloatEmbeddingChunk( + new TextEmbeddingFloatResults.Chunk( textEmbeddingResults.embeddings().get(i).values(), inputs.getInputs().get(i), new ChunkedInference.TextOffset(0, inputs.getInputs().get(i).length()) @@ -132,13 +131,13 @@ private static List translateToChunkedResults(DocumentsOnlyInp return results; } else if (inferenceResults instanceof SparseEmbeddingResults sparseEmbeddingResults) { var inputsAsList = DocumentsOnlyInput.of(inputs).getInputs(); - return ChunkedInferenceEmbeddingSparse.listOf(inputsAsList, sparseEmbeddingResults); + return ChunkedInferenceEmbedding.listOf(inputsAsList, sparseEmbeddingResults); } else if (inferenceResults instanceof ErrorInferenceResults error) { return List.of(new ChunkedInferenceError(error.getException())); } else { String expectedClasses = Strings.format( "One of [%s,%s]", - InferenceTextEmbeddingFloatResults.class.getSimpleName(), + TextEmbeddingFloatResults.class.getSimpleName(), SparseEmbeddingResults.class.getSimpleName() ); throw createInvalidChunkedResultException(expectedClasses, inferenceResults.getWriteableName()); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java index 3fa423c2dae19..5991870489687 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxService.java @@ -310,7 +310,6 @@ protected void doChunkedInfer( var batchedRequests = new EmbeddingRequestChunker( input.getInputs(), EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.FLOAT, model.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); for (var request : batchedRequests) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java index 37add1e264704..71917151623f0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIService.java @@ -268,7 +268,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(model.getServiceSettings().elementType()), jinaaiModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java index 3e40575e42faf..1f50173951dba 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java @@ -113,7 +113,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), MistralConstants.MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.FLOAT, mistralEmbeddingsModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index 94312a39882fd..4651781c3a4dc 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -322,7 +322,6 @@ protected void doChunkedInfer( List batchedRequests = new EmbeddingRequestChunker( inputs.getInputs(), EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.FLOAT, openAiModel.getConfigurations().getChunkingSettings() ).batchRequestsWithListeners(listener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/TextEmbeddingModelValidator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/TextEmbeddingModelValidator.java index 1fe5c684196fe..766d7436d3295 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/TextEmbeddingModelValidator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/validation/TextEmbeddingModelValidator.java @@ -14,8 +14,8 @@ import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.Model; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.inference.results.TextEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults; public class TextEmbeddingModelValidator implements ModelValidator { @@ -33,7 +33,7 @@ public void validate(InferenceService service, Model model, ActionListener embeddingResults) { var serviceSettings = model.getServiceSettings(); var dimensions = serviceSettings.dimensions(); int embeddingSize = getEmbeddingSize(embeddingResults); @@ -58,7 +58,7 @@ private Model postValidate(InferenceService service, Model model, InferenceServi throw new ElasticsearchStatusException( "Validation call did not return expected results type." + "Expected a result of type [" - + InferenceTextEmbeddingFloatResults.NAME + + TextEmbeddingFloatResults.NAME + "] got [" + (results == null ? "null" : results.getWriteableName()) + "]", @@ -67,7 +67,7 @@ private Model postValidate(InferenceService service, Model model, InferenceServi } } - private int getEmbeddingSize(TextEmbedding embeddingResults) { + private int getEmbeddingSize(TextEmbeddingResults embeddingResults) { int embeddingSize; try { embeddingSize = embeddingResults.getFirstEmbeddingSize(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIModel.java new file mode 100644 index 0000000000000..e63a716b96617 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIModel.java @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai; + +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.voyageai.VoyageAIActionVisitor; +import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.settings.ApiKeySecrets; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.Objects; + +public abstract class VoyageAIModel extends Model { + private final SecureString apiKey; + private final VoyageAIRateLimitServiceSettings rateLimitServiceSettings; + protected final URI uri; + + public VoyageAIModel( + ModelConfigurations configurations, + ModelSecrets secrets, + @Nullable ApiKeySecrets apiKeySecrets, + VoyageAIRateLimitServiceSettings rateLimitServiceSettings + ) { + this(configurations, secrets, apiKeySecrets, rateLimitServiceSettings, null); + } + + public VoyageAIModel( + ModelConfigurations configurations, + ModelSecrets secrets, + @Nullable ApiKeySecrets apiKeySecrets, + VoyageAIRateLimitServiceSettings rateLimitServiceSettings, + String url + ) { + super(configurations, secrets); + + this.rateLimitServiceSettings = Objects.requireNonNull(rateLimitServiceSettings); + this.apiKey = ServiceUtils.apiKey(apiKeySecrets); + this.uri = url == null ? null : URI.create(url); + } + + protected VoyageAIModel(VoyageAIModel model, TaskSettings taskSettings) { + super(model, taskSettings); + + this.rateLimitServiceSettings = model.rateLimitServiceSettings(); + this.apiKey = model.apiKey(); + this.uri = model.uri; + } + + protected VoyageAIModel(VoyageAIModel model, ServiceSettings serviceSettings) { + super(model, serviceSettings); + + this.rateLimitServiceSettings = model.rateLimitServiceSettings(); + this.apiKey = model.apiKey(); + this.uri = model.uri; + } + + public SecureString apiKey() { + return apiKey; + } + + public VoyageAIRateLimitServiceSettings rateLimitServiceSettings() { + return rateLimitServiceSettings; + } + + public abstract ExecutableAction accept(VoyageAIActionVisitor creator, Map taskSettings, InputType inputType); + + public URI uri() { + return uri; + } + + public URI buildUri() throws URISyntaxException { + if (uri == null) { + return buildRequestUri(); + } + return uri; + } + + protected abstract URI buildRequestUri() throws URISyntaxException; +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIRateLimitServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIRateLimitServiceSettings.java new file mode 100644 index 0000000000000..a4b325fe2db41 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIRateLimitServiceSettings.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai; + +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; + +public interface VoyageAIRateLimitServiceSettings { + RateLimitSettings rateLimitSettings(); + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIService.java new file mode 100644 index 0000000000000..16659f075c564 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIService.java @@ -0,0 +1,396 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.util.LazyInitializable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInference; +import org.elasticsearch.inference.ChunkingSettings; +import org.elasticsearch.inference.InferenceServiceConfiguration; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.SettingsConfiguration; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.configuration.SettingsConfigurationFieldType; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; +import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; +import org.elasticsearch.xpack.inference.external.action.voyageai.VoyageAIActionCreator; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; +import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.UnifiedChatInput; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.SenderService; +import org.elasticsearch.xpack.inference.services.ServiceComponents; +import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.validation.ModelValidatorBuilder; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankModel; + +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwUnsupportedUnifiedCompletionOperation; + +public class VoyageAIService extends SenderService { + public static final String NAME = "voyageai"; + + private static final String SERVICE_NAME = "Voyage AI"; + private static final EnumSet supportedTaskTypes = EnumSet.of(TaskType.TEXT_EMBEDDING, TaskType.RERANK); + + private static final Integer DEFAULT_BATCH_SIZE = 7; + private static final Map MODEL_BATCH_SIZES = Map.of( + "voyage-multimodal-3", + 7, + "voyage-3-large", + 7, + "voyage-code-3", + 7, + "voyage-3", + 10, + "voyage-3-lite", + 30, + "voyage-finance-2", + 7, + "voyage-law-2", + 7, + "voyage-code-2", + 7, + "voyage-2", + 72, + "voyage-02", + 72 + ); + + public VoyageAIService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { + super(factory, serviceComponents); + } + + @Override + public String name() { + return NAME; + } + + @Override + public void parseRequestConfig( + String inferenceEntityId, + TaskType taskType, + Map config, + ActionListener parsedModelListener + ) { + try { + Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); + Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + + ChunkingSettings chunkingSettings = null; + if (TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap( + removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS) + ); + } + VoyageAIModel model = createModel( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + chunkingSettings, + serviceSettingsMap, + TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), + ConfigurationParseContext.REQUEST + ); + + throwIfNotEmptyMap(config, NAME); + throwIfNotEmptyMap(serviceSettingsMap, NAME); + throwIfNotEmptyMap(taskSettingsMap, NAME); + + parsedModelListener.onResponse(model); + } catch (Exception e) { + parsedModelListener.onFailure(e); + } + } + + private static VoyageAIModel createModelFromPersistent( + String inferenceEntityId, + TaskType taskType, + Map serviceSettings, + Map taskSettings, + ChunkingSettings chunkingSettings, + @Nullable Map secretSettings, + String failureMessage + ) { + return createModel( + inferenceEntityId, + taskType, + serviceSettings, + taskSettings, + chunkingSettings, + secretSettings, + failureMessage, + ConfigurationParseContext.PERSISTENT + ); + } + + private static VoyageAIModel createModel( + String inferenceEntityId, + TaskType taskType, + Map serviceSettings, + Map taskSettings, + ChunkingSettings chunkingSettings, + @Nullable Map secretSettings, + String failureMessage, + ConfigurationParseContext context + ) { + return switch (taskType) { + case TEXT_EMBEDDING -> new VoyageAIEmbeddingsModel( + inferenceEntityId, + NAME, + serviceSettings, + taskSettings, + chunkingSettings, + secretSettings, + context + ); + case RERANK -> new VoyageAIRerankModel(inferenceEntityId, NAME, serviceSettings, taskSettings, secretSettings, context); + default -> throw new ElasticsearchStatusException(failureMessage, RestStatus.BAD_REQUEST); + }; + } + + @Override + public VoyageAIModel parsePersistedConfigWithSecrets( + String inferenceEntityId, + TaskType taskType, + Map config, + Map secrets + ) { + Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); + Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + Map secretSettingsMap = removeFromMapOrThrowIfNull(secrets, ModelSecrets.SECRET_SETTINGS); + + ChunkingSettings chunkingSettings = null; + if (TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMap(config, ModelConfigurations.CHUNKING_SETTINGS)); + } + + return createModelFromPersistent( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + chunkingSettings, + secretSettingsMap, + parsePersistedConfigErrorMsg(inferenceEntityId, NAME) + ); + } + + @Override + public VoyageAIModel parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map config) { + Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); + Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + + ChunkingSettings chunkingSettings = null; + if (TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMap(config, ModelConfigurations.CHUNKING_SETTINGS)); + } + + return createModelFromPersistent( + inferenceEntityId, + taskType, + serviceSettingsMap, + taskSettingsMap, + chunkingSettings, + null, + parsePersistedConfigErrorMsg(inferenceEntityId, NAME) + ); + } + + @Override + public InferenceServiceConfiguration getConfiguration() { + return Configuration.get(); + } + + @Override + public EnumSet supportedTaskTypes() { + return supportedTaskTypes; + } + + @Override + protected void doUnifiedCompletionInfer( + Model model, + UnifiedChatInput inputs, + TimeValue timeout, + ActionListener listener + ) { + throwUnsupportedUnifiedCompletionOperation(NAME); + } + + @Override + public void doInfer( + Model model, + InferenceInputs inputs, + Map taskSettings, + InputType inputType, + TimeValue timeout, + ActionListener listener + ) { + if (model instanceof VoyageAIModel == false) { + listener.onFailure(createInvalidModelException(model)); + return; + } + + VoyageAIModel voyageaiModel = (VoyageAIModel) model; + var actionCreator = new VoyageAIActionCreator(getSender(), getServiceComponents()); + + var action = voyageaiModel.accept(actionCreator, taskSettings, inputType); + action.execute(inputs, timeout, listener); + } + + @Override + protected void doChunkedInfer( + Model model, + DocumentsOnlyInput inputs, + Map taskSettings, + InputType inputType, + TimeValue timeout, + ActionListener> listener + ) { + if (model instanceof VoyageAIModel == false) { + listener.onFailure(createInvalidModelException(model)); + return; + } + + VoyageAIModel voyageaiModel = (VoyageAIModel) model; + var actionCreator = new VoyageAIActionCreator(getSender(), getServiceComponents()); + + List batchedRequests = new EmbeddingRequestChunker( + inputs.getInputs(), + getBatchSize(voyageaiModel), + voyageaiModel.getConfigurations().getChunkingSettings() + ).batchRequestsWithListeners(listener); + + for (var request : batchedRequests) { + var action = voyageaiModel.accept(actionCreator, taskSettings, inputType); + action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); + } + } + + private static int getBatchSize(VoyageAIModel model) { + return MODEL_BATCH_SIZES.getOrDefault(model.getServiceSettings().modelId(), DEFAULT_BATCH_SIZE); + } + + /** + * For text embedding models get the embedding size and + * update the service settings. + * + * @param model The new model + * @param listener The listener + */ + @Override + public void checkModelConfig(Model model, ActionListener listener) { + ModelValidatorBuilder.buildModelValidator(model.getTaskType()).validate(this, model, listener); + } + + @Override + public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { + if (model instanceof VoyageAIEmbeddingsModel embeddingsModel) { + var serviceSettings = embeddingsModel.getServiceSettings(); + var similarityFromModel = serviceSettings.similarity(); + var similarityToUse = similarityFromModel == null ? defaultSimilarity() : similarityFromModel; + var maxInputTokens = serviceSettings.maxInputTokens(); + var dimensionSetByUser = serviceSettings.dimensionsSetByUser(); + + var updatedServiceSettings = new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings( + serviceSettings.getCommonSettings().modelId(), + serviceSettings.getCommonSettings().rateLimitSettings() + ), + serviceSettings.getEmbeddingType(), + similarityToUse, + embeddingSize, + maxInputTokens, + dimensionSetByUser + ); + + return new VoyageAIEmbeddingsModel(embeddingsModel, updatedServiceSettings); + } else { + throw ServiceUtils.invalidModelTypeForUpdateModelWithEmbeddingDetails(model.getClass()); + } + } + + /** + * Return the default similarity measure for the embedding type. + * VoyageAI embeddings are normalized to unit vectors therefore Dot + * Product similarity can be used and is the default for all VoyageAI + * models. + * + * @return The default similarity. + */ + static SimilarityMeasure defaultSimilarity() { + return SimilarityMeasure.DOT_PRODUCT; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.VOYAGE_AI_INTEGRATION_ADDED; + } + + public static class Configuration { + public static InferenceServiceConfiguration get() { + return configuration.getOrCompute(); + } + + private static final LazyInitializable configuration = new LazyInitializable<>( + () -> { + var configurationMap = new HashMap(); + + configurationMap.put( + MODEL_ID, + new SettingsConfiguration.Builder(supportedTaskTypes).setDescription( + "The name of the model to use for the inference task." + ) + .setLabel("Model ID") + .setRequired(true) + .setSensitive(false) + .setUpdatable(false) + .setType(SettingsConfigurationFieldType.STRING) + .build() + ); + + configurationMap.putAll(DefaultSecretSettings.toSettingsConfiguration(supportedTaskTypes)); + configurationMap.putAll(RateLimitSettings.toSettingsConfiguration(supportedTaskTypes)); + + return new InferenceServiceConfiguration.Builder().setService(NAME) + .setName(SERVICE_NAME) + .setTaskTypes(supportedTaskTypes) + .setConfigurations(configurationMap) + .build(); + } + ); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingInt.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceFields.java similarity index 62% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingInt.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceFields.java index 05fc8a3cef1b6..b36f212b61e5d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/EmbeddingInt.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceFields.java @@ -5,8 +5,8 @@ * 2.0. */ -package org.elasticsearch.xpack.core.inference.results; +package org.elasticsearch.xpack.inference.services.voyageai; -public interface EmbeddingInt { - int getSize(); +public class VoyageAIServiceFields { + public static final String TRUNCATION = "truncation"; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceSettings.java new file mode 100644 index 0000000000000..75497d1a4b4f0 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceSettings.java @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString; + +public class VoyageAIServiceSettings extends FilteredXContentObject implements ServiceSettings, VoyageAIRateLimitServiceSettings { + + public static final String NAME = "voyageai_service_settings"; + public static final String MODEL_ID = "model_id"; + private static final Logger logger = LogManager.getLogger(VoyageAIServiceSettings.class); + // See https://docs.voyageai.com/docs/rate-limits + public static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(2_000); + + public static VoyageAIServiceSettings fromMap(Map map, ConfigurationParseContext context) { + ValidationException validationException = new ValidationException(); + + RateLimitSettings rateLimitSettings = RateLimitSettings.of( + map, + DEFAULT_RATE_LIMIT_SETTINGS, + validationException, + VoyageAIService.NAME, + context + ); + + String modelId = extractRequiredString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return new VoyageAIServiceSettings(modelId, rateLimitSettings); + } + + private final String modelId; + private final RateLimitSettings rateLimitSettings; + + public VoyageAIServiceSettings(String modelId, @Nullable RateLimitSettings rateLimitSettings) { + this.modelId = Objects.requireNonNull(modelId); + this.rateLimitSettings = Objects.requireNonNullElse(rateLimitSettings, DEFAULT_RATE_LIMIT_SETTINGS); + } + + public VoyageAIServiceSettings(StreamInput in) throws IOException { + modelId = in.readString(); + rateLimitSettings = new RateLimitSettings(in); + } + + @Override + public RateLimitSettings rateLimitSettings() { + return rateLimitSettings; + } + + @Override + public String modelId() { + return modelId; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + toXContentFragment(builder, params); + + builder.endObject(); + return builder; + } + + public XContentBuilder toXContentFragment(XContentBuilder builder, Params params) throws IOException { + return toXContentFragmentOfExposedFields(builder, params); + } + + @Override + public XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException { + builder.field(MODEL_ID, modelId); + rateLimitSettings.toXContent(builder, params); + + return builder; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.VOYAGE_AI_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(modelId); + rateLimitSettings.writeTo(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VoyageAIServiceSettings that = (VoyageAIServiceSettings) o; + return Objects.equals(modelId, that.modelId) && Objects.equals(rateLimitSettings, that.rateLimitSettings); + } + + @Override + public int hashCode() { + return Objects.hash(modelId, rateLimitSettings); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingType.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingType.java new file mode 100644 index 0000000000000..db13e46b14641 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingType.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.embeddings; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.Locale; +import java.util.Map; + +/** + * Defines the type of embedding that the VoyageAI api should return for a request. + * + *

+ * See api docs for details. + *

+ */ +public enum VoyageAIEmbeddingType { + /** + * Use this when you want to get back the default float embeddings. Valid for all models. + */ + FLOAT(DenseVectorFieldMapper.ElementType.FLOAT, RequestConstants.FLOAT), + /** + * Use this when you want to get back signed int8 embeddings. Valid for only v3 models. + */ + INT8(DenseVectorFieldMapper.ElementType.BYTE, RequestConstants.INT8), + /** + * This is a synonym for INT8 + */ + BYTE(DenseVectorFieldMapper.ElementType.BYTE, RequestConstants.INT8), + /** + * Use this when you want to get back binary embeddings. Valid only for v3 models. + */ + BIT(DenseVectorFieldMapper.ElementType.BIT, RequestConstants.BINARY), + /** + * This is a synonym for BIT + */ + BINARY(DenseVectorFieldMapper.ElementType.BIT, RequestConstants.BINARY); + + private static final class RequestConstants { + private static final String FLOAT = "float"; + private static final String INT8 = "int8"; + private static final String BINARY = "binary"; + } + + private static final Map ELEMENT_TYPE_TO_VOYAGE_EMBEDDING = Map.of( + DenseVectorFieldMapper.ElementType.FLOAT, + FLOAT, + DenseVectorFieldMapper.ElementType.BYTE, + BYTE, + DenseVectorFieldMapper.ElementType.BIT, + BIT + ); + static final EnumSet SUPPORTED_ELEMENT_TYPES = EnumSet.copyOf( + ELEMENT_TYPE_TO_VOYAGE_EMBEDDING.keySet() + ); + + private final DenseVectorFieldMapper.ElementType elementType; + private final String requestString; + + VoyageAIEmbeddingType(DenseVectorFieldMapper.ElementType elementType, String requestString) { + this.elementType = elementType; + this.requestString = requestString; + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + + public String toRequestString() { + return requestString; + } + + public static String toLowerCase(VoyageAIEmbeddingType type) { + return type.toString().toLowerCase(Locale.ROOT); + } + + public static VoyageAIEmbeddingType fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + public static VoyageAIEmbeddingType fromElementType(DenseVectorFieldMapper.ElementType elementType) { + var embedding = ELEMENT_TYPE_TO_VOYAGE_EMBEDDING.get(elementType); + + if (embedding == null) { + var validElementTypes = SUPPORTED_ELEMENT_TYPES.stream() + .map(value -> value.toString().toLowerCase(Locale.ROOT)) + .toArray(String[]::new); + Arrays.sort(validElementTypes); + + throw new IllegalArgumentException( + Strings.format( + "Element type [%s] does not map to a VoyageAI embedding value, must be one of [%s]", + elementType, + String.join(", ", validElementTypes) + ) + ); + } + + return embedding; + } + + public DenseVectorFieldMapper.ElementType toElementType() { + return elementType; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsModel.java new file mode 100644 index 0000000000000..41194f6862a44 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsModel.java @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.embeddings; + +import org.apache.http.client.utils.URIBuilder; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkingSettings; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.voyageai.VoyageAIActionVisitor; +import org.elasticsearch.xpack.inference.external.request.voyageai.VoyageAIUtils; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIModel; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.external.request.voyageai.VoyageAIUtils.HOST; + +public class VoyageAIEmbeddingsModel extends VoyageAIModel { + public static VoyageAIEmbeddingsModel of(VoyageAIEmbeddingsModel model, Map taskSettings, InputType inputType) { + var requestTaskSettings = VoyageAIEmbeddingsTaskSettings.fromMap(taskSettings); + return new VoyageAIEmbeddingsModel( + model, + VoyageAIEmbeddingsTaskSettings.of(model.getTaskSettings(), requestTaskSettings, inputType) + ); + } + + public VoyageAIEmbeddingsModel( + String inferenceId, + String service, + Map serviceSettings, + Map taskSettings, + ChunkingSettings chunkingSettings, + @Nullable Map secrets, + ConfigurationParseContext context + ) { + this( + inferenceId, + service, + VoyageAIEmbeddingsServiceSettings.fromMap(serviceSettings, context), + VoyageAIEmbeddingsTaskSettings.fromMap(taskSettings), + chunkingSettings, + DefaultSecretSettings.fromMap(secrets) + ); + } + + // should only be used for testing + VoyageAIEmbeddingsModel( + String modelId, + String service, + VoyageAIEmbeddingsServiceSettings serviceSettings, + VoyageAIEmbeddingsTaskSettings taskSettings, + ChunkingSettings chunkingSettings, + @Nullable DefaultSecretSettings secretSettings + ) { + super( + new ModelConfigurations(modelId, TaskType.TEXT_EMBEDDING, service, serviceSettings, taskSettings, chunkingSettings), + new ModelSecrets(secretSettings), + secretSettings, + serviceSettings.getCommonSettings() + ); + } + + VoyageAIEmbeddingsModel( + String modelId, + String service, + String url, + VoyageAIEmbeddingsServiceSettings serviceSettings, + VoyageAIEmbeddingsTaskSettings taskSettings, + ChunkingSettings chunkingSettings, + @Nullable DefaultSecretSettings secretSettings + ) { + super( + new ModelConfigurations(modelId, TaskType.TEXT_EMBEDDING, service, serviceSettings, taskSettings, chunkingSettings), + new ModelSecrets(secretSettings), + secretSettings, + serviceSettings.getCommonSettings(), + url + ); + } + + private VoyageAIEmbeddingsModel(VoyageAIEmbeddingsModel model, VoyageAIEmbeddingsTaskSettings taskSettings) { + super(model, taskSettings); + } + + public VoyageAIEmbeddingsModel(VoyageAIEmbeddingsModel model, VoyageAIEmbeddingsServiceSettings serviceSettings) { + super(model, serviceSettings); + } + + @Override + public VoyageAIEmbeddingsServiceSettings getServiceSettings() { + return (VoyageAIEmbeddingsServiceSettings) super.getServiceSettings(); + } + + @Override + public VoyageAIEmbeddingsTaskSettings getTaskSettings() { + return (VoyageAIEmbeddingsTaskSettings) super.getTaskSettings(); + } + + @Override + public DefaultSecretSettings getSecretSettings() { + return (DefaultSecretSettings) super.getSecretSettings(); + } + + @Override + public ExecutableAction accept(VoyageAIActionVisitor visitor, Map taskSettings, InputType inputType) { + return visitor.create(this, taskSettings, inputType); + } + + protected URI buildRequestUri() throws URISyntaxException { + return new URIBuilder().setScheme("https") + .setHost(HOST) + .setPathSegments(VoyageAIUtils.VERSION_1, VoyageAIUtils.EMBEDDINGS_PATH) + .build(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsServiceSettings.java new file mode 100644 index 0000000000000..cc4db278d0e2b --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsServiceSettings.java @@ -0,0 +1,259 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.embeddings; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceSettings; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceFields.DIMENSIONS; +import static org.elasticsearch.xpack.inference.services.ServiceFields.MAX_INPUT_TOKENS; +import static org.elasticsearch.xpack.inference.services.ServiceFields.SIMILARITY; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalEnum; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeAsType; + +public class VoyageAIEmbeddingsServiceSettings extends FilteredXContentObject implements ServiceSettings { + public static final String NAME = "voyageai_embeddings_service_settings"; + static final String DIMENSIONS_SET_BY_USER = "dimensions_set_by_user"; + public static final VoyageAIEmbeddingsServiceSettings EMPTY_SETTINGS = new VoyageAIEmbeddingsServiceSettings( + null, + null, + null, + null, + null, + false + ); + + public static final String EMBEDDING_TYPE = "embedding_type"; + + public static VoyageAIEmbeddingsServiceSettings fromMap(Map map, ConfigurationParseContext context) { + return switch (context) { + case REQUEST -> fromRequestMap(map, context); + case PERSISTENT -> fromPersistentMap(map, context); + }; + } + + private static VoyageAIEmbeddingsServiceSettings fromRequestMap(Map map, ConfigurationParseContext context) { + ValidationException validationException = new ValidationException(); + var commonServiceSettings = VoyageAIServiceSettings.fromMap(map, context); + + VoyageAIEmbeddingType embeddingTypes = parseEmbeddingType(map, context, validationException); + + SimilarityMeasure similarity = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException); + Integer dims = removeAsType(map, DIMENSIONS, Integer.class); + Integer maxInputTokens = removeAsType(map, MAX_INPUT_TOKENS, Integer.class); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return new VoyageAIEmbeddingsServiceSettings(commonServiceSettings, embeddingTypes, similarity, dims, maxInputTokens, dims != null); + } + + private static VoyageAIEmbeddingsServiceSettings fromPersistentMap(Map map, ConfigurationParseContext context) { + ValidationException validationException = new ValidationException(); + var commonServiceSettings = VoyageAIServiceSettings.fromMap(map, context); + + VoyageAIEmbeddingType embeddingTypes = parseEmbeddingType(map, context, validationException); + + SimilarityMeasure similarity = extractSimilarity(map, ModelConfigurations.SERVICE_SETTINGS, validationException); + Integer dims = removeAsType(map, DIMENSIONS, Integer.class); + Integer maxInputTokens = removeAsType(map, MAX_INPUT_TOKENS, Integer.class); + + Boolean dimensionsSetByUser = removeAsType(map, DIMENSIONS_SET_BY_USER, Boolean.class); + if (dimensionsSetByUser == null) { + dimensionsSetByUser = Boolean.FALSE; + } + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return new VoyageAIEmbeddingsServiceSettings( + commonServiceSettings, + embeddingTypes, + similarity, + dims, + maxInputTokens, + dimensionsSetByUser + ); + } + + static VoyageAIEmbeddingType parseEmbeddingType( + Map map, + ConfigurationParseContext context, + ValidationException validationException + ) { + return switch (context) { + case REQUEST, PERSISTENT -> Objects.requireNonNullElse( + extractOptionalEnum( + map, + EMBEDDING_TYPE, + ModelConfigurations.SERVICE_SETTINGS, + VoyageAIEmbeddingType::fromString, + EnumSet.allOf(VoyageAIEmbeddingType.class), + validationException + ), + VoyageAIEmbeddingType.FLOAT + ); + + }; + } + + private final VoyageAIServiceSettings commonSettings; + private final VoyageAIEmbeddingType embeddingType; + private final SimilarityMeasure similarity; + private final Integer dimensions; + private final Integer maxInputTokens; + private final boolean dimensionsSetByUser; + + public VoyageAIEmbeddingsServiceSettings( + VoyageAIServiceSettings commonSettings, + @Nullable VoyageAIEmbeddingType embeddingType, + @Nullable SimilarityMeasure similarity, + @Nullable Integer dimensions, + @Nullable Integer maxInputTokens, + boolean dimensionsSetByUser + ) { + this.commonSettings = commonSettings; + this.similarity = similarity; + this.dimensions = dimensions; + this.maxInputTokens = maxInputTokens; + this.embeddingType = embeddingType; + this.dimensionsSetByUser = dimensionsSetByUser; + } + + public VoyageAIEmbeddingsServiceSettings(StreamInput in) throws IOException { + this.commonSettings = new VoyageAIServiceSettings(in); + this.similarity = in.readOptionalEnum(SimilarityMeasure.class); + this.dimensions = in.readOptionalVInt(); + this.maxInputTokens = in.readOptionalVInt(); + this.embeddingType = Objects.requireNonNullElse(in.readOptionalEnum(VoyageAIEmbeddingType.class), VoyageAIEmbeddingType.FLOAT); + this.dimensionsSetByUser = in.readBoolean(); + } + + public VoyageAIServiceSettings getCommonSettings() { + return commonSettings; + } + + @Override + public SimilarityMeasure similarity() { + return similarity; + } + + @Override + public Integer dimensions() { + return dimensions; + } + + public Integer maxInputTokens() { + return maxInputTokens; + } + + @Override + public String modelId() { + return commonSettings.modelId(); + } + + public VoyageAIEmbeddingType getEmbeddingType() { + return embeddingType; + } + + @Override + public DenseVectorFieldMapper.ElementType elementType() { + return embeddingType == null ? DenseVectorFieldMapper.ElementType.FLOAT : embeddingType.toElementType(); + } + + @Override + public Boolean dimensionsSetByUser() { + return this.dimensionsSetByUser; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder = commonSettings.toXContentFragment(builder, params); + if (similarity != null) { + builder.field(SIMILARITY, similarity); + } + if (dimensions != null) { + builder.field(DIMENSIONS, dimensions); + } + if (maxInputTokens != null) { + builder.field(MAX_INPUT_TOKENS, maxInputTokens); + } + if (embeddingType != null) { + builder.field(EMBEDDING_TYPE, embeddingType); + } + builder.endObject(); + return builder; + } + + @Override + protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException { + commonSettings.toXContentFragmentOfExposedFields(builder, params); + + return builder; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.VOYAGE_AI_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + commonSettings.writeTo(out); + out.writeOptionalEnum(SimilarityMeasure.translateSimilarity(similarity, out.getTransportVersion())); + out.writeOptionalVInt(dimensions); + out.writeOptionalVInt(maxInputTokens); + out.writeOptionalEnum(embeddingType); + out.writeBoolean(dimensionsSetByUser); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VoyageAIEmbeddingsServiceSettings that = (VoyageAIEmbeddingsServiceSettings) o; + return Objects.equals(commonSettings, that.commonSettings) + && Objects.equals(similarity, that.similarity) + && Objects.equals(dimensions, that.dimensions) + && Objects.equals(maxInputTokens, that.maxInputTokens) + && Objects.equals(embeddingType, that.embeddingType) + && Objects.equals(dimensionsSetByUser, that.dimensionsSetByUser); + } + + @Override + public int hashCode() { + return Objects.hash(commonSettings, similarity, dimensions, maxInputTokens, embeddingType, dimensionsSetByUser); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsTaskSettings.java new file mode 100644 index 0000000000000..5d8d282588349 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsTaskSettings.java @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.embeddings; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalBoolean; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalEnum; +import static org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceFields.TRUNCATION; + +/** + * Defines the task settings for the voyageai text embeddings service. + * + *

+ * See api docs for details. + *

+ */ +public class VoyageAIEmbeddingsTaskSettings implements TaskSettings { + + public static final String NAME = "voyageai_embeddings_task_settings"; + public static final VoyageAIEmbeddingsTaskSettings EMPTY_SETTINGS = new VoyageAIEmbeddingsTaskSettings(null, null); + static final String INPUT_TYPE = "input_type"; + static final EnumSet VALID_REQUEST_VALUES = EnumSet.of(InputType.INGEST, InputType.SEARCH); + + public static VoyageAIEmbeddingsTaskSettings fromMap(Map map) { + if (map == null || map.isEmpty()) { + return EMPTY_SETTINGS; + } + + ValidationException validationException = new ValidationException(); + + InputType inputType = extractOptionalEnum( + map, + INPUT_TYPE, + ModelConfigurations.TASK_SETTINGS, + InputType::fromString, + VALID_REQUEST_VALUES, + validationException + ); + Boolean truncation = extractOptionalBoolean(map, TRUNCATION, validationException); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return new VoyageAIEmbeddingsTaskSettings(inputType, truncation); + } + + /** + * Creates a new {@link VoyageAIEmbeddingsTaskSettings} by preferring non-null fields from the provided parameters. + * For the input type, preference is given to requestInputType if it is not null and not UNSPECIFIED. + * Then preference is given to the requestTaskSettings and finally to originalSettings even if the value is null. + * Similarly, for the truncation field preference is given to requestTaskSettings if it is not null and then to + * originalSettings. + * @param originalSettings the settings stored as part of the inference entity configuration + * @param requestTaskSettings the settings passed in within the task_settings field of the request + * @param requestInputType the input type passed in the request parameters + * @return a constructed {@link VoyageAIEmbeddingsTaskSettings} + */ + public static VoyageAIEmbeddingsTaskSettings of( + VoyageAIEmbeddingsTaskSettings originalSettings, + VoyageAIEmbeddingsTaskSettings requestTaskSettings, + InputType requestInputType + ) { + var inputTypeToUse = getValidInputType(originalSettings, requestTaskSettings, requestInputType); + var truncationToUse = getValidTruncation(originalSettings, requestTaskSettings); + + return new VoyageAIEmbeddingsTaskSettings(inputTypeToUse, truncationToUse); + } + + private static InputType getValidInputType( + VoyageAIEmbeddingsTaskSettings originalSettings, + VoyageAIEmbeddingsTaskSettings requestTaskSettings, + InputType requestInputType + ) { + InputType inputTypeToUse = originalSettings.inputType; + + if (VALID_REQUEST_VALUES.contains(requestInputType)) { + inputTypeToUse = requestInputType; + } else if (requestTaskSettings.inputType != null) { + inputTypeToUse = requestTaskSettings.inputType; + } + + return inputTypeToUse; + } + + private static Boolean getValidTruncation( + VoyageAIEmbeddingsTaskSettings originalSettings, + VoyageAIEmbeddingsTaskSettings requestTaskSettings + ) { + return requestTaskSettings.getTruncation() == null ? originalSettings.truncation : requestTaskSettings.getTruncation(); + } + + private final InputType inputType; + private final Boolean truncation; + + public VoyageAIEmbeddingsTaskSettings(StreamInput in) throws IOException { + this(in.readOptionalEnum(InputType.class), in.readOptionalBoolean()); + } + + public VoyageAIEmbeddingsTaskSettings(@Nullable InputType inputType, @Nullable Boolean truncation) { + validateInputType(inputType); + this.inputType = inputType; + this.truncation = truncation; + } + + private static void validateInputType(InputType inputType) { + if (inputType == null) { + return; + } + + assert VALID_REQUEST_VALUES.contains(inputType) : invalidInputTypeMessage(inputType); + } + + @Override + public boolean isEmpty() { + return inputType == null && truncation == null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (inputType != null) { + builder.field(INPUT_TYPE, inputType); + } + + if (truncation != null) { + builder.field(TRUNCATION, truncation); + } + + builder.endObject(); + return builder; + } + + public InputType getInputType() { + return inputType; + } + + public Boolean getTruncation() { + return truncation; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.VOYAGE_AI_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalEnum(inputType); + out.writeOptionalBoolean(truncation); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VoyageAIEmbeddingsTaskSettings that = (VoyageAIEmbeddingsTaskSettings) o; + return Objects.equals(inputType, that.inputType) && Objects.equals(truncation, that.truncation); + } + + @Override + public int hashCode() { + return Objects.hash(inputType, truncation); + } + + public static String invalidInputTypeMessage(InputType inputType) { + return Strings.format("received invalid input type value [%s]", inputType.toString()); + } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + VoyageAIEmbeddingsTaskSettings updatedSettings = VoyageAIEmbeddingsTaskSettings.fromMap(new HashMap<>(newSettings)); + return of(this, updatedSettings, updatedSettings.inputType != null ? updatedSettings.inputType : this.inputType); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankModel.java new file mode 100644 index 0000000000000..57c478962b5f2 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankModel.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.rerank; + +import org.apache.http.client.utils.URIBuilder; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.ModelSecrets; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.voyageai.VoyageAIActionVisitor; +import org.elasticsearch.xpack.inference.external.request.voyageai.VoyageAIUtils; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIModel; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.external.request.voyageai.VoyageAIUtils.HOST; + +public class VoyageAIRerankModel extends VoyageAIModel { + public static VoyageAIRerankModel of(VoyageAIRerankModel model, Map taskSettings) { + var requestTaskSettings = VoyageAIRerankTaskSettings.fromMap(taskSettings); + return new VoyageAIRerankModel(model, VoyageAIRerankTaskSettings.of(model.getTaskSettings(), requestTaskSettings)); + } + + public VoyageAIRerankModel( + String inferenceId, + String service, + Map serviceSettings, + Map taskSettings, + @Nullable Map secrets, + ConfigurationParseContext context + ) { + this( + inferenceId, + service, + VoyageAIRerankServiceSettings.fromMap(serviceSettings, context), + VoyageAIRerankTaskSettings.fromMap(taskSettings), + DefaultSecretSettings.fromMap(secrets) + ); + } + + // should only be used for testing + VoyageAIRerankModel( + String modelId, + String service, + VoyageAIRerankServiceSettings serviceSettings, + VoyageAIRerankTaskSettings taskSettings, + @Nullable DefaultSecretSettings secretSettings + ) { + this(modelId, service, null, serviceSettings, taskSettings, secretSettings); + } + + VoyageAIRerankModel( + String modelId, + String service, + String url, + VoyageAIRerankServiceSettings serviceSettings, + VoyageAIRerankTaskSettings taskSettings, + @Nullable DefaultSecretSettings secretSettings + ) { + super( + new ModelConfigurations(modelId, TaskType.RERANK, service, serviceSettings, taskSettings), + new ModelSecrets(secretSettings), + secretSettings, + serviceSettings.getCommonSettings(), + url + ); + } + + private VoyageAIRerankModel(VoyageAIRerankModel model, VoyageAIRerankTaskSettings taskSettings) { + super(model, taskSettings); + } + + public VoyageAIRerankModel(VoyageAIRerankModel model, VoyageAIRerankServiceSettings serviceSettings) { + super(model, serviceSettings); + } + + @Override + public VoyageAIRerankServiceSettings getServiceSettings() { + return (VoyageAIRerankServiceSettings) super.getServiceSettings(); + } + + @Override + public VoyageAIRerankTaskSettings getTaskSettings() { + return (VoyageAIRerankTaskSettings) super.getTaskSettings(); + } + + @Override + public DefaultSecretSettings getSecretSettings() { + return (DefaultSecretSettings) super.getSecretSettings(); + } + + /** + * Accepts a visitor to create an executable action. The returned action will not return documents in the response. + * @param visitor _ + * @param taskSettings _ + * @param inputType ignored for rerank task + * @return the rerank action + */ + @Override + public ExecutableAction accept(VoyageAIActionVisitor visitor, Map taskSettings, InputType inputType) { + return visitor.create(this, taskSettings); + } + + @Override + protected URI buildRequestUri() throws URISyntaxException { + return new URIBuilder().setScheme("https") + .setHost(HOST) + .setPathSegments(VoyageAIUtils.VERSION_1, VoyageAIUtils.RERANK_PATH) + .build(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankServiceSettings.java new file mode 100644 index 0000000000000..1d3607922c5c2 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankServiceSettings.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.rerank; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.settings.FilteredXContentObject; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIRateLimitServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceSettings; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class VoyageAIRerankServiceSettings extends FilteredXContentObject implements ServiceSettings, VoyageAIRateLimitServiceSettings { + public static final String NAME = "voyageai_rerank_service_settings"; + + private static final Logger logger = LogManager.getLogger(VoyageAIRerankServiceSettings.class); + + public static VoyageAIRerankServiceSettings fromMap(Map map, ConfigurationParseContext context) { + ValidationException validationException = new ValidationException(); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + var commonServiceSettings = VoyageAIServiceSettings.fromMap(map, context); + + return new VoyageAIRerankServiceSettings(commonServiceSettings); + } + + private final VoyageAIServiceSettings commonSettings; + + public VoyageAIRerankServiceSettings(VoyageAIServiceSettings commonSettings) { + this.commonSettings = commonSettings; + } + + public VoyageAIRerankServiceSettings(StreamInput in) throws IOException { + this.commonSettings = new VoyageAIServiceSettings(in); + } + + public VoyageAIServiceSettings getCommonSettings() { + return commonSettings; + } + + @Override + public String modelId() { + return commonSettings.modelId(); + } + + @Override + public RateLimitSettings rateLimitSettings() { + return commonSettings.rateLimitSettings(); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder = commonSettings.toXContentFragment(builder, params); + + builder.endObject(); + return builder; + } + + @Override + protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder builder, Params params) throws IOException { + commonSettings.toXContentFragmentOfExposedFields(builder, params); + return builder; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.VOYAGE_AI_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + commonSettings.writeTo(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VoyageAIRerankServiceSettings that = (VoyageAIRerankServiceSettings) o; + return Objects.equals(commonSettings, that.commonSettings); + } + + @Override + public int hashCode() { + return Objects.hash(commonSettings); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankTaskSettings.java new file mode 100644 index 0000000000000..a5004fde1e17e --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankTaskSettings.java @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.rerank; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.TaskSettings; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalBoolean; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger; +import static org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceFields.TRUNCATION; + +/** + * Defines the task settings for the VoyageAI rerank service. + * + */ +public class VoyageAIRerankTaskSettings implements TaskSettings { + + public static final String NAME = "voyageai_rerank_task_settings"; + public static final String RETURN_DOCUMENTS = "return_documents"; + public static final String TOP_K_DOCS_ONLY = "top_k"; + + public static final VoyageAIRerankTaskSettings EMPTY_SETTINGS = new VoyageAIRerankTaskSettings(null, null, null); + + public static VoyageAIRerankTaskSettings fromMap(Map map) { + ValidationException validationException = new ValidationException(); + + if (map == null || map.isEmpty()) { + return EMPTY_SETTINGS; + } + + Boolean returnDocuments = extractOptionalBoolean(map, RETURN_DOCUMENTS, validationException); + Integer topKDocumentsOnly = extractOptionalPositiveInteger( + map, + TOP_K_DOCS_ONLY, + ModelConfigurations.TASK_SETTINGS, + validationException + ); + + Boolean truncation = extractOptionalBoolean(map, TRUNCATION, validationException); + + if (validationException.validationErrors().isEmpty() == false) { + throw validationException; + } + + return of(topKDocumentsOnly, returnDocuments, truncation); + } + + /** + * Creates a new {@link VoyageAIRerankTaskSettings} by preferring non-null fields from the request settings over the original settings. + * + * @param originalSettings the settings stored as part of the inference entity configuration + * @param requestTaskSettings the settings passed in within the task_settings field of the request + * @return a constructed {@link VoyageAIRerankTaskSettings} + */ + public static VoyageAIRerankTaskSettings of( + VoyageAIRerankTaskSettings originalSettings, + VoyageAIRerankTaskSettings requestTaskSettings + ) { + return new VoyageAIRerankTaskSettings( + requestTaskSettings.getTopKDocumentsOnly() != null + ? requestTaskSettings.getTopKDocumentsOnly() + : originalSettings.getTopKDocumentsOnly(), + requestTaskSettings.getReturnDocuments() != null + ? requestTaskSettings.getReturnDocuments() + : originalSettings.getReturnDocuments(), + requestTaskSettings.getTruncation() != null ? requestTaskSettings.getTruncation() : originalSettings.getTruncation() + + ); + } + + public static VoyageAIRerankTaskSettings of(Integer topKDocumentsOnly, Boolean returnDocuments, Boolean truncation) { + return new VoyageAIRerankTaskSettings(topKDocumentsOnly, returnDocuments, truncation); + } + + private final Integer topKDocumentsOnly; + private final Boolean returnDocuments; + private final Boolean truncation; + + public VoyageAIRerankTaskSettings(StreamInput in) throws IOException { + this(in.readOptionalInt(), in.readOptionalBoolean(), in.readOptionalBoolean()); + } + + public VoyageAIRerankTaskSettings( + @Nullable Integer topKDocumentsOnly, + @Nullable Boolean doReturnDocuments, + @Nullable Boolean truncation + ) { + this.topKDocumentsOnly = topKDocumentsOnly; + this.returnDocuments = doReturnDocuments; + this.truncation = truncation; + } + + @Override + public boolean isEmpty() { + return topKDocumentsOnly == null && returnDocuments == null && truncation == null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (topKDocumentsOnly != null) { + builder.field(TOP_K_DOCS_ONLY, topKDocumentsOnly); + } + if (returnDocuments != null) { + builder.field(RETURN_DOCUMENTS, returnDocuments); + } + if (truncation != null) { + builder.field(TRUNCATION, truncation); + } + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.VOYAGE_AI_INTEGRATION_ADDED; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalInt(topKDocumentsOnly); + out.writeOptionalBoolean(returnDocuments); + out.writeOptionalBoolean(truncation); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VoyageAIRerankTaskSettings that = (VoyageAIRerankTaskSettings) o; + return Objects.equals(topKDocumentsOnly, that.topKDocumentsOnly) + && Objects.equals(returnDocuments, that.returnDocuments) + && Objects.equals(truncation, that.truncation); + } + + @Override + public int hashCode() { + return Objects.hash(truncation, returnDocuments, topKDocumentsOnly); + } + + public Integer getTopKDocumentsOnly() { + return topKDocumentsOnly; + } + + public Boolean getDoesReturnDocuments() { + return returnDocuments; + } + + public Boolean getReturnDocuments() { + return returnDocuments; + } + + public Boolean getTruncation() { + return truncation; + } + + @Override + public TaskSettings updatedTaskSettings(Map newSettings) { + VoyageAIRerankTaskSettings updatedSettings = VoyageAIRerankTaskSettings.fromMap(new HashMap<>(newSettings)); + return VoyageAIRerankTaskSettings.of(this, updatedSettings); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java index 45e22870f1232..17c91b81233fb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/telemetry/InferenceStats.java @@ -16,12 +16,9 @@ import org.elasticsearch.telemetry.metric.MeterRegistry; import org.elasticsearch.xpack.core.inference.action.BaseInferenceActionRequest; +import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Map.entry; public record InferenceStats(LongCounter requestCount, LongHistogram inferenceDuration) { @@ -45,18 +42,16 @@ public static InferenceStats create(MeterRegistry meterRegistry) { ); } - private static Map toMap(Stream> stream) { - return stream.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - public static Map modelAttributes(Model model) { - var stream = Stream.>builder() - .add(entry("service", model.getConfigurations().getService())) - .add(entry("task_type", model.getTaskType().toString())); - if (model.getServiceSettings().modelId() != null) { - stream.add(entry("model_id", model.getServiceSettings().modelId())); + var modelAttributesMap = new HashMap(); + modelAttributesMap.put("service", model.getConfigurations().getService()); + modelAttributesMap.put("task_type", model.getTaskType().toString()); + + if (Objects.nonNull(model.getServiceSettings().modelId())) { + modelAttributesMap.put("model_id", model.getServiceSettings().modelId()); } - return toMap(stream.build()); + + return modelAttributesMap; } public static Map routingAttributes(BaseInferenceActionRequest request, String nodeIdHandlingRequest) { @@ -64,24 +59,18 @@ public static Map routingAttributes(BaseInferenceActionRequest r } public static Map modelAttributes(UnparsedModel model) { - var unknownModelAttributes = Stream.>builder() - .add(entry("service", model.service())) - .add(entry("task_type", model.taskType().toString())) - .build(); - - return toMap(unknownModelAttributes); + return Map.of("service", model.service(), "task_type", model.taskType().toString()); } - public static Map responseAttributes(@Nullable Throwable t) { - var stream = switch (t) { - case null -> Stream.>of(entry("status_code", 200)); - case ElasticsearchStatusException ese -> Stream.>builder() - .add(entry("status_code", ese.status().getStatus())) - .add(entry("error.type", String.valueOf(ese.status().getStatus()))) - .build(); - default -> Stream.>of(entry("error.type", t.getClass().getSimpleName())); - }; + public static Map responseAttributes(@Nullable Throwable throwable) { + if (Objects.isNull(throwable)) { + return Map.of("status_code", 200); + } + + if (throwable instanceof ElasticsearchStatusException ese) { + return Map.of("status_code", ese.status().getStatus(), "error.type", String.valueOf(ese.status().getStatus())); + } - return toMap(stream); + return Map.of("error.type", throwable.getClass().getSimpleName()); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java index 84c3e5cf80b0c..dff740dfc9261 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java @@ -50,7 +50,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.XPackField; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.mapper.SemanticTextField; @@ -652,7 +652,7 @@ public static StaticModel createRandomInstance() { } ChunkedInference getResults(String text) { - return resultMap.getOrDefault(text, new ChunkedInferenceEmbeddingSparse(List.of())); + return resultMap.getOrDefault(text, new ChunkedInferenceEmbedding(List.of())); } void putResult(String text, ChunkedInference result) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java index f0b82f49d4e98..23912b4000d02 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java @@ -10,14 +10,11 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.ChunkedInference; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingByte; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; -import org.elasticsearch.xpack.core.inference.results.InferenceByteEmbedding; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; import org.hamcrest.Matchers; @@ -27,6 +24,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.startsWith; @@ -34,21 +32,19 @@ public class EmbeddingRequestChunkerTests extends ESTestCase { public void testEmptyInput_WordChunker() { - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(List.of(), 100, 100, 10, embeddingType).batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(List.of(), 100, 100, 10).batchRequestsWithListeners(testListener()); assertThat(batches, empty()); } public void testEmptyInput_SentenceChunker() { - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(List.of(), 10, embeddingType, new SentenceBoundaryChunkingSettings(250, 1)) - .batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(List.of(), 10, new SentenceBoundaryChunkingSettings(250, 1)).batchRequestsWithListeners( + testListener() + ); assertThat(batches, empty()); } public void testWhitespaceInput_SentenceChunker() { - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(List.of(" "), 10, embeddingType, new SentenceBoundaryChunkingSettings(250, 1)) + var batches = new EmbeddingRequestChunker(List.of(" "), 10, new SentenceBoundaryChunkingSettings(250, 1)) .batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(1)); assertThat(batches.get(0).batch().inputs(), hasSize(1)); @@ -56,35 +52,30 @@ public void testWhitespaceInput_SentenceChunker() { } public void testBlankInput_WordChunker() { - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(List.of(""), 100, 100, 10, embeddingType).batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(List.of(""), 100, 100, 10).batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(1)); assertThat(batches.get(0).batch().inputs(), hasSize(1)); assertThat(batches.get(0).batch().inputs().get(0), Matchers.is("")); } public void testBlankInput_SentenceChunker() { - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(List.of(""), 10, embeddingType, new SentenceBoundaryChunkingSettings(250, 1)) - .batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(List.of(""), 10, new SentenceBoundaryChunkingSettings(250, 1)).batchRequestsWithListeners( + testListener() + ); assertThat(batches, hasSize(1)); assertThat(batches.get(0).batch().inputs(), hasSize(1)); assertThat(batches.get(0).batch().inputs().get(0), Matchers.is("")); } public void testInputThatDoesNotChunk_WordChunker() { - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(List.of("ABBAABBA"), 100, 100, 10, embeddingType).batchRequestsWithListeners( - testListener() - ); + var batches = new EmbeddingRequestChunker(List.of("ABBAABBA"), 100, 100, 10).batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(1)); assertThat(batches.get(0).batch().inputs(), hasSize(1)); assertThat(batches.get(0).batch().inputs().get(0), Matchers.is("ABBAABBA")); } public void testInputThatDoesNotChunk_SentenceChunker() { - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(List.of("ABBAABBA"), 10, embeddingType, new SentenceBoundaryChunkingSettings(250, 1)) + var batches = new EmbeddingRequestChunker(List.of("ABBAABBA"), 10, new SentenceBoundaryChunkingSettings(250, 1)) .batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(1)); assertThat(batches.get(0).batch().inputs(), hasSize(1)); @@ -93,27 +84,22 @@ public void testInputThatDoesNotChunk_SentenceChunker() { public void testShortInputsAreSingleBatch() { String input = "one chunk"; - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - - var batches = new EmbeddingRequestChunker(List.of(input), 100, 100, 10, embeddingType).batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(List.of(input), 100, 100, 10).batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(1)); assertThat(batches.get(0).batch().inputs(), contains(input)); } public void testMultipleShortInputsAreSingleBatch() { List inputs = List.of("1st small", "2nd small", "3rd small"); - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - - var batches = new EmbeddingRequestChunker(inputs, 100, 100, 10, embeddingType).batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(inputs, 100, 100, 10).batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(1)); - assertEquals(batches.get(0).batch().inputs(), inputs); - var subBatches = batches.get(0).batch().subBatches(); + EmbeddingRequestChunker.BatchRequest batch = batches.getFirst().batch(); + assertEquals(batch.inputs(), inputs); for (int i = 0; i < inputs.size(); i++) { - var subBatch = subBatches.get(i); - assertThat(subBatch.requests().toChunkText(), contains(inputs.get(i))); - assertEquals(0, subBatch.positions().chunkIndex()); - assertEquals(i, subBatch.positions().inputIndex()); - assertEquals(1, subBatch.positions().embeddingCount()); + var request = batch.requests().get(i); + assertThat(request.chunkText(), equalTo(inputs.get(i))); + assertEquals(i, request.inputIndex()); + assertEquals(0, request.chunkIndex()); } } @@ -121,15 +107,12 @@ public void testManyInputsMakeManyBatches() { int maxNumInputsPerBatch = 10; int numInputs = maxNumInputsPerBatch * 3 + 1; // requires 4 batches var inputs = new ArrayList(); - // + for (int i = 0; i < numInputs; i++) { inputs.add("input " + i); } - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(inputs, maxNumInputsPerBatch, 100, 10, embeddingType).batchRequestsWithListeners( - testListener() - ); + var batches = new EmbeddingRequestChunker(inputs, maxNumInputsPerBatch, 100, 10).batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(4)); assertThat(batches.get(0).batch().inputs(), hasSize(maxNumInputsPerBatch)); assertThat(batches.get(1).batch().inputs(), hasSize(maxNumInputsPerBatch)); @@ -146,15 +129,12 @@ public void testManyInputsMakeManyBatches() { assertEquals("input 29", batches.get(2).batch().inputs().get(9)); assertThat(batches.get(3).batch().inputs(), contains("input 30")); - int inputIndex = 0; - var subBatches = batches.get(0).batch().subBatches(); - for (int i = 0; i < batches.size(); i++) { - var subBatch = subBatches.get(i); - assertThat(subBatch.requests().toChunkText(), contains(inputs.get(i))); - assertEquals(0, subBatch.positions().chunkIndex()); - assertEquals(inputIndex, subBatch.positions().inputIndex()); - assertEquals(1, subBatch.positions().embeddingCount()); - inputIndex++; + List requests = batches.get(0).batch().requests(); + for (int i = 0; i < requests.size(); i++) { + EmbeddingRequestChunker.Request request = requests.get(i); + assertThat(request.chunkText(), equalTo(inputs.get(i))); + assertThat(request.inputIndex(), equalTo(i)); + assertThat(request.chunkIndex(), equalTo(0)); } } @@ -166,14 +146,9 @@ public void testChunkingSettingsProvided() { for (int i = 0; i < numInputs; i++) { inputs.add("input " + i); } - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker( - inputs, - maxNumInputsPerBatch, - embeddingType, - ChunkingSettingsTests.createRandomChunkingSettings() - ).batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(inputs, maxNumInputsPerBatch, ChunkingSettingsTests.createRandomChunkingSettings()) + .batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(4)); assertThat(batches.get(0).batch().inputs(), hasSize(maxNumInputsPerBatch)); assertThat(batches.get(1).batch().inputs(), hasSize(maxNumInputsPerBatch)); @@ -190,15 +165,12 @@ public void testChunkingSettingsProvided() { assertEquals("input 29", batches.get(2).batch().inputs().get(9)); assertThat(batches.get(3).batch().inputs(), contains("input 30")); - int inputIndex = 0; - var subBatches = batches.get(0).batch().subBatches(); - for (int i = 0; i < batches.size(); i++) { - var subBatch = subBatches.get(i); - assertThat(subBatch.requests().toChunkText(), contains(inputs.get(i))); - assertEquals(0, subBatch.positions().chunkIndex()); - assertEquals(inputIndex, subBatch.positions().inputIndex()); - assertEquals(1, subBatch.positions().embeddingCount()); - inputIndex++; + List requests = batches.get(0).batch().requests(); + for (int i = 0; i < requests.size(); i++) { + EmbeddingRequestChunker.Request request = requests.get(i); + assertThat(request.chunkText(), equalTo(inputs.get(i))); + assertThat(request.inputIndex(), equalTo(i)); + assertThat(request.chunkIndex(), equalTo(0)); } } @@ -216,63 +188,49 @@ public void testLongInputChunkedOverMultipleBatches() { } List inputs = List.of("1st small", passageBuilder.toString(), "2nd small", "3rd small"); - var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap, embeddingType).batchRequestsWithListeners( - testListener() - ); + var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap).batchRequestsWithListeners(testListener()); + assertThat(batches, hasSize(2)); - { - var batch = batches.get(0).batch(); - assertThat(batch.inputs(), hasSize(batchSize)); - assertEquals(batchSize, batch.size()); - assertThat(batch.subBatches(), hasSize(2)); - { - var subBatch = batch.subBatches().get(0); - assertEquals(0, subBatch.positions().inputIndex()); - assertEquals(0, subBatch.positions().chunkIndex()); - assertEquals(1, subBatch.positions().embeddingCount()); - assertThat(subBatch.requests().toChunkText(), contains("1st small")); - } - { - var subBatch = batch.subBatches().get(1); - assertEquals(1, subBatch.positions().inputIndex()); // 2nd input - assertEquals(0, subBatch.positions().chunkIndex()); // 1st part of the 2nd input - assertEquals(4, subBatch.positions().embeddingCount()); // 4 chunks - assertThat(subBatch.requests().toChunkText().get(0), startsWith("passage_input0 ")); - assertThat(subBatch.requests().toChunkText().get(1), startsWith(" passage_input20 ")); - assertThat(subBatch.requests().toChunkText().get(2), startsWith(" passage_input40 ")); - assertThat(subBatch.requests().toChunkText().get(3), startsWith(" passage_input60 ")); - } + + var batch = batches.get(0).batch(); + assertThat(batch.inputs(), hasSize(batchSize)); + assertThat(batch.requests(), hasSize(batchSize)); + + EmbeddingRequestChunker.Request request = batch.requests().get(0); + assertThat(request.inputIndex(), equalTo(0)); + assertThat(request.chunkIndex(), equalTo(0)); + assertThat(request.chunkText(), equalTo("1st small")); + + for (int requestIndex = 1; requestIndex < 5; requestIndex++) { + request = batch.requests().get(requestIndex); + assertThat(request.inputIndex(), equalTo(1)); + int chunkIndex = requestIndex - 1; + assertThat(request.chunkIndex(), equalTo(chunkIndex)); + assertThat(request.chunkText(), startsWith((chunkIndex == 0 ? "" : " ") + "passage_input" + 20 * chunkIndex)); } - { - var batch = batches.get(1).batch(); - assertThat(batch.inputs(), hasSize(4)); - assertEquals(4, batch.size()); - assertThat(batch.subBatches(), hasSize(3)); - { - var subBatch = batch.subBatches().get(0); - assertEquals(1, subBatch.positions().inputIndex()); // 2nd input - assertEquals(1, subBatch.positions().chunkIndex()); // 2nd part of the 2nd input - assertEquals(2, subBatch.positions().embeddingCount()); - assertThat(subBatch.requests().toChunkText().get(0), startsWith(" passage_input80 ")); - assertThat(subBatch.requests().toChunkText().get(1), startsWith(" passage_input100 ")); - } - { - var subBatch = batch.subBatches().get(1); - assertEquals(2, subBatch.positions().inputIndex()); // 3rd input - assertEquals(0, subBatch.positions().chunkIndex()); // 1st and only part - assertEquals(1, subBatch.positions().embeddingCount()); // 1 chunk - assertThat(subBatch.requests().toChunkText(), contains("2nd small")); - } - { - var subBatch = batch.subBatches().get(2); - assertEquals(3, subBatch.positions().inputIndex()); // 4th input - assertEquals(0, subBatch.positions().chunkIndex()); // 1st and only part - assertEquals(1, subBatch.positions().embeddingCount()); // 1 chunk - assertThat(subBatch.requests().toChunkText(), contains("3rd small")); - } + + batch = batches.get(1).batch(); + assertThat(batch.inputs(), hasSize(4)); + assertThat(batch.requests(), hasSize(4)); + + for (int requestIndex = 0; requestIndex < 2; requestIndex++) { + request = batch.requests().get(requestIndex); + assertThat(request.inputIndex(), equalTo(1)); + int chunkIndex = requestIndex + 4; + assertThat(request.chunkIndex(), equalTo(chunkIndex)); + assertThat(request.chunkText(), startsWith(" passage_input" + 20 * chunkIndex)); } + + request = batch.requests().get(2); + assertThat(request.inputIndex(), equalTo(2)); + assertThat(request.chunkIndex(), equalTo(0)); + assertThat(request.chunkText(), equalTo("2nd small")); + + request = batch.requests().get(3); + assertThat(request.inputIndex(), equalTo(3)); + assertThat(request.chunkIndex(), equalTo(0)); + assertThat(request.chunkText(), equalTo("3rd small")); } public void testMergingListener_Float() { @@ -290,40 +248,39 @@ public void testMergingListener_Float() { List inputs = List.of("1st small", passageBuilder.toString(), "2nd small", "3rd small"); var finalListener = testListener(); - var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap, EmbeddingRequestChunker.EmbeddingType.FLOAT) - .batchRequestsWithListeners(finalListener); + var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap).batchRequestsWithListeners(finalListener); assertThat(batches, hasSize(2)); // 4 inputs in 2 batches { - var embeddings = new ArrayList(); + var embeddings = new ArrayList(); for (int i = 0; i < batchSize; i++) { - embeddings.add(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { randomFloat() })); + embeddings.add(new TextEmbeddingFloatResults.Embedding(new float[] { randomFloat() })); } - batches.get(0).listener().onResponse(new InferenceTextEmbeddingFloatResults(embeddings)); + batches.get(0).listener().onResponse(new TextEmbeddingFloatResults(embeddings)); } { - var embeddings = new ArrayList(); + var embeddings = new ArrayList(); for (int i = 0; i < 4; i++) { // 4 requests in the 2nd batch - embeddings.add(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { randomFloat() })); + embeddings.add(new TextEmbeddingFloatResults.Embedding(new float[] { randomFloat() })); } - batches.get(1).listener().onResponse(new InferenceTextEmbeddingFloatResults(embeddings)); + batches.get(1).listener().onResponse(new TextEmbeddingFloatResults(embeddings)); } assertNotNull(finalListener.results); assertThat(finalListener.results, hasSize(4)); { var chunkedResult = finalListener.results.get(0); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var chunkedFloatResult = (ChunkedInferenceEmbeddingFloat) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedFloatResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedFloatResult.chunks(), hasSize(1)); assertEquals("1st small", chunkedFloatResult.chunks().get(0).matchedText()); } { // this is the large input split in multiple chunks var chunkedResult = finalListener.results.get(1); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var chunkedFloatResult = (ChunkedInferenceEmbeddingFloat) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedFloatResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedFloatResult.chunks(), hasSize(6)); assertThat(chunkedFloatResult.chunks().get(0).matchedText(), startsWith("passage_input0 ")); assertThat(chunkedFloatResult.chunks().get(1).matchedText(), startsWith(" passage_input20 ")); @@ -334,15 +291,15 @@ public void testMergingListener_Float() { } { var chunkedResult = finalListener.results.get(2); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var chunkedFloatResult = (ChunkedInferenceEmbeddingFloat) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedFloatResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedFloatResult.chunks(), hasSize(1)); assertEquals("2nd small", chunkedFloatResult.chunks().get(0).matchedText()); } { var chunkedResult = finalListener.results.get(3); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var chunkedFloatResult = (ChunkedInferenceEmbeddingFloat) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedFloatResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedFloatResult.chunks(), hasSize(1)); assertEquals("3rd small", chunkedFloatResult.chunks().get(0).matchedText()); } @@ -363,40 +320,39 @@ public void testMergingListener_Byte() { List inputs = List.of("1st small", passageBuilder.toString(), "2nd small", "3rd small"); var finalListener = testListener(); - var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap, EmbeddingRequestChunker.EmbeddingType.BYTE) - .batchRequestsWithListeners(finalListener); + var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap).batchRequestsWithListeners(finalListener); assertThat(batches, hasSize(2)); // 4 inputs in 2 batches { - var embeddings = new ArrayList(); + var embeddings = new ArrayList(); for (int i = 0; i < batchSize; i++) { - embeddings.add(new InferenceByteEmbedding(new byte[] { randomByte() })); + embeddings.add(new TextEmbeddingByteResults.Embedding(new byte[] { randomByte() })); } - batches.get(0).listener().onResponse(new InferenceTextEmbeddingByteResults(embeddings)); + batches.get(0).listener().onResponse(new TextEmbeddingByteResults(embeddings)); } { - var embeddings = new ArrayList(); + var embeddings = new ArrayList(); for (int i = 0; i < 4; i++) { // 4 requests in the 2nd batch - embeddings.add(new InferenceByteEmbedding(new byte[] { randomByte() })); + embeddings.add(new TextEmbeddingByteResults.Embedding(new byte[] { randomByte() })); } - batches.get(1).listener().onResponse(new InferenceTextEmbeddingByteResults(embeddings)); + batches.get(1).listener().onResponse(new TextEmbeddingByteResults(embeddings)); } assertNotNull(finalListener.results); assertThat(finalListener.results, hasSize(4)); { var chunkedResult = finalListener.results.get(0); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingByte.class)); - var chunkedByteResult = (ChunkedInferenceEmbeddingByte) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedByteResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedByteResult.chunks(), hasSize(1)); assertEquals("1st small", chunkedByteResult.chunks().get(0).matchedText()); } { // this is the large input split in multiple chunks var chunkedResult = finalListener.results.get(1); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingByte.class)); - var chunkedByteResult = (ChunkedInferenceEmbeddingByte) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedByteResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedByteResult.chunks(), hasSize(6)); assertThat(chunkedByteResult.chunks().get(0).matchedText(), startsWith("passage_input0 ")); assertThat(chunkedByteResult.chunks().get(1).matchedText(), startsWith(" passage_input20 ")); @@ -407,15 +363,15 @@ public void testMergingListener_Byte() { } { var chunkedResult = finalListener.results.get(2); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingByte.class)); - var chunkedByteResult = (ChunkedInferenceEmbeddingByte) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedByteResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedByteResult.chunks(), hasSize(1)); assertEquals("2nd small", chunkedByteResult.chunks().get(0).matchedText()); } { var chunkedResult = finalListener.results.get(3); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingByte.class)); - var chunkedByteResult = (ChunkedInferenceEmbeddingByte) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedByteResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedByteResult.chunks(), hasSize(1)); assertEquals("3rd small", chunkedByteResult.chunks().get(0).matchedText()); } @@ -436,8 +392,7 @@ public void testMergingListener_Sparse() { List inputs = List.of("1st small", "2nd small", "3rd small", passageBuilder.toString()); var finalListener = testListener(); - var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap, EmbeddingRequestChunker.EmbeddingType.SPARSE) - .batchRequestsWithListeners(finalListener); + var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap).batchRequestsWithListeners(finalListener); assertThat(batches, hasSize(3)); // 4 inputs in 3 batches @@ -467,30 +422,30 @@ public void testMergingListener_Sparse() { assertThat(finalListener.results, hasSize(4)); { var chunkedResult = finalListener.results.get(0); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingSparse.class)); - var chunkedSparseResult = (ChunkedInferenceEmbeddingSparse) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedSparseResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedSparseResult.chunks(), hasSize(1)); assertEquals("1st small", chunkedSparseResult.chunks().get(0).matchedText()); } { var chunkedResult = finalListener.results.get(1); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingSparse.class)); - var chunkedSparseResult = (ChunkedInferenceEmbeddingSparse) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedSparseResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedSparseResult.chunks(), hasSize(1)); assertEquals("2nd small", chunkedSparseResult.chunks().get(0).matchedText()); } { var chunkedResult = finalListener.results.get(2); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingSparse.class)); - var chunkedSparseResult = (ChunkedInferenceEmbeddingSparse) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedSparseResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedSparseResult.chunks(), hasSize(1)); assertEquals("3rd small", chunkedSparseResult.chunks().get(0).matchedText()); } { // this is the large input split in multiple chunks var chunkedResult = finalListener.results.get(3); - assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbeddingSparse.class)); - var chunkedSparseResult = (ChunkedInferenceEmbeddingSparse) chunkedResult; + assertThat(chunkedResult, instanceOf(ChunkedInferenceEmbedding.class)); + var chunkedSparseResult = (ChunkedInferenceEmbedding) chunkedResult; assertThat(chunkedSparseResult.chunks(), hasSize(9)); // passage is split into 9 chunks, 10 words each assertThat(chunkedSparseResult.chunks().get(0).matchedText(), startsWith("passage_input0 ")); assertThat(chunkedSparseResult.chunks().get(1).matchedText(), startsWith(" passage_input10 ")); @@ -517,14 +472,13 @@ public void onFailure(Exception e) { } }; - var batches = new EmbeddingRequestChunker(inputs, 10, 100, 0, EmbeddingRequestChunker.EmbeddingType.FLOAT) - .batchRequestsWithListeners(listener); + var batches = new EmbeddingRequestChunker(inputs, 10, 100, 0).batchRequestsWithListeners(listener); assertThat(batches, hasSize(1)); - var embeddings = new ArrayList(); - embeddings.add(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { randomFloat() })); - embeddings.add(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { randomFloat() })); - batches.get(0).listener().onResponse(new InferenceTextEmbeddingFloatResults(embeddings)); + var embeddings = new ArrayList(); + embeddings.add(new TextEmbeddingFloatResults.Embedding(new float[] { randomFloat() })); + embeddings.add(new TextEmbeddingFloatResults.Embedding(new float[] { randomFloat() })); + batches.get(0).listener().onResponse(new TextEmbeddingFloatResults(embeddings)); assertEquals("Error the number of embedding responses [2] does not equal the number of requests [3]", failureMessage.get()); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreatorTests.java index e7543aa6ba9e5..e37e301dc422a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/amazonbedrock/AmazonBedrockActionCreatorTests.java @@ -15,7 +15,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.amazonbedrock.AmazonBedrockMockRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.ChatCompletionInput; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; @@ -52,8 +52,8 @@ public void shutdown() throws IOException { public void testEmbeddingsRequestAction() throws IOException { var serviceComponents = ServiceComponentsTests.createWithEmptySettings(threadPool); - var mockedFloatResults = List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0123F, -0.0123F })); - var mockedResult = new InferenceTextEmbeddingFloatResults(mockedFloatResults); + var mockedFloatResults = List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.0123F, -0.0123F })); + var mockedResult = new TextEmbeddingFloatResults(mockedFloatResults); try (var sender = new AmazonBedrockMockRequestSender()) { sender.enqueue(mockedResult); var creator = new AmazonBedrockActionCreator(sender, serviceComponents, TIMEOUT); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIActionCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIActionCreatorTests.java new file mode 100644 index 0000000000000..50c64468d732a --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIActionCreatorTests.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action.voyageai; + +import org.apache.http.HttpHeaders; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingType; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModelTests; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettings; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettingsTests; +import org.hamcrest.MatcherAssert; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; +import static org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests.createSender; +import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat; +import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class VoyageAIActionCreatorTests extends ESTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + private HttpClientManager clientManager; + + @Before + public void init() throws Exception { + webServer.start(); + threadPool = createThreadPool(inferenceUtilityPool()); + clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); + } + + @After + public void shutdown() throws IOException { + clientManager.close(); + terminate(threadPool); + webServer.close(); + } + + public void testCreate_VoyageAIEmbeddingsModel() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var sender = createSender(senderFactory)) { + sender.start(); + + String responseJson = """ + { + "object": "list", + "data": [{ + "object": "embedding", + "embedding": [ + 0.123, + -0.123 + ], + "index": 0 + }], + "model": "voyage-3-large", + "usage": { + "total_tokens": 123 + } + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, true), + 1024, + 1024, + "model", + VoyageAIEmbeddingType.FLOAT + ); + var actionCreator = new VoyageAIActionCreator(sender, createWithEmptySettings(threadPool)); + var overriddenTaskSettings = VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH); + var action = actionCreator.create(model, overriddenTaskSettings, InputType.UNSPECIFIED); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var result = listener.actionGet(TIMEOUT); + + MatcherAssert.assertThat(result.asMap(), is(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })))); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().getFirst().getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "output_dtype", + "float", + "truncation", + true, + "input_type", + "query", + "output_dimension", + 1024, + "input", + List.of("abc"), + "model", + "model" + ) + ) + ); + } + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIEmbeddingsActionTests.java new file mode 100644 index 0000000000000..3b3397f702a64 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/voyageai/VoyageAIEmbeddingsActionTests.java @@ -0,0 +1,413 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.action.voyageai; + +import org.apache.http.HttpHeaders; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.external.action.ExecutableAction; +import org.elasticsearch.xpack.inference.external.action.SenderExecutableAction; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.external.http.sender.VoyageAIEmbeddingsRequestManager; +import org.elasticsearch.xpack.inference.external.request.voyageai.VoyageAIUtils; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingType; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModelTests; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettings; +import org.hamcrest.MatcherAssert; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.action.ActionUtils.constructFailedToSendRequestMessage; +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; +import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationBinary; +import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationByte; +import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + +public class VoyageAIEmbeddingsActionTests extends ESTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + private HttpClientManager clientManager; + + @Before + public void init() throws Exception { + webServer.start(); + threadPool = createThreadPool(inferenceUtilityPool()); + clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); + } + + @After + public void shutdown() throws IOException { + clientManager.close(); + terminate(threadPool); + webServer.close(); + } + + public void testExecute_ReturnsSuccessfulResponse() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var sender = HttpRequestSenderTests.createSender(senderFactory)) { + sender.start(); + + String responseJson = """ + { + "object": "list", + "data": [{ + "object": "embedding", + "embedding": [ + 0.123, + -0.123 + ], + "index": 0 + }], + "model": "voyage-3-large", + "usage": { + "total_tokens": 123 + } + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var action = createAction( + getUrl(webServer), + "secret", + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, true), + "model", + VoyageAIEmbeddingType.FLOAT, + sender + ); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var result = listener.actionGet(TIMEOUT); + + MatcherAssert.assertThat(result.asMap(), is(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })))); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().getFirst().getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(VoyageAIUtils.REQUEST_SOURCE_HEADER), + equalTo(VoyageAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + equalTo( + Map.of( + "input", + List.of("abc"), + "model", + "model", + "input_type", + "document", + "output_dtype", + "float", + "truncation", + true, + "output_dimension", + 1024 + ) + ) + ); + } + } + + public void testExecute_ReturnsSuccessfulResponse_ForInt8ResponseType() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var sender = HttpRequestSenderTests.createSender(senderFactory)) { + sender.start(); + + String responseJson = """ + { + "object": "list", + "data": [{ + "object": "embedding", + "embedding": [ + 0, + -1 + ], + "index": 0 + }], + "model": "voyage-3-large", + "usage": { + "total_tokens": 123 + } + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var action = createAction( + getUrl(webServer), + "secret", + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, true), + "model", + VoyageAIEmbeddingType.INT8, + sender + ); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var result = listener.actionGet(TIMEOUT); + + assertEquals(buildExpectationByte(List.of(new byte[] { 0, -1 })), result.asMap()); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().getFirst().getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(VoyageAIUtils.REQUEST_SOURCE_HEADER), + equalTo(VoyageAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "input", + List.of("abc"), + "model", + "model", + "input_type", + "document", + "output_dtype", + "int8", + "truncation", + true, + "output_dimension", + 1024 + ) + ) + ); + } + } + + public void testExecute_ReturnsSuccessfulResponse_ForBinaryResponseType() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var sender = HttpRequestSenderTests.createSender(senderFactory)) { + sender.start(); + + String responseJson = """ + { + "object": "list", + "data": [{ + "object": "embedding", + "embedding": [ + 0, + -1 + ], + "index": 0 + }], + "model": "voyage-3-large", + "usage": { + "total_tokens": 123 + } + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var action = createAction( + getUrl(webServer), + "secret", + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, true), + "model", + VoyageAIEmbeddingType.BINARY, + sender + ); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var result = listener.actionGet(TIMEOUT); + + assertEquals(buildExpectationBinary(List.of(new byte[] { 0, -1 })), result.asMap()); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().getFirst().getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(VoyageAIUtils.REQUEST_SOURCE_HEADER), + equalTo(VoyageAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "input", + List.of("abc"), + "model", + "model", + "input_type", + "document", + "output_dtype", + "binary", + "truncation", + true, + "output_dimension", + 1024 + ) + ) + ); + } + } + + public void testExecute_ThrowsElasticsearchException() { + var sender = mock(Sender.class); + doThrow(new ElasticsearchException("failed")).when(sender).send(any(), any(), any(), any()); + + var action = createAction(getUrl(webServer), "secret", VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, "model", null, sender); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + + MatcherAssert.assertThat(thrownException.getMessage(), is("failed")); + } + + public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled() { + var sender = mock(Sender.class); + + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onFailure(new IllegalStateException("failed")); + + return Void.TYPE; + }).when(sender).send(any(), any(), any(), any()); + + var action = createAction(getUrl(webServer), "secret", VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, "model", null, sender); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + + MatcherAssert.assertThat( + thrownException.getMessage(), + is(format("Failed to send VoyageAI embeddings request to [%s]", getUrl(webServer))) + ); + } + + public void testExecute_ThrowsElasticsearchException_WhenSenderOnFailureIsCalled_WhenUrlIsNull() { + var sender = mock(Sender.class); + + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + ActionListener listener = (ActionListener) invocation.getArguments()[2]; + listener.onFailure(new IllegalStateException("failed")); + + return Void.TYPE; + }).when(sender).send(any(), any(), any(), any()); + + var action = createAction(null, "secret", VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, "model", null, sender); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + + MatcherAssert.assertThat(thrownException.getMessage(), is("Failed to send VoyageAI embeddings request")); + } + + public void testExecute_ThrowsException() { + var sender = mock(Sender.class); + doThrow(new IllegalArgumentException("failed")).when(sender).send(any(), any(), any(), any()); + + var action = createAction(getUrl(webServer), "secret", VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, "model", null, sender); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + + MatcherAssert.assertThat( + thrownException.getMessage(), + is(format("Failed to send VoyageAI embeddings request to [%s]", getUrl(webServer))) + ); + } + + public void testExecute_ThrowsExceptionWithNullUrl() { + var sender = mock(Sender.class); + doThrow(new IllegalArgumentException("failed")).when(sender).send(any(), any(), any(), any()); + + var action = createAction(null, "secret", VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, "model", null, sender); + + PlainActionFuture listener = new PlainActionFuture<>(); + action.execute(new DocumentsOnlyInput(List.of("abc")), InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var thrownException = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + + MatcherAssert.assertThat(thrownException.getMessage(), is("Failed to send VoyageAI embeddings request")); + } + + private ExecutableAction createAction( + String url, + String apiKey, + VoyageAIEmbeddingsTaskSettings taskSettings, + @Nullable String modelName, + @Nullable VoyageAIEmbeddingType embeddingType, + Sender sender + ) { + var model = VoyageAIEmbeddingsModelTests.createModel(url, apiKey, taskSettings, 1024, 1024, modelName, embeddingType); + var failedToSendRequestErrorMessage = constructFailedToSendRequestMessage(model.uri(), "VoyageAI embeddings"); + var requestCreator = VoyageAIEmbeddingsRequestManager.of(model, threadPool); + return new SenderExecutableAction(sender, requestCreator, failedToSendRequestErrorMessage); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequestEntityTests.java new file mode 100644 index 0000000000000..66b2287e9cb50 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequestEntityTests.java @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.ServiceFields; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettings; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.is; + +public class VoyageAIEmbeddingsRequestEntityTests extends ESTestCase { + public void testXContent_WritesAllFields_ServiceSettingsDefined() throws IOException { + var entity = new VoyageAIEmbeddingsRequestEntity( + List.of("abc"), + VoyageAIEmbeddingsServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.URL, + "https://www.abc.com", + ServiceFields.SIMILARITY, + SimilarityMeasure.DOT_PRODUCT.toString(), + ServiceFields.DIMENSIONS, + 2048, + ServiceFields.MAX_INPUT_TOKENS, + 512, + VoyageAIServiceSettings.MODEL_ID, + "model", + VoyageAIEmbeddingsServiceSettings.EMBEDDING_TYPE, + "float" + ) + ), + ConfigurationParseContext.PERSISTENT + ), + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), + "model" + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + MatcherAssert.assertThat(xContentResult, is(""" + {"input":["abc"],"model":"model","input_type":"document","output_dimension":2048,"output_dtype":"float"}""")); + } + + public void testXContent_WritesAllFields_ServiceSettingsDefined_Int8() throws IOException { + var entity = new VoyageAIEmbeddingsRequestEntity( + List.of("abc"), + VoyageAIEmbeddingsServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.URL, + "https://www.abc.com", + ServiceFields.SIMILARITY, + SimilarityMeasure.DOT_PRODUCT.toString(), + ServiceFields.DIMENSIONS, + 2048, + ServiceFields.MAX_INPUT_TOKENS, + 512, + VoyageAIServiceSettings.MODEL_ID, + "model", + VoyageAIEmbeddingsServiceSettings.EMBEDDING_TYPE, + "int8" + ) + ), + ConfigurationParseContext.PERSISTENT + ), + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), + "model" + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + MatcherAssert.assertThat(xContentResult, is(""" + {"input":["abc"],"model":"model","input_type":"document","output_dimension":2048,"output_dtype":"int8"}""")); + } + + public void testXContent_WritesAllFields_ServiceSettingsDefined_Binary() throws IOException { + var entity = new VoyageAIEmbeddingsRequestEntity( + List.of("abc"), + VoyageAIEmbeddingsServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.URL, + "https://www.abc.com", + ServiceFields.SIMILARITY, + SimilarityMeasure.DOT_PRODUCT.toString(), + ServiceFields.DIMENSIONS, + 2048, + ServiceFields.MAX_INPUT_TOKENS, + 512, + VoyageAIServiceSettings.MODEL_ID, + "model", + VoyageAIEmbeddingsServiceSettings.EMBEDDING_TYPE, + "binary" + ) + ), + ConfigurationParseContext.PERSISTENT + ), + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), + "model" + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + MatcherAssert.assertThat(xContentResult, is(""" + {"input":["abc"],"model":"model","input_type":"document","output_dimension":2048,"output_dtype":"binary"}""")); + } + + public void testXContent_WritesAllFields_WhenTheyAreDefined() throws IOException { + var entity = new VoyageAIEmbeddingsRequestEntity( + List.of("abc"), + VoyageAIEmbeddingsServiceSettings.EMPTY_SETTINGS, + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), + "model" + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + MatcherAssert.assertThat(xContentResult, is(""" + {"input":["abc"],"model":"model","input_type":"document"}""")); + } + + public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws IOException { + var entity = new VoyageAIEmbeddingsRequestEntity( + List.of("abc"), + VoyageAIEmbeddingsServiceSettings.EMPTY_SETTINGS, + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + "model" + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + MatcherAssert.assertThat(xContentResult, is(""" + {"input":["abc"],"model":"model"}""")); + } + + public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() { + var thrownException = expectThrows( + AssertionError.class, + () -> VoyageAIEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED) + ); + MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]")); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequestTests.java new file mode 100644 index 0000000000000..868849542457c --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIEmbeddingsRequestTests.java @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingType; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModelTests; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettings; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; + +public class VoyageAIEmbeddingsRequestTests extends ESTestCase { + public void testCreateRequest_UrlDefined() throws IOException { + var request = createRequest( + List.of("abc"), + VoyageAIEmbeddingsModelTests.createModel("url", "secret", VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, null, null, "model") + ); + + var httpRequest = request.createHttpRequest(); + MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + MatcherAssert.assertThat(httpPost.getURI().toString(), is("url")); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret")); + MatcherAssert.assertThat( + httpPost.getLastHeader(VoyageAIUtils.REQUEST_SOURCE_HEADER).getValue(), + is(VoyageAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + MatcherAssert.assertThat(requestMap, is(Map.of("input", List.of("abc"), "model", "model", "output_dtype", "float"))); + } + + public void testCreateRequest_AllOptionsDefined() throws IOException { + var request = createRequest( + List.of("abc"), + VoyageAIEmbeddingsModelTests.createModel( + "url", + "secret", + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), + null, + null, + "model" + ) + ); + + var httpRequest = request.createHttpRequest(); + MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + MatcherAssert.assertThat(httpPost.getURI().toString(), is("url")); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret")); + MatcherAssert.assertThat( + httpPost.getLastHeader(VoyageAIUtils.REQUEST_SOURCE_HEADER).getValue(), + is(VoyageAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + MatcherAssert.assertThat( + requestMap, + is(Map.of("input", List.of("abc"), "model", "model", "input_type", "document", "output_dtype", "float")) + ); + } + + public void testCreateRequest_DimensionDefined() throws IOException { + var request = createRequest( + List.of("abc"), + VoyageAIEmbeddingsModelTests.createModel( + "url", + "secret", + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), + null, + 2048, + "model" + ) + ); + + var httpRequest = request.createHttpRequest(); + MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + MatcherAssert.assertThat(httpPost.getURI().toString(), is("url")); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret")); + MatcherAssert.assertThat( + httpPost.getLastHeader(VoyageAIUtils.REQUEST_SOURCE_HEADER).getValue(), + is(VoyageAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "input", + List.of("abc"), + "model", + "model", + "input_type", + "document", + "output_dtype", + "float", + "output_dimension", + 2048 + ) + ) + ); + } + + public void testCreateRequest_EmbeddingTypeDefined() throws IOException { + var request = createRequest( + List.of("abc"), + VoyageAIEmbeddingsModelTests.createModel( + "url", + "secret", + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), + null, + 2048, + "model", + VoyageAIEmbeddingType.BYTE + ) + ); + + var httpRequest = request.createHttpRequest(); + MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + MatcherAssert.assertThat(httpPost.getURI().toString(), is("url")); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret")); + MatcherAssert.assertThat( + httpPost.getLastHeader(VoyageAIUtils.REQUEST_SOURCE_HEADER).getValue(), + is(VoyageAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "input", + List.of("abc"), + "model", + "model", + "input_type", + "document", + "output_dtype", + "int8", + "output_dimension", + 2048 + ) + ) + ); + } + + public void testCreateRequest_InputTypeSearch() throws IOException { + var request = createRequest( + List.of("abc"), + VoyageAIEmbeddingsModelTests.createModel( + "url", + "secret", + new VoyageAIEmbeddingsTaskSettings(InputType.SEARCH, null), + null, + null, + "model" + ) + ); + + var httpRequest = request.createHttpRequest(); + MatcherAssert.assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + MatcherAssert.assertThat(httpPost.getURI().toString(), is("url")); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + MatcherAssert.assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer secret")); + MatcherAssert.assertThat( + httpPost.getLastHeader(VoyageAIUtils.REQUEST_SOURCE_HEADER).getValue(), + is(VoyageAIUtils.ELASTIC_REQUEST_SOURCE) + ); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + MatcherAssert.assertThat( + requestMap, + is(Map.of("input", List.of("abc"), "model", "model", "input_type", "query", "output_dtype", "float")) + ); + } + + public static VoyageAIEmbeddingsRequest createRequest(List input, VoyageAIEmbeddingsModel model) { + return new VoyageAIEmbeddingsRequest(input, model); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRequestTests.java new file mode 100644 index 0000000000000..5244ef83a7c7b --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRequestTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.external.voyageai.VoyageAIAccount; + +import java.net.URI; + +import static org.hamcrest.Matchers.is; + +public class VoyageAIRequestTests extends ESTestCase { + + public void testDecorateWithHeaders() { + var request = new HttpPost("http://www.abc.com"); + + VoyageAIRequest.decorateWithHeaders( + request, + new VoyageAIAccount(URI.create("http://www.abc.com"), new SecureString(new char[] { 'a', 'b', 'c' })) + ); + + assertThat(request.getFirstHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + assertThat(request.getFirstHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer abc")); + assertThat(request.getFirstHeader(VoyageAIUtils.REQUEST_SOURCE_HEADER).getValue(), is(VoyageAIUtils.ELASTIC_REQUEST_SOURCE)); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequestEntityTests.java new file mode 100644 index 0000000000000..ae431b4b7bb13 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequestEntityTests.java @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankTaskSettings; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString; + +public class VoyageAIRerankRequestEntityTests extends ESTestCase { + public void testXContent_SingleRequest_WritesModelAndTopKIfDefined() throws IOException { + var entity = new VoyageAIRerankRequestEntity("query", List.of("abc"), new VoyageAIRerankTaskSettings(8, null, null), "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc" + ], + "top_k": 8 + } + """)); + } + + public void testXContent_SingleRequest_WritesModelAndTopKIfDefined_ReturnDocumentsTrue() throws IOException { + var entity = new VoyageAIRerankRequestEntity("query", List.of("abc"), new VoyageAIRerankTaskSettings(8, true, null), "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc" + ], + "return_documents": true, + "top_k": 8 + } + """)); + } + + public void testXContent_SingleRequest_WritesModelAndTopKIfDefined_ReturnDocumentsFalse() throws IOException { + var entity = new VoyageAIRerankRequestEntity("query", List.of("abc"), new VoyageAIRerankTaskSettings(8, false, null), "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc" + ], + "return_documents": false, + "top_k": 8 + } + """)); + } + + public void testXContent_SingleRequest_WritesModelAndTopKIfDefined_TruncationTrue() throws IOException { + var entity = new VoyageAIRerankRequestEntity("query", List.of("abc"), new VoyageAIRerankTaskSettings(8, false, true), "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc" + ], + "return_documents": false, + "top_k": 8, + "truncation": true + } + """)); + } + + public void testXContent_SingleRequest_WritesModelAndTopKIfDefined_TruncationFalse() throws IOException { + var entity = new VoyageAIRerankRequestEntity("query", List.of("abc"), new VoyageAIRerankTaskSettings(8, false, false), "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc" + ], + "return_documents": false, + "top_k": 8, + "truncation": false + } + """)); + } + + public void testXContent_SingleRequest_DoesNotWriteTopKIfNull() throws IOException { + var entity = new VoyageAIRerankRequestEntity("query", List.of("abc"), null, "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc" + ] + } + """)); + } + + public void testXContent_MultipleRequests_WritesModelAndTopKIfDefined() throws IOException { + var entity = new VoyageAIRerankRequestEntity( + "query", + List.of("abc", "def"), + new VoyageAIRerankTaskSettings(8, null, null), + "model" + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc", + "def" + ], + "top_k": 8 + } + """)); + } + + public void testXContent_MultipleRequests_DoesNotWriteTopKIfNull() throws IOException { + var entity = new VoyageAIRerankRequestEntity("query", List.of("abc", "def"), null, "model"); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model": "model", + "query": "query", + "documents": [ + "abc", + "def" + ] + } + """)); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequestTests.java new file mode 100644 index 0000000000000..a11d259200b98 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIRerankRequestTests.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankModelTests; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; + +public class VoyageAIRerankRequestTests extends ESTestCase { + + private static final String API_KEY = "foo"; + + public void testCreateRequest_WithoutModelSet_And_WithoutTopNSet() throws IOException { + var input = "input"; + var query = "query"; + var modelId = "model"; + + var request = createRequest(query, input, modelId, null); + var httpRequest = request.createHttpRequest(); + + assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer " + API_KEY)); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + + assertThat(requestMap, aMapWithSize(3)); + assertThat(requestMap.get("documents"), is(List.of(input))); + assertThat(requestMap.get("query"), is(query)); + assertThat(requestMap.get("model"), is(modelId)); + } + + public void testCreateRequest_WithTopNSet() throws IOException { + var input = "input"; + var query = "query"; + var topK = 1; + var modelId = "model"; + + var request = createRequest(query, input, modelId, topK); + var httpRequest = request.createHttpRequest(); + + assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer " + API_KEY)); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + + assertThat(requestMap, aMapWithSize(4)); + assertThat(requestMap.get("documents"), is(List.of(input))); + assertThat(requestMap.get("query"), is(query)); + assertThat(requestMap.get("top_k"), is(topK)); + assertThat(requestMap.get("model"), is(modelId)); + } + + public void testCreateRequest_WithModelSet() throws IOException { + var input = "input"; + var query = "query"; + var modelId = "model"; + + var request = createRequest(query, input, modelId, null); + var httpRequest = request.createHttpRequest(); + + assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + assertThat(httpPost.getLastHeader(HttpHeaders.CONTENT_TYPE).getValue(), is(XContentType.JSON.mediaType())); + assertThat(httpPost.getLastHeader(HttpHeaders.AUTHORIZATION).getValue(), is("Bearer " + API_KEY)); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + + assertThat(requestMap, aMapWithSize(3)); + assertThat(requestMap.get("documents"), is(List.of(input))); + assertThat(requestMap.get("query"), is(query)); + assertThat(requestMap.get("model"), is(modelId)); + } + + public void testTruncate_DoesNotTruncate() { + var request = createRequest("query", "input", "null", null); + var truncatedRequest = request.truncate(); + + assertThat(truncatedRequest, sameInstance(request)); + } + + private static VoyageAIRerankRequest createRequest(String query, String input, @Nullable String modelId, @Nullable Integer topK) { + var rerankModel = VoyageAIRerankModelTests.createModel(API_KEY, modelId, topK); + return new VoyageAIRerankRequest(query, List.of(input), rerankModel); + + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIUtilsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIUtilsTests.java new file mode 100644 index 0000000000000..4a8271f4fac88 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/voyageai/VoyageAIUtilsTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.request.voyageai; + +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.is; + +public class VoyageAIUtilsTests extends ESTestCase { + + public void testCreateRequestSourceHeader() { + var requestSourceHeader = VoyageAIUtils.createRequestSourceHeader(); + + assertThat(requestSourceHeader.getName(), is("Request-Source")); + assertThat(requestSourceHeader.getValue(), is("unspecified:elasticsearch")); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntityTests.java index 33fa6a2a542cb..d9db5670611a7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/alibabacloudsearch/AlibabaCloudSearchEmbeddingsResponseEntityTests.java @@ -9,7 +9,7 @@ import org.apache.http.HttpResponse; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.alibabacloudsearch.AlibabaCloudSearchRequest; @@ -50,20 +50,14 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException, URI uri = new URI("mock_uri"); when(request.getURI()).thenReturn(uri); - InferenceTextEmbeddingFloatResults parsedResults = AlibabaCloudSearchEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = AlibabaCloudSearchEmbeddingsResponseEntity.fromResponse( request, new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); assertThat( parsedResults.embeddings(), - is( - List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding( - new float[] { -0.02868066355586052f, 0.022033605724573135f } - ) - ) - ) + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { -0.02868066355586052f, 0.022033605724573135f }))) ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioEmbeddingsResponseEntityTests.java index c2f93554c6b20..1853ed41e4dc8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioEmbeddingsResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/azureaistudio/AzureAiStudioEmbeddingsResponseEntityTests.java @@ -9,7 +9,7 @@ import org.apache.http.HttpResponse; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; @@ -50,14 +50,11 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException { var entity = new AzureAiStudioEmbeddingsResponseEntity(); - var parsedResults = (InferenceTextEmbeddingFloatResults) entity.apply( + var parsedResults = (TextEmbeddingFloatResults) entity.apply( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(List.of(0.014539449F, -0.015288644F)))) - ); + assertThat(parsedResults.embeddings(), is(List.of(TextEmbeddingFloatResults.Embedding.of(List.of(0.014539449F, -0.015288644F))))); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntityTests.java index 42dab0a9021bf..d0093012d8ac4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/cohere/CohereEmbeddingsResponseEntityTests.java @@ -10,10 +10,9 @@ import org.apache.http.HttpResponse; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceByteEmbedding; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingBitResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingBitResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; import org.hamcrest.MatcherAssert; @@ -57,10 +56,10 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException { new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - MatcherAssert.assertThat(parsedResults, instanceOf(InferenceTextEmbeddingFloatResults.class)); + MatcherAssert.assertThat(parsedResults, instanceOf(TextEmbeddingFloatResults.class)); MatcherAssert.assertThat( - ((InferenceTextEmbeddingFloatResults) parsedResults).embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.0018434525F, 0.01777649F }))) + ((TextEmbeddingFloatResults) parsedResults).embeddings(), + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { -0.0018434525F, 0.01777649F }))) ); } @@ -91,14 +90,14 @@ public void testFromResponse_CreatesResultsForASingleItem_ObjectFormat() throws } """; - InferenceTextEmbeddingFloatResults parsedResults = (InferenceTextEmbeddingFloatResults) CohereEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = (TextEmbeddingFloatResults) CohereEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); MatcherAssert.assertThat( parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.0018434525F, 0.01777649F }))) + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { -0.0018434525F, 0.01777649F }))) ); } @@ -135,14 +134,14 @@ public void testFromResponse_UsesTheFirstValidEmbeddingsEntry() throws IOExcepti } """; - InferenceTextEmbeddingFloatResults parsedResults = (InferenceTextEmbeddingFloatResults) CohereEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = (TextEmbeddingFloatResults) CohereEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); MatcherAssert.assertThat( parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.0018434525F, 0.01777649F }))) + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { -0.0018434525F, 0.01777649F }))) ); } @@ -179,12 +178,15 @@ public void testFromResponse_UsesTheFirstValidEmbeddingsEntryInt8_WithInvalidFir } """; - InferenceTextEmbeddingByteResults parsedResults = (InferenceTextEmbeddingByteResults) CohereEmbeddingsResponseEntity.fromResponse( + TextEmbeddingByteResults parsedResults = (TextEmbeddingByteResults) CohereEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - MatcherAssert.assertThat(parsedResults.embeddings(), is(List.of(new InferenceByteEmbedding(new byte[] { (byte) -1, (byte) 0 })))); + MatcherAssert.assertThat( + parsedResults.embeddings(), + is(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) -1, (byte) 0 }))) + ); } public void testFromResponse_ParsesBytes() throws IOException { @@ -214,12 +216,15 @@ public void testFromResponse_ParsesBytes() throws IOException { } """; - InferenceTextEmbeddingByteResults parsedResults = (InferenceTextEmbeddingByteResults) CohereEmbeddingsResponseEntity.fromResponse( + TextEmbeddingByteResults parsedResults = (TextEmbeddingByteResults) CohereEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - MatcherAssert.assertThat(parsedResults.embeddings(), is(List.of(new InferenceByteEmbedding(new byte[] { (byte) -1, (byte) 0 })))); + MatcherAssert.assertThat( + parsedResults.embeddings(), + is(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) -1, (byte) 0 }))) + ); } public void testFromResponse_ParsesBytes_FromBinaryEmbeddingsEntry() throws IOException { @@ -252,14 +257,14 @@ public void testFromResponse_ParsesBytes_FromBinaryEmbeddingsEntry() throws IOEx } """; - InferenceTextEmbeddingBitResults parsedResults = (InferenceTextEmbeddingBitResults) CohereEmbeddingsResponseEntity.fromResponse( + TextEmbeddingBitResults parsedResults = (TextEmbeddingBitResults) CohereEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); MatcherAssert.assertThat( parsedResults.embeddings(), - is(List.of(new InferenceByteEmbedding(new byte[] { (byte) -55, (byte) 74, (byte) 101, (byte) 67, (byte) 83 }))) + is(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) -55, (byte) 74, (byte) 101, (byte) 67, (byte) 83 }))) ); } @@ -292,7 +297,7 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException } """; - InferenceTextEmbeddingFloatResults parsedResults = (InferenceTextEmbeddingFloatResults) CohereEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = (TextEmbeddingFloatResults) CohereEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -301,8 +306,8 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException parsedResults.embeddings(), is( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.0018434525F, 0.01777649F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.123F, 0.123F }) + new TextEmbeddingFloatResults.Embedding(new float[] { -0.0018434525F, 0.01777649F }), + new TextEmbeddingFloatResults.Embedding(new float[] { -0.123F, 0.123F }) ) ) ); @@ -339,7 +344,7 @@ public void testFromResponse_CreatesResultsForMultipleItems_ObjectFormat() throw } """; - InferenceTextEmbeddingFloatResults parsedResults = (InferenceTextEmbeddingFloatResults) CohereEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = (TextEmbeddingFloatResults) CohereEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -348,8 +353,8 @@ public void testFromResponse_CreatesResultsForMultipleItems_ObjectFormat() throw parsedResults.embeddings(), is( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.0018434525F, 0.01777649F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.123F, 0.123F }) + new TextEmbeddingFloatResults.Embedding(new float[] { -0.0018434525F, 0.01777649F }), + new TextEmbeddingFloatResults.Embedding(new float[] { -0.123F, 0.123F }) ) ) ); @@ -392,7 +397,7 @@ public void testFromResponse_CreatesResultsForMultipleItems_ObjectFormat_Binary( } """; - InferenceTextEmbeddingBitResults parsedResults = (InferenceTextEmbeddingBitResults) CohereEmbeddingsResponseEntity.fromResponse( + TextEmbeddingBitResults parsedResults = (TextEmbeddingBitResults) CohereEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -401,8 +406,8 @@ public void testFromResponse_CreatesResultsForMultipleItems_ObjectFormat_Binary( parsedResults.embeddings(), is( List.of( - new InferenceByteEmbedding(new byte[] { (byte) -55, (byte) 74, (byte) 101, (byte) 67 }), - new InferenceByteEmbedding(new byte[] { (byte) 34, (byte) -64, (byte) 97, (byte) 65, (byte) -42 }) + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) -55, (byte) 74, (byte) 101, (byte) 67 }), + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 34, (byte) -64, (byte) 97, (byte) 65, (byte) -42 }) ) ) ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntityTests.java index 170395e8af919..b58828dc24aa0 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googleaistudio/GoogleAiStudioEmbeddingsResponseEntityTests.java @@ -9,7 +9,7 @@ import org.apache.http.HttpResponse; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; @@ -36,15 +36,12 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException { } """; - InferenceTextEmbeddingFloatResults parsedResults = GoogleAiStudioEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = GoogleAiStudioEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(List.of(-0.00606332F, 0.058092743F)))) - ); + assertThat(parsedResults.embeddings(), is(List.of(TextEmbeddingFloatResults.Embedding.of(List.of(-0.00606332F, 0.058092743F))))); } public void testFromResponse_CreatesResultsForMultipleItems() throws IOException { @@ -67,7 +64,7 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException } """; - InferenceTextEmbeddingFloatResults parsedResults = GoogleAiStudioEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = GoogleAiStudioEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -76,8 +73,8 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException parsedResults.embeddings(), is( List.of( - InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(List.of(-0.00606332F, 0.058092743F)), - InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(List.of(0.030681048F, 0.01714732F)) + TextEmbeddingFloatResults.Embedding.of(List.of(-0.00606332F, 0.058092743F)), + TextEmbeddingFloatResults.Embedding.of(List.of(0.030681048F, 0.01714732F)) ) ) ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiEmbeddingsResponseEntityTests.java index 39bf08a21a76b..e0acd65c419fc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiEmbeddingsResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/googlevertexai/GoogleVertexAiEmbeddingsResponseEntityTests.java @@ -9,7 +9,7 @@ import org.apache.http.HttpResponse; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; @@ -42,15 +42,12 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException { } """; - InferenceTextEmbeddingFloatResults parsedResults = GoogleVertexAiEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = GoogleVertexAiEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(List.of(-0.123F, 0.123F)))) - ); + assertThat(parsedResults.embeddings(), is(List.of(TextEmbeddingFloatResults.Embedding.of(List.of(-0.123F, 0.123F))))); } public void testFromResponse_CreatesResultsForMultipleItems() throws IOException { @@ -85,7 +82,7 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException } """; - InferenceTextEmbeddingFloatResults parsedResults = GoogleVertexAiEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = GoogleVertexAiEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -94,8 +91,8 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException parsedResults.embeddings(), is( List.of( - InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(List.of(-0.123F, 0.123F)), - InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(List.of(-0.456F, 0.456F)) + TextEmbeddingFloatResults.Embedding.of(List.of(-0.123F, 0.123F)), + TextEmbeddingFloatResults.Embedding.of(List.of(-0.456F, 0.456F)) ) ) ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntityTests.java index 6f06a32f19a68..4ba6def4516bd 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/huggingface/HuggingFaceEmbeddingsResponseEntityTests.java @@ -10,7 +10,7 @@ import org.apache.http.HttpResponse; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; @@ -32,14 +32,14 @@ public void testFromResponse_CreatesResultsForASingleItem_ArrayFormat() throws I ] """; - InferenceTextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); assertThat( parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }))) + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.014539449F, -0.015288644F }))) ); } @@ -55,14 +55,14 @@ public void testFromResponse_CreatesResultsForASingleItem_ObjectFormat() throws } """; - InferenceTextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); assertThat( parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }))) + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.014539449F, -0.015288644F }))) ); } @@ -80,7 +80,7 @@ public void testFromResponse_CreatesResultsForMultipleItems_ArrayFormat() throws ] """; - InferenceTextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -89,8 +89,8 @@ public void testFromResponse_CreatesResultsForMultipleItems_ArrayFormat() throws parsedResults.embeddings(), is( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0123F, -0.0123F }) + new TextEmbeddingFloatResults.Embedding(new float[] { 0.014539449F, -0.015288644F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.0123F, -0.0123F }) ) ) ); @@ -112,7 +112,7 @@ public void testFromResponse_CreatesResultsForMultipleItems_ObjectFormat() throw } """; - InferenceTextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -121,8 +121,8 @@ public void testFromResponse_CreatesResultsForMultipleItems_ObjectFormat() throw parsedResults.embeddings(), is( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0123F, -0.0123F }) + new TextEmbeddingFloatResults.Embedding(new float[] { 0.014539449F, -0.015288644F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.0123F, -0.0123F }) ) ) ); @@ -255,15 +255,12 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsInt_ArrayFormat() throw ] """; - InferenceTextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 1.0F }))) - ); + assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 1.0F })))); } public void testFromResponse_SucceedsWhenEmbeddingValueIsInt_ObjectFormat() throws IOException { @@ -277,15 +274,12 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsInt_ObjectFormat() thro } """; - InferenceTextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 1.0F }))) - ); + assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 1.0F })))); } public void testFromResponse_SucceedsWhenEmbeddingValueIsLong_ArrayFormat() throws IOException { @@ -297,15 +291,12 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsLong_ArrayFormat() thro ] """; - InferenceTextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 4.0294965E10F }))) - ); + assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 4.0294965E10F })))); } public void testFromResponse_SucceedsWhenEmbeddingValueIsLong_ObjectFormat() throws IOException { @@ -319,15 +310,12 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsLong_ObjectFormat() thr } """; - InferenceTextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = HuggingFaceEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 4.0294965E10F }))) - ); + assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 4.0294965E10F })))); } public void testFromResponse_FailsWhenEmbeddingValueIsAnObject_ObjectFormat() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxEmbeddingsResponseEntityTests.java index ae39f19b83176..b122eafe06a54 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxEmbeddingsResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/ibmwatsonx/IbmWatsonxEmbeddingsResponseEntityTests.java @@ -9,7 +9,7 @@ import org.apache.http.HttpResponse; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; @@ -36,15 +36,12 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException { } """; - InferenceTextEmbeddingFloatResults parsedResults = IbmWatsonxEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = IbmWatsonxEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(List.of(-0.00606332F, 0.058092743F)))) - ); + assertThat(parsedResults.embeddings(), is(List.of(TextEmbeddingFloatResults.Embedding.of(List.of(-0.00606332F, 0.058092743F))))); } public void testFromResponse_CreatesResultsForMultipleItems() throws IOException { @@ -69,7 +66,7 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException } """; - InferenceTextEmbeddingFloatResults parsedResults = IbmWatsonxEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = IbmWatsonxEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -78,8 +75,8 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException parsedResults.embeddings(), is( List.of( - InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(List.of(-0.00606332F, 0.058092743F)), - InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding.of(List.of(0.030681048F, 0.01714732F)) + TextEmbeddingFloatResults.Embedding.of(List.of(-0.00606332F, 0.058092743F)), + TextEmbeddingFloatResults.Embedding.of(List.of(0.030681048F, 0.01714732F)) ) ) ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java index 7dbb9d5441a4a..a5f7c4eadae28 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/jinaai/JinaAIEmbeddingsResponseEntityTests.java @@ -10,7 +10,7 @@ import org.apache.http.HttpResponse; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; @@ -44,14 +44,14 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException { } """; - InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); assertThat( parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }))) + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.014539449F, -0.015288644F }))) ); } @@ -85,7 +85,7 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException } """; - InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -94,8 +94,8 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException parsedResults.embeddings(), is( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0123F, -0.0123F }) + new TextEmbeddingFloatResults.Embedding(new float[] { 0.014539449F, -0.015288644F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.0123F, -0.0123F }) ) ) ); @@ -259,15 +259,12 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsInt() throws IOExceptio } """; - InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 1.0F }))) - ); + assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 1.0F })))); } public void testFromResponse_SucceedsWhenEmbeddingValueIsLong() throws IOException { @@ -291,15 +288,12 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsLong() throws IOExcepti } """; - InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 4.0294965E10F }))) - ); + assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 4.0294965E10F })))); } public void testFromResponse_FailsWhenEmbeddingValueIsAnObject() { @@ -378,7 +372,7 @@ public void testFieldsInDifferentOrderServer() throws IOException { } }"""; - InferenceTextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = JinaAIEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), response.getBytes(StandardCharsets.UTF_8)) ); @@ -387,9 +381,9 @@ public void testFieldsInDifferentOrderServer() throws IOException { parsedResults.embeddings(), is( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.9F, 0.5F, 0.3F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.1F, 0.5F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.5F, 0.5F }) + new TextEmbeddingFloatResults.Embedding(new float[] { -0.9F, 0.5F, 0.3F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.1F, 0.5F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.5F, 0.5F }) ) ) ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntityTests.java index 8f5bd95126fb7..b5300f852e726 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/openai/OpenAiEmbeddingsResponseEntityTests.java @@ -10,7 +10,7 @@ import org.apache.http.HttpResponse; import org.elasticsearch.common.ParsingException; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.request.Request; @@ -44,14 +44,14 @@ public void testFromResponse_CreatesResultsForASingleItem() throws IOException { } """; - InferenceTextEmbeddingFloatResults parsedResults = OpenAiEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = OpenAiEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); assertThat( parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }))) + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.014539449F, -0.015288644F }))) ); } @@ -85,7 +85,7 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException } """; - InferenceTextEmbeddingFloatResults parsedResults = OpenAiEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = OpenAiEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); @@ -94,8 +94,8 @@ public void testFromResponse_CreatesResultsForMultipleItems() throws IOException parsedResults.embeddings(), is( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.014539449F, -0.015288644F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0123F, -0.0123F }) + new TextEmbeddingFloatResults.Embedding(new float[] { 0.014539449F, -0.015288644F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.0123F, -0.0123F }) ) ) ); @@ -259,15 +259,12 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsInt() throws IOExceptio } """; - InferenceTextEmbeddingFloatResults parsedResults = OpenAiEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = OpenAiEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 1.0F }))) - ); + assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 1.0F })))); } public void testFromResponse_SucceedsWhenEmbeddingValueIsLong() throws IOException { @@ -291,15 +288,12 @@ public void testFromResponse_SucceedsWhenEmbeddingValueIsLong() throws IOExcepti } """; - InferenceTextEmbeddingFloatResults parsedResults = OpenAiEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = OpenAiEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) ); - assertThat( - parsedResults.embeddings(), - is(List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 4.0294965E10F }))) - ); + assertThat(parsedResults.embeddings(), is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 4.0294965E10F })))); } public void testFromResponse_FailsWhenEmbeddingValueIsAnObject() { @@ -379,7 +373,7 @@ public void testFieldsInDifferentOrderServer() throws IOException { } }"""; - InferenceTextEmbeddingFloatResults parsedResults = OpenAiEmbeddingsResponseEntity.fromResponse( + TextEmbeddingFloatResults parsedResults = OpenAiEmbeddingsResponseEntity.fromResponse( mock(Request.class), new HttpResult(mock(HttpResponse.class), response.getBytes(StandardCharsets.UTF_8)) ); @@ -388,9 +382,9 @@ public void testFieldsInDifferentOrderServer() throws IOException { parsedResults.embeddings(), is( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.9F, 0.5F, 0.3F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.1F, 0.5F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.5F, 0.5F }) + new TextEmbeddingFloatResults.Embedding(new float[] { -0.9F, 0.5F, 0.3F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.1F, 0.5F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.5F, 0.5F }) ) ) ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIEmbeddingsResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIEmbeddingsResponseEntityTests.java new file mode 100644 index 0000000000000..f91afc665f6de --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIEmbeddingsResponseEntityTests.java @@ -0,0 +1,432 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.voyageai; + +import org.apache.http.HttpResponse; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentParseException; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.request.voyageai.VoyageAIEmbeddingsRequest; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModelTests.createModel; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class VoyageAIEmbeddingsResponseEntityTests extends ESTestCase { + public void testFromResponse_CreatesResultsForASingleItem() throws IOException { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.014539449, + -0.015288644 + ] + } + ], + "model": "voyage-3-large", + "usage": { + "total_tokens": 8 + } + } + """; + + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest( + List.of("abc", "def"), + createModel("url", "api_key", null, "voyage-3-large") + ); + + InferenceServiceResults parsedResults = VoyageAIEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + ((TextEmbeddingFloatResults) parsedResults).embeddings(), + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.014539449F, -0.015288644F }))) + ); + } + + public void testFromResponse_CreatesResultsForMultipleItems() throws IOException { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.014539449, + -0.015288644 + ] + }, + { + "object": "embedding", + "index": 1, + "embedding": [ + 0.0123, + -0.0123 + ] + } + ], + "model": "voyage-3-large", + "usage": { + "total_tokens": 8 + } + } + """; + + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest( + List.of("abc", "def"), + createModel("url", "api_key", null, "voyage-3-large") + ); + + InferenceServiceResults parsedResults = VoyageAIEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + ((TextEmbeddingFloatResults) parsedResults).embeddings(), + is( + List.of( + new TextEmbeddingFloatResults.Embedding(new float[] { 0.014539449F, -0.015288644F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.0123F, -0.0123F }) + ) + ) + ); + } + + public void testFromResponse_FailsWhenDataFieldIsNotPresent() { + String responseJson = """ + { + "object": "list", + "not_data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.014539449, + -0.015288644 + ] + } + ], + "model": "voyage-3-large", + "usage": { + "total_tokens": 8 + } + } + """; + + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest( + List.of("abc", "def"), + createModel("url", "api_key", null, "voyage-3-large") + ); + + var thrownException = expectThrows( + java.lang.IllegalArgumentException.class, + () -> VoyageAIEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertThat(thrownException.getMessage(), is("Required [data]")); + } + + public void testFromResponse_FailsWhenDataFieldNotAnArray() { + String responseJson = """ + { + "object": "list", + "data": { + "test": { + "object": "embedding", + "index": 0, + "embedding": [ + 0.014539449, + -0.015288644 + ] + } + }, + "model": "voyage-3-large", + "usage": { + "total_tokens": 8 + } + } + """; + + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest( + List.of("abc", "def"), + createModel("url", "api_key", null, "voyage-3-large") + ); + + var thrownException = expectThrows( + XContentParseException.class, + () -> VoyageAIEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertThat(thrownException.getMessage(), containsString("[EmbeddingFloatResult] failed to parse field [data]")); + } + + public void testFromResponse_FailsWhenEmbeddingsDoesNotExist() { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embeddingzzz": [ + 0.014539449, + -0.015288644 + ] + } + ], + "model": "voyage-3-large", + "usage": { + "total_tokens": 8 + } + } + """; + + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest( + List.of("abc", "def"), + createModel("url", "api_key", null, "voyage-3-large") + ); + + var thrownException = expectThrows( + XContentParseException.class, + () -> VoyageAIEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertThat(thrownException.getMessage(), containsString("[EmbeddingFloatResult] failed to parse field [data]")); + } + + public void testFromResponse_FailsWhenEmbeddingValueIsAString() { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + "abc" + ] + } + ], + "model": "voyage-3-large", + "usage": { + "total_tokens": 8 + } + } + """; + + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest( + List.of("abc", "def"), + createModel("url", "api_key", null, "voyage-3-large") + ); + + var thrownException = expectThrows( + XContentParseException.class, + () -> VoyageAIEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertThat(thrownException.getMessage(), is("[8:15] [EmbeddingFloatResult] failed to parse field [data]")); + } + + public void testFromResponse_SucceedsWhenEmbeddingValueIsInt() throws IOException { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 1 + ] + } + ], + "model": "voyage-3-large", + "usage": { + "total_tokens": 8 + } + } + """; + + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest( + List.of("abc", "def"), + createModel("url", "api_key", null, "voyage-3-large") + ); + + InferenceServiceResults parsedResults = VoyageAIEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + ((TextEmbeddingFloatResults) parsedResults).embeddings(), + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 1.0F }))) + ); + } + + public void testFromResponse_SucceedsWhenEmbeddingValueIsLong() throws IOException { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 40294967295 + ] + } + ], + "model": "voyage-3-large", + "usage": { + "total_tokens": 8 + } + } + """; + + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest( + List.of("abc", "def"), + createModel("url", "api_key", null, "voyage-3-large") + ); + + InferenceServiceResults parsedResults = VoyageAIEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat( + ((TextEmbeddingFloatResults) parsedResults).embeddings(), + is(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 4.0294965E10F }))) + ); + } + + public void testFromResponse_FailsWhenEmbeddingValueIsAnObject() { + String responseJson = """ + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + {} + ] + } + ], + "model": "voyage-3-large", + "usage": { + "total_tokens": 8 + } + } + """; + + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest( + List.of("abc", "def"), + createModel("url", "api_key", null, "voyage-3-large") + ); + + var thrownException = expectThrows( + XContentParseException.class, + () -> VoyageAIEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ) + ); + + assertThat(thrownException.getMessage(), is("[8:15] [EmbeddingFloatResult] failed to parse field [data]")); + } + + public void testFieldsInDifferentOrderServer() throws IOException { + // The fields of the objects in the data array are reordered + String response = """ + { + "object": "list", + "model": "voyage-3-large", + "data": [ + { + "embedding": [ + -0.9, + 0.5, + 0.3 + ], + "index": 0, + "object": "embedding" + }, + { + "index": 0, + "embedding": [ + 0.1, + 0.5 + ], + "object": "embedding" + }, + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.5, + 0.5 + ] + } + ], + "usage": { + "total_tokens": 0 + } + }"""; + + VoyageAIEmbeddingsRequest request = new VoyageAIEmbeddingsRequest( + List.of("abc", "def"), + createModel("url", "api_key", null, "voyage-3-large") + ); + + InferenceServiceResults parsedResults = VoyageAIEmbeddingsResponseEntity.fromResponse( + request, + new HttpResult(mock(HttpResponse.class), response.getBytes(StandardCharsets.UTF_8)) + ); + + assertThat(parsedResults, instanceOf(TextEmbeddingFloatResults.class)); + + assertThat( + ((TextEmbeddingFloatResults) parsedResults).embeddings(), + is( + List.of( + new TextEmbeddingFloatResults.Embedding(new float[] { -0.9F, 0.5F, 0.3F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.1F, 0.5F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.5F, 0.5F }) + ) + ) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIErrorResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIErrorResponseEntityTests.java new file mode 100644 index 0000000000000..bb5c1da90d776 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIErrorResponseEntityTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.voyageai; + +import org.apache.http.HttpResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.ErrorResponse; +import org.hamcrest.MatcherAssert; + +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class VoyageAIErrorResponseEntityTests extends ESTestCase { + public void testFromResponse() { + String message = "\"input\" length 2049 is larger than the largest allowed size 2048"; + String escapedMessage = message.replace("\\", "\\\\").replace("\"", "\\\""); + String responseJson = Strings.format(""" + { + "detail": "%s" + } + """, escapedMessage); + + ErrorResponse errorResponse = VoyageAIErrorResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + assertNotNull(errorResponse); + MatcherAssert.assertThat(errorResponse.getErrorMessage(), is(message)); + } + + public void testFromResponse_noMessage() { + String responseJson = """ + { + "error": "abc" + } + """; + + ErrorResponse errorResponse = VoyageAIErrorResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseJson.getBytes(StandardCharsets.UTF_8)) + ); + MatcherAssert.assertThat(errorResponse, is(ErrorResponse.UNDEFINED_ERROR)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIRerankResponseEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIRerankResponseEntityTests.java new file mode 100644 index 0000000000000..5c7aa7a80ad8f --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/voyageai/VoyageAIRerankResponseEntityTests.java @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.voyageai; + +import org.apache.http.HttpResponse; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; + +public class VoyageAIRerankResponseEntityTests extends ESTestCase { + + public void testResponseLiteral() throws IOException { + String responseLiteral = """ + { + "object": "list", + "model": "model", + "data": [ + { + "index": 2, + "relevance_score": 0.98005307 + }, + { + "index": 3, + "relevance_score": 0.27904198 + }, + { + "index": 0, + "relevance_score": 0.10194652 + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + InferenceServiceResults parsedResults = VoyageAIRerankResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseLiteral.getBytes(StandardCharsets.UTF_8)) + ); + + MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class)); + List expected = responseLiteralDocs(); + for (int i = 0; i < ((RankedDocsResults) parsedResults).getRankedDocs().size(); i++) { + assertEquals(((RankedDocsResults) parsedResults).getRankedDocs().get(i).index(), expected.get(i).index()); + } + } + + public void testGeneratedResponse() throws IOException { + int numDocs = randomIntBetween(1, 10); + + List expected = new ArrayList<>(numDocs); + StringBuilder responseBuilder = new StringBuilder(); + + responseBuilder.append("{"); + responseBuilder.append("\"model\": \"model\","); + responseBuilder.append("\"object\": \"list\","); + responseBuilder.append("\"data\": ["); + List indices = linear(numDocs); + List scores = linearFloats(numDocs); + for (int i = 0; i < numDocs; i++) { + int index = indices.remove(randomInt(indices.size() - 1)); + + responseBuilder.append("{"); + responseBuilder.append("\"index\":").append(index).append(","); + responseBuilder.append("\"relevance_score\":").append(scores.get(i).toString()).append("}"); + expected.add(new RankedDocsResults.RankedDoc(index, scores.get(i), null)); + if (i < numDocs - 1) { + responseBuilder.append(","); + } + } + responseBuilder.append("],"); + responseBuilder.append("\"usage\": {"); + responseBuilder.append("\"total_tokens\": 15}"); + responseBuilder.append("}"); + + InferenceServiceResults parsedResults = VoyageAIRerankResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseBuilder.toString().getBytes(StandardCharsets.UTF_8)) + ); + MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class)); + for (int i = 0; i < ((RankedDocsResults) parsedResults).getRankedDocs().size(); i++) { + assertEquals(((RankedDocsResults) parsedResults).getRankedDocs().get(i).index(), expected.get(i).index()); + } + } + + private ArrayList responseLiteralDocs() { + var list = new ArrayList(); + + list.add(new RankedDocsResults.RankedDoc(2, 0.98005307F, null)); + list.add(new RankedDocsResults.RankedDoc(3, 0.27904198F, null)); + list.add(new RankedDocsResults.RankedDoc(0, 0.10194652F, null)); + return list; + } + + public void testResponseLiteralWithDocuments() throws IOException { + String responseLiteralWithDocuments = """ + { + "object": "list", + "model": "model", + "data": [ + { + "document": "Washington, D.C..", + "index": 2, + "relevance_score": 0.98005307 + }, + { + "document": "Capital punishment has existed in the United States since beforethe United States was a country. ", + "index": 3, + "relevance_score": 0.27904198 + }, + { + "document": "Carson City is the capital city of the American state of Nevada.", + "index": 0, + "relevance_score": 0.10194652 + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + InferenceServiceResults parsedResults = VoyageAIRerankResponseEntity.fromResponse( + new HttpResult(mock(HttpResponse.class), responseLiteralWithDocuments.getBytes(StandardCharsets.UTF_8)) + ); + + MatcherAssert.assertThat(parsedResults, instanceOf(RankedDocsResults.class)); + MatcherAssert.assertThat(((RankedDocsResults) parsedResults).getRankedDocs(), is(responseLiteralDocsWithText)); + } + + private final List responseLiteralDocsWithText = List.of( + new RankedDocsResults.RankedDoc(2, 0.98005307F, "Washington, D.C.."), + new RankedDocsResults.RankedDoc( + 3, + 0.27904198F, + "Capital punishment has existed in the United States since beforethe United States was a country. " + ), + new RankedDocsResults.RankedDoc(0, 0.10194652F, "Carson City is the capital city of the American state of Nevada.") + ); + + private ArrayList linear(int n) { + ArrayList list = new ArrayList<>(); + for (int i = 0; i <= n; i++) { + list.add(i); + } + return list; + } + + // creates a list of doubles of monotonically decreasing magnitude + private ArrayList linearFloats(int n) { + ArrayList list = new ArrayList<>(); + float startValue = 1.0f; + float decrement = startValue / n + 1; + for (int i = 0; i <= n; i++) { + list.add(startValue - (i * decrement)); + } + return list; + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/voyageai/VoyageAIResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/voyageai/VoyageAIResponseHandlerTests.java new file mode 100644 index 0000000000000..d032116d9a894 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/voyageai/VoyageAIResponseHandlerTests.java @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.voyageai; + +import org.apache.http.Header; +import org.apache.http.HeaderElement; +import org.apache.http.HttpResponse; +import org.apache.http.StatusLine; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.elasticsearch.xpack.inference.external.http.retry.RetryException; +import org.elasticsearch.xpack.inference.external.request.Request; +import org.hamcrest.MatcherAssert; + +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.core.Is.is; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class VoyageAIResponseHandlerTests extends ESTestCase { + public void testCheckForFailureStatusCode_DoesNotThrowForStatusCodesBetween200And299() { + callCheckForFailureStatusCode(randomIntBetween(200, 299), "id"); + } + + public void testCheckForFailureStatusCode_ThrowsFor503() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(503, "id")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received a server error status code for request from inference entity id [id] status [503]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST)); + } + + public void testCheckForFailureStatusCode_ThrowsFor500_WithShouldRetryTrue() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(500, "id")); + assertTrue(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received a server error status code for request from inference entity id [id] status [500]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST)); + } + + public void testCheckForFailureStatusCode_ThrowsFor429_WithShouldRetryTrue() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(429, "id")); + assertTrue(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received a rate limit status code for request from inference entity id [id] status [429]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.TOO_MANY_REQUESTS)); + } + + public void testCheckForFailureStatusCode_ThrowsFor400() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(400, "id")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received an input validation error response for request from inference entity id [id] status [400]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST)); + } + + public void testCheckForFailureStatusCode_ThrowsFor400_InputsTooLarge() { + var exception = expectThrows( + RetryException.class, + () -> callCheckForFailureStatusCode(400, "\"input\" length 2049 is larger than the largest allowed size 2048", "id") + ); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString("Received an input validation error response for request from inference entity id [id] status [400]") + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.BAD_REQUEST)); + } + + public void testCheckForFailureStatusCode_ThrowsFor401() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(401, "inferenceEntityId")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat( + exception.getCause().getMessage(), + containsString( + "Received an authentication error status code for request from inference entity id [inferenceEntityId] status [401]" + ) + ); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.UNAUTHORIZED)); + } + + public void testCheckForFailureStatusCode_ThrowsFor402() { + var exception = expectThrows(RetryException.class, () -> callCheckForFailureStatusCode(402, "inferenceEntityId")); + assertFalse(exception.shouldRetry()); + MatcherAssert.assertThat(exception.getCause().getMessage(), containsString("Payment required")); + MatcherAssert.assertThat(((ElasticsearchStatusException) exception.getCause()).status(), is(RestStatus.PAYMENT_REQUIRED)); + } + + private static void callCheckForFailureStatusCode(int statusCode, String modelId) { + callCheckForFailureStatusCode(statusCode, null, modelId); + } + + private static void callCheckForFailureStatusCode(int statusCode, @Nullable String errorMessage, String modelId) { + var statusLine = mock(StatusLine.class); + when(statusLine.getStatusCode()).thenReturn(statusCode); + + var httpResponse = mock(HttpResponse.class); + when(httpResponse.getStatusLine()).thenReturn(statusLine); + var header = mock(Header.class); + when(header.getElements()).thenReturn(new HeaderElement[] {}); + when(httpResponse.getFirstHeader(anyString())).thenReturn(header); + + String escapedErrorMessage = errorMessage != null ? errorMessage.replace("\\", "\\\\").replace("\"", "\\\"") : errorMessage; + + String responseJson = Strings.format(""" + { + "detail": "%s" + } + """, escapedErrorMessage); + + var mockRequest = mock(Request.class); + when(mockRequest.getInferenceEntityId()).thenReturn(modelId); + var httpResult = new HttpResult(httpResponse, errorMessage == null ? new byte[] {} : responseJson.getBytes(StandardCharsets.UTF_8)); + var handler = new VoyageAIResponseHandler("", (request, result) -> null); + + handler.checkForFailureStatusCode(mockRequest, httpResult); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java index 404713581eddd..9eb10cfd9f1a8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java @@ -21,9 +21,10 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingByte; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; import org.elasticsearch.xpack.core.utils.FloatConversionUtils; import org.elasticsearch.xpack.inference.model.TestModel; @@ -169,50 +170,44 @@ public void testModelSettingsValidation() { assertThat(ex.getMessage(), containsString("required [element_type] field is missing")); } - public static ChunkedInferenceEmbeddingByte randomChunkedInferenceEmbeddingByte(Model model, List inputs) { - List chunks = new ArrayList<>(); + public static ChunkedInferenceEmbedding randomChunkedInferenceEmbeddingByte(Model model, List inputs) { + List chunks = new ArrayList<>(); for (String input : inputs) { byte[] values = new byte[model.getServiceSettings().dimensions()]; for (int j = 0; j < values.length; j++) { values[j] = randomByte(); } - chunks.add( - new ChunkedInferenceEmbeddingByte.ByteEmbeddingChunk(values, input, new ChunkedInference.TextOffset(0, input.length())) - ); + chunks.add(new TextEmbeddingByteResults.Chunk(values, input, new ChunkedInference.TextOffset(0, input.length()))); } - return new ChunkedInferenceEmbeddingByte(chunks); + return new ChunkedInferenceEmbedding(chunks); } - public static ChunkedInferenceEmbeddingFloat randomChunkedInferenceEmbeddingFloat(Model model, List inputs) { - List chunks = new ArrayList<>(); + public static ChunkedInferenceEmbedding randomChunkedInferenceEmbeddingFloat(Model model, List inputs) { + List chunks = new ArrayList<>(); for (String input : inputs) { float[] values = new float[model.getServiceSettings().dimensions()]; for (int j = 0; j < values.length; j++) { values[j] = randomFloat(); } - chunks.add( - new ChunkedInferenceEmbeddingFloat.FloatEmbeddingChunk(values, input, new ChunkedInference.TextOffset(0, input.length())) - ); + chunks.add(new TextEmbeddingFloatResults.Chunk(values, input, new ChunkedInference.TextOffset(0, input.length()))); } - return new ChunkedInferenceEmbeddingFloat(chunks); + return new ChunkedInferenceEmbedding(chunks); } - public static ChunkedInferenceEmbeddingSparse randomChunkedInferenceEmbeddingSparse(List inputs) { + public static ChunkedInferenceEmbedding randomChunkedInferenceEmbeddingSparse(List inputs) { return randomChunkedInferenceEmbeddingSparse(inputs, true); } - public static ChunkedInferenceEmbeddingSparse randomChunkedInferenceEmbeddingSparse(List inputs, boolean withFloats) { - List chunks = new ArrayList<>(); + public static ChunkedInferenceEmbedding randomChunkedInferenceEmbeddingSparse(List inputs, boolean withFloats) { + List chunks = new ArrayList<>(); for (String input : inputs) { var tokens = new ArrayList(); for (var token : input.split("\\s+")) { tokens.add(new WeightedToken(token, withFloats ? randomFloat() : randomIntBetween(1, 255))); } - chunks.add( - new ChunkedInferenceEmbeddingSparse.SparseEmbeddingChunk(tokens, input, new ChunkedInference.TextOffset(0, input.length())) - ); + chunks.add(new SparseEmbeddingResults.Chunk(tokens, input, new ChunkedInference.TextOffset(0, input.length()))); } - return new ChunkedInferenceEmbeddingSparse(chunks); + return new ChunkedInferenceEmbedding(chunks); } public static SemanticTextField randomSemanticText( @@ -302,7 +297,7 @@ public static ChunkedInference toChunkedResult( ) { switch (field.inference().modelSettings().taskType()) { case SPARSE_EMBEDDING -> { - List chunks = new ArrayList<>(); + List chunks = new ArrayList<>(); for (var entry : field.inference().chunks().entrySet()) { String entryField = entry.getKey(); List entryChunks = entry.getValue(); @@ -313,13 +308,13 @@ public static ChunkedInference toChunkedResult( String matchedText = matchedTextIt.next(); ChunkedInference.TextOffset offset = createOffset(useLegacyFormat, chunk, matchedText); var tokens = parseWeightedTokens(chunk.rawEmbeddings(), field.contentType()); - chunks.add(new ChunkedInferenceEmbeddingSparse.SparseEmbeddingChunk(tokens, matchedText, offset)); + chunks.add(new SparseEmbeddingResults.Chunk(tokens, matchedText, offset)); } } - return new ChunkedInferenceEmbeddingSparse(chunks); + return new ChunkedInferenceEmbedding(chunks); } case TEXT_EMBEDDING -> { - List chunks = new ArrayList<>(); + List chunks = new ArrayList<>(); for (var entry : field.inference().chunks().entrySet()) { String entryField = entry.getKey(); List entryChunks = entry.getValue(); @@ -334,16 +329,10 @@ public static ChunkedInference toChunkedResult( field.inference().modelSettings().dimensions(), field.contentType() ); - chunks.add( - new ChunkedInferenceEmbeddingFloat.FloatEmbeddingChunk( - FloatConversionUtils.floatArrayOf(values), - matchedText, - offset - ) - ); + chunks.add(new TextEmbeddingFloatResults.Chunk(FloatConversionUtils.floatArrayOf(values), matchedText, offset)); } } - return new ChunkedInferenceEmbeddingFloat(chunks); + return new ChunkedInferenceEmbedding(chunks); } default -> throw new AssertionError("Invalid task_type: " + field.inference().modelSettings().taskType().name()); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java index 9a3b4eff1958a..451f5fb6ad7ba 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java @@ -50,8 +50,8 @@ import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.XPackClientPlugin; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; @@ -300,7 +300,7 @@ private InferenceAction.Response generateTextEmbeddingInferenceResponse() { Arrays.fill(inference, 1.0); MlTextEmbeddingResults textEmbeddingResults = new MlTextEmbeddingResults(DEFAULT_RESULTS_FIELD, inference, false); - return new InferenceAction.Response(InferenceTextEmbeddingFloatResults.of(List.of(textEmbeddingResults))); + return new InferenceAction.Response(TextEmbeddingFloatResults.of(List.of(textEmbeddingResults))); } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java index 61d28fae402ba..91eb4a5b9af99 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rest/BaseInferenceActionTests.java @@ -22,8 +22,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.action.InferenceActionProxy; -import org.elasticsearch.xpack.core.inference.results.InferenceByteEmbedding; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; import org.junit.Before; import java.util.HashMap; @@ -149,7 +148,7 @@ public void testUses3SecondTimeoutFromParams() { static InferenceAction.Response createResponse() { return new InferenceAction.Response( - new InferenceTextEmbeddingByteResults(List.of(new InferenceByteEmbedding(new byte[] { (byte) -1 }))) + new TextEmbeddingByteResults(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) -1 }))) ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/InferenceTextEmbeddingBitResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingBitResultsTests.java similarity index 56% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/InferenceTextEmbeddingBitResultsTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingBitResultsTests.java index 45b9627371575..8d3ead44f2335 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/InferenceTextEmbeddingBitResultsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingBitResultsTests.java @@ -10,8 +10,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceByteEmbedding; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingBitResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingBitResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import java.io.IOException; @@ -21,19 +21,19 @@ import static org.hamcrest.Matchers.is; -public class InferenceTextEmbeddingBitResultsTests extends AbstractWireSerializingTestCase { - public static InferenceTextEmbeddingBitResults createRandomResults() { +public class TextEmbeddingBitResultsTests extends AbstractWireSerializingTestCase { + public static TextEmbeddingBitResults createRandomResults() { int embeddings = randomIntBetween(1, 10); - List embeddingResults = new ArrayList<>(embeddings); + List embeddingResults = new ArrayList<>(embeddings); for (int i = 0; i < embeddings; i++) { embeddingResults.add(createRandomEmbedding()); } - return new InferenceTextEmbeddingBitResults(embeddingResults); + return new TextEmbeddingBitResults(embeddingResults); } - private static InferenceByteEmbedding createRandomEmbedding() { + private static TextEmbeddingByteResults.Embedding createRandomEmbedding() { int columns = randomIntBetween(1, 10); byte[] bytes = new byte[columns]; @@ -41,11 +41,11 @@ private static InferenceByteEmbedding createRandomEmbedding() { bytes[i] = randomByte(); } - return new InferenceByteEmbedding(bytes); + return new TextEmbeddingByteResults.Embedding(bytes); } public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOException { - var entity = new InferenceTextEmbeddingBitResults(List.of(new InferenceByteEmbedding(new byte[] { (byte) 23 }))); + var entity = new TextEmbeddingBitResults(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23 }))); String xContentResult = Strings.toString(entity, true, true); assertThat(xContentResult, is(""" @@ -61,8 +61,11 @@ public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOE } public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws IOException { - var entity = new InferenceTextEmbeddingBitResults( - List.of(new InferenceByteEmbedding(new byte[] { (byte) 23 }), new InferenceByteEmbedding(new byte[] { (byte) 24 })) + var entity = new TextEmbeddingBitResults( + List.of( + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23 }), + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 24 }) + ) ); String xContentResult = Strings.toString(entity, true, true); @@ -84,10 +87,10 @@ public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws I } public void testTransformToCoordinationFormat() { - var results = new InferenceTextEmbeddingBitResults( + var results = new TextEmbeddingBitResults( List.of( - new InferenceByteEmbedding(new byte[] { (byte) 23, (byte) 24 }), - new InferenceByteEmbedding(new byte[] { (byte) 25, (byte) 26 }) + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23, (byte) 24 }), + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 25, (byte) 26 }) ) ).transformToCoordinationFormat(); @@ -95,41 +98,41 @@ public void testTransformToCoordinationFormat() { results, is( List.of( - new MlTextEmbeddingResults(InferenceTextEmbeddingBitResults.TEXT_EMBEDDING_BITS, new double[] { 23F, 24F }, false), - new MlTextEmbeddingResults(InferenceTextEmbeddingBitResults.TEXT_EMBEDDING_BITS, new double[] { 25F, 26F }, false) + new MlTextEmbeddingResults(TextEmbeddingBitResults.TEXT_EMBEDDING_BITS, new double[] { 23F, 24F }, false), + new MlTextEmbeddingResults(TextEmbeddingBitResults.TEXT_EMBEDDING_BITS, new double[] { 25F, 26F }, false) ) ) ); } @Override - protected Writeable.Reader instanceReader() { - return InferenceTextEmbeddingBitResults::new; + protected Writeable.Reader instanceReader() { + return TextEmbeddingBitResults::new; } @Override - protected InferenceTextEmbeddingBitResults createTestInstance() { + protected TextEmbeddingBitResults createTestInstance() { return createRandomResults(); } @Override - protected InferenceTextEmbeddingBitResults mutateInstance(InferenceTextEmbeddingBitResults instance) throws IOException { + protected TextEmbeddingBitResults mutateInstance(TextEmbeddingBitResults instance) throws IOException { // if true we reduce the embeddings list by a random amount, if false we add an embedding to the list if (randomBoolean()) { // -1 to remove at least one item from the list int end = randomInt(instance.embeddings().size() - 1); - return new InferenceTextEmbeddingBitResults(instance.embeddings().subList(0, end)); + return new TextEmbeddingBitResults(instance.embeddings().subList(0, end)); } else { - List embeddings = new ArrayList<>(instance.embeddings()); + List embeddings = new ArrayList<>(instance.embeddings()); embeddings.add(createRandomEmbedding()); - return new InferenceTextEmbeddingBitResults(embeddings); + return new TextEmbeddingBitResults(embeddings); } } public static Map buildExpectationByte(List> embeddings) { return Map.of( - InferenceTextEmbeddingBitResults.TEXT_EMBEDDING_BITS, - embeddings.stream().map(embedding -> Map.of(InferenceByteEmbedding.EMBEDDING, embedding)).toList() + TextEmbeddingBitResults.TEXT_EMBEDDING_BITS, + embeddings.stream().map(embedding -> Map.of(TextEmbeddingByteResults.Embedding.EMBEDDING, embedding)).toList() ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/InferenceTextEmbeddingByteResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java similarity index 56% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/InferenceTextEmbeddingByteResultsTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java index d932f36fb25a7..aaa823fda0723 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/InferenceTextEmbeddingByteResultsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingByteResultsTests.java @@ -10,8 +10,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceByteEmbedding; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import java.io.IOException; @@ -21,19 +20,19 @@ import static org.hamcrest.Matchers.is; -public class InferenceTextEmbeddingByteResultsTests extends AbstractWireSerializingTestCase { - public static InferenceTextEmbeddingByteResults createRandomResults() { +public class TextEmbeddingByteResultsTests extends AbstractWireSerializingTestCase { + public static TextEmbeddingByteResults createRandomResults() { int embeddings = randomIntBetween(1, 10); - List embeddingResults = new ArrayList<>(embeddings); + List embeddingResults = new ArrayList<>(embeddings); for (int i = 0; i < embeddings; i++) { embeddingResults.add(createRandomEmbedding()); } - return new InferenceTextEmbeddingByteResults(embeddingResults); + return new TextEmbeddingByteResults(embeddingResults); } - private static InferenceByteEmbedding createRandomEmbedding() { + private static TextEmbeddingByteResults.Embedding createRandomEmbedding() { int columns = randomIntBetween(1, 10); byte[] bytes = new byte[columns]; @@ -41,11 +40,11 @@ private static InferenceByteEmbedding createRandomEmbedding() { bytes[i] = randomByte(); } - return new InferenceByteEmbedding(bytes); + return new TextEmbeddingByteResults.Embedding(bytes); } public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOException { - var entity = new InferenceTextEmbeddingByteResults(List.of(new InferenceByteEmbedding(new byte[] { (byte) 23 }))); + var entity = new TextEmbeddingByteResults(List.of(new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23 }))); String xContentResult = Strings.toString(entity, true, true); assertThat(xContentResult, is(""" @@ -61,8 +60,11 @@ public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOE } public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws IOException { - var entity = new InferenceTextEmbeddingByteResults( - List.of(new InferenceByteEmbedding(new byte[] { (byte) 23 }), new InferenceByteEmbedding(new byte[] { (byte) 24 })) + var entity = new TextEmbeddingByteResults( + List.of( + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23 }), + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 24 }) + ) ); String xContentResult = Strings.toString(entity, true, true); @@ -84,10 +86,10 @@ public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws I } public void testTransformToCoordinationFormat() { - var results = new InferenceTextEmbeddingByteResults( + var results = new TextEmbeddingByteResults( List.of( - new InferenceByteEmbedding(new byte[] { (byte) 23, (byte) 24 }), - new InferenceByteEmbedding(new byte[] { (byte) 25, (byte) 26 }) + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 23, (byte) 24 }), + new TextEmbeddingByteResults.Embedding(new byte[] { (byte) 25, (byte) 26 }) ) ).transformToCoordinationFormat(); @@ -95,41 +97,41 @@ public void testTransformToCoordinationFormat() { results, is( List.of( - new MlTextEmbeddingResults(InferenceTextEmbeddingByteResults.TEXT_EMBEDDING_BYTES, new double[] { 23F, 24F }, false), - new MlTextEmbeddingResults(InferenceTextEmbeddingByteResults.TEXT_EMBEDDING_BYTES, new double[] { 25F, 26F }, false) + new MlTextEmbeddingResults(TextEmbeddingByteResults.TEXT_EMBEDDING_BYTES, new double[] { 23F, 24F }, false), + new MlTextEmbeddingResults(TextEmbeddingByteResults.TEXT_EMBEDDING_BYTES, new double[] { 25F, 26F }, false) ) ) ); } @Override - protected Writeable.Reader instanceReader() { - return InferenceTextEmbeddingByteResults::new; + protected Writeable.Reader instanceReader() { + return TextEmbeddingByteResults::new; } @Override - protected InferenceTextEmbeddingByteResults createTestInstance() { + protected TextEmbeddingByteResults createTestInstance() { return createRandomResults(); } @Override - protected InferenceTextEmbeddingByteResults mutateInstance(InferenceTextEmbeddingByteResults instance) throws IOException { + protected TextEmbeddingByteResults mutateInstance(TextEmbeddingByteResults instance) throws IOException { // if true we reduce the embeddings list by a random amount, if false we add an embedding to the list if (randomBoolean()) { // -1 to remove at least one item from the list int end = randomInt(instance.embeddings().size() - 1); - return new InferenceTextEmbeddingByteResults(instance.embeddings().subList(0, end)); + return new TextEmbeddingByteResults(instance.embeddings().subList(0, end)); } else { - List embeddings = new ArrayList<>(instance.embeddings()); + List embeddings = new ArrayList<>(instance.embeddings()); embeddings.add(createRandomEmbedding()); - return new InferenceTextEmbeddingByteResults(embeddings); + return new TextEmbeddingByteResults(embeddings); } } public static Map buildExpectationByte(List> embeddings) { return Map.of( - InferenceTextEmbeddingByteResults.TEXT_EMBEDDING_BYTES, - embeddings.stream().map(embedding -> Map.of(InferenceByteEmbedding.EMBEDDING, embedding)).toList() + TextEmbeddingByteResults.TEXT_EMBEDDING_BYTES, + embeddings.stream().map(embedding -> Map.of(TextEmbeddingByteResults.Embedding.EMBEDDING, embedding)).toList() ); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java index 56bd690a9cdbf..77cbeaea68c39 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/TextEmbeddingResultsTests.java @@ -10,9 +10,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceByteEmbedding; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import java.io.IOException; @@ -22,32 +21,30 @@ import static org.hamcrest.Matchers.is; -public class TextEmbeddingResultsTests extends AbstractWireSerializingTestCase { - public static InferenceTextEmbeddingFloatResults createRandomResults() { +public class TextEmbeddingResultsTests extends AbstractWireSerializingTestCase { + public static TextEmbeddingFloatResults createRandomResults() { int embeddings = randomIntBetween(1, 10); - List embeddingResults = new ArrayList<>(embeddings); + List embeddingResults = new ArrayList<>(embeddings); for (int i = 0; i < embeddings; i++) { embeddingResults.add(createRandomEmbedding()); } - return new InferenceTextEmbeddingFloatResults(embeddingResults); + return new TextEmbeddingFloatResults(embeddingResults); } - private static InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding createRandomEmbedding() { + private static TextEmbeddingFloatResults.Embedding createRandomEmbedding() { int columns = randomIntBetween(1, 10); float[] floats = new float[columns]; for (int i = 0; i < columns; i++) { floats[i] = randomFloat(); } - return new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(floats); + return new TextEmbeddingFloatResults.Embedding(floats); } public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOException { - var entity = new InferenceTextEmbeddingFloatResults( - List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.1F })) - ); + var entity = new TextEmbeddingFloatResults(List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.1F }))); String xContentResult = Strings.toString(entity, true, true); assertThat(xContentResult, is(""" @@ -63,10 +60,10 @@ public void testToXContent_CreatesTheRightFormatForASingleEmbedding() throws IOE } public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws IOException { - var entity = new InferenceTextEmbeddingFloatResults( + var entity = new TextEmbeddingFloatResults( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.1F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.2F }) + new TextEmbeddingFloatResults.Embedding(new float[] { 0.1F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.2F }) ) ); @@ -90,10 +87,10 @@ public void testToXContent_CreatesTheRightFormatForMultipleEmbeddings() throws I } public void testTransformToCoordinationFormat() { - var results = new InferenceTextEmbeddingFloatResults( + var results = new TextEmbeddingFloatResults( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.1F, 0.2F }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.3F, 0.4F }) + new TextEmbeddingFloatResults.Embedding(new float[] { 0.1F, 0.2F }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.3F, 0.4F }) ) ).transformToCoordinationFormat(); @@ -101,49 +98,49 @@ public void testTransformToCoordinationFormat() { results, is( List.of( - new MlTextEmbeddingResults(InferenceTextEmbeddingFloatResults.TEXT_EMBEDDING, new double[] { 0.1F, 0.2F }, false), - new MlTextEmbeddingResults(InferenceTextEmbeddingFloatResults.TEXT_EMBEDDING, new double[] { 0.3F, 0.4F }, false) + new MlTextEmbeddingResults(TextEmbeddingFloatResults.TEXT_EMBEDDING, new double[] { 0.1F, 0.2F }, false), + new MlTextEmbeddingResults(TextEmbeddingFloatResults.TEXT_EMBEDDING, new double[] { 0.3F, 0.4F }, false) ) ) ); } @Override - protected Writeable.Reader instanceReader() { - return InferenceTextEmbeddingFloatResults::new; + protected Writeable.Reader instanceReader() { + return TextEmbeddingFloatResults::new; } @Override - protected InferenceTextEmbeddingFloatResults createTestInstance() { + protected TextEmbeddingFloatResults createTestInstance() { return createRandomResults(); } @Override - protected InferenceTextEmbeddingFloatResults mutateInstance(InferenceTextEmbeddingFloatResults instance) throws IOException { + protected TextEmbeddingFloatResults mutateInstance(TextEmbeddingFloatResults instance) throws IOException { // if true we reduce the embeddings list by a random amount, if false we add an embedding to the list if (randomBoolean()) { // -1 to remove at least one item from the list int end = randomInt(instance.embeddings().size() - 1); - return new InferenceTextEmbeddingFloatResults(instance.embeddings().subList(0, end)); + return new TextEmbeddingFloatResults(instance.embeddings().subList(0, end)); } else { - List embeddings = new ArrayList<>(instance.embeddings()); + List embeddings = new ArrayList<>(instance.embeddings()); embeddings.add(createRandomEmbedding()); - return new InferenceTextEmbeddingFloatResults(embeddings); + return new TextEmbeddingFloatResults(embeddings); } } public static Map buildExpectationFloat(List embeddings) { - return Map.of( - InferenceTextEmbeddingFloatResults.TEXT_EMBEDDING, - embeddings.stream().map(InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding::new).toList() - ); + return Map.of(TextEmbeddingFloatResults.TEXT_EMBEDDING, embeddings.stream().map(TextEmbeddingFloatResults.Embedding::new).toList()); } public static Map buildExpectationByte(List embeddings) { return Map.of( - InferenceTextEmbeddingByteResults.TEXT_EMBEDDING_BYTES, - embeddings.stream().map(InferenceByteEmbedding::new).toList() + TextEmbeddingByteResults.TEXT_EMBEDDING_BYTES, + embeddings.stream().map(TextEmbeddingByteResults.Embedding::new).toList() ); } + public static Map buildExpectationBinary(List embeddings) { + return Map.of("text_embedding_bits", embeddings.stream().map(TextEmbeddingByteResults.Embedding::new).toList()); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java index e3df0f0b5a2e1..ddaea0dc3c9a1 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ServiceUtilsTests.java @@ -19,10 +19,10 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; -import org.elasticsearch.xpack.inference.results.InferenceTextEmbeddingByteResultsTests; +import org.elasticsearch.xpack.inference.results.TextEmbeddingByteResultsTests; import org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests; import java.util.EnumSet; @@ -911,7 +911,7 @@ public void testGetEmbeddingSize_ReturnsError_WhenTextEmbeddingResults_IsEmpty() doAnswer(invocation -> { ActionListener listener = invocation.getArgument(7); - listener.onResponse(new InferenceTextEmbeddingFloatResults(List.of())); + listener.onResponse(new TextEmbeddingFloatResults(List.of())); return Void.TYPE; }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); @@ -933,7 +933,7 @@ public void testGetEmbeddingSize_ReturnsError_WhenTextEmbeddingByteResults_IsEmp doAnswer(invocation -> { ActionListener listener = invocation.getArgument(7); - listener.onResponse(new InferenceTextEmbeddingByteResults(List.of())); + listener.onResponse(new TextEmbeddingByteResults(List.of())); return Void.TYPE; }).when(service).infer(any(), any(), any(), anyBoolean(), any(), any(), any(), any()); @@ -967,7 +967,7 @@ public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingResults() { var size = listener.actionGet(TIMEOUT); - assertThat(size, is(textEmbedding.embeddings().get(0).getSize())); + assertThat(size, is(textEmbedding.embeddings().get(0).values().length)); } public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingByteResults() { @@ -976,7 +976,7 @@ public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingByteResults() { var model = mock(Model.class); when(model.getTaskType()).thenReturn(TaskType.TEXT_EMBEDDING); - var textEmbedding = InferenceTextEmbeddingByteResultsTests.createRandomResults(); + var textEmbedding = TextEmbeddingByteResultsTests.createRandomResults(); doAnswer(invocation -> { ActionListener listener = invocation.getArgument(7); @@ -990,7 +990,7 @@ public void testGetEmbeddingSize_ReturnsSize_ForTextEmbeddingByteResults() { var size = listener.actionGet(TIMEOUT); - assertThat(size, is(textEmbedding.embeddings().get(0).getSize())); + assertThat(size, is(textEmbedding.embeddings().get(0).values().length)); } private static Map modifiableMap(Map aMap) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java index 1ca50d1887ee1..c4c6b69b117bc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/AlibabaCloudSearchServiceTests.java @@ -29,9 +29,9 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.alibabacloudsearch.AlibabaCloudSearchActionVisitor; @@ -43,9 +43,6 @@ import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import org.elasticsearch.xpack.inference.results.SparseEmbeddingResultsTests; import org.elasticsearch.xpack.inference.services.ServiceFields; -import org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion.AlibabaCloudSearchCompletionModelTests; -import org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion.AlibabaCloudSearchCompletionServiceSettingsTests; -import org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion.AlibabaCloudSearchCompletionTaskSettingsTests; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModel; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsModelTests; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.embeddings.AlibabaCloudSearchEmbeddingsServiceSettingsTests; @@ -274,8 +271,8 @@ public void doInfer( TimeValue timeout, ActionListener listener ) { - InferenceTextEmbeddingFloatResults results = new InferenceTextEmbeddingFloatResults( - List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { -0.028680f, 0.022033f })) + TextEmbeddingFloatResults results = new TextEmbeddingFloatResults( + List.of(new TextEmbeddingFloatResults.Embedding(new float[] { -0.028680f, 0.022033f })) ); listener.onResponse(results); @@ -380,35 +377,6 @@ public void testChunkedInfer_SparseEmbeddingChunkingSettingsNotSet() throws IOEx testChunkedInfer(TaskType.SPARSE_EMBEDDING, null); } - public void testChunkedInfer_InvalidTaskType() throws IOException { - var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); - - try (var service = new AlibabaCloudSearchService(senderFactory, createWithEmptySettings(threadPool))) { - var model = AlibabaCloudSearchCompletionModelTests.createModel( - randomAlphaOfLength(10), - TaskType.COMPLETION, - AlibabaCloudSearchCompletionServiceSettingsTests.createRandom(), - AlibabaCloudSearchCompletionTaskSettingsTests.createRandom(), - null - ); - - PlainActionFuture> listener = new PlainActionFuture<>(); - try { - service.chunkedInfer( - model, - null, - List.of("foo", "bar"), - new HashMap<>(), - InputType.INGEST, - InferenceAction.Request.DEFAULT_TIMEOUT, - listener - ); - } catch (Exception e) { - assertThat(e, instanceOf(IllegalArgumentException.class)); - } - } - } - private void testChunkedInfer(TaskType taskType, ChunkingSettings chunkingSettings) throws IOException { var input = List.of("foo", "bar"); @@ -423,12 +391,14 @@ private void testChunkedInfer(TaskType taskType, ChunkingSettings chunkingSettin var results = listener.actionGet(TIMEOUT); assertThat(results, instanceOf(List.class)); assertThat(results, hasSize(2)); - var firstResult = results.get(0); - if (TaskType.TEXT_EMBEDDING.equals(taskType)) { - assertThat(firstResult, instanceOf(ChunkedInferenceEmbeddingFloat.class)); - } else if (TaskType.SPARSE_EMBEDDING.equals(taskType)) { - assertThat(firstResult, instanceOf(ChunkedInferenceEmbeddingSparse.class)); - } + var firstResult = results.getFirst(); + assertThat(firstResult, instanceOf(ChunkedInferenceEmbedding.class)); + Class expectedClass = switch (taskType) { + case TEXT_EMBEDDING -> TextEmbeddingFloatResults.Chunk.class; + case SPARSE_EMBEDDING -> SparseEmbeddingResults.Chunk.class; + default -> null; + }; + assertThat(((ChunkedInferenceEmbedding) firstResult).chunks().getFirst(), instanceOf(expectedClass)); } } @@ -552,10 +522,10 @@ private AlibabaCloudSearchModel createEmbeddingsModel( ) { public ExecutableAction accept(AlibabaCloudSearchActionVisitor visitor, Map taskSettings, InputType inputType) { return (inferenceInputs, timeout, listener) -> { - InferenceTextEmbeddingFloatResults results = new InferenceTextEmbeddingFloatResults( + TextEmbeddingFloatResults results = new TextEmbeddingFloatResults( List.of( - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0123f, -0.0123f }), - new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.0456f, -0.0456f }) + new TextEmbeddingFloatResults.Embedding(new float[] { 0.0123f, -0.0123f }), + new TextEmbeddingFloatResults.Embedding(new float[] { 0.0456f, -0.0456f }) ) ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java index 6505c280c295a..970dab45731bd 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java @@ -35,8 +35,8 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.Utils; import org.elasticsearch.xpack.inference.external.amazonbedrock.AmazonBedrockMockRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; @@ -965,8 +965,8 @@ public void testInfer_SendsRequest_ForEmbeddingsModel() throws IOException { try (var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool))) { try (var requestSender = (AmazonBedrockMockRequestSender) amazonBedrockFactory.createSender()) { - var results = new InferenceTextEmbeddingFloatResults( - List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.123F, 0.678F })) + var results = new TextEmbeddingFloatResults( + List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.123F, 0.678F })) ); requestSender.enqueue(results); @@ -1051,8 +1051,8 @@ public void testCheckModelConfig_IncludesMaxTokens_ForEmbeddingsModel() throws I try (var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool))) { try (var requestSender = (AmazonBedrockMockRequestSender) amazonBedrockFactory.createSender()) { - var results = new InferenceTextEmbeddingFloatResults( - List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.123F, 0.678F })) + var results = new TextEmbeddingFloatResults( + List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.123F, 0.678F })) ); requestSender.enqueue(results); @@ -1110,8 +1110,8 @@ public void testCheckModelConfig_HasSimilarity_ForEmbeddingsModel() throws IOExc try (var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool))) { try (var requestSender = (AmazonBedrockMockRequestSender) amazonBedrockFactory.createSender()) { - var results = new InferenceTextEmbeddingFloatResults( - List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.123F, 0.678F })) + var results = new TextEmbeddingFloatResults( + List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.123F, 0.678F })) ); requestSender.enqueue(results); @@ -1169,8 +1169,8 @@ public void testCheckModelConfig_ThrowsIfEmbeddingSizeDoesNotMatchValueSetByUser try (var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool))) { try (var requestSender = (AmazonBedrockMockRequestSender) amazonBedrockFactory.createSender()) { - var results = new InferenceTextEmbeddingFloatResults( - List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.123F, 0.678F })) + var results = new TextEmbeddingFloatResults( + List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.123F, 0.678F })) ); requestSender.enqueue(results); @@ -1218,8 +1218,8 @@ public void testCheckModelConfig_ReturnsNewModelReference_AndDoesNotSendDimensio try (var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool))) { try (var requestSender = (AmazonBedrockMockRequestSender) amazonBedrockFactory.createSender()) { - var results = new InferenceTextEmbeddingFloatResults( - List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.123F, 0.678F })) + var results = new TextEmbeddingFloatResults( + List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.123F, 0.678F })) ); requestSender.enqueue(results); @@ -1428,14 +1428,14 @@ private void testChunkedInfer(AmazonBedrockEmbeddingsModel model) throws IOExcep try (var service = new AmazonBedrockService(factory, amazonBedrockFactory, createWithEmptySettings(threadPool))) { try (var requestSender = (AmazonBedrockMockRequestSender) amazonBedrockFactory.createSender()) { { - var mockResults1 = new InferenceTextEmbeddingFloatResults( - List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.123F, 0.678F })) + var mockResults1 = new TextEmbeddingFloatResults( + List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.123F, 0.678F })) ); requestSender.enqueue(mockResults1); } { - var mockResults2 = new InferenceTextEmbeddingFloatResults( - List.of(new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(new float[] { 0.223F, 0.278F })) + var mockResults2 = new TextEmbeddingFloatResults( + List.of(new TextEmbeddingFloatResults.Embedding(new float[] { 0.223F, 0.278F })) ); requestSender.enqueue(mockResults2); } @@ -1454,18 +1454,28 @@ private void testChunkedInfer(AmazonBedrockEmbeddingsModel model) throws IOExcep var results = listener.actionGet(TIMEOUT); assertThat(results, hasSize(2)); { - assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("abc", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 0.123F, 0.678F }, floatResult.chunks().get(0).embedding(), 0.0f); + assertThat(floatResult.chunks().get(0), instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 0.123F, 0.678F }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } { - assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1); + assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(1); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("xyz", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 0.223F, 0.278F }, floatResult.chunks().get(0).embedding(), 0.0f); + assertThat(floatResult.chunks().get(0), instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 0.223F, 0.278F }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java index cebea7901b956..045789a92bf38 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java @@ -36,7 +36,8 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -1201,18 +1202,28 @@ private void testChunkedInfer(AzureAiStudioEmbeddingsModel model) throws IOExcep var results = listener.actionGet(TIMEOUT); assertThat(results, hasSize(2)); { - assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("foo", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 0.0123f, -0.0123f }, floatResult.chunks().get(0).embedding(), 0.0f); + assertThat(floatResult.chunks().get(0), instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 0.0123f, -0.0123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } { - assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1); + assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(1); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("bar", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 1.0123f, -1.0123f }, floatResult.chunks().get(0).embedding(), 0.0f); + assertThat(floatResult.chunks().get(0), instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 1.0123f, -1.0123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } assertThat(webServer.requests(), hasSize(1)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java index e67a5dac0e7c2..e58a7049ef872 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java @@ -35,7 +35,8 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -1350,18 +1351,28 @@ private void testChunkedInfer(AzureOpenAiEmbeddingsModel model) throws IOExcepti var results = listener.actionGet(TIMEOUT); assertThat(results, hasSize(2)); { - assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("foo", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding(), 0.0f); + assertThat(floatResult.chunks().get(0), instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 0.123f, -0.123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } { - assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1); + assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(1); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("bar", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 1.123f, -1.123f }, floatResult.chunks().get(0).embedding(), 0.0f); + assertThat(floatResult.chunks().get(0), instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 1.123f, -1.123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } assertThat(webServer.requests(), hasSize(1)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index 90e5dc6890c45..8549871d67ffd 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -36,8 +36,9 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingByte; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -1458,18 +1459,26 @@ private void testChunkedInfer(CohereEmbeddingsModel model) throws IOException { var results = listener.actionGet(TIMEOUT); assertThat(results, hasSize(2)); { - assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("foo", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding(), 0.0f); + assertArrayEquals( + new float[] { 0.123f, -0.123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } { - assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1); + assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(1); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("bar", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding(), 0.0f); + assertArrayEquals( + new float[] { 0.223f, -0.223f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } MatcherAssert.assertThat(webServer.requests(), hasSize(1)); @@ -1549,18 +1558,20 @@ public void testChunkedInfer_BatchesCalls_Bytes() throws IOException { var results = listener.actionGet(TIMEOUT); assertThat(results, hasSize(2)); { - assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingByte.class)); - var floatResult = (ChunkedInferenceEmbeddingByte) results.get(0); - assertThat(floatResult.chunks(), hasSize(1)); - assertEquals("foo", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new byte[] { 23, -23 }, floatResult.chunks().get(0).embedding()); + assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var byteResult = (ChunkedInferenceEmbedding) results.get(0); + assertThat(byteResult.chunks(), hasSize(1)); + assertEquals("foo", byteResult.chunks().get(0).matchedText()); + assertThat(byteResult.chunks().get(0), instanceOf(TextEmbeddingByteResults.Chunk.class)); + assertArrayEquals(new byte[] { 23, -23 }, ((TextEmbeddingByteResults.Chunk) byteResult.chunks().get(0)).embedding()); } { - assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingByte.class)); - var byteResult = (ChunkedInferenceEmbeddingByte) results.get(1); + assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var byteResult = (ChunkedInferenceEmbedding) results.get(1); assertThat(byteResult.chunks(), hasSize(1)); assertEquals("bar", byteResult.chunks().get(0).matchedText()); - assertArrayEquals(new byte[] { 24, -24 }, byteResult.chunks().get(0).embedding()); + assertThat(byteResult.chunks().get(0), instanceOf(TextEmbeddingByteResults.Chunk.class)); + assertArrayEquals(new byte[] { 24, -24 }, ((TextEmbeddingByteResults.Chunk) byteResult.chunks().get(0)).embedding()); } MatcherAssert.assertThat(webServer.requests(), hasSize(1)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java index f6d722da3f4e0..b3f8579903885 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -36,7 +36,8 @@ import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; import org.elasticsearch.xpack.core.ml.search.WeightedToken; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; @@ -553,13 +554,13 @@ public void testChunkedInfer_PassesThrough() throws IOException { ); var results = listener.actionGet(TIMEOUT); - assertThat(results.get(0), instanceOf(ChunkedInferenceEmbeddingSparse.class)); - var sparseResult = (ChunkedInferenceEmbeddingSparse) results.get(0); + assertThat(results.get(0), instanceOf(ChunkedInferenceEmbedding.class)); + var sparseResult = (ChunkedInferenceEmbedding) results.get(0); assertThat( sparseResult.chunks(), is( List.of( - new ChunkedInferenceEmbeddingSparse.SparseEmbeddingChunk( + new SparseEmbeddingResults.Chunk( List.of(new WeightedToken("hello", 2.1259406f), new WeightedToken("greet", 1.7073475f)), "input text", new ChunkedInference.TextOffset(0, "input text".length()) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index d1ce79b863c61..e7e654b599fe6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -43,9 +43,10 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.action.util.QueryPage; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceError; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.action.GetDeploymentStatsAction; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; @@ -67,7 +68,6 @@ import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests; -import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.chunking.WordBoundaryChunkingSettings; import org.elasticsearch.xpack.inference.services.ServiceFields; import org.hamcrest.Matchers; @@ -100,7 +100,6 @@ import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.NAME; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.OLD_ELSER_SERVICE_NAME; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -894,21 +893,23 @@ private void testChunkInfer_e5(ChunkingSettings chunkingSettings) throws Interru var gotResults = new AtomicBoolean(); var resultsListener = ActionListener.>wrap(chunkedResponse -> { assertThat(chunkedResponse, hasSize(2)); - assertThat(chunkedResponse.get(0), instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var result1 = (ChunkedInferenceEmbeddingFloat) chunkedResponse.get(0); + assertThat(chunkedResponse.get(0), instanceOf(ChunkedInferenceEmbedding.class)); + var result1 = (ChunkedInferenceEmbedding) chunkedResponse.get(0); assertThat(result1.chunks(), hasSize(1)); + assertThat(result1.chunks().get(0), instanceOf(TextEmbeddingFloatResults.Chunk.class)); assertArrayEquals( ((MlTextEmbeddingResults) mlTrainedModelResults.get(0)).getInferenceAsFloat(), - result1.chunks().get(0).embedding(), + ((TextEmbeddingFloatResults.Chunk) result1.chunks().get(0)).embedding(), 0.0001f ); assertEquals("foo", result1.chunks().get(0).matchedText()); - assertThat(chunkedResponse.get(1), instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var result2 = (ChunkedInferenceEmbeddingFloat) chunkedResponse.get(1); + assertThat(chunkedResponse.get(1), instanceOf(ChunkedInferenceEmbedding.class)); + var result2 = (ChunkedInferenceEmbedding) chunkedResponse.get(1); assertThat(result2.chunks(), hasSize(1)); + assertThat(result2.chunks().get(0), instanceOf(TextEmbeddingFloatResults.Chunk.class)); assertArrayEquals( ((MlTextEmbeddingResults) mlTrainedModelResults.get(1)).getInferenceAsFloat(), - result2.chunks().get(0).embedding(), + ((TextEmbeddingFloatResults.Chunk) result2.chunks().get(0)).embedding(), 0.0001f ); assertEquals("bar", result2.chunks().get(0).matchedText()); @@ -969,18 +970,20 @@ private void testChunkInfer_Sparse(ChunkingSettings chunkingSettings) throws Int var resultsListener = ActionListener.>wrap(chunkedResponse -> { assertThat(chunkedResponse, hasSize(2)); - assertThat(chunkedResponse.get(0), instanceOf(ChunkedInferenceEmbeddingSparse.class)); - var result1 = (ChunkedInferenceEmbeddingSparse) chunkedResponse.get(0); + assertThat(chunkedResponse.get(0), instanceOf(ChunkedInferenceEmbedding.class)); + var result1 = (ChunkedInferenceEmbedding) chunkedResponse.get(0); + assertThat(result1.chunks().get(0), instanceOf(SparseEmbeddingResults.Chunk.class)); assertEquals( ((TextExpansionResults) mlTrainedModelResults.get(0)).getWeightedTokens(), - result1.chunks().get(0).weightedTokens() + ((SparseEmbeddingResults.Chunk) result1.chunks().get(0)).weightedTokens() ); assertEquals("foo", result1.chunks().get(0).matchedText()); - assertThat(chunkedResponse.get(1), instanceOf(ChunkedInferenceEmbeddingSparse.class)); - var result2 = (ChunkedInferenceEmbeddingSparse) chunkedResponse.get(1); + assertThat(chunkedResponse.get(1), instanceOf(ChunkedInferenceEmbedding.class)); + var result2 = (ChunkedInferenceEmbedding) chunkedResponse.get(1); + assertThat(result2.chunks().get(0), instanceOf(SparseEmbeddingResults.Chunk.class)); assertEquals( ((TextExpansionResults) mlTrainedModelResults.get(1)).getWeightedTokens(), - result2.chunks().get(0).weightedTokens() + ((SparseEmbeddingResults.Chunk) result2.chunks().get(0)).weightedTokens() ); assertEquals("bar", result2.chunks().get(0).matchedText()); gotResults.set(true); @@ -1039,18 +1042,20 @@ private void testChunkInfer_Elser(ChunkingSettings chunkingSettings) throws Inte var gotResults = new AtomicBoolean(); var resultsListener = ActionListener.>wrap(chunkedResponse -> { assertThat(chunkedResponse, hasSize(2)); - assertThat(chunkedResponse.get(0), instanceOf(ChunkedInferenceEmbeddingSparse.class)); - var result1 = (ChunkedInferenceEmbeddingSparse) chunkedResponse.get(0); + assertThat(chunkedResponse.get(0), instanceOf(ChunkedInferenceEmbedding.class)); + var result1 = (ChunkedInferenceEmbedding) chunkedResponse.get(0); + assertThat(result1.chunks().get(0), instanceOf(SparseEmbeddingResults.Chunk.class)); assertEquals( ((TextExpansionResults) mlTrainedModelResults.get(0)).getWeightedTokens(), - result1.chunks().get(0).weightedTokens() + ((SparseEmbeddingResults.Chunk) result1.chunks().get(0)).weightedTokens() ); assertEquals("foo", result1.chunks().get(0).matchedText()); - assertThat(chunkedResponse.get(1), instanceOf(ChunkedInferenceEmbeddingSparse.class)); - var result2 = (ChunkedInferenceEmbeddingSparse) chunkedResponse.get(1); + assertThat(chunkedResponse.get(1), instanceOf(ChunkedInferenceEmbedding.class)); + var result2 = (ChunkedInferenceEmbedding) chunkedResponse.get(1); + assertThat(result2.chunks().get(0), instanceOf(SparseEmbeddingResults.Chunk.class)); assertEquals( ((TextExpansionResults) mlTrainedModelResults.get(1)).getWeightedTokens(), - result2.chunks().get(0).weightedTokens() + ((SparseEmbeddingResults.Chunk) result2.chunks().get(0)).weightedTokens() ); assertEquals("bar", result2.chunks().get(0).matchedText()); gotResults.set(true); @@ -1219,8 +1224,8 @@ public void testChunkingLargeDocument() throws InterruptedException { var gotResults = new AtomicBoolean(); var resultsListener = ActionListener.>wrap(chunkedResponse -> { assertThat(chunkedResponse, hasSize(1)); - assertThat(chunkedResponse.get(0), instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var sparseResults = (ChunkedInferenceEmbeddingFloat) chunkedResponse.get(0); + assertThat(chunkedResponse.get(0), instanceOf(ChunkedInferenceEmbedding.class)); + var sparseResults = (ChunkedInferenceEmbedding) chunkedResponse.get(0); assertThat(sparseResults.chunks(), hasSize(numChunks)); gotResults.set(true); @@ -1544,41 +1549,6 @@ public void testModelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic() } } - public void testEmbeddingTypeFromTaskTypeAndSettings() { - assertEquals( - EmbeddingRequestChunker.EmbeddingType.SPARSE, - ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( - TaskType.SPARSE_EMBEDDING, - new ElasticsearchInternalServiceSettings(1, 1, "foo", null, null) - ) - ); - assertEquals( - EmbeddingRequestChunker.EmbeddingType.FLOAT, - ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( - TaskType.TEXT_EMBEDDING, - new MultilingualE5SmallInternalServiceSettings(1, 1, "foo", null) - ) - ); - - var e1 = expectThrows( - ElasticsearchStatusException.class, - () -> ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( - TaskType.COMPLETION, - new ElasticsearchInternalServiceSettings(1, 1, "foo", null, null) - ) - ); - assertThat(e1.getMessage(), containsString("Chunking is not supported for task type [completion]")); - - var e2 = expectThrows( - ElasticsearchStatusException.class, - () -> ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( - TaskType.RERANK, - new ElasticsearchInternalServiceSettings(1, 1, "foo", null, null) - ) - ); - assertThat(e2.getMessage(), containsString("Chunking is not supported for task type [rerank]")); - } - public void testIsDefaultId() { var service = createService(mock(Client.class)); assertTrue(service.isDefaultId(".elser-2-elasticsearch")); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java index d0760a583df29..9828a4f21ab51 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/googleaistudio/GoogleAiStudioServiceTests.java @@ -36,7 +36,8 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -877,20 +878,32 @@ private void testChunkedInfer(String modelId, String apiKey, GoogleAiStudioEmbed // first result { - assertThat(results.get(0), instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(results.get(0), instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); assertEquals(input.get(0), floatResult.chunks().get(0).matchedText()); - assertTrue(Arrays.equals(new float[] { 0.0123f, -0.0123f }, floatResult.chunks().get(0).embedding())); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertTrue( + Arrays.equals( + new float[] { 0.0123f, -0.0123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding() + ) + ); } // second result { - assertThat(results.get(1), instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1); + assertThat(results.get(1), instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(1); assertThat(floatResult.chunks(), hasSize(1)); assertEquals(input.get(1), floatResult.chunks().get(0).matchedText()); - assertTrue(Arrays.equals(new float[] { 0.0456f, -0.0456f }, floatResult.chunks().get(0).embedding())); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertTrue( + Arrays.equals( + new float[] { 0.0456f, -0.0456f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding() + ) + ); } assertThat(webServer.requests(), hasSize(1)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java index 53e7c6c25fd47..1050ac137be8d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java @@ -24,7 +24,8 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingSparse; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -102,13 +103,13 @@ public void testChunkedInfer_CallsInfer_Elser_ConvertsFloatResponse() throws IOE ); var result = listener.actionGet(TIMEOUT).get(0); - assertThat(result, instanceOf(ChunkedInferenceEmbeddingSparse.class)); - var sparseResult = (ChunkedInferenceEmbeddingSparse) result; + assertThat(result, instanceOf(ChunkedInferenceEmbedding.class)); + var sparseResult = (ChunkedInferenceEmbedding) result; assertThat( sparseResult.chunks(), is( List.of( - new ChunkedInferenceEmbeddingSparse.SparseEmbeddingChunk( + new SparseEmbeddingResults.Chunk( List.of(new WeightedToken(".", 0.13315596f)), "abc", new ChunkedInference.TextOffset(0, "abc".length()) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java index f3137d7011cec..b9e7cda1461cc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java @@ -34,7 +34,8 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -783,12 +784,17 @@ public void testChunkedInfer_CallsInfer_TextEmbedding_ConvertsFloatResponse() th ); var result = listener.actionGet(TIMEOUT).get(0); - assertThat(result, CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var embeddingResult = (ChunkedInferenceEmbeddingFloat) result; + assertThat(result, CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var embeddingResult = (ChunkedInferenceEmbedding) result; assertThat(embeddingResult.chunks(), hasSize(1)); assertThat(embeddingResult.chunks().get(0).matchedText(), is("abc")); assertThat(embeddingResult.chunks().get(0).offset(), is(new ChunkedInference.TextOffset(0, "abc".length()))); - assertArrayEquals(new float[] { -0.0123f, 0.0123f }, embeddingResult.chunks().get(0).embedding(), 0.001f); + assertThat(embeddingResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { -0.0123f, 0.0123f }, + ((TextEmbeddingFloatResults.Chunk) embeddingResult.chunks().get(0)).embedding(), + 0.001f + ); assertThat(webServer.requests(), hasSize(1)); assertNull(webServer.requests().get(0).getUri().getQuery()); assertThat( @@ -833,11 +839,16 @@ public void testChunkedInfer() throws IOException { var results = listener.actionGet(TIMEOUT); assertThat(results, hasSize(1)); { - assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("abc", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding(), 0.0f); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 0.123f, -0.123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } assertThat(webServer.requests(), hasSize(1)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java index 99b7b3868b7f4..74d055d44363d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/IbmWatsonxServiceTests.java @@ -35,7 +35,8 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.common.Truncator; import org.elasticsearch.xpack.inference.external.action.ibmwatsonx.IbmWatsonxActionCreator; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; @@ -729,20 +730,32 @@ private void testChunkedInfer_Batches(ChunkingSettings chunkingSettings) throws // first result { - assertThat(results.get(0), instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(results.get(0), instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); assertEquals(input.get(0), floatResult.chunks().get(0).matchedText()); - assertTrue(Arrays.equals(new float[] { 0.0123f, -0.0123f }, floatResult.chunks().get(0).embedding())); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertTrue( + Arrays.equals( + new float[] { 0.0123f, -0.0123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding() + ) + ); } // second result { - assertThat(results.get(1), instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1); + assertThat(results.get(1), instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(1); assertThat(floatResult.chunks(), hasSize(1)); assertEquals(input.get(1), floatResult.chunks().get(0).matchedText()); - assertTrue(Arrays.equals(new float[] { 0.0456f, -0.0456f }, floatResult.chunks().get(0).embedding())); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertTrue( + Arrays.equals( + new float[] { 0.0456f, -0.0456f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding() + ) + ); } assertThat(webServer.requests(), hasSize(1)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java index 2aeb0447f9c78..3387b8b73978f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/jinaai/JinaAIServiceTests.java @@ -35,7 +35,8 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -1797,18 +1798,28 @@ private void test_Embedding_ChunkedInfer_BatchesCalls(JinaAIEmbeddingsModel mode var results = listener.actionGet(TIMEOUT); assertThat(results, hasSize(2)); { - assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("foo", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding(), 0.0f); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 0.123f, -0.123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } { - assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1); + assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(1); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("bar", floatResult.chunks().get(0).matchedText()); - assertArrayEquals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding(), 0.0f); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 0.223f, -0.223f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding(), + 0.0f + ); } MatcherAssert.assertThat(webServer.requests(), hasSize(1)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java index 95ac2cde0e31b..6acafd59272ef 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/mistral/MistralServiceTests.java @@ -34,7 +34,8 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.ModelConfigurationsTests; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; @@ -680,16 +681,28 @@ public void testChunkedInfer(MistralEmbeddingsModel model) throws IOException { assertThat(results, hasSize(2)); { - assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); - assertTrue(Arrays.equals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding())); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertTrue( + Arrays.equals( + new float[] { 0.123f, -0.123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding() + ) + ); } { - assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1); + assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(1); assertThat(floatResult.chunks(), hasSize(1)); - assertTrue(Arrays.equals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding())); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertTrue( + Arrays.equals( + new float[] { 0.223f, -0.223f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding() + ) + ); } assertThat(webServer.requests(), hasSize(1)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index b31cdf4f9d592..13782a538f1f9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -36,7 +36,8 @@ import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbeddingFloat; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.UnifiedChatCompletionException; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; @@ -1799,18 +1800,30 @@ private void testChunkedInfer(OpenAiEmbeddingsModel model) throws IOException { var results = listener.actionGet(TIMEOUT); assertThat(results, hasSize(2)); { - assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(0); + assertThat(results.get(0), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("foo", floatResult.chunks().get(0).matchedText()); - assertTrue(Arrays.equals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding())); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertTrue( + Arrays.equals( + new float[] { 0.123f, -0.123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding() + ) + ); } { - assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbeddingFloat.class)); - var floatResult = (ChunkedInferenceEmbeddingFloat) results.get(1); + assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(1); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("bar", floatResult.chunks().get(0).matchedText()); - assertTrue(Arrays.equals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding())); + assertThat(floatResult.chunks().get(0), Matchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertTrue( + Arrays.equals( + new float[] { 0.223f, -0.223f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().get(0)).embedding() + ) + ); } assertThat(webServer.requests(), hasSize(1)); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/TextEmbeddingModelValidatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/TextEmbeddingModelValidatorTests.java index d608b42841305..d596d53ba5104 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/TextEmbeddingModelValidatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/validation/TextEmbeddingModelValidatorTests.java @@ -14,12 +14,12 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingByteResults; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.EmptyTaskSettingsTests; import org.elasticsearch.xpack.inference.ModelConfigurationsTests; -import org.elasticsearch.xpack.inference.results.InferenceTextEmbeddingByteResultsTests; import org.elasticsearch.xpack.inference.results.SparseEmbeddingResultsTests; +import org.elasticsearch.xpack.inference.results.TextEmbeddingByteResultsTests; import org.junit.Before; import org.mockito.Mock; @@ -89,7 +89,7 @@ public void testValidate_ServiceReturnsNonTextEmbeddingResults() { } public void testValidate_RetrievingEmbeddingSizeThrowsIllegalStateException() { - InferenceTextEmbeddingFloatResults results = new InferenceTextEmbeddingFloatResults(List.of()); + TextEmbeddingFloatResults results = new TextEmbeddingFloatResults(List.of()); when(mockServiceSettings.dimensionsSetByUser()).thenReturn(true); when(mockServiceSettings.dimensions()).thenReturn(randomNonNegativeInt()); @@ -102,7 +102,7 @@ public void testValidate_RetrievingEmbeddingSizeThrowsIllegalStateException() { } public void testValidate_DimensionsSetByUserDoNotEqualEmbeddingSize() { - InferenceTextEmbeddingByteResults results = InferenceTextEmbeddingByteResultsTests.createRandomResults(); + TextEmbeddingByteResults results = TextEmbeddingByteResultsTests.createRandomResults(); var dimensions = randomValueOtherThan(results.getFirstEmbeddingSize(), ESTestCase::randomNonNegativeInt); when(mockServiceSettings.dimensionsSetByUser()).thenReturn(true); @@ -126,7 +126,7 @@ public void testValidate_DimensionsNotSetByUser() { } private void mockSuccessfulValidation(Boolean dimensionsSetByUser) { - InferenceTextEmbeddingByteResults results = InferenceTextEmbeddingByteResultsTests.createRandomResults(); + TextEmbeddingByteResults results = TextEmbeddingByteResultsTests.createRandomResults(); when(mockModel.getConfigurations()).thenReturn(ModelConfigurationsTests.createRandomInstance()); when(mockModel.getTaskSettings()).thenReturn(EmptyTaskSettingsTests.createRandom()); when(mockServiceSettings.dimensionsSetByUser()).thenReturn(dimensionsSetByUser); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceSettingsTests.java new file mode 100644 index 0000000000000..09d890bd21f67 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceSettingsTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.is; + +public class VoyageAIServiceSettingsTests extends AbstractWireSerializingTestCase { + + public static VoyageAIServiceSettings createRandomWithNonNullUrl() { + return createRandom(); + } + + /** + * The created settings can have a url set to null. + */ + public static VoyageAIServiceSettings createRandom() { + var model = randomAlphaOfLength(15); + + return new VoyageAIServiceSettings(model, RateLimitSettingsTests.createRandom()); + } + + public void testFromMap() { + var model = "model"; + var serviceSettings = VoyageAIServiceSettings.fromMap( + new HashMap<>(Map.of(VoyageAIServiceSettings.MODEL_ID, model)), + ConfigurationParseContext.REQUEST + ); + + MatcherAssert.assertThat(serviceSettings, is(new VoyageAIServiceSettings(model, null))); + } + + public void testFromMap_WithRateLimit() { + var model = "model"; + var serviceSettings = VoyageAIServiceSettings.fromMap( + new HashMap<>( + Map.of( + VoyageAIServiceSettings.MODEL_ID, + model, + RateLimitSettings.FIELD_NAME, + new HashMap<>(Map.of(RateLimitSettings.REQUESTS_PER_MINUTE_FIELD, 3)) + ) + ), + ConfigurationParseContext.REQUEST + ); + + MatcherAssert.assertThat(serviceSettings, is(new VoyageAIServiceSettings(model, new RateLimitSettings(3)))); + } + + public void testFromMap_WhenUsingModelId() { + var model = "model"; + var serviceSettings = VoyageAIServiceSettings.fromMap( + new HashMap<>(Map.of(VoyageAIServiceSettings.MODEL_ID, model)), + ConfigurationParseContext.PERSISTENT + ); + + MatcherAssert.assertThat(serviceSettings, is(new VoyageAIServiceSettings(model, null))); + } + + public void testXContent_WritesModelId() throws IOException { + var entity = new VoyageAIServiceSettings("model", new RateLimitSettings(1)); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + entity.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, is(""" + {"model_id":"model","rate_limit":{"requests_per_minute":1}}""")); + } + + @Override + protected Writeable.Reader instanceReader() { + return VoyageAIServiceSettings::new; + } + + @Override + protected VoyageAIServiceSettings createTestInstance() { + return createRandomWithNonNullUrl(); + } + + @Override + protected VoyageAIServiceSettings mutateInstance(VoyageAIServiceSettings instance) throws IOException { + return randomValueOtherThan(instance, VoyageAIServiceSettingsTests::createRandom); + } + + public static Map getServiceSettingsMap(String model) { + var map = new HashMap(); + + map.put(VoyageAIServiceSettings.MODEL_ID, model); + + return map; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceTests.java new file mode 100644 index 0000000000000..6a0428e962f52 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/VoyageAIServiceTests.java @@ -0,0 +1,1982 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai; + +import org.apache.http.HttpHeaders; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInference; +import org.elasticsearch.inference.ChunkingSettings; +import org.elasticsearch.inference.InferenceServiceConfiguration; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.ModelConfigurations; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.results.ChunkedInferenceEmbedding; +import org.elasticsearch.xpack.core.inference.results.TextEmbeddingFloatResults; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.external.http.sender.Sender; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModel; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsModelTests; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettings; +import org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettingsTests; +import org.elasticsearch.xpack.inference.services.voyageai.rerank.VoyageAIRerankModelTests; +import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; +import static org.elasticsearch.xpack.inference.Utils.getInvalidModel; +import static org.elasticsearch.xpack.inference.Utils.getPersistedConfigMap; +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettings; +import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettingsMap; +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; +import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat; +import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; +import static org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettingsTests.getSecretSettingsMap; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +public class VoyageAIServiceTests extends ESTestCase { + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + private HttpClientManager clientManager; + + @Before + public void init() throws Exception { + webServer.start(); + threadPool = createThreadPool(inferenceUtilityPool()); + clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); + } + + @After + public void shutdown() throws IOException { + clientManager.close(); + terminate(threadPool); + webServer.close(); + } + + public void testParseRequestConfig_CreatesAVoyageAIEmbeddingsModel() throws IOException { + try (var service = createVoyageAIService()) { + ActionListener modelListener = ActionListener.wrap(model -> { + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null))); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, e -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST), + getSecretSettingsMap("secret") + ), + modelListener + ); + + } + } + + public void testParseRequestConfig_CreatesAVoyageAIEmbeddingsModelWhenChunkingSettingsProvided() throws IOException { + try (var service = createVoyageAIService()) { + ActionListener modelListener = ActionListener.wrap(model -> { + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, e -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST), + createRandomChunkingSettingsMap(), + getSecretSettingsMap("secret") + ), + modelListener + ); + + } + } + + public void testParseRequestConfig_CreatesAVoyageAIEmbeddingsModelWhenChunkingSettingsNotProvided() throws IOException { + try (var service = createVoyageAIService()) { + ActionListener modelListener = ActionListener.wrap(model -> { + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, e -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST), + getSecretSettingsMap("secret") + ), + modelListener + ); + + } + } + + public void testParseRequestConfig_OptionalTaskSettings() throws IOException { + try (var service = createVoyageAIService()) { + + ActionListener modelListener = ActionListener.wrap(model -> { + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), equalTo(VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, e -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap(VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), getSecretSettingsMap("secret")), + modelListener + ); + + } + } + + public void testParseRequestConfig_ThrowsUnsupportedTaskType() throws IOException { + try (var service = createVoyageAIService()) { + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "The [voyageai] service does not support task type [sparse_embedding]" + ); + + service.parseRequestConfig( + "id", + TaskType.SPARSE_EMBEDDING, + getRequestConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + getSecretSettingsMap("secret") + ), + failureListener + ); + } + } + + private static ActionListener getModelListenerForException(Class exceptionClass, String expectedMessage) { + return ActionListener.wrap((model) -> fail("Model parsing should have failed"), e -> { + MatcherAssert.assertThat(e, instanceOf(exceptionClass)); + MatcherAssert.assertThat(e.getMessage(), is(expectedMessage)); + }); + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInConfig() throws IOException { + try (var service = createVoyageAIService()) { + var config = getRequestConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + getSecretSettingsMap("secret") + ); + config.put("extra_key", "value"); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [voyageai] service" + ); + service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener); + } + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInServiceSettingsMap() throws IOException { + try (var service = createVoyageAIService()) { + var serviceSettings = VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"); + serviceSettings.put("extra_key", "value"); + + var config = getRequestConfigMap( + serviceSettings, + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + getSecretSettingsMap("secret") + ); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [voyageai] service" + ); + service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener); + } + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInTaskSettingsMap() throws IOException { + try (var service = createVoyageAIService()) { + var taskSettingsMap = VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST); + taskSettingsMap.put("extra_key", "value"); + + var config = getRequestConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + taskSettingsMap, + getSecretSettingsMap("secret") + ); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [voyageai] service" + ); + service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener); + + } + } + + public void testParseRequestConfig_ThrowsWhenAnExtraKeyExistsInSecretSettingsMap() throws IOException { + try (var service = createVoyageAIService()) { + var secretSettingsMap = getSecretSettingsMap("secret"); + secretSettingsMap.put("extra_key", "value"); + + var config = getRequestConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + secretSettingsMap + ); + + var failureListener = getModelListenerForException( + ElasticsearchStatusException.class, + "Model configuration contains settings [{extra_key=value}] unknown to the [voyageai] service" + ); + service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesAVoyageAIEmbeddingsModel() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings((InputType) null, null))); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesAVoyageAIEmbeddingsModelWhenChunkingSettingsProvided() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + createRandomChunkingSettingsMap(), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings((InputType) null, null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesAVoyageAIEmbeddingsModelWhenChunkingSettingsNotProvided() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings((InputType) null, null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfigWithSecrets_ThrowsErrorTryingToParseInvalidModel() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("oldmodel"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + getSecretSettingsMap("secret") + ); + + var thrownException = expectThrows( + ElasticsearchStatusException.class, + () -> service.parsePersistedConfigWithSecrets( + "id", + TaskType.SPARSE_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ) + ); + + MatcherAssert.assertThat( + thrownException.getMessage(), + is("Failed to parse stored model [id] for [voyageai] service, please delete and add the service again") + ); + } + } + + public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInConfig() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH), + getSecretSettingsMap("secret") + ); + persistedConfig.config().put("extra_key", "value"); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings(InputType.SEARCH, null))); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfigWithSecrets_DoesNotThrowWhenAnExtraKeyExistsInSecretsSettings() throws IOException { + try (var service = createVoyageAIService()) { + var secretSettingsMap = getSecretSettingsMap("secret"); + secretSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + secretSettingsMap + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInSecrets() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + getSecretSettingsMap("secret") + ); + persistedConfig.secrets().put("extra_key", "value"); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings((InputType) null, null))); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInServiceSettings() throws IOException { + try (var service = createVoyageAIService()) { + var serviceSettingsMap = VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"); + serviceSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap( + serviceSettingsMap, + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty(), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfigWithSecrets_NotThrowWhenAnExtraKeyExistsInTaskSettings() throws IOException { + try (var service = createVoyageAIService()) { + var taskSettingsMap = VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH); + taskSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + taskSettingsMap, + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings(InputType.SEARCH, null))); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfig_CreatesAVoyageAIEmbeddingsModel() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null) + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings((InputType) null, null))); + assertNull(embeddingsModel.getSecretSettings()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfig_CreatesAVoyageAIEmbeddingsModelWhenChunkingSettingsProvided() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null), + createRandomChunkingSettingsMap() + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings((InputType) null, null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertNull(embeddingsModel.getSecretSettings()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfig_CreatesAVoyageAIEmbeddingsModelWhenChunkingSettingsNotProvided() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null) + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings((InputType) null, null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertNull(embeddingsModel.getSecretSettings()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfig_ThrowsErrorTryingToParseInvalidModel() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model_old"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty() + ); + + var thrownException = expectThrows( + ElasticsearchStatusException.class, + () -> service.parsePersistedConfig("id", TaskType.SPARSE_EMBEDDING, persistedConfig.config()) + ); + + MatcherAssert.assertThat( + thrownException.getMessage(), + is("Failed to parse stored model [id] for [voyageai] service, please delete and add the service again") + ); + } + } + + public void testParsePersistedConfig_CreatesAVoyageAIEmbeddingsModelWithoutUrl() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(null) + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings((InputType) null, null))); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_DoesNotThrowWhenAnExtraKeyExistsInConfig() throws IOException { + try (var service = createVoyageAIService()) { + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMapEmpty() + ); + persistedConfig.config().put("extra_key", "value"); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS)); + assertNull(embeddingsModel.getSecretSettings()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInServiceSettings() throws IOException { + try (var service = createVoyageAIService()) { + var serviceSettingsMap = VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"); + serviceSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap( + serviceSettingsMap, + VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.SEARCH) + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings(InputType.SEARCH, null))); + assertNull(embeddingsModel.getSecretSettings()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testParsePersistedConfig_NotThrowWhenAnExtraKeyExistsInTaskSettings() throws IOException { + try (var service = createVoyageAIService()) { + var taskSettingsMap = VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap(InputType.INGEST); + taskSettingsMap.put("extra_key", "value"); + + var persistedConfig = getPersistedConfigMap( + VoyageAIEmbeddingsServiceSettingsTests.getServiceSettingsMap("model"), + taskSettingsMap + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + MatcherAssert.assertThat(model, instanceOf(VoyageAIEmbeddingsModel.class)); + + var embeddingsModel = (VoyageAIEmbeddingsModel) model; + assertNull(embeddingsModel.uri()); + MatcherAssert.assertThat(embeddingsModel.buildUri().toString(), is("https://api.voyageai.com/v1/embeddings")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null))); + assertNull(embeddingsModel.getSecretSettings()); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + public void testInfer_ThrowsErrorWhenModelIsNotVoyageAIModel() throws IOException { + var sender = mock(Sender.class); + + var factory = mock(HttpRequestSender.Factory.class); + when(factory.createSender()).thenReturn(sender); + + var mockModel = getInvalidModel("model_id", "service_name"); + + try (var service = new VoyageAIService(factory, createWithEmptySettings(threadPool))) { + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + mockModel, + null, + List.of(""), + false, + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); + MatcherAssert.assertThat( + thrownException.getMessage(), + is("The internal model was invalid, please delete the service [service_name] with id [model_id] and add it again.") + ); + + verify(factory, times(1)).createSender(); + verify(sender, times(1)).start(); + } + + verify(sender, times(1)).close(); + verifyNoMoreInteractions(factory); + verifyNoMoreInteractions(sender); + } + + public void testCheckModelConfig_UpdatesDimensions() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "voyage-3-large", + "object": "list", + "usage": { + "total_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 1, + "voyage-3-large" + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.checkModelConfig(model, listener); + var result = listener.actionGet(TIMEOUT); + + MatcherAssert.assertThat( + result, + // the dimension is set to 2 because there are 2 embeddings returned from the mock server + is( + VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 2, + "voyage-3-large" + ) + ) + ); + } + } + + public void testCheckModelConfig_UpdatesSimilarityToDotProduct_WhenItIsNull() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "voyage-3-large", + "object": "list", + "usage": { + "total_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 1, + "voyage-3-large", + (SimilarityMeasure) null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.checkModelConfig(model, listener); + var result = listener.actionGet(TIMEOUT); + + MatcherAssert.assertThat( + result, + // the dimension is set to 2 because there are 2 embeddings returned from the mock server + is( + VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 2, + "voyage-3-large", + SimilarityMeasure.DOT_PRODUCT + ) + ) + ); + } + } + + public void testCheckModelConfig_DoesNotUpdateSimilarity_WhenItIsSpecifiedAsCosine() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "voyage-3-large", + "object": "list", + "usage": { + "total_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 1, + "voyage-3-large", + SimilarityMeasure.COSINE + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.checkModelConfig(model, listener); + var result = listener.actionGet(TIMEOUT); + + MatcherAssert.assertThat( + result, + // the dimension is set to 2 because there are 2 embeddings returned from the mock server + is( + VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 10, + 2, + "voyage-3-large", + SimilarityMeasure.COSINE + ) + ) + ); + } + } + + public void testUpdateModelWithEmbeddingDetails_NullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(null); + } + + public void testUpdateModelWithEmbeddingDetails_NonNullSimilarityInOriginalModel() throws IOException { + testUpdateModelWithEmbeddingDetails_Successful(randomFrom(SimilarityMeasure.values())); + } + + private void testUpdateModelWithEmbeddingDetails_Successful(SimilarityMeasure similarityMeasure) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + var embeddingSize = randomNonNegativeInt(); + var model = VoyageAIEmbeddingsModelTests.createModel( + randomAlphaOfLength(10), + randomAlphaOfLength(10), + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + randomNonNegativeInt(), + randomNonNegativeInt(), + randomAlphaOfLength(10), + similarityMeasure + ); + + Model updatedModel = service.updateModelWithEmbeddingDetails(model, embeddingSize); + + SimilarityMeasure expectedSimilarityMeasure = similarityMeasure == null + ? VoyageAIService.defaultSimilarity() + : similarityMeasure; + assertEquals(expectedSimilarityMeasure, updatedModel.getServiceSettings().similarity()); + assertEquals(embeddingSize, updatedModel.getServiceSettings().dimensions().intValue()); + } + } + + public void testInfer_Embedding_UnauthorisedResponse() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "detail": "Unauthorized" + } + """; + webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson)); + + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 1024, + 1024, + "model", + (SimilarityMeasure) null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + false, + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var error = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + MatcherAssert.assertThat(error.getMessage(), containsString("Received an authentication error status code for request")); + MatcherAssert.assertThat(error.getMessage(), containsString("Error message: [Unauthorized]")); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + } + } + + public void testInfer_Rerank_UnauthorisedResponse() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "detail": "Unauthorized" + } + """; + webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson)); + + var model = VoyageAIRerankModelTests.createModel(getUrl(webServer), "model", 1024, false, false); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + "query", + List.of("candidate1", "candidate2"), + false, + new HashMap<>(), + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var error = expectThrows(ElasticsearchException.class, () -> listener.actionGet(TIMEOUT)); + MatcherAssert.assertThat(error.getMessage(), containsString("Received an authentication error status code for request")); + MatcherAssert.assertThat(error.getMessage(), containsString("Error message: [Unauthorized]")); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + } + } + + public void testInfer_Embedding_Get_Response_Ingest() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "voyage-3-large", + "object": "list", + "usage": { + "total_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 1024, + 1024, + "voyage-3-large", + (SimilarityMeasure) null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + false, + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + + assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap()); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().getFirst().getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "input", + List.of("abc"), + "model", + "voyage-3-large", + "input_type", + "document", + "output_dtype", + "float", + "output_dimension", + 1024 + ) + ) + ); + } + } + + public void testInfer_Embedding_Get_Response_Search() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "voyage-3-large", + "object": "list", + "usage": { + "total_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 1024, + 1024, + "voyage-3-large", + (SimilarityMeasure) null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + false, + new HashMap<>(), + InputType.SEARCH, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + + assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap()); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().getFirst().getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "input", + List.of("abc"), + "model", + "voyage-3-large", + "input_type", + "query", + "output_dtype", + "float", + "output_dimension", + 1024 + ) + ) + ); + } + } + + public void testInfer_Embedding_Get_Response_clustering() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + {"model":"voyage-3-large","object":"list","usage":{"total_tokens":5}, + "data":[{"object":"embedding","index":0,"embedding":[0.123, -0.123]}]} + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 1024, + 1024, + "voyage-3-large", + (SimilarityMeasure) null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + false, + new HashMap<>(), + InputType.CLUSTERING, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + + assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap()); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().getFirst().getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is(Map.of("input", List.of("abc"), "model", "voyage-3-large", "output_dtype", "float", "output_dimension", 1024)) + ); + } + } + + public void testInfer_Embedding_Get_Response_NullInputType() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "voyage-3-large", + "object": "list", + "usage": { + "total_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + 1024, + 1024, + "voyage-3-large", + (SimilarityMeasure) null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer(model, null, List.of("abc"), false, new HashMap<>(), null, InferenceAction.Request.DEFAULT_TIMEOUT, listener); + + var result = listener.actionGet(TIMEOUT); + + assertEquals(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })), result.asMap()); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().getFirst().getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is(Map.of("input", List.of("abc"), "model", "voyage-3-large", "output_dtype", "float", "output_dimension", 1024)) + ); + } + } + + public void testInfer_Rerank_Get_Response_NoReturnDocuments_NoTopN() throws IOException { + String responseJson = """ + { + "model": "model", + "object": "list", + "data": [ + { + "index": 2, + "relevance_score": 0.98005307 + }, + { + "index": 1, + "relevance_score": 0.27904198 + }, + { + "index": 0, + "relevance_score": 0.10194652 + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + var model = VoyageAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", null, false, false); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + "query", + List.of("candidate1", "candidate2", "candidate3"), + false, + new HashMap<>(), + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + var resultAsMap = result.asMap(); + assertThat( + resultAsMap, + is( + Map.of( + "rerank", + List.of( + Map.of("ranked_doc", Map.of("index", 2, "relevance_score", 0.98005307F)), + Map.of("ranked_doc", Map.of("index", 1, "relevance_score", 0.27904198F)), + Map.of("ranked_doc", Map.of("index", 0, "relevance_score", 0.10194652F)) + ) + ) + ) + ); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "query", + "query", + "documents", + List.of("candidate1", "candidate2", "candidate3"), + "model", + "model", + "return_documents", + false, + "truncation", + false + ) + ) + ); + + } + } + + public void testInfer_Rerank_Get_Response_NoReturnDocuments_TopN() throws IOException { + String responseJson = """ + { + "object": "list", + "model": "model", + "data": [ + { + "index": 2, + "relevance_score": 0.98005307 + }, + { + "index": 1, + "relevance_score": 0.27904198 + }, + { + "index": 0, + "relevance_score": 0.10194652 + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + var model = VoyageAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", 3, false, false); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + "query", + List.of("candidate1", "candidate2", "candidate3", "candidate4"), + false, + new HashMap<>(), + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + var resultAsMap = result.asMap(); + assertThat( + resultAsMap, + is( + Map.of( + "rerank", + List.of( + Map.of("ranked_doc", Map.of("index", 2, "relevance_score", 0.98005307F)), + Map.of("ranked_doc", Map.of("index", 1, "relevance_score", 0.27904198F)), + Map.of("ranked_doc", Map.of("index", 0, "relevance_score", 0.10194652F)) + ) + ) + ) + ); + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "query", + "query", + "documents", + List.of("candidate1", "candidate2", "candidate3", "candidate4"), + "model", + "model", + "return_documents", + false, + "top_k", + 3, + "truncation", + false + ) + ) + ); + + } + + } + + public void testInfer_Rerank_Get_Response_ReturnDocumentsNull_NoTopN() throws IOException { + String responseJson = """ + { + "object": "list", + "model": "model", + "data": [ + { + "index": 2, + "relevance_score": 0.98005307, + "document": "candidate3" + }, + { + "index": 1, + "relevance_score": 0.27904198, + "document": "candidate2" + }, + { + "index": 0, + "relevance_score": 0.10194652, + "document": "candidate1" + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + var model = VoyageAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", null, null, null); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + "query", + List.of("candidate1", "candidate2", "candidate3"), + false, + new HashMap<>(), + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + var resultAsMap = result.asMap(); + assertThat( + resultAsMap, + is( + Map.of( + "rerank", + List.of( + Map.of("ranked_doc", Map.of("text", "candidate3", "index", 2, "relevance_score", 0.98005307F)), + Map.of("ranked_doc", Map.of("text", "candidate2", "index", 1, "relevance_score", 0.27904198F)), + Map.of("ranked_doc", Map.of("text", "candidate1", "index", 0, "relevance_score", 0.10194652F)) + ) + ) + ) + ); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is(Map.of("query", "query", "documents", List.of("candidate1", "candidate2", "candidate3"), "model", "model")) + ); + + } + + } + + public void testInfer_Rerank_Get_Response_ReturnDocuments_TopN() throws IOException { + String responseJson = """ + { + "object": "list", + "model": "model", + "data": [ + { + "index": 2, + "relevance_score": 0.98005307, + "document": "candidate3" + }, + { + "index": 1, + "relevance_score": 0.27904198, + "document": "candidate2" + }, + { + "index": 0, + "relevance_score": 0.10194652, + "document": "candidate1" + } + ], + "usage": { + "total_tokens": 15 + } + } + """; + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + var model = VoyageAIRerankModelTests.createModel(getUrl(webServer), "secret", "model", 3, true, true); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + "query", + List.of("candidate1", "candidate2", "candidate3", "candidate4"), + false, + new HashMap<>(), + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + var resultAsMap = result.asMap(); + assertThat( + resultAsMap, + is( + Map.of( + "rerank", + List.of( + Map.of("ranked_doc", Map.of("text", "candidate3", "index", 2, "relevance_score", 0.98005307F)), + Map.of("ranked_doc", Map.of("text", "candidate2", "index", 1, "relevance_score", 0.27904198F)), + Map.of("ranked_doc", Map.of("text", "candidate1", "index", 0, "relevance_score", 0.10194652F)) + ) + ) + ) + ); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "query", + "query", + "documents", + List.of("candidate1", "candidate2", "candidate3", "candidate4"), + "model", + "model", + "return_documents", + true, + "top_k", + 3, + "truncation", + true + ) + ) + ); + + } + + } + + public void testInfer_Embedding_DoesNotSetInputType_WhenNotPresentInTaskSettings_AndUnspecifiedIsPassedInRequest() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + { + "model": "voyage-3-large", + "object": "list", + "usage": { + "total_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), + 1024, + 1024, + "voyage-3-large", + (SimilarityMeasure) null + ); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + false, + new HashMap<>(), + InputType.UNSPECIFIED, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT); + + MatcherAssert.assertThat(result.asMap(), Matchers.is(buildExpectationFloat(List.of(new float[] { 0.123F, -0.123F })))); + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().getFirst().getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is( + Map.of( + "input", + List.of("abc"), + "model", + "voyage-3-large", + "input_type", + "document", + "output_dtype", + "float", + "output_dimension", + 1024 + ) + ) + ); + } + } + + public void test_Embedding_ChunkedInfer_BatchesCallsChunkingSettingsSet() throws IOException { + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new VoyageAIEmbeddingsTaskSettings((InputType) null, null), + createRandomChunkingSettings(), + 1024, + 1024, + "voyage-3-large" + ); + + test_Embedding_ChunkedInfer_BatchesCalls(model); + } + + public void test_Embedding_ChunkedInfer_ChunkingSettingsNotSet() throws IOException { + var model = VoyageAIEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new VoyageAIEmbeddingsTaskSettings((InputType) null, null), + null, + 1024, + 1024, + "voyage-3-large" + ); + + test_Embedding_ChunkedInfer_BatchesCalls(model); + } + + private void test_Embedding_ChunkedInfer_BatchesCalls(VoyageAIEmbeddingsModel model) throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new VoyageAIService(senderFactory, createWithEmptySettings(threadPool))) { + + // Batching will call the service with 2 input + String responseJson = """ + { + "model": "voyage-3-large", + "object": "list", + "usage": { + "total_tokens": 5 + }, + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + 0.123, + -0.123 + ] + }, + { + "object": "embedding", + "index": 1, + "embedding": [ + 0.223, + -0.223 + ] + } + ] + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + PlainActionFuture> listener = new PlainActionFuture<>(); + // 2 input + service.chunkedInfer( + model, + null, + List.of("foo", "bar"), + new HashMap<>(), + InputType.UNSPECIFIED, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var results = listener.actionGet(TIMEOUT); + assertThat(results, hasSize(2)); + { + assertThat(results.getFirst(), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.getFirst(); + assertThat(floatResult.chunks(), hasSize(1)); + assertEquals("foo", floatResult.chunks().getFirst().matchedText()); + assertThat(floatResult.chunks().getFirst(), CoreMatchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 0.123f, -0.123f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().getFirst()).embedding(), + 0.0f + ); + } + { + assertThat(results.get(1), CoreMatchers.instanceOf(ChunkedInferenceEmbedding.class)); + var floatResult = (ChunkedInferenceEmbedding) results.get(1); + assertThat(floatResult.chunks(), hasSize(1)); + assertEquals("bar", floatResult.chunks().getFirst().matchedText()); + assertThat(floatResult.chunks().getFirst(), CoreMatchers.instanceOf(TextEmbeddingFloatResults.Chunk.class)); + assertArrayEquals( + new float[] { 0.223f, -0.223f }, + ((TextEmbeddingFloatResults.Chunk) floatResult.chunks().getFirst()).embedding(), + 0.0f + ); + } + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().getFirst().getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().getFirst().getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().getFirst().getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().getFirst().getBody()); + MatcherAssert.assertThat( + requestMap, + is(Map.of("input", List.of("foo", "bar"), "model", "voyage-3-large", "output_dtype", "float", "output_dimension", 1024)) + ); + } + } + + public void testDefaultSimilarity() { + assertEquals(SimilarityMeasure.DOT_PRODUCT, VoyageAIService.defaultSimilarity()); + } + + @SuppressWarnings("checkstyle:LineLength") + public void testGetConfiguration() throws Exception { + try (var service = createVoyageAIService()) { + String content = XContentHelper.stripWhitespace(""" + { + "service": "voyageai", + "name": "Voyage AI", + "task_types": ["text_embedding", "rerank"], + "configurations": { + "model_id": { + "description": "The name of the model to use for the inference task.", + "label": "Model ID", + "required": true, + "sensitive": false, + "updatable": false, + "type": "str", + "supported_task_types": ["text_embedding", "rerank"] + }, + "api_key": { + "description": "API Key for the provider you're connecting to.", + "label": "API Key", + "required": true, + "sensitive": true, + "updatable": true, + "type": "str", + "supported_task_types": ["text_embedding", "rerank"] + }, + "rate_limit.requests_per_minute": { + "description": "Minimize the number of rate limit errors.", + "label": "Rate Limit", + "required": false, + "sensitive": false, + "updatable": false, + "type": "int", + "supported_task_types": ["text_embedding", "rerank"] + } + } + } + """); + InferenceServiceConfiguration configuration = InferenceServiceConfiguration.fromXContentBytes( + new BytesArray(content), + XContentType.JSON + ); + boolean humanReadable = true; + BytesReference originalBytes = toShuffledXContent(configuration, XContentType.JSON, ToXContent.EMPTY_PARAMS, humanReadable); + InferenceServiceConfiguration serviceConfiguration = service.getConfiguration(); + assertToXContentEquivalent( + originalBytes, + toXContent(serviceConfiguration, XContentType.JSON, humanReadable), + XContentType.JSON + ); + } + } + + public void testDoesNotSupportsStreaming() throws IOException { + try (var service = new VoyageAIService(mock(), createWithEmptySettings(mock()))) { + assertFalse(service.canStream(TaskType.COMPLETION)); + assertFalse(service.canStream(TaskType.ANY)); + } + } + + private Map getRequestConfigMap( + Map serviceSettings, + Map taskSettings, + Map chunkingSettings, + Map secretSettings + ) { + var requestConfigMap = getRequestConfigMap(serviceSettings, taskSettings, secretSettings); + requestConfigMap.put(ModelConfigurations.CHUNKING_SETTINGS, chunkingSettings); + + return requestConfigMap; + } + + private Map getRequestConfigMap( + Map serviceSettings, + Map taskSettings, + Map secretSettings + ) { + var builtServiceSettings = new HashMap<>(); + builtServiceSettings.putAll(serviceSettings); + builtServiceSettings.putAll(secretSettings); + + return new HashMap<>( + Map.of(ModelConfigurations.SERVICE_SETTINGS, builtServiceSettings, ModelConfigurations.TASK_SETTINGS, taskSettings) + ); + } + + private Map getRequestConfigMap(Map serviceSettings, Map secretSettings) { + var builtServiceSettings = new HashMap<>(); + builtServiceSettings.putAll(serviceSettings); + builtServiceSettings.putAll(secretSettings); + + return new HashMap<>(Map.of(ModelConfigurations.SERVICE_SETTINGS, builtServiceSettings)); + } + + private VoyageAIService createVoyageAIService() { + return new VoyageAIService(mock(HttpRequestSender.Factory.class), createWithEmptySettings(threadPool)); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsModelTests.java new file mode 100644 index 0000000000000..e497e606b0689 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsModelTests.java @@ -0,0 +1,209 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.embeddings; + +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkingSettings; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceSettings; +import org.hamcrest.MatcherAssert; + +import java.util.Map; + +import static org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettingsTests.getTaskSettingsMap; +import static org.hamcrest.Matchers.is; + +public class VoyageAIEmbeddingsModelTests extends ESTestCase { + + public void testOverrideWith_DoesNotOverrideAndModelRemainsEqual_WhenSettingsAreEmpty_AndInputTypeIsInvalid() { + var model = createModel("url", "api_key", null, null, "model"); + + var overriddenModel = VoyageAIEmbeddingsModel.of(model, Map.of(), InputType.UNSPECIFIED); + MatcherAssert.assertThat(overriddenModel, is(model)); + } + + public void testOverrideWith_DoesNotOverrideAndModelRemainsEqual_WhenSettingsAreNull_AndInputTypeIsInvalid() { + var model = createModel("url", "api_key", null, null, "model"); + + var overriddenModel = VoyageAIEmbeddingsModel.of(model, null, InputType.UNSPECIFIED); + MatcherAssert.assertThat(overriddenModel, is(model)); + } + + public void testOverrideWith_SetsInputTypeToIngest_WhenTheFieldIsNullInModelTaskSettings_AndNullInRequestTaskSettings() { + var model = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings((InputType) null, null), null, null, "model"); + + var overriddenModel = VoyageAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.INGEST); + var expectedModel = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public void testOverrideWith_SetsInputType_FromRequest_IfValid_OverridingStoredTaskSettings() { + var model = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), null, null, "model"); + + var overriddenModel = VoyageAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.SEARCH); + var expectedModel = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings(InputType.SEARCH, null), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public void testOverrideWith_SetsInputType_FromRequest_IfValid_OverridingRequestTaskSettings() { + var model = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings((InputType) null, null), null, null, "model"); + + var overriddenModel = VoyageAIEmbeddingsModel.of(model, getTaskSettingsMap(InputType.INGEST), InputType.SEARCH); + var expectedModel = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings(InputType.SEARCH, null), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public void testOverrideWith_OverridesInputType_WithRequestTaskSettingsSearch_WhenRequestInputTypeIsInvalid() { + var model = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), null, null, "model"); + + var overriddenModel = VoyageAIEmbeddingsModel.of(model, getTaskSettingsMap(InputType.SEARCH), InputType.UNSPECIFIED); + var expectedModel = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings(InputType.SEARCH, null), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public void testOverrideWith_DoesNotSetInputType_FromRequest_IfInputTypeIsInvalid() { + var model = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings((InputType) null, null), null, null, "model"); + + var overriddenModel = VoyageAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.UNSPECIFIED); + var expectedModel = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings((InputType) null, null), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public void testOverrideWith_DoesNotSetInputType_WhenRequestTaskSettingsIsNull_AndRequestInputTypeIsInvalid() { + var model = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), null, null, "model"); + + var overriddenModel = VoyageAIEmbeddingsModel.of(model, getTaskSettingsMap(null), InputType.UNSPECIFIED); + var expectedModel = createModel("url", "api_key", new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, null), null, null, "model"); + MatcherAssert.assertThat(overriddenModel, is(expectedModel)); + } + + public static VoyageAIEmbeddingsModel createModel(String url, String apiKey, @Nullable Integer tokenLimit, @Nullable String model) { + return createModel(url, apiKey, VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, tokenLimit, null, model); + } + + public static VoyageAIEmbeddingsModel createModel( + String url, + String apiKey, + @Nullable Integer tokenLimit, + @Nullable Integer dimensions, + String model + ) { + return createModel(url, apiKey, VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, tokenLimit, dimensions, model); + } + + public static VoyageAIEmbeddingsModel createModel( + String url, + String apiKey, + VoyageAIEmbeddingsTaskSettings taskSettings, + ChunkingSettings chunkingSettings, + @Nullable Integer tokenLimit, + @Nullable Integer dimensions, + String model + ) { + return new VoyageAIEmbeddingsModel( + "id", + "service", + url, + new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings(model, null), + VoyageAIEmbeddingType.FLOAT, + SimilarityMeasure.DOT_PRODUCT, + dimensions, + tokenLimit, + false + ), + taskSettings, + chunkingSettings, + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + + public static VoyageAIEmbeddingsModel createModel( + String url, + String apiKey, + VoyageAIEmbeddingsTaskSettings taskSettings, + @Nullable Integer tokenLimit, + @Nullable Integer dimensions, + String model + ) { + return new VoyageAIEmbeddingsModel( + "id", + "service", + url, + new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings(model, null), + VoyageAIEmbeddingType.FLOAT, + SimilarityMeasure.DOT_PRODUCT, + dimensions, + tokenLimit, + false + ), + taskSettings, + null, + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + + public static VoyageAIEmbeddingsModel createModel( + String url, + String apiKey, + VoyageAIEmbeddingsTaskSettings taskSettings, + @Nullable Integer tokenLimit, + @Nullable Integer dimensions, + String model, + VoyageAIEmbeddingType embeddingType + ) { + return new VoyageAIEmbeddingsModel( + "id", + "service", + url, + new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings(model, null), + embeddingType, + SimilarityMeasure.DOT_PRODUCT, + dimensions, + tokenLimit, + false + ), + taskSettings, + null, + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + + public static VoyageAIEmbeddingsModel createModel( + String url, + String apiKey, + VoyageAIEmbeddingsTaskSettings taskSettings, + @Nullable Integer tokenLimit, + @Nullable Integer dimensions, + String model, + @Nullable SimilarityMeasure similarityMeasure + ) { + return new VoyageAIEmbeddingsModel( + "id", + "service", + url, + new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings(model, null), + VoyageAIEmbeddingType.FLOAT, + similarityMeasure, + dimensions, + tokenLimit, + false + ), + taskSettings, + null, + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsServiceSettingsTests.java new file mode 100644 index 0000000000000..0bedd0d60c25c --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsServiceSettingsTests.java @@ -0,0 +1,327 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.embeddings; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.inference.SimilarityMeasure; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; +import org.elasticsearch.xpack.inference.InferenceNamedWriteablesProvider; +import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; +import org.elasticsearch.xpack.inference.services.ServiceFields; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceSettingsTests; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsServiceSettings.DIMENSIONS_SET_BY_USER; +import static org.hamcrest.Matchers.is; + +public class VoyageAIEmbeddingsServiceSettingsTests extends AbstractWireSerializingTestCase { + public static VoyageAIEmbeddingsServiceSettings createRandom() { + SimilarityMeasure similarityMeasure = SimilarityMeasure.DOT_PRODUCT; + Integer dims = 1024; + Integer maxInputTokens = randomBoolean() ? null : randomIntBetween(128, 256); + Boolean dimensionSetByUser = randomBoolean(); + + var commonSettings = VoyageAIServiceSettingsTests.createRandom(); + + return new VoyageAIEmbeddingsServiceSettings( + commonSettings, + VoyageAIEmbeddingType.FLOAT, + similarityMeasure, + dims, + maxInputTokens, + dimensionSetByUser + ); + } + + public void testFromMap() { + var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); + var dims = 1536; + var maxInputTokens = 512; + var model = "model"; + var serviceSettings = VoyageAIEmbeddingsServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.SIMILARITY, + similarity, + ServiceFields.DIMENSIONS, + dims, + ServiceFields.MAX_INPUT_TOKENS, + maxInputTokens, + VoyageAIServiceSettings.MODEL_ID, + model + ) + ), + ConfigurationParseContext.PERSISTENT + ); + + MatcherAssert.assertThat( + serviceSettings, + is( + new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings(model, null), + VoyageAIEmbeddingType.FLOAT, + SimilarityMeasure.DOT_PRODUCT, + dims, + maxInputTokens, + false + ) + ) + ); + } + + public void testFromMap_WithModelId() { + var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); + var maxInputTokens = 512; + var model = "model"; + var serviceSettings = VoyageAIEmbeddingsServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.SIMILARITY, + similarity, + ServiceFields.MAX_INPUT_TOKENS, + maxInputTokens, + VoyageAIServiceSettings.MODEL_ID, + model + ) + ), + ConfigurationParseContext.REQUEST + ); + + MatcherAssert.assertThat( + serviceSettings, + is( + new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings(model, null), + VoyageAIEmbeddingType.FLOAT, + SimilarityMeasure.DOT_PRODUCT, + null, + maxInputTokens, + false + ) + ) + ); + } + + public void testFromMap_WithModelId_WithDimensions() { + var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); + var dims = 1536; + var maxInputTokens = 512; + var model = "model"; + var serviceSettings = VoyageAIEmbeddingsServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.SIMILARITY, + similarity, + ServiceFields.DIMENSIONS, + dims, + ServiceFields.MAX_INPUT_TOKENS, + maxInputTokens, + VoyageAIServiceSettings.MODEL_ID, + model + ) + ), + ConfigurationParseContext.REQUEST + ); + + MatcherAssert.assertThat( + serviceSettings, + is( + new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings(model, null), + VoyageAIEmbeddingType.FLOAT, + SimilarityMeasure.DOT_PRODUCT, + dims, + maxInputTokens, + true + ) + ) + ); + } + + public void testFromMap_DimensionsSetByUserIsFalseInRequestContext() { + var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); + var maxInputTokens = 512; + var model = "model"; + var serviceSettings = VoyageAIEmbeddingsServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.SIMILARITY, + similarity, + DIMENSIONS_SET_BY_USER, + true, + ServiceFields.MAX_INPUT_TOKENS, + maxInputTokens, + VoyageAIServiceSettings.MODEL_ID, + model + ) + ), + ConfigurationParseContext.REQUEST + ); + + MatcherAssert.assertThat( + serviceSettings, + is( + new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings(model, null), + VoyageAIEmbeddingType.FLOAT, + SimilarityMeasure.DOT_PRODUCT, + null, + maxInputTokens, + false + ) + ) + ); + } + + public void testFromMap_DimensionsSetByUserIsSetInPersistentContext() { + var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); + var maxInputTokens = 512; + var model = "model"; + var dimensionsSetByUser = randomBoolean(); + var serviceSettings = VoyageAIEmbeddingsServiceSettings.fromMap( + new HashMap<>( + Map.of( + ServiceFields.SIMILARITY, + similarity, + DIMENSIONS_SET_BY_USER, + dimensionsSetByUser, + ServiceFields.MAX_INPUT_TOKENS, + maxInputTokens, + VoyageAIServiceSettings.MODEL_ID, + model + ) + ), + ConfigurationParseContext.PERSISTENT + ); + + MatcherAssert.assertThat( + serviceSettings, + is( + new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings(model, null), + VoyageAIEmbeddingType.FLOAT, + SimilarityMeasure.DOT_PRODUCT, + null, + maxInputTokens, + dimensionsSetByUser + ) + ) + ); + } + + public void testFromMap_InvalidSimilarity_ThrowsError() { + var similarity = "by_size"; + var thrownException = expectThrows( + ValidationException.class, + () -> VoyageAIEmbeddingsServiceSettings.fromMap( + new HashMap<>(Map.of(VoyageAIServiceSettings.MODEL_ID, "model", ServiceFields.SIMILARITY, similarity)), + ConfigurationParseContext.PERSISTENT + ) + ); + + MatcherAssert.assertThat( + thrownException.getMessage(), + is( + "Validation Failed: 1: [service_settings] Invalid value [by_size] received. [similarity] " + + "must be one of [cosine, dot_product, l2_norm];" + ) + ); + } + + @SuppressWarnings("checkstyle:LineLength") + public void testToXContent_WritesAllValues() throws IOException { + var serviceSettings = new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings("model", new RateLimitSettings(3)), + VoyageAIEmbeddingType.FLOAT, + SimilarityMeasure.COSINE, + 5, + 10, + false + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + serviceSettings.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + assertThat( + xContentResult, + is( + """ + {"model_id":"model",""" + + """ + "rate_limit":{"requests_per_minute":3},"similarity":"cosine","dimensions":5,"max_input_tokens":10,"embedding_type":"float"}""" + ) + ); + } + + @SuppressWarnings("checkstyle:LineLength") + public void testToXContent_WritesAllValues_DimensionSetByUser() throws IOException { + var serviceSettings = new VoyageAIEmbeddingsServiceSettings( + new VoyageAIServiceSettings("model", new RateLimitSettings(3)), + VoyageAIEmbeddingType.FLOAT, + SimilarityMeasure.COSINE, + 5, + 10, + true + ); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + serviceSettings.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + assertThat( + xContentResult, + is( + """ + {"model_id":"model",""" + + """ + "rate_limit":{"requests_per_minute":3},"similarity":"cosine","dimensions":5,"max_input_tokens":10,"embedding_type":"float"}""" + ) + ); + } + + @Override + protected Writeable.Reader instanceReader() { + return VoyageAIEmbeddingsServiceSettings::new; + } + + @Override + protected VoyageAIEmbeddingsServiceSettings createTestInstance() { + return createRandom(); + } + + @Override + protected VoyageAIEmbeddingsServiceSettings mutateInstance(VoyageAIEmbeddingsServiceSettings instance) throws IOException { + return randomValueOtherThan(instance, VoyageAIEmbeddingsServiceSettingsTests::createRandom); + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + List entries = new ArrayList<>(); + entries.addAll(new MlInferenceNamedXContentProvider().getNamedWriteables()); + entries.addAll(InferenceNamedWriteablesProvider.getNamedWriteables()); + return new NamedWriteableRegistry(entries); + } + + public static Map getServiceSettingsMap(String model) { + return new HashMap<>(VoyageAIServiceSettingsTests.getServiceSettingsMap(model)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsTaskSettingsTests.java new file mode 100644 index 0000000000000..f3d85749e8e29 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/embeddings/VoyageAIEmbeddingsTaskSettingsTests.java @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.embeddings; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceFields; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.InputTypeTests.randomWithIngestAndSearch; +import static org.elasticsearch.xpack.inference.services.voyageai.embeddings.VoyageAIEmbeddingsTaskSettings.VALID_REQUEST_VALUES; +import static org.hamcrest.Matchers.is; + +public class VoyageAIEmbeddingsTaskSettingsTests extends AbstractWireSerializingTestCase { + + public static VoyageAIEmbeddingsTaskSettings createRandom() { + var inputType = randomBoolean() ? randomWithIngestAndSearch() : null; + var truncation = randomBoolean(); + + return new VoyageAIEmbeddingsTaskSettings(inputType, truncation); + } + + public void testIsEmpty() { + var randomSettings = createRandom(); + var stringRep = Strings.toString(randomSettings); + assertEquals(stringRep, randomSettings.isEmpty(), stringRep.equals("{}")); + } + + public void testUpdatedTaskSettings_NotUpdated_UseInitialSettings() { + var initialSettings = createRandom(); + var newSettings = new VoyageAIEmbeddingsTaskSettings((InputType) null, null); + Map newSettingsMap = new HashMap<>(); + VoyageAIEmbeddingsTaskSettings updatedSettings = (VoyageAIEmbeddingsTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + assertEquals(initialSettings.getInputType(), updatedSettings.getInputType()); + } + + public void testUpdatedTaskSettings_Updated_UseNewSettings() { + var initialSettings = createRandom(); + var newSettings = new VoyageAIEmbeddingsTaskSettings(randomWithIngestAndSearch(), randomBoolean()); + Map newSettingsMap = new HashMap<>(); + newSettingsMap.put(VoyageAIEmbeddingsTaskSettings.INPUT_TYPE, newSettings.getInputType().toString()); + VoyageAIEmbeddingsTaskSettings updatedSettings = (VoyageAIEmbeddingsTaskSettings) initialSettings.updatedTaskSettings( + Collections.unmodifiableMap(newSettingsMap) + ); + assertEquals(newSettings.getInputType(), updatedSettings.getInputType()); + } + + public void testFromMap_CreatesEmptySettings_WhenAllFieldsAreNull() { + MatcherAssert.assertThat( + VoyageAIEmbeddingsTaskSettings.fromMap(new HashMap<>(Map.of())), + is(new VoyageAIEmbeddingsTaskSettings((InputType) null, null)) + ); + } + + public void testFromMap_CreatesEmptySettings_WhenMapIsNull() { + MatcherAssert.assertThat( + VoyageAIEmbeddingsTaskSettings.fromMap(null), + is(new VoyageAIEmbeddingsTaskSettings((InputType) null, null)) + ); + } + + public void testFromMap_CreatesSettings_WhenAllFieldsOfSettingsArePresent() { + MatcherAssert.assertThat( + VoyageAIEmbeddingsTaskSettings.fromMap( + new HashMap<>( + Map.of(VoyageAIEmbeddingsTaskSettings.INPUT_TYPE, InputType.INGEST.toString(), VoyageAIServiceFields.TRUNCATION, false) + ) + ), + is(new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, false)) + ); + } + + public void testFromMap_ReturnsFailure_WhenInputTypeIsInvalid() { + var exception = expectThrows( + ValidationException.class, + () -> VoyageAIEmbeddingsTaskSettings.fromMap( + new HashMap<>(Map.of(VoyageAIEmbeddingsTaskSettings.INPUT_TYPE, "abc", VoyageAIServiceFields.TRUNCATION, false)) + ) + ); + + MatcherAssert.assertThat( + exception.getMessage(), + is( + Strings.format( + "Validation Failed: 1: [task_settings] Invalid value [abc] received. [input_type] must be one of [%s];", + getValidValuesSortedAndCombined(VALID_REQUEST_VALUES) + ) + ) + ); + } + + public void testFromMap_ReturnsFailure_WhenTruncationIsInvalid() { + var exception = expectThrows( + ValidationException.class, + () -> VoyageAIEmbeddingsTaskSettings.fromMap( + new HashMap<>( + Map.of(VoyageAIEmbeddingsTaskSettings.INPUT_TYPE, InputType.INGEST.toString(), VoyageAIServiceFields.TRUNCATION, "abc") + ) + ) + ); + + MatcherAssert.assertThat( + exception.getMessage(), + is("Validation Failed: 1: field [truncation] is not of the expected type. The value [abc] cannot be converted to a [Boolean];") + ); + } + + public void testFromMap_ReturnsFailure_WhenInputTypeIsUnspecified() { + var exception = expectThrows( + ValidationException.class, + () -> VoyageAIEmbeddingsTaskSettings.fromMap( + new HashMap<>(Map.of(VoyageAIEmbeddingsTaskSettings.INPUT_TYPE, InputType.UNSPECIFIED.toString())) + ) + ); + + MatcherAssert.assertThat( + exception.getMessage(), + is( + Strings.format( + "Validation Failed: 1: [task_settings] Invalid value [unspecified] received. [input_type] must be one of [%s];", + getValidValuesSortedAndCombined(VALID_REQUEST_VALUES) + ) + ) + ); + } + + private static > String getValidValuesSortedAndCombined(EnumSet validValues) { + var validValuesAsStrings = validValues.stream().map(value -> value.toString().toLowerCase(Locale.ROOT)).toArray(String[]::new); + Arrays.sort(validValuesAsStrings); + + return String.join(", ", validValuesAsStrings); + } + + public void testXContent_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() { + var thrownException = expectThrows(AssertionError.class, () -> new VoyageAIEmbeddingsTaskSettings(InputType.UNSPECIFIED, null)); + MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]")); + } + + public void testOf_KeepsOriginalValuesWhenRequestSettingsAreNull_AndRequestInputTypeIsInvalid() { + var taskSettings = new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, false); + var overriddenTaskSettings = VoyageAIEmbeddingsTaskSettings.of( + taskSettings, + VoyageAIEmbeddingsTaskSettings.EMPTY_SETTINGS, + InputType.UNSPECIFIED + ); + MatcherAssert.assertThat(overriddenTaskSettings, is(taskSettings)); + } + + public void testOf_UsesRequestTaskSettings() { + var taskSettings = new VoyageAIEmbeddingsTaskSettings((InputType) null, null); + var overriddenTaskSettings = VoyageAIEmbeddingsTaskSettings.of( + taskSettings, + new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, true), + InputType.UNSPECIFIED + ); + + MatcherAssert.assertThat(overriddenTaskSettings, is(new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, true))); + } + + public void testOf_UsesRequestTaskSettings_AndRequestInputType() { + var taskSettings = new VoyageAIEmbeddingsTaskSettings(InputType.SEARCH, true); + var overriddenTaskSettings = VoyageAIEmbeddingsTaskSettings.of( + taskSettings, + new VoyageAIEmbeddingsTaskSettings((InputType) null, null), + InputType.INGEST + ); + + MatcherAssert.assertThat(overriddenTaskSettings, is(new VoyageAIEmbeddingsTaskSettings(InputType.INGEST, true))); + } + + @Override + protected Writeable.Reader instanceReader() { + return VoyageAIEmbeddingsTaskSettings::new; + } + + @Override + protected VoyageAIEmbeddingsTaskSettings createTestInstance() { + return createRandom(); + } + + @Override + protected VoyageAIEmbeddingsTaskSettings mutateInstance(VoyageAIEmbeddingsTaskSettings instance) throws IOException { + return randomValueOtherThan(instance, VoyageAIEmbeddingsTaskSettingsTests::createRandom); + } + + public static Map getTaskSettingsMapEmpty() { + return new HashMap<>(); + } + + public static Map getTaskSettingsMap(@Nullable InputType inputType) { + var map = new HashMap(); + + if (inputType != null) { + map.put(VoyageAIEmbeddingsTaskSettings.INPUT_TYPE, inputType.toString()); + } + + return map; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankModelTests.java new file mode 100644 index 0000000000000..d9ceca107f9e4 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankModelTests.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.rerank; + +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceSettings; + +public class VoyageAIRerankModelTests { + public static VoyageAIRerankModel createModel(String apiKey, String modelId, @Nullable Integer topK, @Nullable Boolean truncation) { + return new VoyageAIRerankModel( + "id", + "service", + ESTestCase.randomAlphaOfLength(10), + new VoyageAIRerankServiceSettings(new VoyageAIServiceSettings(modelId, null)), + new VoyageAIRerankTaskSettings(topK, null, truncation), + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + + public static VoyageAIRerankModel createModel(String apiKey, String modelId, @Nullable Integer topK) { + return new VoyageAIRerankModel( + "id", + "service", + ESTestCase.randomAlphaOfLength(10), + new VoyageAIRerankServiceSettings(new VoyageAIServiceSettings(modelId, null)), + new VoyageAIRerankTaskSettings(topK, null, null), + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + + public static VoyageAIRerankModel createModel(String modelId, @Nullable Integer topK) { + return new VoyageAIRerankModel( + "id", + "service", + ESTestCase.randomAlphaOfLength(10), + new VoyageAIRerankServiceSettings(new VoyageAIServiceSettings(modelId, null)), + new VoyageAIRerankTaskSettings(topK, null, null), + new DefaultSecretSettings(ESTestCase.randomSecureStringOfLength(8)) + ); + } + + public static VoyageAIRerankModel createModel(String modelId, @Nullable Integer topK, Boolean returnDocuments, Boolean truncation) { + return new VoyageAIRerankModel( + "id", + "service", + ESTestCase.randomAlphaOfLength(10), + new VoyageAIRerankServiceSettings(new VoyageAIServiceSettings(modelId, null)), + new VoyageAIRerankTaskSettings(topK, returnDocuments, truncation), + new DefaultSecretSettings(ESTestCase.randomSecureStringOfLength(8)) + ); + } + + public static VoyageAIRerankModel createModel( + String url, + String modelId, + @Nullable Integer topK, + Boolean returnDocuments, + Boolean truncation + ) { + return new VoyageAIRerankModel( + "id", + "service", + url, + new VoyageAIRerankServiceSettings(new VoyageAIServiceSettings(modelId, null)), + new VoyageAIRerankTaskSettings(topK, returnDocuments, truncation), + new DefaultSecretSettings(ESTestCase.randomSecureStringOfLength(8)) + ); + } + + public static VoyageAIRerankModel createModel( + String url, + String apiKey, + String modelId, + @Nullable Integer topK, + Boolean returnDocuments, + Boolean truncation + ) { + return new VoyageAIRerankModel( + "id", + "service", + url, + new VoyageAIRerankServiceSettings(new VoyageAIServiceSettings(modelId, null)), + new VoyageAIRerankTaskSettings(topK, returnDocuments, truncation), + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankServiceSettingsTests.java new file mode 100644 index 0000000000000..7891d5cc7cca0 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankServiceSettingsTests.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.rerank; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceSettings; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceSettingsTests; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString; + +public class VoyageAIRerankServiceSettingsTests extends AbstractBWCWireSerializationTestCase { + public static VoyageAIRerankServiceSettings createRandom() { + return new VoyageAIRerankServiceSettings( + new VoyageAIServiceSettings(randomAlphaOfLength(10), RateLimitSettingsTests.createRandom()) + ); + } + + public void testToXContent_WritesAllValues() throws IOException { + var url = "http://www.abc.com"; + var model = "model"; + + var serviceSettings = new VoyageAIRerankServiceSettings(new VoyageAIServiceSettings(model, null)); + + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + serviceSettings.toXContent(builder, null); + String xContentResult = Strings.toString(builder); + + assertThat(xContentResult, equalToIgnoringWhitespaceInJsonString(""" + { + "model_id":"model", + "rate_limit": { + "requests_per_minute": 2000 + } + } + """)); + } + + @Override + protected Writeable.Reader instanceReader() { + return VoyageAIRerankServiceSettings::new; + } + + @Override + protected VoyageAIRerankServiceSettings createTestInstance() { + return createRandom(); + } + + @Override + protected VoyageAIRerankServiceSettings mutateInstance(VoyageAIRerankServiceSettings instance) throws IOException { + return randomValueOtherThan(instance, VoyageAIRerankServiceSettingsTests::createRandom); + } + + @Override + protected VoyageAIRerankServiceSettings mutateInstanceForVersion(VoyageAIRerankServiceSettings instance, TransportVersion version) { + return instance; + } + + public static Map getServiceSettingsMap(@Nullable String model) { + return new HashMap<>(VoyageAIServiceSettingsTests.getServiceSettingsMap(model)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankTaskSettingsTests.java new file mode 100644 index 0000000000000..02c8f9ae677ef --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/voyageai/rerank/VoyageAIRerankTaskSettingsTests.java @@ -0,0 +1,162 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.voyageai.rerank; + +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.inference.services.voyageai.VoyageAIServiceFields; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; + +public class VoyageAIRerankTaskSettingsTests extends AbstractWireSerializingTestCase { + + public static VoyageAIRerankTaskSettings createRandom() { + var returnDocuments = randomBoolean() ? randomBoolean() : null; + var topNDocsOnly = randomBoolean() ? randomIntBetween(1, 10) : null; + var truncation = randomBoolean() ? randomBoolean() : null; + + return new VoyageAIRerankTaskSettings(topNDocsOnly, returnDocuments, truncation); + } + + public void testFromMap_WithInvalidTruncation_ThrowsValidationException() { + Map taskMap = Map.of( + VoyageAIRerankTaskSettings.RETURN_DOCUMENTS, + true, + VoyageAIRerankTaskSettings.TOP_K_DOCS_ONLY, + 5, + VoyageAIServiceFields.TRUNCATION, + "invalid" + ); + var thrownException = expectThrows(ValidationException.class, () -> VoyageAIRerankTaskSettings.fromMap(new HashMap<>(taskMap))); + assertThat(thrownException.getMessage(), containsString("field [truncation] is not of the expected type")); + } + + public void testFromMap_WithValidValues_ReturnsSettings() { + Map taskMap = Map.of( + VoyageAIRerankTaskSettings.RETURN_DOCUMENTS, + true, + VoyageAIRerankTaskSettings.TOP_K_DOCS_ONLY, + 5, + VoyageAIServiceFields.TRUNCATION, + true + ); + var settings = VoyageAIRerankTaskSettings.fromMap(new HashMap<>(taskMap)); + assertTrue(settings.getReturnDocuments()); + assertEquals(5, settings.getTopKDocumentsOnly().intValue()); + assertTrue(settings.getTruncation()); + } + + public void testFromMap_WithNullValues_ReturnsSettingsWithNulls() { + var settings = VoyageAIRerankTaskSettings.fromMap(Map.of()); + assertNull(settings.getReturnDocuments()); + assertNull(settings.getTopKDocumentsOnly()); + assertNull(settings.getTruncation()); + } + + public void testFromMap_WithInvalidReturnDocuments_ThrowsValidationException() { + Map taskMap = Map.of( + VoyageAIRerankTaskSettings.RETURN_DOCUMENTS, + "invalid", + VoyageAIRerankTaskSettings.TOP_K_DOCS_ONLY, + 5 + ); + var thrownException = expectThrows(ValidationException.class, () -> VoyageAIRerankTaskSettings.fromMap(new HashMap<>(taskMap))); + assertThat(thrownException.getMessage(), containsString("field [return_documents] is not of the expected type")); + } + + public void testFromMap_WithInvalidTopNDocsOnly_ThrowsValidationException() { + Map taskMap = Map.of( + VoyageAIRerankTaskSettings.RETURN_DOCUMENTS, + true, + VoyageAIRerankTaskSettings.TOP_K_DOCS_ONLY, + "invalid" + ); + var thrownException = expectThrows(ValidationException.class, () -> VoyageAIRerankTaskSettings.fromMap(new HashMap<>(taskMap))); + assertThat( + thrownException.getMessage(), + containsString("field [top_k] is not of the expected type. The value [invalid] cannot be converted to a [Integer];") + ); + } + + public void testUpdatedTaskSettings_WithEmptyMap_ReturnsSameSettings() { + var initialSettings = new VoyageAIRerankTaskSettings(5, true, true); + VoyageAIRerankTaskSettings updatedSettings = (VoyageAIRerankTaskSettings) initialSettings.updatedTaskSettings(Map.of()); + assertEquals(initialSettings, updatedSettings); + } + + public void testUpdatedTaskSettings_WithNewReturnDocuments_ReturnsUpdatedSettings() { + var initialSettings = new VoyageAIRerankTaskSettings(5, true, true); + Map newSettings = Map.of(VoyageAIRerankTaskSettings.RETURN_DOCUMENTS, false); + VoyageAIRerankTaskSettings updatedSettings = (VoyageAIRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings); + assertFalse(updatedSettings.getReturnDocuments()); + assertTrue(updatedSettings.getTruncation()); + assertEquals(initialSettings.getTopKDocumentsOnly(), updatedSettings.getTopKDocumentsOnly()); + } + + public void testUpdatedTaskSettings_WithNewTopNDocsOnly_ReturnsUpdatedSettings() { + var initialSettings = new VoyageAIRerankTaskSettings(5, true, true); + Map newSettings = Map.of(VoyageAIRerankTaskSettings.TOP_K_DOCS_ONLY, 7); + VoyageAIRerankTaskSettings updatedSettings = (VoyageAIRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings); + assertTrue(updatedSettings.getTruncation()); + assertEquals(7, updatedSettings.getTopKDocumentsOnly().intValue()); + assertEquals(initialSettings.getReturnDocuments(), updatedSettings.getReturnDocuments()); + } + + public void testUpdatedTaskSettings_WithMultipleNewValues_ReturnsUpdatedSettings() { + var initialSettings = new VoyageAIRerankTaskSettings(5, true, true); + Map newSettings = Map.of( + VoyageAIRerankTaskSettings.RETURN_DOCUMENTS, + false, + VoyageAIRerankTaskSettings.TOP_K_DOCS_ONLY, + 7 + ); + VoyageAIRerankTaskSettings updatedSettings = (VoyageAIRerankTaskSettings) initialSettings.updatedTaskSettings(newSettings); + assertTrue(updatedSettings.getTruncation()); + assertFalse(updatedSettings.getReturnDocuments()); + assertEquals(7, updatedSettings.getTopKDocumentsOnly().intValue()); + } + + @Override + protected Writeable.Reader instanceReader() { + return VoyageAIRerankTaskSettings::new; + } + + @Override + protected VoyageAIRerankTaskSettings createTestInstance() { + return createRandom(); + } + + @Override + protected VoyageAIRerankTaskSettings mutateInstance(VoyageAIRerankTaskSettings instance) throws IOException { + return randomValueOtherThan(instance, VoyageAIRerankTaskSettingsTests::createRandom); + } + + public static Map getTaskSettingsMapEmpty() { + return new HashMap<>(); + } + + public static Map getTaskSettingsMap(@Nullable Integer topNDocumentsOnly, Boolean returnDocuments) { + var map = new HashMap(); + + if (topNDocumentsOnly != null) { + map.put(VoyageAIRerankTaskSettings.TOP_K_DOCS_ONLY, topNDocumentsOnly.toString()); + } + + if (returnDocuments != null) { + map.put(VoyageAIRerankTaskSettings.RETURN_DOCUMENTS, returnDocuments.toString()); + } + + return map; + } +} diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java index 03839b7e05d71..bfb539a514e1e 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java @@ -23,9 +23,13 @@ import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.ingest.PutPipelineTransportAction; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -38,7 +42,9 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.ingest.common.IngestCommonPlugin; @@ -47,6 +53,18 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.StartILMRequest; +import org.elasticsearch.xpack.core.ilm.StopILMRequest; +import org.elasticsearch.xpack.core.ilm.action.GetStatusAction; +import org.elasticsearch.xpack.core.ilm.action.ILMActions; +import org.elasticsearch.xpack.core.ilm.action.PutLifecycleRequest; +import org.elasticsearch.xpack.ilm.IndexLifecycle; import org.elasticsearch.xpack.migrate.MigratePlugin; import org.elasticsearch.xpack.migrate.MigrateTemplateRegistry; import org.junit.Before; @@ -57,6 +75,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; import static java.lang.Boolean.parseBoolean; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; @@ -93,10 +112,22 @@ protected Collection> nodePlugins() { ReindexPlugin.class, MockTransportService.TestPlugin.class, DataStreamsPlugin.class, - IngestCommonPlugin.class + IngestCommonPlugin.class, + IndexLifecycle.class, + LocalStateCompositeXPackPlugin.class ); } + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal, otherSettings)) + .put(LifecycleSettings.LIFECYCLE_POLL_INTERVAL, "1s") + // This just generates less churn and makes it easier to read the log file if needed + .put(LifecycleSettings.LIFECYCLE_HISTORY_INDEX_ENABLED, false) + .build(); + } + private static String DATA_STREAM_MAPPING = """ { "dynamic": true, @@ -588,6 +619,135 @@ public void testTsdbStartEndSet() throws Exception { assertHitCount(prepareSearch(destIndex).setSize(0), 1); } + public void testIndexLifecycleSettingNotCopied() throws Exception { + Map phases = Map.of( + "hot", + new Phase( + "hot", + TimeValue.ZERO, + Map.of( + "rollover", + new org.elasticsearch.xpack.core.ilm.RolloverAction(null, null, null, 1L, null, null, null, null, null, null) + ) + ) + ); + + var policyName = "my-policy"; + LifecyclePolicy policy = new LifecyclePolicy(policyName, phases); + PutLifecycleRequest putLifecycleRequest = new PutLifecycleRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, policy); + assertAcked(client().execute(ILMActions.PUT, putLifecycleRequest).actionGet()); + + // create data stream with a document and wait for ILM to roll it over + var dataStream = createDataStream(policyName); + createDocument(dataStream); + + assertAcked(safeGet(client().execute(ILMActions.START, new StartILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)))); + assertBusy(() -> { + var getIndexResponse = safeGet(indicesAdmin().getIndex(new GetIndexRequest(TEST_REQUEST_TIMEOUT).indices(dataStream))); + assertTrue(getIndexResponse.indices().length >= 2); + }); + stopILM(); + + var dataStreams = safeGet( + indicesAdmin().execute( + GetDataStreamAction.INSTANCE, + new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStream }) + ) + ).getDataStreams(); + + assertFalse(dataStreams.isEmpty()); + String writeIndex = dataStreams.get(0).getDataStream().getWriteIndex().getName(); + List indices = dataStreams.get(0).getDataStream().getIndices().stream().map(Index::getName).toList(); + assertTrue(indices.size() >= 2); + + for (var backingIndex : indices) { + if (backingIndex.equals(writeIndex) == false) { + var destIndex = safeGet( + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(backingIndex)) + ).getDestIndex(); + var settingsResponse = safeGet( + indicesAdmin().getSettings(new GetSettingsRequest(TEST_REQUEST_TIMEOUT).indices(backingIndex, destIndex)) + ); + assertEquals(policyName, settingsResponse.getSetting(backingIndex, IndexMetadata.LIFECYCLE_NAME)); + assertNull(settingsResponse.getSetting(destIndex, IndexMetadata.LIFECYCLE_NAME)); + } + } + } + + private void stopILM() throws Exception { + assertAcked(safeGet(client().execute(ILMActions.STOP, new StopILMRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)))); + assertBusy(() -> { + var statusResponse = safeGet( + client().execute(GetStatusAction.INSTANCE, new AcknowledgedRequest.Plain(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT)) + ); + assertEquals(OperationMode.STOPPED, statusResponse.getMode()); + }); + } + + private String createDataStream(String ilmPolicy) throws Exception { + String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault()); + + Settings settings = ilmPolicy != null ? Settings.builder().put(IndexMetadata.LIFECYCLE_NAME, ilmPolicy).build() : null; + + String mapping = """ + { + "properties": { + "@timestamp": { + "type":"date" + }, + "data":{ + "type":"keyword" + } + } + } + """; + Template idxTemplate = new Template(settings, new CompressedXContent(mapping), null); + + ComposableIndexTemplate template = ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName + "*")) + .template(idxTemplate) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build(); + + assertAcked( + client().execute( + TransportPutComposableIndexTemplateAction.TYPE, + new TransportPutComposableIndexTemplateAction.Request(dataStreamName + "_template").indexTemplate(template) + ) + ); + assertAcked( + client().execute( + CreateDataStreamAction.INSTANCE, + new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName) + ) + ); + return dataStreamName; + } + + private long createDocument(String dataStreamName) throws Exception { + // Get some randomized but reasonable timestamps on the data since not all of it is guaranteed to arrive in order. + long timeSeed = System.currentTimeMillis(); + long timestamp = randomLongBetween(timeSeed - TimeUnit.HOURS.toMillis(5), timeSeed); + safeGet( + client().index( + new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE) + .source( + JsonXContent.contentBuilder() + .startObject() + .field("@timestamp", timestamp) + .field("data", randomAlphaOfLength(25)) + .endObject() + ) + ) + ); + safeGet( + indicesAdmin().refresh( + new RefreshRequest(".ds-" + dataStreamName + "*").indicesOptions(IndicesOptions.lenientExpandOpenHidden()) + ) + ); + return timestamp; + } + private static void cleanupMetadataBlocks(String index) { var settings = Settings.builder() .putNull(IndexMetadata.SETTING_READ_ONLY) diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java index 31fdcbe074c13..2f20290d58048 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -254,6 +254,8 @@ private void createIndex( var settingsOverride = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) + // remove lifecycle so that ILM does not start processing before the index is added to data stream + .putNull(IndexMetadata.LIFECYCLE_NAME) .build(); var request = new CreateIndexFromSourceAction.Request( diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java index 38ab0275f62c1..f901e811f70b9 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java @@ -15,6 +15,10 @@ import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; import org.elasticsearch.action.support.CountDownActionListener; @@ -23,8 +27,10 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamAction; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; @@ -215,9 +221,8 @@ private void maybeProcessNextIndex( SubscribableListener.newForked( l -> client.execute(ReindexDataStreamIndexAction.INSTANCE, reindexDataStreamIndexRequest, l) ) - .andThen( - (l, result) -> updateDataStream(sourceDataStream, index.getName(), result.getDestIndex(), l, parentTaskId) - ) + .andThen((l, result) -> updateDataStream(sourceDataStream, index.getName(), result.getDestIndex(), l, parentTaskId)) + .andThen((l, newIndex) -> copySettings(index.getName(), newIndex, l, parentTaskId)) .andThen(l -> deleteIndex(index.getName(), parentTaskId, l)) .addListener(ActionListener.wrap(unused -> { reindexDataStreamTask.reindexSucceeded(index.getName()); @@ -234,7 +239,7 @@ private void updateDataStream( String dataStream, String oldIndex, String newIndex, - ActionListener listener, + ActionListener listener, TaskId parentTaskId ) { ModifyDataStreamsAction.Request modifyDataStreamRequest = new ModifyDataStreamsAction.Request( @@ -243,7 +248,28 @@ private void updateDataStream( List.of(DataStreamAction.removeBackingIndex(dataStream, oldIndex), DataStreamAction.addBackingIndex(dataStream, newIndex)) ); modifyDataStreamRequest.setParentTask(parentTaskId); - client.execute(ModifyDataStreamsAction.INSTANCE, modifyDataStreamRequest, listener); + client.execute(ModifyDataStreamsAction.INSTANCE, modifyDataStreamRequest, listener.map(ingored -> newIndex)); + } + + /** + * Copy lifecycle name from the old index to the new index, so that ILM can now process the new index. + * If the new index has a lifecycle name before it is swapped into the data stream, ILM will try, and fail, to process + * the new index. For this reason, lifecycle is not set until after the new index has been added to the data stream. + */ + private void copySettings(String oldIndex, String newIndex, ActionListener listener, TaskId parentTaskId) { + var getSettingsRequest = new GetSettingsRequest(TimeValue.MAX_VALUE).indices(oldIndex); + getSettingsRequest.setParentTask(parentTaskId); + client.execute(GetSettingsAction.INSTANCE, getSettingsRequest, listener.delegateFailure((delegate, response) -> { + String lifecycleName = response.getSetting(oldIndex, IndexMetadata.LIFECYCLE_NAME); + if (lifecycleName != null) { + var settings = Settings.builder().put(IndexMetadata.LIFECYCLE_NAME, lifecycleName).build(); + var updateSettingsRequest = new UpdateSettingsRequest(settings, newIndex); + updateSettingsRequest.setParentTask(parentTaskId); + client.execute(TransportUpdateSettingsAction.TYPE, updateSettingsRequest, delegate); + } else { + delegate.onResponse(null); + } + })); } private void deleteIndex(String indexName, TaskId parentTaskId, ActionListener listener) { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java index c86596f237227..6e98f4814218f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.bytes.BytesReference; @@ -29,8 +28,6 @@ import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.metrics.Max; -import org.elasticsearch.search.profile.SearchProfileResults; -import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -57,7 +54,6 @@ import java.io.IOException; import java.util.ArrayDeque; -import java.util.ArrayList; import java.util.Deque; import java.util.List; import java.util.Map; @@ -253,39 +249,12 @@ private Client mockClient() { when(threadpool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); when(client.threadPool()).thenReturn(threadpool); - Supplier withHits = () -> new SearchResponse( - SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), - InternalAggregations.from(List.of(new Max(DestinationIndex.INCREMENTAL_ID, 1, DocValueFormat.RAW, Map.of()))), - new Suggest(new ArrayList<>()), - false, - false, - new SearchProfileResults(Map.of()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); - Supplier withNoHits = () -> new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - // Simulate completely null aggs - null, - new Suggest(new ArrayList<>()), - false, - false, - new SearchProfileResults(Map.of()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); + Supplier withHits = () -> SearchResponseUtils.response( + SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f) + ) + .aggregations(InternalAggregations.from(List.of(new Max(DestinationIndex.INCREMENTAL_ID, 1, DocValueFormat.RAW, Map.of())))) + .build(); + Supplier withNoHits = () -> SearchResponseUtils.successfulResponse(SearchHits.EMPTY_WITH_TOTAL_HITS); when(client.search(any())).thenReturn(response(withHits)).thenReturn(response(withNoHits)); return client; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java index 7a513f12bf302..8115c0f8e42a2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/utils/persistence/ResultsPersisterServiceTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; @@ -79,31 +78,10 @@ public class ResultsPersisterServiceTests extends ESTestCase { // Constants for searchWithRetry tests private static final SearchRequest SEARCH_REQUEST = new SearchRequest("my-index"); - public static final SearchResponse SEARCH_RESPONSE_SUCCESS = SearchResponseUtils.emptyWithTotalHits( - null, - 1, - 1, - 0, - 1L, - ShardSearchFailure.EMPTY_ARRAY, - null - ); - public static final SearchResponse SEARCH_RESPONSE_FAILURE = new SearchResponse( - SearchHits.EMPTY_WITHOUT_TOTAL_HITS, - null, - null, - false, - null, - null, - 1, - null, - 1, - 0, - 0, - 1L, - ShardSearchFailure.EMPTY_ARRAY, - null - ); + public static final SearchResponse SEARCH_RESPONSE_SUCCESS = SearchResponseUtils.successfulResponse(SearchHits.EMPTY_WITH_TOTAL_HITS); + public static final SearchResponse SEARCH_RESPONSE_FAILURE = SearchResponseUtils.response(SearchHits.EMPTY_WITHOUT_TOTAL_HITS) + .shards(1, 0, 0) + .build(); // Constants for bulkIndexWithRetry tests private static final IndexRequest INDEX_REQUEST_SUCCESS = new IndexRequest("my-index").id("success") diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java index 399e33c1871e7..88abbd9d89244 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java @@ -44,6 +44,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.Aggregator; @@ -561,43 +562,16 @@ public void testMismatch() throws IOException { iw.addDocument(singleton(new NumericDocValuesField("number", 3))); }, filterBuilder, new MappedFieldType[] { fieldType }, new MappedFieldType[] { fieldType }); - // TODO SearchResponse.Clusters is not public, using null for now. Should fix upstream. MultiSearchResponse.Item unrolledItem = new MultiSearchResponse.Item( - new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - InternalAggregations.from(Collections.singletonList(responses.get(0))), - null, - false, - false, - null, - 1, - null, - 1, - 1, - 0, - 10, - null, - null - ), + SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS) + .aggregations(InternalAggregations.from(responses.get(0))) + .build(), null ); MultiSearchResponse.Item rolledItem = new MultiSearchResponse.Item( - new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - InternalAggregations.from(Collections.singletonList(responses.get(1))), - null, - false, - false, - null, - 1, - null, - 1, - 1, - 0, - 10, - null, - null - ), + SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS) + .aggregations(InternalAggregations.from(responses.get(1))) + .build(), null ); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index 34f1f2f97a328..76f81801135f4 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.Rounding; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.TimeValue; @@ -44,6 +43,7 @@ import org.elasticsearch.index.query.SearchExecutionContextHelper; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; @@ -868,22 +868,7 @@ protected void doNextSearch(long waitTimeInNanos, ActionListener } ActionListener.respondAndRelease( listener, - new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - InternalAggregations.from(Collections.singletonList(result)), - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ) + SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS).aggregations(InternalAggregations.from(result)).build() ); } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java index ad5e6a0cf9b40..40b6741f09bad 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.composite.InternalComposite; import org.elasticsearch.test.ESTestCase; @@ -75,26 +76,9 @@ protected void doNextSearch(long waitTimeInNanos, ActionListener when(composite.getBuckets()).thenReturn(List.of()); when(composite.getName()).thenReturn(AGGREGATION_NAME); - InternalAggregations aggs = InternalAggregations.from(List.of(composite)); - ActionListener.respondAndRelease( nextPhase, - new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - aggs, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - new ShardSearchFailure[0], - null - ) + SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS).aggregations(InternalAggregations.from(composite)).build() ); } @@ -426,26 +410,11 @@ protected void doNextSearch(long waitTimeInNanos, ActionListener }); when(composite.getName()).thenReturn(AGGREGATION_NAME); - InternalAggregations aggs = InternalAggregations.from(List.of(composite)); - ActionListener.respondAndRelease( nextPhase, - new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - aggs, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ) + SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS) + .aggregations(InternalAggregations.from(composite)) + .build() ); } @@ -595,24 +564,9 @@ public void testUnknownKey() throws Exception { when(composite.getBuckets()).thenReturn(List.of(bucket)); when(composite.getName()).thenReturn(RollupField.NAME); - InternalAggregations aggs = InternalAggregations.from(List.of(composite)); - - return new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - aggs, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ); + return SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS) + .aggregations(InternalAggregations.from(composite)) + .build(); }; Function bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100); @@ -681,23 +635,9 @@ public void testFailureWhileStopping() throws Exception { when(composite.getBuckets()).thenReturn(List.of(bucket)); when(composite.getName()).thenReturn(RollupField.NAME); - InternalAggregations aggs = InternalAggregations.from(List.of(composite)); - return new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - aggs, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ); + return SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS) + .aggregations(InternalAggregations.from(composite)) + .build(); }; Function bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100); @@ -813,24 +753,9 @@ public void testBulkFailure() throws Exception { when(composite.getName()).thenReturn(RollupField.NAME); when(composite.getBuckets()).thenReturn(List.of(bucket)); - InternalAggregations aggs = InternalAggregations.from(List.of(composite)); - - return new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - aggs, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ); + return SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS) + .aggregations(InternalAggregations.from(composite)) + .build(); }; Function bulkFunction = bulkRequest -> { diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java index 500b796e62660..df3392cfe9e18 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java @@ -10,17 +10,14 @@ import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; -import org.apache.lucene.search.TotalHits; import org.elasticsearch.TransportVersion; import org.elasticsearch.action.admin.cluster.remote.RemoteClusterNodesAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchShardsRequest; import org.elasticsearch.action.search.SearchShardsResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchShardsAction; import org.elasticsearch.client.Request; @@ -29,6 +26,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.VersionInformation; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -37,7 +35,7 @@ import org.elasticsearch.core.ReleasableRef; import org.elasticsearch.core.Tuple; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.test.transport.MockTransportService; @@ -1218,22 +1216,7 @@ private static MockTransportService startTransport( ); try ( var searchResponseRef = ReleasableRef.of( - new SearchResponse( - SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN), - InternalAggregations.EMPTY, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 100, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) + SearchResponseUtils.successfulResponse(SearchHits.empty(Lucene.TOTAL_HITS_EQUAL_TO_ZERO, Float.NaN)) ) ) { channel.sendResponse(searchResponseRef.get()); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/ScrollHelperIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/ScrollHelperIntegTests.java index 79cf0cb9f7987..655143f1cb16b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/ScrollHelperIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/ScrollHelperIntegTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; @@ -20,6 +19,7 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.security.ScrollHelper; @@ -32,6 +32,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import static org.elasticsearch.action.support.ActionTestUtils.assertNoSuccessListener; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; @@ -87,26 +88,13 @@ public void testFetchAllByEntityWithBrokenScroll() { ActionListener listener = (ActionListener) invocation.getArguments()[1]; ActionListener.respondAndRelease( listener, - new SearchResponse( + SearchResponseUtils.response( SearchHits.unpooled( new SearchHit[] { SearchHit.unpooled(1), SearchHit.unpooled(2) }, new TotalHits(3, TotalHits.Relation.EQUAL_TO), 1 - ), - null, - null, - false, - false, - null, - 1, - scrollId, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) + ) + ).scrollId(scrollId).build() ); return null; }; @@ -117,17 +105,7 @@ public void testFetchAllByEntityWithBrokenScroll() { doAnswer(returnResponse).when(client).searchScroll(any(), any()); AtomicReference failure = new AtomicReference<>(); - ScrollHelper.fetchAllByEntity(client, request, new ActionListener>() { - @Override - public void onResponse(Collection response) { - fail("This shouldn't succeed."); - } - - @Override - public void onFailure(Exception e) { - failure.set(e); - } - }, Function.identity()); + ScrollHelper.fetchAllByEntity(client, request, assertNoSuccessListener(failure::set), Function.identity()); assertNotNull("onFailure wasn't called", failure.get()); assertEquals( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java index 269f8cb0471fc..5dc71de2f4db7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlInvalidateSessionActionTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.TransportClearScrollAction; import org.elasticsearch.action.search.TransportSearchAction; @@ -53,6 +52,7 @@ import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.client.NoOpClient; @@ -203,22 +203,7 @@ protected void try { ActionListener.respondAndRelease( listener, - (Response) new SearchResponse( - searchHits, - null, - null, - false, - false, - null, - 1, - "_scrollId1", - 1, - 1, - 0, - 1, - null, - null - ) + (Response) SearchResponseUtils.response(searchHits).scrollId("_scrollId1").build() ); } finally { searchHits.decRef(); @@ -227,22 +212,7 @@ protected void assertThat(request, instanceOf(SearchScrollRequest.class)); ActionListener.respondAndRelease( listener, - (Response) new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - null, - null, - false, - false, - null, - 1, - "_scrollId1", - 1, - 1, - 0, - 1, - null, - null - ) + (Response) SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS).scrollId("_scrollId1").build() ); } else if (TransportClearScrollAction.NAME.equals(action.name())) { assertThat(request, instanceOf(ClearScrollRequest.class)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index c7632943b63b1..7d06f90ca95d4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -62,6 +62,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.telemetry.metric.MeterRegistry; @@ -478,7 +479,7 @@ public void testApiKeysOwnerRealmIdentifier() throws Exception { builder.map(apiKeySourceDoc); searchHits[1].sourceRef(BytesReference.bytes(builder)); } - return new SearchResponse( + return SearchResponseUtils.successfulResponse( SearchHits.unpooled( searchHits, new TotalHits(searchHits.length, TotalHits.Relation.EQUAL_TO), @@ -486,20 +487,7 @@ public void testApiKeysOwnerRealmIdentifier() throws Exception { null, null, null - ), - null, - null, - false, - null, - null, - 0, - randomAlphaOfLengthBetween(3, 8), - 1, - 1, - 0, - 10, - null, - null + ) ); }; doAnswer(invocation -> { @@ -637,7 +625,7 @@ public void testInvalidateApiKeysWillSetInvalidatedFlagAndRecordTimestamp() { } ActionListener.respondAndRelease( listener, - new SearchResponse( + SearchResponseUtils.successfulResponse( SearchHits.unpooled( new SearchHit[] { searchHit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), @@ -645,20 +633,7 @@ public void testInvalidateApiKeysWillSetInvalidatedFlagAndRecordTimestamp() { null, null, null - ), - null, - null, - false, - null, - null, - 0, - randomAlphaOfLengthBetween(3, 8), - 1, - 1, - 0, - 10, - null, - null + ) ) ); return null; @@ -734,7 +709,7 @@ public void testInvalidateApiKeysWithSkippedCrossClusterKeysAndNullType() { } ActionListener.respondAndRelease( listener, - new SearchResponse( + SearchResponseUtils.successfulResponse( SearchHits.unpooled( new SearchHit[] { searchHit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), @@ -742,20 +717,7 @@ public void testInvalidateApiKeysWithSkippedCrossClusterKeysAndNullType() { null, null, null - ), - null, - null, - false, - null, - null, - 0, - randomAlphaOfLengthBetween(3, 8), - 1, - 1, - 0, - 10, - null, - null + ) ) ); return null; @@ -1078,7 +1040,7 @@ public void testCrossClusterApiKeyUsageStats() { final ActionListener listener = invocationOnMock.getArgument(1); ActionListener.respondAndRelease( listener, - new SearchResponse( + SearchResponseUtils.successfulResponse( SearchHits.unpooled( searchHits.toArray(SearchHit[]::new), new TotalHits(searchHits.size(), TotalHits.Relation.EQUAL_TO), @@ -1086,20 +1048,7 @@ public void testCrossClusterApiKeyUsageStats() { null, null, null - ), - null, - null, - false, - null, - null, - 0, - randomAlphaOfLengthBetween(3, 8), - 1, - 1, - 0, - 10, - null, - null + ) ) ); return null; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java index 33d3e6783b9e6..e13bfbddd6406 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/service/IndexServiceAccountTokenStoreTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.FilterClient; @@ -42,6 +41,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.threadpool.ThreadPool; @@ -271,21 +271,8 @@ public void testFindTokensFor() { .toArray(SearchHit[]::new); ActionListener.respondAndRelease( l, - new SearchResponse( - SearchHits.unpooled(hits, new TotalHits(nhits, TotalHits.Relation.EQUAL_TO), randomFloat(), null, null, null), - null, - null, - false, - null, - null, - 0, - randomAlphaOfLengthBetween(3, 8), - 1, - 1, - 0, - 10, - null, - null + SearchResponseUtils.successfulResponse( + SearchHits.unpooled(hits, new TotalHits(nhits, TotalHits.Relation.EQUAL_TO), randomFloat(), null, null, null) ) ); } else if (r instanceof ClearScrollRequest) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index ca84a9189d90a..206c6379eaff3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.script.mustache.MustacheScriptEngine; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ToXContent; @@ -352,21 +353,8 @@ private void doAnswerWithSearchResult(Client client, ExpressionRoleMapping mappi } ActionListener.respondAndRelease( listener, - new SearchResponse( - SearchHits.unpooled(new SearchHit[] { searchHit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), randomFloat()), - null, - null, - false, - null, - null, - 0, - randomAlphaOfLengthBetween(3, 8), - 1, - 1, - 0, - 10, - null, - null + SearchResponseUtils.successfulResponse( + SearchHits.unpooled(new SearchHit[] { searchHit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), randomFloat()) ) ); return null; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java index ed3949450cb9f..0c09b4f4eb325 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreTests.java @@ -43,6 +43,7 @@ import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; @@ -930,7 +931,7 @@ private SearchHit[] buildHits(List sourcePrivile private static SearchResponse buildSearchResponse(SearchHit[] hits) { var searchHits = new SearchHits(hits, new TotalHits(hits.length, TotalHits.Relation.EQUAL_TO), 0f); try { - return new SearchResponse(searchHits.asUnpooled(), null, null, false, false, null, 1, "_scrollId1", 1, 1, 0, 1, null, null); + return SearchResponseUtils.successfulResponse(searchHits.asUnpooled()); } finally { searchHits.decRef(); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProviderTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProviderTests.java index 468a14bc1db12..15a681d093739 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProviderTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/TimeBasedCheckpointProviderTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; @@ -25,6 +24,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; @@ -342,22 +342,7 @@ public SingleGroupSource get() { } private static SearchResponse newSearchResponse(long totalHits) { - return new SearchResponse( - SearchHits.empty(new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), 0), - null, - null, - false, - false, - null, - 0, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ); + return SearchResponseUtils.successfulResponse(SearchHits.empty(new TotalHits(totalHits, TotalHits.Relation.EQUAL_TO), 0)); } @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java index eeef51bcbcb06..b9380ade1238b 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java @@ -35,8 +35,7 @@ import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.profile.SearchProfileResults; -import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.common.notifications.Level; @@ -236,26 +235,7 @@ protected void onAbort() { @Override void doGetInitialProgress(SearchRequest request, ActionListener responseListener) { - ActionListener.respondAndRelease( - responseListener, - new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - // Simulate completely null aggs - null, - new Suggest(Collections.emptyList()), - false, - false, - new SearchProfileResults(Collections.emptyMap()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) - ); + ActionListener.respondAndRelease(responseListener, SearchResponseUtils.successfulResponse(SearchHits.EMPTY_WITH_TOTAL_HITS)); } @Override @@ -387,23 +367,7 @@ public void testDoProcessAggNullCheck() { null, null ); - SearchResponse searchResponse = new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - // Simulate completely null aggs - null, - new Suggest(Collections.emptyList()), - false, - false, - new SearchProfileResults(Collections.emptyMap()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); + SearchResponse searchResponse = SearchResponseUtils.successfulResponse(SearchHits.EMPTY_WITH_TOTAL_HITS); try { AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); Function searchFunction = searchRequest -> searchResponse; @@ -516,22 +480,8 @@ public void testRetentionPolicyDeleteByQueryThrowsIrrecoverable() throws Excepti null ); - final SearchResponse searchResponse = new SearchResponse( - SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), - // Simulate completely null aggs - null, - new Suggest(Collections.emptyList()), - false, - false, - new SearchProfileResults(Collections.emptyMap()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY + final SearchResponse searchResponse = SearchResponseUtils.successfulResponse( + SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f) ); try { AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); @@ -611,22 +561,8 @@ public void testRetentionPolicyDeleteByQueryThrowsTemporaryProblem() throws Exce null ); - final SearchResponse searchResponse = new SearchResponse( - SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), - // Simulate completely null aggs - null, - new Suggest(Collections.emptyList()), - false, - false, - new SearchProfileResults(Collections.emptyMap()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY + final SearchResponse searchResponse = SearchResponseUtils.successfulResponse( + SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f) ); try { AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); @@ -709,22 +645,8 @@ public void testFailureCounterIsResetOnSuccess() throws Exception { null ); - final SearchResponse searchResponse = new SearchResponse( - SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), - // Simulate completely null aggs - null, - new Suggest(Collections.emptyList()), - false, - false, - new SearchProfileResults(Collections.emptyMap()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY + final SearchResponse searchResponse = SearchResponseUtils.successfulResponse( + SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f) ); try { AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); @@ -954,22 +876,8 @@ private MockedTransformIndexer createMockIndexer( } private static Function returnHit() { - return request -> new SearchResponse( - SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), - // Simulate completely null aggs - null, - new Suggest(Collections.emptyList()), - false, - false, - new SearchProfileResults(Collections.emptyMap()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY + return request -> SearchResponseUtils.successfulResponse( + SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f) ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java index e3a4ef118f611..4306e6dd8bf4f 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; @@ -27,8 +26,7 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.profile.SearchProfileResults; -import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.test.junit.annotations.TestIssueLogging; @@ -87,22 +85,8 @@ public class TransformIndexerStateTests extends ESTestCase { - private static final SearchResponse ONE_HIT_SEARCH_RESPONSE = new SearchResponse( - SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), - // Simulate completely null aggs - null, - new Suggest(Collections.emptyList()), - false, - false, - new SearchProfileResults(Collections.emptyMap()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY + private static final SearchResponse ONE_HIT_SEARCH_RESPONSE = SearchResponseUtils.successfulResponse( + SearchHits.unpooled(new SearchHit[] { SearchHit.unpooled(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f) ); private Client client; diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index 644518538638d..9e352c0e89032 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -28,8 +27,7 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.profile.SearchProfileResults; -import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.threadpool.TestThreadPool; @@ -81,22 +79,8 @@ public class TransformIndexerTests extends ESTestCase { - private static final SearchResponse ONE_HIT_SEARCH_RESPONSE = new SearchResponse( - new SearchHits(new SearchHit[] { new SearchHit(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), - // Simulate completely null aggs - null, - new Suggest(Collections.emptyList()), - false, - false, - new SearchProfileResults(Collections.emptyMap()), - 1, - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY + private static final SearchResponse ONE_HIT_SEARCH_RESPONSE = SearchResponseUtils.successfulResponse( + new SearchHits(new SearchHit[] { new SearchHit(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f) ); private Client client; diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java index a774a202f333b..d6551c990e9d3 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/CompositeBucketsChangeCollectorTests.java @@ -8,10 +8,10 @@ package org.elasticsearch.xpack.transform.transforms.pivot; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.composite.InternalComposite; @@ -109,24 +109,10 @@ public void testTermsFieldCollector() throws IOException { return compositeBuckets; }); - InternalAggregations aggs = InternalAggregations.from(Collections.singletonList(composite)); - - SearchResponse response = new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - aggs, - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ); + + SearchResponse response = SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS) + .aggregations(InternalAggregations.from(composite)) + .build(); try { collector.processSearchResponse(response); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/DateHistogramFieldCollectorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/DateHistogramFieldCollectorTests.java index a70bf930a7d5d..20bdce2fccca0 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/DateHistogramFieldCollectorTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/DateHistogramFieldCollectorTests.java @@ -8,10 +8,10 @@ package org.elasticsearch.xpack.transform.transforms.pivot; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation; @@ -23,9 +23,9 @@ import org.elasticsearch.xpack.transform.transforms.Function.ChangeCollector; import org.junit.Before; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.hamcrest.CoreMatchers.instanceOf; @@ -171,22 +171,8 @@ private static QueryBuilder buildFilterQuery(ChangeCollector collector) { } private static SearchResponse buildSearchResponse(SingleValue minTimestamp, SingleValue maxTimestamp) { - return new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - InternalAggregations.from(Arrays.asList(minTimestamp, maxTimestamp)), - null, - false, - null, - null, - 1, - null, - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ); + return SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS) + .aggregations(InternalAggregations.from(List.of(minTimestamp, maxTimestamp))) + .build(); } - } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java index 0a030d26016f7..37817bc261be5 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.aggregations.InternalAggregations; import org.elasticsearch.search.aggregations.bucket.composite.InternalComposite; import org.elasticsearch.test.ESTestCase; @@ -351,22 +352,7 @@ public void testPreviewForCompositeAggregation() throws Exception { } private static SearchResponse searchResponseFromAggs(InternalAggregations aggs) { - return new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - aggs, - null, - false, - null, - null, - 1, - null, - 10, - 5, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - null - ); + return SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS).aggregations(aggs).shards(10, 5, 0).build(); } private class MyMockClient extends NoOpClient { @@ -397,22 +383,10 @@ protected void } ActionListener.respondAndRelease( listener, - (Response) new SearchResponse( - SearchHits.EMPTY_WITH_TOTAL_HITS, - null, - null, - false, - null, - null, - 1, - null, - 10, - searchFailures.size() > 0 ? 0 : 5, - 0, - 0, - searchFailures.toArray(new ShardSearchFailure[searchFailures.size()]), - null - ) + (Response) SearchResponseUtils.response(SearchHits.EMPTY_WITH_TOTAL_HITS) + .shards(10, searchFailures.isEmpty() ? 5 : 0, 0) + .shardFailures(searchFailures) + .build() ); return; } diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java index d97b0bd81a101..a15f18fb5c987 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java @@ -8,10 +8,10 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; @@ -104,22 +104,9 @@ public void testExecuteAccessHits() throws Exception { hit.score(1f); hit.shard(new SearchShardTarget("a", new ShardId("a", "indexUUID", 0), null)); - SearchResponse response = new SearchResponse( - SearchHits.unpooled(new SearchHit[] { hit }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1f), - null, - null, - false, - false, - null, - 1, - "", - 3, - 3, - 0, - 500L, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); + SearchResponse response = SearchResponseUtils.response( + SearchHits.unpooled(new SearchHit[] { hit }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1f) + ).shards(3, 3, 0).build(); try { WatchExecutionContext ctx = mockExecutionContext("_watch_name", new Payload.XContent(response, ToXContent.EMPTY_PARAMS)); assertThat(condition.execute(ctx).met(), is(true)); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java index 24a4eede1b20d..89d10564c8b99 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherServiceTests.java @@ -216,25 +216,7 @@ void stopExecutor() {} SearchHits searchHits = SearchHits.unpooled(hits, new TotalHits(count, TotalHits.Relation.EQUAL_TO), 1.0f); doAnswer(invocation -> { ActionListener listener = (ActionListener) invocation.getArguments()[2]; - ActionListener.respondAndRelease( - listener, - new SearchResponse( - searchHits, - null, - null, - false, - false, - null, - 1, - "scrollId", - 1, - 1, - 0, - 10, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) - ); + ActionListener.respondAndRelease(listener, SearchResponseUtils.response(searchHits).scrollId("scrollId").build()); return null; }).when(client).execute(eq(TransportSearchAction.TYPE), any(SearchRequest.class), anyActionListener()); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java index 776f649300aa4..1cdb6debfbb80 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java @@ -240,22 +240,9 @@ public void testFindTriggeredWatchesGoodCase() { hit.sourceRef(source); ActionListener.respondAndRelease( listener, - new SearchResponse( - SearchHits.unpooled(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f), - null, - null, - false, - null, - null, - 1, - "_scrollId1", - 1, - 1, - 0, - 1, - null, - null - ) + SearchResponseUtils.response( + SearchHits.unpooled(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f) + ).scrollId("_scrollId1").build() ); } else if (request.scrollId().equals("_scrollId1")) { ActionListener.respondAndRelease(listener, SearchResponseUtils.emptyWithTotalHits("_scrollId2", 1, 1, 0, 1, null, null));