Merge branch 'main' into dependabot/gradle/plugins/repository-gcs/com.google.cloud-google-cloud-storage-2.60.0

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

Author: sandeshkr419

CHANGELOG.md

@@ -28,6 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add async periodic flush task support for pull-based ingestion ([#19878](https://github.com/opensearch-project/OpenSearch/pull/19878))
 - Add support for context aware segments ([#19098](https://github.com/opensearch-project/OpenSearch/pull/19098))
 - Implement GRPC FunctionScoreQuery ([#19888](https://github.com/opensearch-project/OpenSearch/pull/19888))
+- Allow the truncate filter in normalizers ([#19778](https://github.com/opensearch-project/OpenSearch/issues/19778))
 - Support pull-based ingestion message mappers and raw payload support ([#19765](https://github.com/opensearch-project/OpenSearch/pull/19765)]
 
 ### Changed
@@ -105,6 +106,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.google.api:gax-httpjson` from 2.69.0 to 2.72.1 ([#19943](https://github.com/opensearch-project/OpenSearch/pull/19943))
 - Update Hadoop to 3.4.2 and enable security (Kerberos) integration tests under JDK-24 and above  ([#19952](https://github.com/opensearch-project/OpenSearch/pull/19952))
 - Bump `com.google.cloud:google-cloud-storage` from 2.55.0 to 2.60.0 ([#20023](https://github.com/opensearch-project/OpenSearch/pull/20023))
+- Bump `commons-cli:commons-cli` from 1.10.0 to 1.11.0 ([#20022](https://github.com/opensearch-project/OpenSearch/pull/20022))
 - Bump `com.squareup.okio:okio` from 3.16.0 to 3.16.3 ([#20025](https://github.com/opensearch-project/OpenSearch/pull/20025))
 
 ### Deprecated

modules/analysis-common/src/main/java/org/opensearch/analysis/common/TruncateTokenFilterFactory.java

@@ -38,8 +38,9 @@
 import org.opensearch.env.Environment;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.analysis.AbstractTokenFilterFactory;
+import org.opensearch.index.analysis.NormalizingTokenFilterFactory;
 
-public class TruncateTokenFilterFactory extends AbstractTokenFilterFactory {
+public class TruncateTokenFilterFactory extends AbstractTokenFilterFactory implements NormalizingTokenFilterFactory {
 
     private final int length;
 

modules/analysis-common/src/test/java/org/opensearch/analysis/common/TruncateTokenFilterTests.java

@@ -0,0 +1,88 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.analysis.common;
+
+import org.apache.lucene.util.BytesRef;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.env.Environment;
+import org.opensearch.index.analysis.AnalysisTestsHelper;
+import org.opensearch.index.analysis.NamedAnalyzer;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.test.OpenSearchTokenStreamTestCase;
+
+import java.io.IOException;
+
+public class TruncateTokenFilterTests extends OpenSearchTokenStreamTestCase {
+
+    public void testFilter() throws IOException {
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.truncate.type", "truncate")
+            .put("index.analysis.filter.truncate.length", 3)
+            .put("index.analysis.analyzer.my_analyzer.type", "custom")
+            .put("index.analysis.analyzer.my_analyzer.tokenizer", "whitespace")
+            .putList("index.analysis.analyzer.my_analyzer.filter", "truncate")
+            .putList("index.analysis.normalizer.my_normalizer.filter", "truncate")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(
+            settings,
+            new CommonAnalysisModulePlugin()
+        );
+        NamedAnalyzer analyzer = analysis.indexAnalyzers.get("my_analyzer");
+        assertNotNull(analyzer);
+
+        assertTokenStreamContents(analyzer.tokenStream("foo", "a bb ccc dddd"), new String[] { "a", "bb", "ccc", "ddd" });
+    }
+
+    public void testNormalizer() throws IOException {
+        Settings settings = Settings.builder()
+            .put("index.analysis.filter.truncate.type", "truncate")
+            .put("index.analysis.filter.truncate.length", 3)
+            .put("index.analysis.normalizer.my_normalizer.type", "custom")
+            .putList("index.analysis.normalizer.my_normalizer.filter", "truncate")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(
+            settings,
+            new CommonAnalysisModulePlugin()
+        );
+        assertNull(analysis.indexAnalyzers.get("my_normalizer"));
+        NamedAnalyzer normalizer = analysis.indexAnalyzers.getNormalizer("my_normalizer");
+        assertNotNull(normalizer);
+        assertEquals("my_normalizer", normalizer.name());
+        assertEquals(new BytesRef("a"), normalizer.normalize("foo", "a"));
+        assertEquals(new BytesRef("bb"), normalizer.normalize("foo", "bb"));
+        assertEquals(new BytesRef("ccc"), normalizer.normalize("foo", "ccc"));
+        assertEquals(new BytesRef("ddd"), normalizer.normalize("foo", "dddd"));
+    }
+}

modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/40_token_filters.yml

@@ -824,6 +824,11 @@
                   my_truncate:
                     type: truncate
                     length: 3
+                normalizer:
+                  my_truncate_norm:
+                    type: custom
+                    filter:
+                      - my_truncate
     - do:
         indices.analyze:
           index: test
@@ -833,6 +838,14 @@
             filter:    [my_truncate]
     - length: { tokens: 1 }
     - match:  { tokens.0.token: foo }
+    - do:
+        indices.analyze:
+          index: test
+          body:
+            text:      foobar
+            normalizer: my_truncate_norm
+    - length: { tokens: 1 }
+    - match:  { tokens.0.token: foo }
 
 ---
 "pattern_capture":

plugins/repository-hdfs/build.gradle

@@ -66,7 +66,7 @@ dependencies {
   api "com.google.code.gson:gson:${versions.gson}"
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
-  api 'commons-cli:commons-cli:1.10.0'
+  api 'commons-cli:commons-cli:1.11.0'
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api 'commons-collections:commons-collections:3.2.2'
   api "org.apache.commons:commons-compress:${versions.commonscompress}"

plugins/repository-hdfs/licenses/commons-cli-1.10.0.jar.sha1

@@ -0,0 +1 @@
+a461452d3e31bebf2706323f8738ec44b19c96e1