[Backport 2.x] limit the max value of cluster.max_shards_per_node to avoid int overflow

CHANGELOG.md

@@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix handling of Short and Byte data types in ScriptProcessor ingest pipeline ([#14379](https://github.com/opensearch-project/OpenSearch/issues/14379))
 - Switch to iterative version of WKT format parser ([#14086](https://github.com/opensearch-project/OpenSearch/pull/14086))
+- Fix the computed max shards of cluster to avoid int overflow ([#14155](https://github.com/opensearch-project/OpenSearch/pull/14155))
 
 ### Security
 

server/src/main/java/org/opensearch/indices/ShardLimitValidator.java

@@ -261,14 +261,15 @@ static Optional<String> checkShardLimit(
             return Optional.empty();
         }
 
+        int computedMaxShards = (int) Math.min(Integer.MAX_VALUE, (long) maxShardsPerNodeSetting * nodeCount);
         int maxShardsInCluster = maxShardsPerClusterSetting;
         if (maxShardsInCluster == -1) {
-            maxShardsInCluster = maxShardsPerNodeSetting * nodeCount;
+            maxShardsInCluster = computedMaxShards;
         } else {
-            maxShardsInCluster = Math.min(maxShardsInCluster, maxShardsPerNodeSetting * nodeCount);
+            maxShardsInCluster = Math.min(maxShardsInCluster, computedMaxShards);
         }
 
-        int currentOpenShards = state.getMetadata().getTotalOpenIndexShards();
+        long currentOpenShards = state.getMetadata().getTotalOpenIndexShards();
         if ((currentOpenShards + newShards) > maxShardsInCluster) {
             String errorMessage = "this action would add ["
                 + newShards

server/src/test/java/org/opensearch/indices/ShardLimitValidatorTests.java

@@ -214,6 +214,25 @@ public void testNonSystemIndexCreationFailsWithMaxShardLimitOnCluster() {
         );
     }
 
+    public void testComputedMaxShardsOfClusterIntOverFlow() {
+        final int maxShardLimitPerNode = 500_000_000;
+        ClusterState state = createClusterForShardLimitTest(15, 1, 1);
+        Optional<String> errorMessage = ShardLimitValidator.checkShardLimit(2, state, maxShardLimitPerNode, -1);
+        assertFalse(errorMessage.isPresent());
+
+        errorMessage = ShardLimitValidator.checkShardLimit(Integer.MAX_VALUE - 1, state, maxShardLimitPerNode, -1);
+        assertEquals(
+            "this action would add ["
+                + (Integer.MAX_VALUE - 1)
+                + "] total shards, but this cluster currently has ["
+                + 2
+                + "]/["
+                + Integer.MAX_VALUE
+                + "] maximum shards open",
+            errorMessage.get()
+        );
+    }
+
     public void testNonSystemIndexCreationPassesWithMaxShardLimitOnCluster() {
         final int maxShardLimitOnCluster = 5;
         Settings limitOnlySettings = Settings.builder()