Force nodes to down and power_save when stopping the cluster

If a cluster is stopped while a node is powering-up (alloc#-idle#),
node is kept in the powering-up state on cluster start.

This makes the node unavailable for the entire ResumeTimeout which is 60 minutes.
Slurm is ignoring the transition to power_down if we don't put the node to down first.

Copy Changelog from 2.11 branch.

Signed-off-by: Enrico Usai <usai@amazon.com>

Author: Enrico Usai

CHANGELOG.md

@@ -7,14 +7,25 @@ This file is used to list changes made in each version of the aws-parallelcluste
 ------
 
 **CHANGES**
-
 - Drop support for SGE and Torque schedulers.
 - Use tags prefix `parallelcluster:`.
 - Run Slurm command `scontrol` with sudo because clustermgtd is run as cluster admin user (not root).
 - Implement `computemgtd` self-termination via `shutdown` command instead of calling TerminateInstances.
 - Implement scaling protection mechanism with Slurm scheduler: compute fleet is automatically set to 'PROTECTED' state
   in case recurrent failures are encountered when provisioning nodes.
 
+2.11.2
+-----
+
+**BUG FIXES**
+- Slurm: fix issue that prevented powering-up nodes to be correctly reset after a stop and start of the cluster.
+
+2.11.1
+-----
+
+**CHANGES**
+- There were no notable changes for this version.
+
 2.11.0
 -----
 

src/common/schedulers/slurm_commands.py

@@ -138,7 +138,7 @@ def update_all_partitions(state, reset_node_addrs_hostname):
                 log.info(f"Setting partition {part.name} state from {part.state} to {state}")
                 if reset_node_addrs_hostname:
                     log.info(f"Resetting partition nodes {part.nodenames}")
-                    reset_nodes(part.nodenames, state="power_down", reason="stopping cluster")
+                    set_nodes_down_and_power_save(part.nodenames, reason="stopping cluster")
                 partition_to_update.append(part.name)
         succeeded_partitions = update_partitions(partition_to_update, state)
         return succeeded_partitions == partition_to_update

tests/common/schedulers/test_slurm_commands.py

@@ -535,7 +535,7 @@ def test_update_partitions(
             ],
             PartitionStatus.INACTIVE,
             True,
-            [call("node-3,node-4", reason="stopping cluster", state="power_down")],
+            [call("node-3,node-4", reason="stopping cluster")],
             ["part-2"],
             ["part-2"],
             True,
@@ -548,8 +548,8 @@ def test_update_partitions(
             PartitionStatus.INACTIVE,
             True,
             [
-                call("node-1,node-2", reason="stopping cluster", state="power_down"),
-                call("node-3,node-4", reason="stopping cluster", state="power_down"),
+                call("node-1,node-2", reason="stopping cluster"),
+                call("node-3,node-4", reason="stopping cluster"),
             ],
             ["part-1", "part-2"],
             ["part-1", "part-2"],
@@ -603,7 +603,9 @@ def test_update_all_partitions(
     expected_results,
     mocker,
 ):
-    reset_node_spy = mocker.patch("common.schedulers.slurm_commands.reset_nodes", auto_spec=True)
+    set_nodes_down_and_power_save_spy = mocker.patch(
+        "common.schedulers.slurm_commands.set_nodes_down_and_power_save", auto_spec=True
+    )
     update_partitions_spy = mocker.patch(
         "common.schedulers.slurm_commands.update_partitions", return_value=mock_succeeded_partitions, auto_spec=True
     )
@@ -613,7 +615,7 @@ def test_update_all_partitions(
     assert_that(update_all_partitions(state, reset_node_addrs_hostname=reset_node_info)).is_equal_to(expected_results)
     get_part_spy.assert_called_with(get_all_nodes=True)
     if expected_reset_nodes_calls:
-        reset_node_spy.assert_has_calls(expected_reset_nodes_calls)
+        set_nodes_down_and_power_save_spy.assert_has_calls(expected_reset_nodes_calls)
     else:
-        reset_node_spy.assert_not_called()
+        set_nodes_down_and_power_save_spy.assert_not_called()
     update_partitions_spy.assert_called_with(partitions_to_update, state)