Don't use port open check to determine if reboot completed. Fixes #856.

nixops/backends/__init__.py

@@ -3,6 +3,7 @@
 import os
 import re
 import subprocess
+import time
 
 import nixops.util
 import nixops.resources
@@ -182,16 +183,53 @@ def reboot(self, hard=False):
  reboot_command = "systemctl reboot"
  self.run_command(reboot_command, check=False)
  self.state = self.STARTING
+ # Note we always reset the connection to ensure the SSH control master
+ # connection is stopped.
+ # While an orderly `reboot` should make the remote kernel send RST to
+ # all TCP connections, we don't want to rely on that here.
  self.ssh.reset()
 
  def reboot_sync(self, hard=False):
  """Reboot this machine and wait until it's up again."""
+
+ # To check when the machine has finished rebooting in a race-free
+ # manner, we compare the output of `last reboot` before and after
+ # the reboot. Once the output has changed, the reboot is done.
+ def get_last_reboot_output():
+ # Note `last reboot` does not exist on older OSs like
+ # the Hetzner rescue system, but that doesn't matter much
+ # because all we care about is when the output of the
+ # command invocation changes.
+ # We use timeout=10 so that the user gets some sense
+ # of progress, as reboots can take a long time.
+ return self.run_command('last reboot --time-format iso | head -n 1', capture_stdout=True, timeout=10).rstrip()
+
+ # The server might be off.
+ # In that case, we just set `pre_reboot_last_reboot_output` to some string
+ # so that it changes when it comes up.
+ try:
+ pre_reboot_last_reboot_output = get_last_reboot_output()
+ except nixops.ssh_util.SSHConnectionFailed:
+ self.log("server is unreachable, so reboot completion detection may not be accurate if it's a network problem")
+ pre_reboot_last_reboot_output = "server is off or couldn't be contacted"
+
  self.reboot(hard=hard)
- self.log_start("waiting for the machine to finish rebooting...")
- nixops.util.wait_for_tcp_port(self.get_ssh_name(), self.ssh_port, open=False, callback=lambda: self.log_continue("."))
- self.log_continue("[down]")
- nixops.util.wait_for_tcp_port(self.get_ssh_name(), self.ssh_port, callback=lambda: self.log_continue("."))
- self.log_end("[up]")
+
+ self.log_start("waiting for reboot to complete...")
+ while True:
+ last_reboot_output = None
+ try:
+ last_reboot_output = get_last_reboot_output()
+ except (nixops.ssh_util.SSHConnectionFailed, nixops.ssh_util.SSHCommandFailed):
+ # We accept this because the machine might be down,
+ # and show an 'x' as progress indicator in that case.
+ self.log_continue("x")
+ if last_reboot_output is not None and last_reboot_output != pre_reboot_last_reboot_output:
+ break
+ self.log_continue(".")
+ time.sleep(1)
+ self.log_end("done.")
+
  self.state = self.UP
  self.ssh_pinged = True
  self._ssh_pinged_this_time = True

nixops/backends/azure_vm.py

@@ -953,7 +953,8 @@ def reboot(self, hard=False):
  self.log("sending hard reset to Azure machine...")
  self.cmc().virtual_machines.restart(self.resource_group, self.machine_name)
  self.state = self.STARTING
- self.ssh.reset()
+ if reset:
+ self.ssh.reset()
  else:
  MachineState.reboot(self, hard=hard)
  self.ssh_pinged = False

nixops/backends/hetzner.py

@@ -318,6 +318,10 @@ def reboot(self, hard=False):
  server.reboot('hard')
  self.log_end("done.")
  self.state = self.STARTING
+ # Note we always reset the connection because on a hard reboot, the
+ # other side may not have a chance to terminate the SSH connection,
+ # (send TCP RST), which could lead to the control master process
+ # still being alive and hanging until the TCP connection times out.
  self.ssh.reset()
  else:
  MachineState.reboot(self, hard=hard)