feat(multi-node): added a reconnection strategy so agents auto-reconn…

…ect to master when disconnected

README.md

@@ -454,6 +454,7 @@ To run Isaiah, you will need to set the following environment variables in a `.e
 | `MASTER_HOST`           | `string`  | For multi-node deployments only. The host used to reach the Master node, specifying the IP address or the hostname, and the port if applicable (e.g. my-server.tld:3000). | Empty        |
 | `MASTER_SECRET`         | `string`  | For multi-node deployments only. The secret password used to authenticate on the Master node. Note that it should equal the `AUTHENTICATION_SECRET` setting on the Master node. | Empty        |
 | `AGENT_NAME`            | `string`  | For multi-node deployments only. The name associated with the Agent node as it is displayed on the web interface. It should be unique for each Agent. | Empty        |
+| `AGENT_REGISTRATION_RETRY_DELAY`  | `integer`  | For multi-node deployments only. The delay (in seconds) between reconnection attempts when the connection to the Master node was lost. | 30        |
 | `MULTI_HOST_ENABLED`    | `boolean` | Whether Isaiah should be run in multi-host mode. When enabled, make sure to have your `docker_hosts` file next to the executable. | False        |
 | `FORWARD_PROXY_AUTHENTICATION_ENABLED`    | `boolean` | Whether Isaiah should accept authentication headers from a forward proxy. | False        |
 | `FORWARD_PROXY_AUTHENTICATION_HEADER_KEY` | `string` | The name of the authentication header sent by the forward proxy after a succesful authentication. | Remote-User        |

app/default.env

@@ -7,6 +7,7 @@ SERVER_CHUNKED_COMMUNICATION_ENABLED="FALSE"
 SERVER_CHUNKED_COMMUNICATION_SIZE="50"
 
 SERVER_ROLE="Master"
+AGENT_REGISTRATION_RETRY_DELAY="30"
 
 AUTHENTICATION_ENABLED="TRUE"
 AUTHENTICATION_SECRET="one-very-long-and-mysterious-secret"

app/main.go

@@ -323,6 +323,11 @@ func main() {
 
 	// When current node is an agent, perform agent registration procedure with the master node
 	if _os.GetEnv("SERVER_ROLE") == "Agent" {
+		hasRegisteredSuccessfullyAtLeastOnce := false
+		lastRegistrationAttemptAt := time.Now().Unix()
+		retryDelay := _strconv.ParseInt(_os.GetEnv("AGENT_REGISTRATION_RETRY_DELAY"), 10, 64)
+
+	agentRegistration:
 		log.Print("Initiating registration with master node")
 
 		var response ui.Notification
@@ -333,7 +338,21 @@ func main() {
 		if err != nil {
 			log.Print("Error establishing connection to the master node")
 			log.Print(err)
-			return
+
+			if hasRegisteredSuccessfullyAtLeastOnce {
+				currentAttemptAt := time.Now().Unix()
+				nextAttemptDelay := retryDelay - (currentAttemptAt - lastRegistrationAttemptAt)
+
+				if nextAttemptDelay > 0 {
+					log.Printf("New attempt in %d seconds", nextAttemptDelay)
+					time.Sleep(time.Duration(nextAttemptDelay) * time.Second)
+				}
+
+				lastRegistrationAttemptAt = time.Now().Unix()
+				goto agentRegistration
+			} else {
+				return
+			}
 		}
 
 		if _os.GetEnv("MASTER_SECRET") != "" {
@@ -357,7 +376,7 @@ func main() {
 			}
 
 			if response.Type != ui.TypeSuccess {
-				log.Print("Authentication with master node unsuccesful")
+				log.Print("Authentication with master node unsuccessful")
 				log.Print("Please check your MASTER_SECRET setting and restart")
 				return
 			}
@@ -403,22 +422,26 @@ func main() {
 			return
 		}
 		if response.Type != ui.TypeSuccess {
-			log.Print("Registration with master node unsuccesful")
+			log.Print("Registration with master node unsuccessful")
 			log.Print("Please check your settings and connectivity")
 			log.Printf("Error : %s", response.Content["Message"])
 			return
 		}
 
 		log.Print("Connection with master node is established")
+		hasRegisteredSuccessfullyAtLeastOnce = true
 
 		// Workaround : Create a tweaked reimplementation of melody.Session to reuse existing code
 		session := _session.Create(connection)
 
 		// 6. Process the commands as they are received
+		masterConnectionLost := false
 		for {
 			_, message, err := connection.ReadMessage()
 			if err != nil {
 				log.Print(err)
+				log.Print("Connection with master node was lost, will reconnect")
+				masterConnectionLost = true
 				break
 			}
 
@@ -444,7 +467,11 @@ func main() {
 			}
 		}
 
-		return
+		if masterConnectionLost {
+			goto agentRegistration
+		} else {
+			return
+		}
 	}
 
 	// When current node is master, start the HTTP server