Checking that replaced_ip is a real ip in the ring

Netflix · Mar 22, 2018 · 7b8d8be · 7b8d8be
1 parent b64dd32
commit 7b8d8be
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 5 deletions.
diff --git a/...ss-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java b/...ss-extensions/src/main/java/com/netflix/priam/cassandra/extensions/PriamStartupAgent.java
@@ -93,8 +93,10 @@ private void setPriamProperties()
         	{
         		System.setProperty("cassandra.replace_token", token);
         	} else 
-        	{	
-               System.setProperty("cassandra.replace_address", replacedIp);
+        	{
+                if (StringUtils.isNotBlank(replacedIp)) {
+                    System.setProperty("cassandra.replace_address", replacedIp);
+                }
         	}
         }
 

diff --git a/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java b/priam/src/main/java/com/netflix/priam/identity/token/DeadTokenRetriever.java
@@ -113,7 +113,7 @@ public PriamInstance get() throws Exception {
 
             //find the replaced IP
             this.replacedIp = findReplaceIp(allIds, markAsDead.getToken(), markAsDead.getDC());
-            if (this.replacedIp == null)
+            if (replacedIp == null && confirmReplacedIp(allIds, markAsDead.getHostIP(), markAsDead.getDC()))
                 this.replacedIp = markAsDead.getHostIP();
 
             String payLoad = markAsDead.getToken();
@@ -155,7 +155,7 @@ private String findReplaceIp(List<PriamInstance> allIds, String token, String lo
         return null;
     }
 
-    private String getIp(String host, String token) throws ParseException {
+    private JSONObject getGossipInfo(String host) throws ParseException {
         ClientConfig config = new DefaultClientConfig();
         Client client = Client.create(config);
         String baseURI = getBaseURI(host);
@@ -172,7 +172,7 @@ private String getIp(String host, String token) throws ParseException {
 
             textEntity = clientResp.getEntity(String.class);
 
-            logger.info("Respond from calling gossipinfo on host[{}] and token[{}] : {}", host, token, textEntity);
+            logger.info("Respond from calling gossipinfo on host[{}] : {}", host, textEntity);
 
             if (StringUtils.isEmpty(textEntity))
                 return null;
@@ -186,6 +186,12 @@ private String getIp(String host, String token) throws ParseException {
 
         JSONObject jsonObject = (JSONObject) obj;
 
+        return jsonObject;
+    }
+
+    private String getIp(String host, String token) throws ParseException {
+        JSONObject jsonObject = getGossipInfo(host);
+
         Iterator iter = jsonObject.keySet().iterator();
 
         while (iter.hasNext()) {
@@ -205,6 +211,25 @@ private String getIp(String host, String token) throws ParseException {
         return null;
     }
 
+    private boolean confirmReplacedIp(List<PriamInstance> allIds, String replacedIp, String location) {
+        for (PriamInstance ins : allIds) {
+            if (replacedIp.equalsIgnoreCase(ins.getHostIP()) || !ins.getDC().equals(location)) { //avoid using dead instance and other regions' instances
+                continue;
+            }
+
+            try {
+                JSONObject jsonObject = getGossipInfo(ins.getHostName());
+                if (jsonObject != null && jsonObject.keySet().contains(replacedIp)) {
+                    logger.info("Confirmed the IP to replace: {}", replacedIp);
+                    return true;
+                }
+            } catch (ParseException e) {
+            }
+        }
+
+        return false;
+    }
+
     private String getBaseURI(String host) {
         return "http://" + host + ":8080/";
     }