Fixing issues in deployment, including #666, #667 and #668

Netflix · Mar 19, 2018 · 588e54b · 588e54b
1 parent f291f28
commit 588e54b
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 23 deletions.
diff --git a/build.gradle b/build.gradle
@@ -2,7 +2,7 @@ plugins {
     id 'nebula.netflixoss' version '5.0.0'
 }
 
-ext.githubProjectName = 'Priam'
+ext.githubProjectName = 'Priam-parent'
 
 allprojects {
     apply plugin: 'nebula.netflixoss'

diff --git a/priam/src/main/java/com/netflix/priam/SimpleDBConfigSource.java b/priam/src/main/java/com/netflix/priam/SimpleDBConfigSource.java
@@ -47,15 +47,22 @@
 public final class SimpleDBConfigSource extends AbstractConfigSource {
     private static final Logger logger = LoggerFactory.getLogger(SimpleDBConfigSource.class.getName());
 
-    private static final String DOMAIN = "PriamProperties";
-    private static String ALL_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'";
+    private static final String DEFAULT_DOMAIN = "PriamProperties";
+    private static String ALL_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s'";
 
     private final Map<String, String> data = Maps.newConcurrentMap();
     private final ICredential provider;
-
+    private final String domain;
+
     @Inject
     public SimpleDBConfigSource(final ICredential provider) {
         this.provider = provider;
+        String configuredDomain = System.getProperty("priam.sdb.properties.domain");
+        if (configuredDomain == null) {
+            domain = DEFAULT_DOMAIN;
+        } else {
+            domain = configuredDomain;
+        }
     }
 
     @Override
@@ -68,8 +75,9 @@ public void intialize(final String asgName, final String region) {
         String nextToken = null;
         String appid = asgName.lastIndexOf('-') > 0 ? asgName.substring(0, asgName.indexOf('-')) : asgName;
         logger.info("appid used to fetch properties is: {}", appid);
+        logger.info("domain used to fetch properties is: {}", domain);
         do {
-            SelectRequest request = new SelectRequest(String.format(ALL_QUERY, appid));
+            SelectRequest request = new SelectRequest(String.format(ALL_QUERY, domain, appid));
             request.setNextToken(nextToken);
             SelectResult result = simpleDBClient.select(request);
             nextToken = result.getNextToken();

diff --git a/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java b/priam/src/main/java/com/netflix/priam/aws/SDBInstanceData.java
@@ -28,11 +28,16 @@
 
 import java.util.*;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * DAO for handling Instance identity information such as token, zone, region
  */
 @Singleton
 public class SDBInstanceData {
+    private static final Logger logger = LoggerFactory.getLogger(SDBInstanceData.class.getName());
+
     public static class Attributes {
         public final static String APP_ID = "appId";
         public final static String ID = "id";
@@ -45,17 +50,24 @@ public static class Attributes {
         public final static String HOSTNAME = "hostname";
     }
 
-    public static final String DOMAIN = "InstanceIdentity";
-    public static final String ALL_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s'";
-    public static final String INSTANCE_QUERY = "select * from " + DOMAIN + " where " + Attributes.APP_ID + "='%s' and " + Attributes.LOCATION + "='%s' and " + Attributes.ID + "='%d'";
+    public static final String DEFAULT_DOMAIN = "InstanceIdentity";
+    public static final String ALL_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s'";
+    public static final String INSTANCE_QUERY = "select * from `%s` where " + Attributes.APP_ID + "='%s' and " + Attributes.LOCATION + "='%s' and " + Attributes.ID + "='%d'";
 
     private final ICredential provider;
     private final IConfiguration configuration;
-
+    private final String domain;
+
     @Inject
     public SDBInstanceData(ICredential provider, IConfiguration configuration) {
         this.provider = provider;
         this.configuration = configuration;
+        String configuredDomain = System.getProperty("priam.sdb.instanceidentity.domain");
+        if (configuredDomain == null) {
+            domain = DEFAULT_DOMAIN;
+        } else {
+            domain = configuredDomain;
+        }
     }
 
     /**
@@ -67,7 +79,9 @@ public SDBInstanceData(ICredential provider, IConfiguration configuration) {
      */
     public PriamInstance getInstance(String app, String dc, int id) {
         AmazonSimpleDB simpleDBClient = getSimpleDBClient();
-        SelectRequest request = new SelectRequest(String.format(INSTANCE_QUERY, app, dc, id));
+        String query = String.format(INSTANCE_QUERY, domain, app, dc, id);
+        logger.info("Fetching instance data using query {}", query);
+        SelectRequest request = new SelectRequest(query);
         SelectResult result = simpleDBClient.select(request);
         if (result.getItems().size() == 0)
             return null;
@@ -85,7 +99,9 @@ public Set<PriamInstance> getAllIds(String app) {
         Set<PriamInstance> inslist = new HashSet<PriamInstance>();
         String nextToken = null;
         do {
-            SelectRequest request = new SelectRequest(String.format(ALL_QUERY, app));
+            String query = String.format(ALL_QUERY, domain, app);
+            logger.info("Fetching IDs using query {}", query);
+            SelectRequest request = new SelectRequest(query);
             request.setNextToken(nextToken);
             SelectResult result = simpleDBClient.select(request);
             nextToken = result.getNextToken();
@@ -106,7 +122,7 @@ public Set<PriamInstance> getAllIds(String app) {
      */
     public void createInstance(PriamInstance instance) throws AmazonServiceException {
         AmazonSimpleDB simpleDBClient = getSimpleDBClient();
-        PutAttributesRequest putReq = new PutAttributesRequest(DOMAIN, getKey(instance), createAttributesToRegister(instance));
+        PutAttributesRequest putReq = new PutAttributesRequest(domain, getKey(instance), createAttributesToRegister(instance));
         simpleDBClient.putAttributes(putReq);
     }
 
@@ -118,7 +134,7 @@ public void createInstance(PriamInstance instance) throws AmazonServiceException
      */
     public void registerInstance(PriamInstance instance) throws AmazonServiceException {
         AmazonSimpleDB simpleDBClient = getSimpleDBClient();
-        PutAttributesRequest putReq = new PutAttributesRequest(DOMAIN, getKey(instance), createAttributesToRegister(instance));
+        PutAttributesRequest putReq = new PutAttributesRequest(domain, getKey(instance), createAttributesToRegister(instance));
         UpdateCondition expected = new UpdateCondition();
         expected.setName(Attributes.INSTANCE_ID);
         expected.setExists(false);
@@ -134,7 +150,7 @@ public void registerInstance(PriamInstance instance) throws AmazonServiceExcepti
      */
     public void deregisterInstance(PriamInstance instance) throws AmazonServiceException {
         AmazonSimpleDB simpleDBClient = getSimpleDBClient();
-        DeleteAttributesRequest delReq = new DeleteAttributesRequest(DOMAIN, getKey(instance), createAttributesToDeRegister(instance));
+        DeleteAttributesRequest delReq = new DeleteAttributesRequest(domain, getKey(instance), createAttributesToDeRegister(instance));
         simpleDBClient.deleteAttributes(delReq);
     }
 

diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamConfigSource.java b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamConfigSource.java
@@ -34,7 +34,7 @@ public PriamConfigSource(final SimpleDBConfigSource simpleDBConfigSource,
         // this order was based off PriamConfigurations loading.  W/e loaded last could override, but with Composite, first
         // has the highest priority.
         super(simpleDBConfigSource,
-                propertiesConfigSource,
-                systemPropertiesConfigSource);
+                systemPropertiesConfigSource,
+                propertiesConfigSource);
     }
 }
diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java b/priam/src/main/java/com/netflix/priam/defaultimpl/PriamGuiceModule.java
@@ -26,6 +26,7 @@
 import com.netflix.priam.aws.auth.EC2RoleAssumptionCredential;
 import com.netflix.priam.aws.auth.IS3Credential;
 import com.netflix.priam.aws.auth.S3RoleAssumptionCredential;
+import com.netflix.priam.aws.IAMCredential;
 import com.netflix.priam.backup.BackupFileSystemContext;
 import com.netflix.priam.backup.IBackupFileSystem;
 import com.netflix.priam.backup.IBackupMetrics;
@@ -63,7 +64,7 @@ protected void configure() {
         bind(IFileCryptography.class).annotatedWith(Names.named("filecryptoalgorithm")).to(PgpCryptography.class);
         bind(ICredentialGeneric.class).annotatedWith(Names.named("gcscredential")).to(GcsCredential.class);
         bind(ICredentialGeneric.class).annotatedWith(Names.named("pgpcredential")).to(PgpCredential.class);
-        bind(ICredential.class).to(ClearCredential.class);
+        bind(ICredential.class).to(IAMCredential.class);
         bind(IDeadTokenRetriever.class).to(DeadTokenRetriever.class);
         bind(IPreGeneratedTokenRetriever.class).to(PreGeneratedTokenRetriever.class);
         bind(INewTokenRetriever.class).to(NewTokenRetriever.class);

diff --git a/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java b/priam/src/main/java/com/netflix/priam/tuner/StandardTuner.java
@@ -57,8 +57,10 @@ public void writeAllProperties(String yamlLocation, String hostname, String seed
         map.put("rpc_port", config.getThriftPort());
         map.put("start_native_transport", config.isNativeTransportEnabled());
         map.put("native_transport_port", config.getNativeTransportPort());
-        map.put("listen_address", hostname);
-        map.put("rpc_address", hostname);
+        if (hostname != null) {
+            map.put("listen_address", hostname);
+            map.put("rpc_address", hostname);
+        }
         //Dont bootstrap in restore mode
         if (!Restore.isRestoreEnabled(config)) {
             map.put("auto_bootstrap", config.getAutoBoostrap());

diff --git a/priam/src/main/resources/Priam.properties b/priam/src/main/resources/Priam.properties
@@ -10,19 +10,27 @@ priam.backup.retention=
 priam.backup.threads=2
 priam.bootcluster=
 priam.cache.location=/var/lib/cassandra/saved_caches
-priam.cass.home=/mnt/cassandra
+priam.cass.home=/etc/cassandra
 priam.cass.manual.start.enable=
 priam.cass.process=
-priam.cass.startscript=/mnt/cassandra/bin/cassandra
-priam.cass.stopscript=/mnt/cassandra/bin/cassandra
+priam.cass.startscript=/etc/init.d/cassandra start 
+priam.cass.stopscript=/etc/init.d/cassandra stop
 priam.clustername=cass_cluster
 priam.commitlog.location=/var/lib/cassandra/commitlog
 priam.compaction.throughput=
 priam.data.location=/var/lib/cassandra/data
 priam.direct.memory.size.m1.large=1G
 priam.endpoint_snitch=org.apache.cassandra.locator.Ec2Snitch
 priam.heap.newgen.size.m1.large=2G
-priam.heap.size.m1.large=4G
+priam.heap.size.m1.large=1G
+priam.heap.newgen.size.t2.small=1G
+priam.heap.size.t2.small=1G
+priam.heap.newgen.size.t2.micro=512M
+priam.heap.size.t2.micro=512M
+priam.heap.newgen.size.m3.medium=1G
+priam.heap.size.m3.medium=1G
+priam.heap.newgen.size.m3.large=1G
+priam.heap.size.m3.large=1G
 priam.hint.delay=
 priam.hint.window=
 priam.jmx.port=7199
@@ -53,3 +61,4 @@ priam.thrift.port=9160
 priam.upload.throttle=
 priam.yamlLocation=
 priam.zones.available=
+priam.nativeTransport.enabled=true