Skip to content

Commit

Permalink
Merge pull request #713 from OpenSourcePolicyCenter/bhard/fabfile_dep…
Browse files Browse the repository at this point in the history
…loy_simplification

Improvements to Create and Update scripts
  • Loading branch information
brittainhard authored Oct 26, 2017
2 parents 2fa70f4 + 175968d commit f2b2ad9
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 313 deletions.
1 change: 0 additions & 1 deletion deploy/fab/ec2-deploy.sh

This file was deleted.

53 changes: 23 additions & 30 deletions deploy/fab/ec2-fabfile.py → deploy/fab/ec2_fabfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,10 @@
DEPLOYMENT_VERSIONS_ARGS = deploy_versions_cli(ip_address='skip')
if not 'OSPC_ANACONDA_TOKEN' in os.environ:
raise ValueError('OSPC_ANACONDA_TOKEN must be defined in env vars')
check_unmodified()
#based on https://github.com/ContinuumIO/wakari-deploy/blob/master/ami_creation/fabfile.py

if not os.environ.get("ALLOW_UNCOMMITTED", None) == 'True':
check_unmodified()

ssh_transport = logging.getLogger("ssh.transport")
ssh_transport.disabled = True
log = logging.getLogger(__name__)
Expand All @@ -52,15 +54,6 @@
# log.addHandler(handler)
log.addHandler(console)

# ami-65ce000e
# ami-849e0c84
# ami-1998e77c
# ami-af5591eb
# ami-9f92edfa
# ami-1d4b1f78
#AMI_ID = 'ami-1d4b1f78' #SSD in us-east-1
#AMI_ID = 'ami-dfcab0b5' #SSD in us-east-1
#AMI_ID = 'ami-dfcab0b5' #SSD in us-east-1
AMI_ID = 'ami-6c276206' #SSD in us-east-1

use_latest_key = 'True' == os.environ.get('USE_LATEST_KEY', False)
Expand Down Expand Up @@ -90,16 +83,14 @@
env.timeout = 50

KEYNAME = "{}-aei-dropq-{}".format(AMI_ID,os.environ.get('USER', 'ospc'))
#KEYNAME = "keypair-ospc"
#SECURITY_GROUP = 'launch-wizard-26'
#SECURITY_GROUP = 'dropq-security-group'
KEYFILE = "./" + KEYNAME + ".pem"
SECURITY_GROUP = 'dropq-iam-group'
NODE_COUNT = int(os.environ.get('WORKER_NODE_COUNT', 1))

def create_box():
old_ids = set(i.id for i in ec2.get_only_instances())
machine = ec2.run_instances(AMI_ID, key_name=KEYNAME, min_count=NODE_COUNT, max_count=NODE_COUNT,
security_groups=[SECURITY_GROUP,], instance_type=os.environ.get('EC2_INSTANCE_TYPE', 'm3.medium'))
security_groups=[SECURITY_GROUP,], instance_type=os.environ.get('EC2_INSTANCE_TYPE', 'r3.large'))
new_instances = [i for i in ec2.get_only_instances() if i.id not in old_ids]
for new_instance in new_instances:
print("new instance:", new_instance.id)
Expand Down Expand Up @@ -186,6 +177,16 @@ def test_ssh2():
'''
run('pwd')

def add_public_keys():
# Add the necessary public key information to the instance.
from boto.s3.connection import S3Connection
conn = S3Connection(os.environ.get('AWS_ID'), os.environ.get('AWS_SECRET'))
bucket = conn.get_bucket("taxbrain-deploy-assets")

k = bucket.get_key("keys.txt")
keyfile = k.get_contents_as_string()
run("echo $'%s' >> ~/.ssh/authorized_keys" % keyfile)

def test_ssh(instance, key_file):
# needed to convert from unicode to ascii?
key_file = str(key_file)
Expand Down Expand Up @@ -309,36 +310,26 @@ def wait_for_login_prompt(instance, retry_count=20):
public_dns_name = 'ec2-{}.compute-1.amazonaws.com'.format(ip_address.replace('.','-'))
connect_to_existing_machine(ip_address, key_filename)
else:
if not use_latest_key:
raise ValueError()
key_filename = create_keypair()

else:
key_filename = "./latest.pem"

instances = create_box()
public_dns_names = []
ip_addresses = []
for instance in instances:
print("trying this instance :", instance)
subprocess.check_output(['cp', key_filename,
os.path.join(os.path.dirname(key_filename), 'ec2-{}.pem'.format(instance.ip_address))])
subprocess.check_output(['cp', KEYFILE,
os.path.join(os.path.dirname(KEYFILE), 'ec2-{}.pem'.format(instance.ip_address))])
public_dns_names.append(instance.public_dns_name)
ip_addresses.append(instance.ip_address)

if 'quitafterec2spinup' in sys.argv:
quit()

#import pdb;pdb.set_trace()
#test_all_ssh(instances, key_filename)

key_file = str(key_filename)
fqdns = [str(instance.public_dns_name) for instance in instances]
env.hosts = [fqdn for fqdn in fqdns]
env.user = 'ubuntu'
env.password = ''
env.key_file = key_file
env.key_filename = key_file
env.key_file = KEYFILE
env.key_filename = KEYFILE
# forward ssh agent -- equivalent of ssh -A
env.forward_agent = True

Expand All @@ -359,6 +350,7 @@ def wait_for_login_prompt(instance, retry_count=20):


execute(test_ssh2)
execute(add_public_keys)
execute(apt_installs)
execute(install_deploy_repo)
execute(install_ogusa_repo)
Expand All @@ -368,7 +360,8 @@ def wait_for_login_prompt(instance, retry_count=20):
execute(reset_server)

for instance in instances:
ssh_command = 'ssh -i {key} ubuntu@{ip} "'.format(ip=instance.ip_address, key=key_filename)
ssh_command = 'ssh -i {key} ubuntu@{ip} "'.format(ip=instance.ip_address,
key=KEYNAME)
with open("log_{}.log".format(instance.ip_address), 'w') as f:
f.write(ssh_command)
print(ssh_command)
Expand Down
Loading

0 comments on commit f2b2ad9

Please sign in to comment.