Develop to master - clean up OpsWorks replacement

README.md

@@ -2,13 +2,13 @@
 
 Author: qol.development@smartservice.qld.gov.au
 
-**Full one click deployment of Datashared AWS OpsWorks Stack via Ansible**
+**Full one click deployment of CKAN AWS Stack via Ansible**
 
 This stands up the www.data.qld.gov.au and www.publications.qld.gov.au aws stacks using:
 * SSM
 * RDS
 * Redis Cluster
-* OpsWorks
+* EC2 Autoscaling
 * Cloudfront with Lambda@Edge
 * and may more features
 
@@ -25,7 +25,7 @@ This stands up the www.data.qld.gov.au and www.publications.qld.gov.au aws stack
 For the system to work with updates to the lambda function, you must;
 * first add a new version to cloudfront-lambdaAtEdge.cfn.yml which references the changed lambda function (there is no need for a new lambda function)
 * export the new version from said cloudformation template
-* update the cloudfront.yml ansible script to load the new version property name. 
+* update the cloudfront.yml ansible script to load the new version property name.
 * you can delete previous versions after a successful real, do note that cloudfront will hold onto a lambda function and versions until its 'replication' finishes.
 
 **QOL 2019 update**
@@ -60,7 +60,7 @@ Common issues during set up are as follows:
 
 It's assumed that you:
 
-* have a pretty good working knowledge of AWS, CloudFormation, OpsWorks, and CKAN and its requirements such as Solr and Postgres. Those will be necessary to troubleshoot builds when you haven't provided the correct parameters or some other obstacle gets in your way.
+* have a pretty good working knowledge of AWS, CloudFormation, EC2 Autoscaling, and CKAN and its requirements such as Solr and Postgres. Those will be necessary to troubleshoot builds when you haven't provided the correct parameters or some other obstacle gets in your way.
 * have built, installed and successfully run CKAN manually on some kind of single node configuration. If not, this stack isn't designed to be something to cut your teeth on. It's been designed to be relatively foolproof, but not completely so.
 * know your way around the Linux command line reasonably well and know how to deal with error logs, dependency conflicts etc.
 
@@ -118,7 +118,7 @@ and automated system maintenance.
 
 Our hope and expectation is that it benefits the wider Public Data community and progresses the Open Data ideal.
 
-Current AWS costs for 2 CKAN applications by 4 envirionments is just shy of 3k USD a month. 
+Current AWS costs for 2 CKAN applications by 4 environments is just shy of 3k USD a month.
 
 ## TODO ##
 Make requirements-dev look up vars/shared-${app}.var.yml and test all environment plugins

build-CKAN.sh

@@ -38,6 +38,8 @@ run-shared-resource-playbooks () {
 run-deployment () {
   run-playbook "chef-json"
   ./chef-deploy.sh datashades::ckanweb-setup,datashades::ckanweb-deploy,datashades::ckanweb-configure $INSTANCE_NAME $ENVIRONMENT web & WEB_PID=$!
+  # Check if the web deployment immediately failed
+  kill -0 $WEB_PID
   PARALLEL=1 ./chef-deploy.sh datashades::ckanbatch-setup,datashades::ckanbatch-deploy,datashades::ckanbatch-configure $INSTANCE_NAME $ENVIRONMENT batch & BATCH_PID=$!
   wait $WEB_PID
   wait $BATCH_PID

chef-deploy.sh

@@ -146,18 +146,32 @@ deploy () {
       # double-check that instance is still running
       INSTANCE_STATE=$(aws ec2 describe-instances --filters Name=instance-id,Values=$instance --query "Reservations[].Instances[0].State.Name" --output text)
       if [ "$INSTANCE_STATE" != "running" ]; then continue; fi
-      if [ "$ASG_NAME" != "" ] && (aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].Instances[?InstanceId=='$instance'].InstanceId" --output text |grep "$instance" >/dev/null); then
+      if [ "$ASG_NAME" != "" ] && (aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].Instances[?InstanceId=='$instance' && LifecycleState=='InService'].InstanceId" --output text |grep "$instance" >/dev/null); then
+        IN_ASG="true"
+        # Check if the group is already at minimum capacity
+        CAPACITIES=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-name $ASG_NAME --query "AutoScalingGroups[0].{min: MinSize, desired: DesiredCapacity}" --output text)
+        CAPACITY_1=`echo $CAPACITIES | awk '{print $1}'`
+        CAPACITY_2=`echo $CAPACITIES | awk '{print $2}'`
+        if [ "$CAPACITY_1" = "$CAPACITY_2" ]; then
+          debug "Capacity is at minimum ($CAPACITY_1 = $CAPACITY_2), new instance will be started"
+          DECREMENT_BEHAVIOUR="--no-should-decrement-desired-capacity"
+        else
+          DECREMENT_BEHAVIOUR="--should-decrement-desired-capacity"
+        fi
         # Instances in standby will not get traffic nor health checks, allowing us to update them without interruption
-        OUTPUT=$(aws autoscaling enter-standby --auto-scaling-group-name "$ASG_NAME" --should-decrement-desired-capacity --instance-ids $instance --query "Activities[].Description" --output text)
+        OUTPUT=$(aws autoscaling enter-standby --auto-scaling-group-name "$ASG_NAME" $DECREMENT_BEHAVIOUR --instance-ids $instance --query "Activities[].Description" --output text)
         debug "$OUTPUT"
       elif [ "$ELB_NAME" != "" ]; then
         OUTPUT=$(aws elb deregister-instances-from-load-balancer --load-balancer-name "$ELB_NAME" --instances "$instance" --query "Instances[].InstanceId" --output text)
         debug "Deregistered instance $instance from load balancer $ELB_NAME, resulting registered instances: $OUTPUT"
       fi
       DEPLOYMENT_ID=$(aws ssm send-command --document-name "AWS-ApplyChefRecipes" --document-version "\$DEFAULT" --instance-ids $instance --parameters '{'"$CHEF_SOURCE"',"RunList":["'"$RUN_LIST"'"],"JsonAttributesSources":[""],"JsonAttributesContent":[""],"ChefClientVersion":["14"],"ChefClientArguments":[""],"WhyRun":["False"],"ComplianceSeverity":["None"],"ComplianceType":["Custom:Chef"],"ComplianceReportBucket":[""]}' --timeout-seconds 3600 --max-concurrency "50" --max-errors "0" --output-s3-bucket-name "osssio-ckan-web-logs" --output-s3-key-prefix "run_command" --region ap-southeast-2 --query "Command.CommandId" --output text)
-      wait_for_deployment $DEPLOYMENT_ID
-      DEPLOYMENT_SUCCESS=$?
-      if [ "$ASG_NAME" != "" ]; then
+      DEPLOYMENT_SUCCESS=0
+      wait_for_deployment $DEPLOYMENT_ID || DEPLOYMENT_SUCCESS=$?
+      if [ "$IN_ASG" = "true" ]; then
+        # reactivate the instance if we put it into standby
+        # NB If it was in standby before we started, then we will deploy to it
+        # but leave it in standby.
         OUTPUT=$(aws autoscaling exit-standby --auto-scaling-group-name "$ASG_NAME" --instance-ids $instance --query "Activities[].Description" --output text)
         debug "$OUTPUT"
       elif [ "$ELB_NAME" != "" ]; then

files/instanceSetupLambda.js

@@ -113,7 +113,7 @@ exports.handler = async (event) => {
       SourceType: [cookbookType],
       SourceInfo: [sourceInfo],
       RunList: [runList],
-      ChefClientVersion: ["14"],
+      ChefClientVersion: ["None"],
       WhyRun: ["False"],
       ComplianceSeverity: ["None"],
       ComplianceType: ["Custom:Chef"]

templates/3_tier_vpc.yml

@@ -1168,12 +1168,12 @@ Resources:
   PrivateRouteTable:
     Properties:
       Tags:
-      - Key: Name
-        Value: !Sub "${VPCNamePrefix}Vpc-${Environment}-PrivateRoutes"
+        - Key: Name
+          Value: !Sub "${VPCNamePrefix}Vpc-${Environment}-PrivateRoutes"
       VpcId:
         Ref: VPC
     Type: AWS::EC2::RouteTable
-## private RouteTableB to E
+  ## private RouteTableB to E
   PrivateNATGatewayRouteB:
     Condition: 2PlusAZsNatGateways
     DependsOn: NATGatewayB
@@ -1584,7 +1584,6 @@ Resources:
       SubnetId:
         Ref: WebSubnetE
     Type: AWS::EC2::SubnetRouteTableAssociation
-  #Allow s3 root for listing as well as get set on folders
   S3Endpoint:
     Type: "AWS::EC2::VPCEndpoint"
     Properties:

templates/Datashades-OpsWorks-CKAN-Extensions.cfn.yml.j2

@@ -1,10 +1,6 @@
 ---
 AWSTemplateFormatVersion: '2010-09-09'
-Description: |-
-  Creates OpsWorks Applications for CKAN Stack extensions.
-  Current extension list:
-  Legacy theme
-  Queensland Government extension
+Description: Creates metadata needed to deploy CKAN Stack extensions.
 
 Parameters:
   Environment:

templates/Datashades-OpsWorks-CKAN-Instances.cfn.yml.j2

@@ -1,6 +1,6 @@
 ---
 AWSTemplateFormatVersion: '2010-09-09'
-Description: 'Creates instances for OpsWorks CKAN NFS Stack.'
+Description: 'Creates server instances for a CKAN Stack.'
 
 Parameters:
   ApplicationName:
@@ -113,15 +113,16 @@ Parameters:
   BatchImageId:
     Description: The Amazon Machine Image ID to use for launching batch instances. Defaults to Amazon Linux 2.
     Type: String
-    Default: "ami-03b836d87d294e89e"
+    # Customised image based on Amazon Linux 2, preinstalling some basics
+    Default: "ami-0d71fe73adf7a9887"
   WebImageId:
     Description: The Amazon Machine Image ID to use for launching web instances. Defaults to Amazon Linux 2.
     Type: String
-    Default: "ami-03b836d87d294e89e"
+    Default: "ami-0d71fe73adf7a9887"
   SolrImageId:
     Description: The Amazon Machine Image ID to use for launching Solr instances. Defaults to Amazon Linux 2.
     Type: String
-    Default: "ami-03b836d87d294e89e"
+    Default: "ami-0d71fe73adf7a9887"
   DefaultEC2Key:
     Description: Select an existing SSH key
     Type: AWS::EC2::KeyPair::KeyName
@@ -166,24 +167,28 @@ Resources:
                 echo '/dev/sdi /mnt/local_data xfs defaults,nofail 0 2' >> /etc/fstab
                 mount -a
               fi
+              if ! (yum install chef); then
+                for i in `seq 1 5`; do
+                  yum install -y libxcrypt-compat "https://packages.chef.io/files/stable/chef/18.4.12/el/7/chef-18.4.12-1.el7.x86_64.rpm" && break
+                  sleep 5
+                done
+              fi
+              REGION="--region ${AWS::Region}"
               metadata_token=`curl -X PUT -H "X-aws-ec2-metadata-token-ttl-seconds: 60" http://169.254.169.254/latest/api/token` && \
               INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $metadata_token" http://169.254.169.254/latest/meta-data/instance-id) && \
-              aws ec2 create-tags --region "${AWS::Region}" --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-$INSTANCE_ID"
-              FUNCTION_NAME=$(aws ssm get-parameter --region "${AWS::Region}" --name "/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/setup_function_name" --query "Parameter.Value" --output text)
-              aws lambda invoke --region "${AWS::Region}" --function-name "$FUNCTION_NAME" --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "setup"}' /var/log/instance-setup.log.`date '+%s'`
-
-{% if layer == 'Web' %}
-{% set minInstanceCount = (item.template_parameters['WebEC2Count'] | default('2') | int) - 1 %}
-{% else %}
-{% set minInstanceCount = 1 %}
-{% endif %}
+              aws ec2 create-tags $REGION --resources $INSTANCE_ID --tags "Key=Name,Value=${ApplicationName}_${Environment}-{{ layer }}-$INSTANCE_ID"
+              FUNCTION_NAME=$(aws ssm get-parameter $REGION --name "/config/CKAN/${Environment}/app/${ApplicationId}/cookbook/setup_function_name" --query "Parameter.Value" --output text)
+              if (aws --version |grep -o 'aws-cli/[2-9]'); then
+                PAYLOAD_FORMAT="--cli-binary-format raw-in-base64-out"
+              fi
+              aws lambda invoke $REGION --function-name "$FUNCTION_NAME" $PAYLOAD_FORMAT --payload '{"EC2InstanceId": "'$INSTANCE_ID'", "phase": "setup"}' /var/log/instance-setup.log.`date '+%s'`
 
   {{ layer }}ScalingGroup:
     Type: AWS::AutoScaling::AutoScalingGroup
     Properties:
       AutoScalingGroupName: !Sub "${Environment}-${ApplicationName}-{{ layer }}-ASG"
       DesiredCapacity: !Ref {{ layer }}EC2Count
-      MinSize: {{ minInstanceCount }}
+      MinSize: !Ref {{ layer }}EC2Count
       MaxSize: 6
       LaunchTemplate:
         LaunchTemplateId: !Ref {{ layer }}LaunchTemplate
@@ -209,7 +214,17 @@ Resources:
           Value: {{ layer|lower }}
           PropagateAtLaunch: true
 
-{% if item.tags["PowerManaged"] == "Yes" %}
+  {{ layer }}DynamicScalingPolicy:
+    Type: AWS::AutoScaling::ScalingPolicy
+    Properties:
+      AutoScalingGroupName: !Ref {{ layer }}ScalingGroup
+      PolicyType: TargetTrackingScaling
+      TargetTrackingConfiguration:
+        PredefinedMetricSpecification:
+          PredefinedMetricType: ASGAverageCPUUtilization
+        TargetValue: 50
+
+{% if item.tags["PowerManaged"] == "Yes" and layer != 'Solr' %}
   {{ layer }}ScalingIn:
     Type: AWS::AutoScaling::ScheduledAction
     Properties:
@@ -224,7 +239,7 @@ Resources:
     Properties:
       AutoScalingGroupName: !Ref {{ layer }}ScalingGroup
       DesiredCapacity: !Ref {{ layer }}EC2Count
-      MinSize: {{ minInstanceCount }}
+      MinSize: !Ref {{ layer }}EC2Count
       MaxSize: 6
       Recurrence: "0 20 * * *"
 {% endif %}