From 1cc9c73e4b373a055aaecc35db0265d59337a323 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Thu, 24 Oct 2019 23:21:50 +0200
Subject: [PATCH 1/9] RDS: Add DBSubnet resource, VPC security groups

This commit enables EC2 RDS resource to use:

- DBSubnets
- VPC security groups

As a result, it adds a DBSubnet resource which consists of at least 2
VPC subnets containing at least each AZ of your region.

It enables support of RDS in a VPC.

This makes it possible to use RDS in a VPC setup, i.e. in EC2-VPC.
---
 nix/default.nix                             |   1 +
 nix/ec2-rds-dbinstance.nix                  |  20 ++-
 nix/ec2-rds-subnet-group.nix                |  44 +++++++
 nix/vpc.nix                                 |   0
 nixopsaws/backends/ec2.py                   |  10 +-
 nixopsaws/resources/__init__.py             |   1 +
 nixopsaws/resources/ec2_rds_dbinstance.py   |  80 ++++++++++--
 nixopsaws/resources/ec2_rds_subnet_group.py | 133 ++++++++++++++++++++
 8 files changed, 278 insertions(+), 11 deletions(-)
 create mode 100644 nix/ec2-rds-subnet-group.nix
 mode change 100755 => 100644 nix/vpc.nix
 create mode 100644 nixopsaws/resources/ec2_rds_subnet_group.py

diff --git a/nix/default.nix b/nix/default.nix
index fd71ca4f..0302ab32 100644
--- a/nix/default.nix
+++ b/nix/default.nix
@@ -18,6 +18,7 @@
     ebsVolumes = evalResources ./ebs-volume.nix (zipAttrs resourcesByType.ebsVolumes or []);
     elasticIPs = evalResources ./elastic-ip.nix (zipAttrs resourcesByType.elasticIPs or []);
     rdsDbInstances = evalResources ./ec2-rds-dbinstance.nix (zipAttrs resourcesByType.rdsDbInstances or []);
+    rdsDbSubnetGroups = evalResources ./ec2-rds-subnet-group.nix (zipAttrs resourcesByType.rdsDbSubnetGroups or []);
     rdsDbSecurityGroups = evalResources ./ec2-rds-dbsecurity-group.nix (zipAttrs resourcesByType.rdsDbSecurityGroups or []);
     route53RecordSets = evalResources ./route53-recordset.nix (zipAttrs resourcesByType.route53RecordSets or []);
     elasticFileSystems = evalResources ./elastic-file-system.nix (zipAttrs resourcesByType.elasticFileSystems or []);
diff --git a/nix/ec2-rds-dbinstance.nix b/nix/ec2-rds-dbinstance.nix
index b95b740f..eb5023a3 100644
--- a/nix/ec2-rds-dbinstance.nix
+++ b/nix/ec2-rds-dbinstance.nix
@@ -86,10 +86,28 @@ with import ./lib.nix lib;
       type = types.listOf (types.either types.str (resource "ec2-rds-security-group"));
       apply = map (x: if builtins.isString x then x else "res-" + x._name);
       description = ''
-        List of names of DBSecurityGroup to authorize on this DBInstance.
+        List of names of DBSecurityGroup to authorize on this DBInstance, use it if you are not a VPC or are using EC2-Classic.
       '';
     };
 
+    vpcSecurityGroupIds = mkOption {
+      default = null; # default to the default security group of the DB subnet.
+      type = types.nullOr (types.listOf (types.either types.str (resource "ec2-security-group")));
+      apply = map (x: if builtins.isString x then x else "res-" + x._name);
+      description = ''
+        List of names of VPCSecurityGroupMembership to authorize on this DBInstance, use this if you are in an VPC and not on EC2-Classic. Not applicable for Amazon Aurora.
+        '';
+    };
+
+    dbSubnetGroup = mkOption {
+      default = null;
+      type = types.nullOr (types.either types.str (resource "ec2-rds-db-subnet-group"));
+      apply = group: if (builtins.isString group || group == null) then group else "res-" + group._name;
+      description = ''
+        A name for a DBSubnetGroup, they must contain at least one subnet in each availability zone in the AWS region.
+        '';
+    };
+
   };
 
   config._type = "ec2-rds-dbinstance";
diff --git a/nix/ec2-rds-subnet-group.nix b/nix/ec2-rds-subnet-group.nix
new file mode 100644
index 00000000..9903f6c8
--- /dev/null
+++ b/nix/ec2-rds-subnet-group.nix
@@ -0,0 +1,44 @@
+{ config, lib, uuid, name, ... }:
+
+with lib;
+with import ./lib.nix lib;
+
+{
+  options = {
+    name = mkOption {
+      default = "nixops-${uuid}-${name}";
+      type = types.str;
+      description = ''
+        Name of the RDS DB subnet group.
+      '';
+    };
+
+    description = mkOption {
+      type = types.str;
+      description = ''
+        Description of the RDS DB subnet group.
+      ''; 
+    };
+
+    region = mkOption {
+      type = types.str;
+      description = "Amazon RDS DB subnet group region.";
+    };
+
+    accessKeyId = mkOption {
+      default = "";
+      type = types.str;
+      description = "The AWS Access Key ID.";
+    };
+
+    subnetIds = mkOption {
+      default = [];
+      type = types.listOf (types.either types.str (resource "vpc-subnet"));
+      apply = map (x: if builtins.isString x then x else "res-" + x._name + "." + x._type + ".subnetId");
+      description = "List of VPC subnets to use for this DB subnet group (must contain at least a subnet for each AZ), it can be a VPC Subnet resource or a string representing the ID of the VPC Subnet.";
+    };
+    
+  };
+
+  config._type = "ec2-rds-db-subnet-group";
+}
diff --git a/nix/vpc.nix b/nix/vpc.nix
old mode 100755
new mode 100644
diff --git a/nixopsaws/backends/ec2.py b/nixopsaws/backends/ec2.py
index 3258e73e..1662bfe0 100644
--- a/nixopsaws/backends/ec2.py
+++ b/nixopsaws/backends/ec2.py
@@ -6,6 +6,7 @@
 import time
 import math
 import shutil
+import socket
 import calendar
 import boto.ec2
 import boto.ec2.blockdevicemapping
@@ -14,6 +15,7 @@
 from nixops.nix_expr import Function, Call, RawValue
 from nixopsaws.resources.ebs_volume import EBSVolumeState
 from nixopsaws.resources.elastic_ip import ElasticIPState
+from nixopsaws.resources.ec2_rds_dbinstance import EC2RDSDbInstanceState
 import nixopsaws.resources.ec2_common
 import nixopsaws.resources
 from nixops.util import device_name_to_boto_expected, device_name_stored_to_real, device_name_user_entered_to_stored
@@ -271,8 +273,14 @@ def resource_id(self):
 
 
     def address_to(self, m):
-        if isinstance(m, EC2State): # FIXME: only if we're in the same region
+        # FIXME: enforce region/VPC constraints
+        if isinstance(m, EC2State):
             return m.private_ipv4
+        if isinstance(m, EC2RDSDbInstanceState) and m.rds_dbinstance_endpoint is not None:
+            # FIXME: it returns a hostname, need DNS resolution in the VPC/region_
+            # FIXME: support IPv6 stack using getaddrinfo rather.
+            hostname, _ = m.rds_dbinstance_endpoint.split(":")
+            return socket.gethostbyname(hostname)
         return MachineState.address_to(self, m)
 
 
diff --git a/nixopsaws/resources/__init__.py b/nixopsaws/resources/__init__.py
index 20773100..3d2f6862 100644
--- a/nixopsaws/resources/__init__.py
+++ b/nixopsaws/resources/__init__.py
@@ -10,6 +10,7 @@
 import ec2_placement_group
 import ec2_rds_dbinstance
 import ec2_rds_dbsecurity_group
+import ec2_rds_subnet_group
 import ec2_security_group
 import efs_common
 import elastic_file_system
diff --git a/nixopsaws/resources/ec2_rds_dbinstance.py b/nixopsaws/resources/ec2_rds_dbinstance.py
index 8ac3bdc9..026d0917 100644
--- a/nixopsaws/resources/ec2_rds_dbinstance.py
+++ b/nixopsaws/resources/ec2_rds_dbinstance.py
@@ -7,8 +7,11 @@
 import nixops.util
 import nixopsaws.ec2_utils
 import time
+import sys
 from uuid import uuid4
 
+# TODO: port this file to Boto3 using https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/rds.html
+
 class EC2RDSDbInstanceDefinition(nixops.resources.ResourceDefinition):
     """Definition of an EC2 RDS Database Instance."""
 
@@ -37,6 +40,14 @@ def __init__(self, xml):
             for sg_str in sg.findall("string"):
                sg_name =  sg_str.get("value")
                self.rds_dbinstance_security_groups.append(sg_name)
+        self.rds_dbinstance_vpc_security_group_ids = []
+        for sg in xml.findall("attrs/attr[@name='vpcSecurityGroupIds']/list"):
+            for sg_str in sg.findall("string"):
+                sg_name = sg_str.get("value")
+                self.rds_dbinstance_vpc_security_group_ids.append(sg_name)
+
+        self.rds_dbinstance_db_subnet_group = xml.find("attrs/attr[@name='dbSubnetGroup']/string").get("value")
+
         # TODO: implement remainder of boto.rds.RDSConnection.create_dbinstance parameters
 
         # common params
@@ -62,6 +73,8 @@ class EC2RDSDbInstanceState(nixops.resources.ResourceState):
     rds_dbinstance_endpoint = nixops.util.attr_property("ec2.rdsEndpoint", None)
     rds_dbinstance_multi_az = nixops.util.attr_property("ec2.multiAZ", False)
     rds_dbinstance_security_groups = nixops.util.attr_property("ec2.securityGroups", [], "json")
+    rds_dbinstance_db_subnet_group  = nixops.util.attr_property("ec2.rdsDbSubnetGroup", None)
+    rds_dbinstance_vpc_security_group_ids = nixops.util.attr_property("ec2.vpcSecurityGroupIds", [], "json")
 
     requires_reboot_attrs = ('rds_dbinstance_id', 'rds_dbinstance_allocated_storage',
         'rds_dbinstance_instance_class', 'rds_dbinstance_master_password')
@@ -91,7 +104,14 @@ def resource_id(self):
 
     def create_after(self, resources, defn):
         return {r for r in resources if
-                isinstance(r, nixopsaws.resources.ec2_rds_dbsecurity_group.EC2RDSDbSecurityGroupState)}
+                isinstance(r, nixopsaws.resources.ec2_rds_dbsecurity_group.EC2RDSDbSecurityGroupState)
+                or isinstance(r, nixopsaws.resources.ec2_rds_subnet_group.EC2RDSSubnetGroupState)
+                or isinstance(r, nixopsaws.resources.ec2_security_group.EC2SecurityGroupState)}
+
+    def destroy_before(self, resources):
+        return {r for r in resources if
+                isinstance(r, nixopsaws.resources.ec2_rds_subnet_group.EC2RDSSubnetGroupState)
+                or isinstance(r, nixopsaws.resources.ec2_security_group.EC2SecurityGroupState)}
 
     def _connect(self):
         if self._conn: return
@@ -137,6 +157,7 @@ def _try_fetch_dbinstance(self, instance_id):
     def _diff_defn(self, defn):
         attrs = ('region', 'rds_dbinstance_port', 'rds_dbinstance_engine', 'rds_dbinstance_multi_az',
             'rds_dbinstance_instance_class', 'rds_dbinstance_db_name', 'rds_dbinstance_master_username',
+            'rds_dbinstance_db_subnet_group', 'rds_dbinstance_vpc_security_group_ids',
             'rds_dbinstance_master_password', 'rds_dbinstance_allocated_storage', 'rds_dbinstance_security_groups')
 
         def get_state_attr(attr):
@@ -150,6 +171,10 @@ def get_state_attr(attr):
         def get_defn_attr(attr):
             if attr == 'rds_dbinstance_security_groups':
                 return self.fetch_security_group_resources(defn.rds_dbinstance_security_groups)
+            elif attr == 'rds_dbinstance_vpc_security_group_ids':
+                return self.fetch_vpc_security_group_resources(defn.rds_dbinstance_vpc_security_group_ids)
+            elif attr == 'rds_dbinstance_db_subnet_group':
+                return self.get_db_subnet_group_name(defn.rds_dbinstance_db_subnet_group)
             else:
                 return getattr(defn, attr)
 
@@ -170,7 +195,7 @@ def _wait_for_dbinstance(self, dbinstance, state='available'):
                 break
             time.sleep(6)
 
-    def _copy_dbinstance_attrs(self, dbinstance, security_groups):
+    def _copy_dbinstance_attrs(self, dbinstance, security_groups=None, vpc_security_group_ids=None):
         with self.depl._db:
             self.rds_dbinstance_id = dbinstance.id
             self.rds_dbinstance_allocated_storage = int(dbinstance.allocated_storage)
@@ -179,8 +204,10 @@ def _copy_dbinstance_attrs(self, dbinstance, security_groups):
             self.rds_dbinstance_engine = dbinstance.engine
             self.rds_dbinstance_multi_az = dbinstance.multi_az
             self.rds_dbinstance_port = int(dbinstance.endpoint[1])
-            self.rds_dbinstance_endpoint = "%s:%d"% dbinstance.endpoint
+            self.rds_dbinstance_endpoint = "%s:%d" % dbinstance.endpoint
+            self.rds_dbinstance_db_subnet_group = dbinstance.subnet_group.name
             self.rds_dbinstance_security_groups = security_groups
+            self.rds_dbinstance_vpc_security_group_ids = vpc_security_group_ids
 
     def _to_boto_kwargs(self, attrs):
         attr_to_kwarg = {
@@ -188,7 +215,9 @@ def _to_boto_kwargs(self, attrs):
             'rds_dbinstance_master_password': 'master_password',
             'rds_dbinstance_instance_class': 'instance_class',
             'rds_dbinstance_multi_az': 'multi_az',
-            'rds_dbinstance_security_groups': 'security_groups'
+            'rds_dbinstance_security_groups': 'security_groups',
+            'rds_dbinstance_db_subnet_group': 'db_subnet_group_name',
+            'rds_dbinstance_vpc_security_group_ids': 'vpc_security_groups'
         }
         return { attr_to_kwarg[attr] : attrs[attr] for attr in attrs.keys() }
 
@@ -206,6 +235,27 @@ def fetch_security_group_resources(self, config):
                 security_groups.append(sg)
         return security_groups
 
+    def get_db_subnet_group_name(self, group):
+        if isinstance(group, boto.rds.DBSubnetGroup):
+            return group.name
+
+        if group.startswith("res-"):
+            res = self.depl.get_typed_resource(group[4:].split(".")[0], "ec2-rds-subnet-group")
+            return res.resource_id
+        else:
+            return group
+
+    def fetch_vpc_security_group_resources(self, config):
+        vpc_sgs = []
+        for sg in config:
+            if sg.startswith("res-"):
+                res = self.depl.get_typed_resource(sg[4:].split('.')[0], "ec2-security-group")
+                vpc_sgs.append(res.security_group_id) # TODO: replace me by physical_spec/physical_id whatever.
+            else:
+                vpc_sgs.append(sg)
+
+        return vpc_sgs
+
     def create(self, defn, check, allow_reboot, allow_recreate):
         with self.depl._db:
             self.access_key_id = defn.access_key_id or nixopsaws.ec2_utils.get_access_key_id()
@@ -254,13 +304,20 @@ def create(self, defn, check, allow_reboot, allow_recreate):
                 if not dbinstance and (self.state == self.MISSING or self.state == self.UNKNOWN):
                     self.logger.log("creating RDS database instance ‘{0}’ (this may take a while)...".format(defn.rds_dbinstance_id))
                     # create a new dbinstance with desired config
-                    security_groups = self.fetch_security_group_resources(defn.rds_dbinstance_security_groups)
+                    extra = {}
+                    if defn.rds_dbinstance_security_groups:
+                       extra['security_groups'] = self.fetch_security_group_resources(defn.rds_dbinstance_security_groups)
+                    elif defn.rds_dbinstance_vpc_security_group_ids:
+                        extra['vpc_security_groups'] = self.fetch_vpc_security_group_resources(defn.rds_dbinstance_vpc_security_group_ids)
+                    if defn.rds_dbinstance_db_subnet_group:
+                        extra['db_subnet_group_name'] = self.get_db_subnet_group_name(defn.rds_dbinstance_db_subnet_group)
+
                     dbinstance = self._conn.create_dbinstance(defn.rds_dbinstance_id,
                         defn.rds_dbinstance_allocated_storage, defn.rds_dbinstance_instance_class,
                         defn.rds_dbinstance_master_username, defn.rds_dbinstance_master_password,
                         port=defn.rds_dbinstance_port, engine=defn.rds_dbinstance_engine,
                         db_name=defn.rds_dbinstance_db_name, multi_az=defn.rds_dbinstance_multi_az,
-                        security_groups=security_groups)
+                        **extra)
 
                     self.state = self.STARTING
                     self._wait_for_dbinstance(dbinstance)
@@ -269,7 +326,9 @@ def create(self, defn, check, allow_reboot, allow_recreate):
                 self.access_key_id = defn.access_key_id or nixopsaws.ec2_utils.get_access_key_id()
                 self.rds_dbinstance_db_name = defn.rds_dbinstance_db_name
                 self.rds_dbinstance_master_password = defn.rds_dbinstance_master_password
-                self._copy_dbinstance_attrs(dbinstance, defn.rds_dbinstance_security_groups)
+                self._copy_dbinstance_attrs(dbinstance,
+                        security_groups=defn.rds_dbinstance_security_groups,
+                        vpc_security_group_ids=defn.rds_dbinstance_vpc_security_group_ids)
                 self.state = self.UP
 
         with self.depl._db:
@@ -296,10 +355,12 @@ def create(self, defn, check, allow_reboot, allow_recreate):
                 # Ugly hack to prevent from waiting on state
                 # 'modifying' on sg change as that looks like it's an
                 # immediate change in RDS.
-                if not (len(boto_kwargs) == 2 and 'security_groups' in boto_kwargs):
+                if not (len(boto_kwargs) == 2 and ('security_groups' in boto_kwargs or 'vpc_security_groups' in boto_kwargs)):
                     self._wait_for_dbinstance(dbinstance, state="modifying")
                 self._wait_for_dbinstance(dbinstance)
-                self._copy_dbinstance_attrs(dbinstance, defn.rds_dbinstance_security_groups)
+                self._copy_dbinstance_attrs(dbinstance,
+                        security_groups=defn.rds_dbinstance_security_groups,
+                        vpc_security_group_ids=defn.rds_dbinstance_vpc_security_group_ids)
 
 
     def after_activation(self, defn):
@@ -329,6 +390,7 @@ def destroy(self, wipe=False):
                         break
                     self.log_continue("[{0}] ".format(dbinstance.status))
                     time.sleep(6)
+                    dbinstance.update(validate=False)
 
             else:
                 self.logger.log("RDS instance `{0}` does not exist, skipping.".format(self.rds_dbinstance_id))
diff --git a/nixopsaws/resources/ec2_rds_subnet_group.py b/nixopsaws/resources/ec2_rds_subnet_group.py
new file mode 100644
index 00000000..db50f0e2
--- /dev/null
+++ b/nixopsaws/resources/ec2_rds_subnet_group.py
@@ -0,0 +1,133 @@
+import boto3
+import botocore.exceptions
+import botocore.errorfactory
+
+import nixops.util
+import nixops.resources
+import nixopsaws.ec2_utils
+from nixopsaws.resources.ec2_common import EC2CommonState
+from nixops.diff import Diff, Handler
+
+class EC2RDSSubnetGroupDefinition(nixops.resources.ResourceDefinition):
+
+    @classmethod
+    def get_type(cls):
+        return "ec2-rds-subnet-group"
+
+    @classmethod
+    def get_resource_type(cls):
+        return "rdsDbSubnetGroups"
+
+    def show_type(self):
+        return "{0}".format(self.get_type())
+
+class EC2RDSSubnetGroupState(nixops.resources.DiffEngineResourceState, EC2CommonState):
+
+    state = nixops.util.attr_property("state", nixops.resources.ResourceState.MISSING, int)
+    access_key_id = nixops.util.attr_property("accessKeyId", None)
+    _reserved_keys = EC2CommonState.COMMON_EC2_RESERVED
+
+    @classmethod
+    def get_type(cls):
+        return "ec2-rds-subnet-group"
+
+    def __init__(self, depl, name, id):
+        nixops.resources.DiffEngineResourceState.__init__(self, depl, name, id)
+        self.subnetIds = self._state.get('subnetIds', [])
+        self.handle_create_rds_db_subnet = Handler(
+            ['name', 'region', 'description', 'subnetIds'],
+            handle=self.realize_create_subnet)
+
+    def show_type(self):
+        s = super(EC2RDSSubnetGroupState, self).show_type()
+        if self._state.get('region') is not None:
+            return "{0} [{1}]".format(s, self._state.get('region'))
+        else:
+            return s
+
+    @property
+    def resource_id(self):
+        return self._state.get('name', None)
+
+    def prefix_definition(self, attrs):
+        return {('resources', 'rdsDbSubnetGroups'): attrs}
+
+    def create_after(self, resources, defn):
+        return {r for r in resources
+                if isinstance(r, nixopsaws.resources.vpc_subnet.VPCSubnetState)}
+
+    def _check(self):
+        if self._state.get('name', None) is None:
+            return
+        if self.state == self.UP:
+            try:
+                response = self.get_client("rds").describe_db_subnet_groups(
+                        DBSubnetGroupName=self.resource_id)
+            except botocore.exceptions.ClientError as error:
+                if error.response['Error']['Code'] == 'DBSubnetGroupNotFound':
+                    self.warn("RDS db subnet group {} not found, performing destroy to sync the state ...".format(self._state['name']))
+                    self._destroy()
+                    return
+                else:
+                    raise error
+
+            subnetIds = []
+            for subnetGroup in response['DBSubnetGroups']:
+                subnetIds.extend((subnet['SubnetIdentifier'] for subnet in subnetGroup['Subnets']))
+
+            with self.depl._db:
+                self._state['subnetIds'] = subnetIds
+
+    def realize_create_subnet(self, allow_recreate):
+        config = self.get_defn()
+        if self.state == self.UP:
+            if not allow_recreate:
+                raise Exception(
+                    "RDS subnet group {} definition changed and it needs to be recreated"
+                    " use --allow-recreate if you want to create a new one"
+                    .format(self._state['name']))
+
+            self.warn("RDS subnet group definition changed, recreating ...")
+            self._destroy()
+            self.reset_client() # FIXME: ideally this should be detected automatically
+
+        self.log("creating RDS subnet group {}".format(config['name']))
+        self._state['region'] = config['region']
+        subnet_ids = config['subnetIds']
+        for idx, subnet_id in enumerate(subnet_ids):
+            if subnet_id.startswith("res-"):
+                res = self.depl.get_typed_resource(subnet_id[4:].split(".")[0], "vpc-subnet")
+                subnet_ids[idx] = res._state['subnetId']
+        self.get_client("rds").create_db_subnet_group(
+            DBSubnetGroupName=config['name'],
+            DBSubnetGroupDescription=config['description'],
+            SubnetIds=subnet_ids,
+            Tags=[{ 'Key': 'Generator', 'Value': 'Automatically generated by NixOps'}]) # FIXME: is it a good practice? shouldn't this go into the Nix definition file and be propagated through defaults here?
+        with self.depl._db:
+            self.state = self.UP
+            self._state['name'] = config['name']
+            self._state['description'] = config['description']
+            self._state['subnetIds'] = subnet_ids
+
+    def _destroy(self):
+        if self.state != self.UP:
+            return
+        self.log("destroying rds db subnet group {}".format(self.resource_id))
+        try:
+            self.get_client("rds").delete_db_subnet_group(DBSubnetGroupName=self.resource_id)
+        except botocore.exceptions.ClientError as error:
+            if error.response['Error']['Code'] == 'DBSubnetGroupNotFound':
+                self.warn("rds subnet group {} already deleted".format(self._state['name']))
+            else:
+                raise error
+
+        with self.depl._db:
+            self.state = self.MISSING
+            self._state['name'] = None
+            self._state['region'] = None
+            self._state['description'] = None
+            self._state['subnetIds'] = None
+
+    def destroy(self, wipe=True):
+        self._destroy()
+        return True

From 410217696efea8bdf272a13f2a9966bd747a511c Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Wed, 30 Oct 2019 20:14:20 +0100
Subject: [PATCH 2/9] example: Add RDS with VPC

---
 examples/ec2-rds-with-vpc.nix | 71 +++++++++++++++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 examples/ec2-rds-with-vpc.nix

diff --git a/examples/ec2-rds-with-vpc.nix b/examples/ec2-rds-with-vpc.nix
new file mode 100644
index 00000000..f44ab736
--- /dev/null
+++ b/examples/ec2-rds-with-vpc.nix
@@ -0,0 +1,71 @@
+let
+  region = "us-east-1";
+  accessKeyId = "AKIA...";
+in
+  {
+    network.description = "NixOps RDS in a VPC Testing";
+  # A VPC.
+  resources.vpc.private = {
+    inherit region accessKeyId;
+    enableDnsSupport = true;
+    enableDnsHostnames = true;
+    cidrBlock = "10.0.0.0/16";
+  };
+
+  # 2 VPC at least.
+  resources.vpcSubnets = {
+    db-a = { resources, ... }: {
+      inherit region accessKeyId;
+      vpcId=resources.vpc.private;
+      cidrBlock="10.0.0.0/19";
+      zone="us-east-1a";
+    };
+    db-b = { resources, ... }: {
+      inherit region accessKeyId;
+      vpcId=resources.vpc.private;
+      zone="us-east-1c";
+      cidrBlock="10.0.32.0/19";
+    };
+  };
+
+
+  resources.ec2SecurityGroups = {
+    database = { resources, lib, ... }:
+    {
+      inherit region accessKeyId;
+      vpcId = resources.vpc.private;
+      rules = [
+        {
+          sourceIp = "10.0.0.0/16";
+          fromPort = 5432;
+          toPort = 5432;
+        }
+      ];
+    };
+  };
+
+  resources.rdsDbSubnetGroups.db-subnet =
+    {resources, ...}:
+    {
+      inherit region accessKeyId;
+      description = "RDS test subnet";
+      subnetIds = (map (key: resources.vpcSubnets.${"db-" + key}) ["a" "b"]);
+    };
+
+    resources.rdsDbInstances.test-rds-instance =
+      { resources, ... }:
+      {
+        inherit region accessKeyId;
+        id = "test-multi-az";
+        instanceClass = "db.r3.large";
+        allocatedStorage = 30;
+        masterUsername = "administrator";
+        masterPassword = "testing123";
+        port = 5432;
+        engine = "postgres";
+        dbName = "testNixOps";
+        multiAZ = true;
+        vpcSecurityGroups = [ resources.ec2SecurityGroups.database ];
+        dbSubnetGroup = resources.rdsDbSubnetGroups.db-subnet.name;
+      };
+    }

From bb99cb93e776910bffc0d53fa796c165975d44ed Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Fri, 1 Nov 2019 17:31:31 +0100
Subject: [PATCH 3/9] PR: Address review

---
 examples/ec2-rds-with-vpc.nix |  2 +-
 nix/ec2-rds-dbinstance.nix    |  9 ++++++---
 nix/ec2-rds-subnet-group.nix  | 16 +++++-----------
 3 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/examples/ec2-rds-with-vpc.nix b/examples/ec2-rds-with-vpc.nix
index f44ab736..e179081d 100644
--- a/examples/ec2-rds-with-vpc.nix
+++ b/examples/ec2-rds-with-vpc.nix
@@ -65,7 +65,7 @@ in
         engine = "postgres";
         dbName = "testNixOps";
         multiAZ = true;
-        vpcSecurityGroups = [ resources.ec2SecurityGroups.database ];
+        vpcSecurityGroupIds = [ resources.ec2SecurityGroups.database ];
         dbSubnetGroup = resources.rdsDbSubnetGroups.db-subnet.name;
       };
     }
diff --git a/nix/ec2-rds-dbinstance.nix b/nix/ec2-rds-dbinstance.nix
index eb5023a3..663b640d 100644
--- a/nix/ec2-rds-dbinstance.nix
+++ b/nix/ec2-rds-dbinstance.nix
@@ -86,7 +86,8 @@ with import ./lib.nix lib;
       type = types.listOf (types.either types.str (resource "ec2-rds-security-group"));
       apply = map (x: if builtins.isString x then x else "res-" + x._name);
       description = ''
-        List of names of DBSecurityGroup to authorize on this DBInstance, use it if you are not a VPC or are using EC2-Classic.
+        List of names of DBSecurityGroup to authorize on this DBInstance.
+        Use it if you are not a VPC or are using EC2-Classic.
       '';
     };
 
@@ -95,7 +96,8 @@ with import ./lib.nix lib;
       type = types.nullOr (types.listOf (types.either types.str (resource "ec2-security-group")));
       apply = map (x: if builtins.isString x then x else "res-" + x._name);
       description = ''
-        List of names of VPCSecurityGroupMembership to authorize on this DBInstance, use this if you are in an VPC and not on EC2-Classic. Not applicable for Amazon Aurora.
+        List of names of VPCSecurityGroupMembership to authorize on this DBInstance.
+        Use this if you are in an VPC and not on EC2-Classic. Not applicable for Amazon Aurora.
         '';
     };
 
@@ -104,7 +106,8 @@ with import ./lib.nix lib;
       type = types.nullOr (types.either types.str (resource "ec2-rds-db-subnet-group"));
       apply = group: if (builtins.isString group || group == null) then group else "res-" + group._name;
       description = ''
-        A name for a DBSubnetGroup, they must contain at least one subnet in each availability zone in the AWS region.
+        A name for a DBSubnetGroup.
+        They must contain at least one subnet in each availability zone in the AWS region.
         '';
     };
 
diff --git a/nix/ec2-rds-subnet-group.nix b/nix/ec2-rds-subnet-group.nix
index 9903f6c8..645eca6c 100644
--- a/nix/ec2-rds-subnet-group.nix
+++ b/nix/ec2-rds-subnet-group.nix
@@ -20,22 +20,16 @@ with import ./lib.nix lib;
       ''; 
     };
 
-    region = mkOption {
-      type = types.str;
-      description = "Amazon RDS DB subnet group region.";
-    };
-
-    accessKeyId = mkOption {
-      default = "";
-      type = types.str;
-      description = "The AWS Access Key ID.";
-    };
+    imports = [ ./common-ec2-options.nix ];
 
     subnetIds = mkOption {
       default = [];
       type = types.listOf (types.either types.str (resource "vpc-subnet"));
       apply = map (x: if builtins.isString x then x else "res-" + x._name + "." + x._type + ".subnetId");
-      description = "List of VPC subnets to use for this DB subnet group (must contain at least a subnet for each AZ), it can be a VPC Subnet resource or a string representing the ID of the VPC Subnet.";
+      description = ''List of VPC subnets to use for this DB subnet group
+        (must contain at least a subnet for each AZ).
+        It can be a VPC Subnet resource or a string representing the ID of the VPC Subnet.
+      '';
     };
     
   };

From 0d34b022b2f2911d64bb00d08c6b9cdce74fe104 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Fri, 1 Nov 2019 17:32:44 +0100
Subject: [PATCH 4/9] PR: Add a RDS non-error state as per PsyanticY's remark

---
 nixopsaws/resources/ec2_rds_dbinstance.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixopsaws/resources/ec2_rds_dbinstance.py b/nixopsaws/resources/ec2_rds_dbinstance.py
index 026d0917..b3a9a6dd 100644
--- a/nixopsaws/resources/ec2_rds_dbinstance.py
+++ b/nixopsaws/resources/ec2_rds_dbinstance.py
@@ -189,7 +189,7 @@ def _wait_for_dbinstance(self, dbinstance, state='available'):
         while True:
             dbinstance.update()
             self.log_continue("[{0}] ".format(dbinstance.status))
-            if dbinstance.status not in {'creating', 'backing-up', 'available', 'modifying'}:
+            if dbinstance.status not in {'creating', 'backing-up', 'available', 'modifying', 'configuring-enhanced-monitoring'}:
                 raise Exception("RDS database instance ‘{0}’ in an error state (state is ‘{1}’)".format(dbinstance.id, dbinstance.status))
             if dbinstance.status == state:
                 break

From ce79966220240e4b61504342171611867eb87a9d Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Fri, 1 Nov 2019 20:14:11 +0100
Subject: [PATCH 5/9] PR: Fix import of common EC2 auth options & reindent the
 example

---
 examples/ec2-rds-with-vpc.nix | 48 ++++++++++++++++-------------------
 nix/ec2-rds-subnet-group.nix  |  3 +--
 2 files changed, 23 insertions(+), 28 deletions(-)

diff --git a/examples/ec2-rds-with-vpc.nix b/examples/ec2-rds-with-vpc.nix
index e179081d..be75863f 100644
--- a/examples/ec2-rds-with-vpc.nix
+++ b/examples/ec2-rds-with-vpc.nix
@@ -2,8 +2,8 @@ let
   region = "us-east-1";
   accessKeyId = "AKIA...";
 in
-  {
-    network.description = "NixOps RDS in a VPC Testing";
+{
+  network.description = "NixOps RDS in a VPC Testing";
   # A VPC.
   resources.vpc.private = {
     inherit region accessKeyId;
@@ -44,28 +44,24 @@ in
     };
   };
 
-  resources.rdsDbSubnetGroups.db-subnet =
-    {resources, ...}:
-    {
-      inherit region accessKeyId;
-      description = "RDS test subnet";
-      subnetIds = (map (key: resources.vpcSubnets.${"db-" + key}) ["a" "b"]);
-    };
+  resources.rdsDbSubnetGroups.db-subnet = {resources, ...}: {
+    inherit region accessKeyId;
+    description = "RDS test subnet";
+    subnetIds = (map (key: resources.vpcSubnets.${"db-" + key}) ["a" "b"]);
+  };
 
-    resources.rdsDbInstances.test-rds-instance =
-      { resources, ... }:
-      {
-        inherit region accessKeyId;
-        id = "test-multi-az";
-        instanceClass = "db.r3.large";
-        allocatedStorage = 30;
-        masterUsername = "administrator";
-        masterPassword = "testing123";
-        port = 5432;
-        engine = "postgres";
-        dbName = "testNixOps";
-        multiAZ = true;
-        vpcSecurityGroupIds = [ resources.ec2SecurityGroups.database ];
-        dbSubnetGroup = resources.rdsDbSubnetGroups.db-subnet.name;
-      };
-    }
+  resources.rdsDbInstances.test-rds-instance = { resources, ... }: {
+    inherit region accessKeyId;
+    id = "test-multi-az";
+    instanceClass = "db.r3.large";
+    allocatedStorage = 30;
+    masterUsername = "administrator";
+    masterPassword = "testing123";
+    port = 5432;
+    engine = "postgres";
+    dbName = "testNixOps";
+    multiAZ = true;
+    vpcSecurityGroupIds = [ resources.ec2SecurityGroups.database ];
+    dbSubnetGroup = resources.rdsDbSubnetGroups.db-subnet.name;
+  };
+}
diff --git a/nix/ec2-rds-subnet-group.nix b/nix/ec2-rds-subnet-group.nix
index 645eca6c..151517ca 100644
--- a/nix/ec2-rds-subnet-group.nix
+++ b/nix/ec2-rds-subnet-group.nix
@@ -4,6 +4,7 @@ with lib;
 with import ./lib.nix lib;
 
 {
+  imports = [ ./common-ec2-auth-options ];
   options = {
     name = mkOption {
       default = "nixops-${uuid}-${name}";
@@ -20,8 +21,6 @@ with import ./lib.nix lib;
       ''; 
     };
 
-    imports = [ ./common-ec2-options.nix ];
-
     subnetIds = mkOption {
       default = [];
       type = types.listOf (types.either types.str (resource "vpc-subnet"));

From 648249f403bad032c9678488df525f716ab6de32 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Fri, 1 Nov 2019 20:15:34 +0100
Subject: [PATCH 6/9] PR: Rewrite example's comments

---
 examples/ec2-rds-with-vpc.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/ec2-rds-with-vpc.nix b/examples/ec2-rds-with-vpc.nix
index be75863f..a66e7be7 100644
--- a/examples/ec2-rds-with-vpc.nix
+++ b/examples/ec2-rds-with-vpc.nix
@@ -4,7 +4,6 @@ let
 in
 {
   network.description = "NixOps RDS in a VPC Testing";
-  # A VPC.
   resources.vpc.private = {
     inherit region accessKeyId;
     enableDnsSupport = true;
@@ -12,7 +11,8 @@ in
     cidrBlock = "10.0.0.0/16";
   };
 
-  # 2 VPC at least.
+  # You cannot create a db subnet without at least 2 VPC subnets which spans across all AZ of your region.
+  # In this case, us-east-1a / us-east-1c for example.
   resources.vpcSubnets = {
     db-a = { resources, ... }: {
       inherit region accessKeyId;

From 9b2de6fecbba40f41b53fbbc5c9f7b7c60750c2c Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Fri, 1 Nov 2019 20:16:16 +0100
Subject: [PATCH 7/9] PR: Remove destroy_before hook

---
 nixopsaws/resources/ec2_rds_dbinstance.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/nixopsaws/resources/ec2_rds_dbinstance.py b/nixopsaws/resources/ec2_rds_dbinstance.py
index b3a9a6dd..692634ec 100644
--- a/nixopsaws/resources/ec2_rds_dbinstance.py
+++ b/nixopsaws/resources/ec2_rds_dbinstance.py
@@ -108,11 +108,6 @@ def create_after(self, resources, defn):
                 or isinstance(r, nixopsaws.resources.ec2_rds_subnet_group.EC2RDSSubnetGroupState)
                 or isinstance(r, nixopsaws.resources.ec2_security_group.EC2SecurityGroupState)}
 
-    def destroy_before(self, resources):
-        return {r for r in resources if
-                isinstance(r, nixopsaws.resources.ec2_rds_subnet_group.EC2RDSSubnetGroupState)
-                or isinstance(r, nixopsaws.resources.ec2_security_group.EC2SecurityGroupState)}
-
     def _connect(self):
         if self._conn: return
         (access_key_id, secret_access_key) = nixopsaws.ec2_utils.fetch_aws_secret_key(self.access_key_id)

From 2063833fd702d87688a2a5f054f1e9c2ad0ac60b Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Fri, 1 Nov 2019 20:58:48 +0100
Subject: [PATCH 8/9] PR: Post-test fixes

---
 nix/ec2-rds-dbinstance.nix   | 2 +-
 nix/ec2-rds-subnet-group.nix | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/nix/ec2-rds-dbinstance.nix b/nix/ec2-rds-dbinstance.nix
index 663b640d..f5da458b 100644
--- a/nix/ec2-rds-dbinstance.nix
+++ b/nix/ec2-rds-dbinstance.nix
@@ -82,7 +82,7 @@ with import ./lib.nix lib;
     };
 
     securityGroups = mkOption {
-      default = [ "default" ];
+      default = [];
       type = types.listOf (types.either types.str (resource "ec2-rds-security-group"));
       apply = map (x: if builtins.isString x then x else "res-" + x._name);
       description = ''
diff --git a/nix/ec2-rds-subnet-group.nix b/nix/ec2-rds-subnet-group.nix
index 151517ca..e00560dc 100644
--- a/nix/ec2-rds-subnet-group.nix
+++ b/nix/ec2-rds-subnet-group.nix
@@ -4,7 +4,8 @@ with lib;
 with import ./lib.nix lib;
 
 {
-  imports = [ ./common-ec2-auth-options ];
+  imports = [ ./common-ec2-auth-options.nix ];
+
   options = {
     name = mkOption {
       default = "nixops-${uuid}-${name}";
@@ -30,7 +31,6 @@ with import ./lib.nix lib;
         It can be a VPC Subnet resource or a string representing the ID of the VPC Subnet.
       '';
     };
-    
   };
 
   config._type = "ec2-rds-db-subnet-group";

From 7f112b7a76b87cffdb9e0d9e859ae51de394fda4 Mon Sep 17 00:00:00 2001
From: Raito Bezarius <masterancpp@gmail.com>
Date: Sun, 10 Nov 2019 16:42:12 +0100
Subject: [PATCH 9/9] Fix: default to [] for vpcSecurityGroupIds and do not
 assume db subnet group is available in XML description

---
 nix/ec2-rds-dbinstance.nix                | 2 +-
 nixopsaws/resources/ec2_rds_dbinstance.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/nix/ec2-rds-dbinstance.nix b/nix/ec2-rds-dbinstance.nix
index f5da458b..353d3795 100644
--- a/nix/ec2-rds-dbinstance.nix
+++ b/nix/ec2-rds-dbinstance.nix
@@ -92,7 +92,7 @@ with import ./lib.nix lib;
     };
 
     vpcSecurityGroupIds = mkOption {
-      default = null; # default to the default security group of the DB subnet.
+      default = []; # default to the default security group of the DB subnet.
       type = types.nullOr (types.listOf (types.either types.str (resource "ec2-security-group")));
       apply = map (x: if builtins.isString x then x else "res-" + x._name);
       description = ''
diff --git a/nixopsaws/resources/ec2_rds_dbinstance.py b/nixopsaws/resources/ec2_rds_dbinstance.py
index 692634ec..86d1fd02 100644
--- a/nixopsaws/resources/ec2_rds_dbinstance.py
+++ b/nixopsaws/resources/ec2_rds_dbinstance.py
@@ -46,7 +46,8 @@ def __init__(self, xml):
                 sg_name = sg_str.get("value")
                 self.rds_dbinstance_vpc_security_group_ids.append(sg_name)
 
-        self.rds_dbinstance_db_subnet_group = xml.find("attrs/attr[@name='dbSubnetGroup']/string").get("value")
+        db_subnet_group_node = xml.find("attrs/attr[@name='dbSubnetGroup']/string")
+        self.rds_dbinstance_db_subnet_group = db_subnet_group_node.get("value") if db_subnet_group_node is not None else None
 
         # TODO: implement remainder of boto.rds.RDSConnection.create_dbinstance parameters