diff --git a/the-basic-mq/README.md b/the-basic-mq/README.md index 3ccda648..b9351a47 100644 --- a/the-basic-mq/README.md +++ b/the-basic-mq/README.md @@ -144,3 +144,4 @@ then set the SOCKS Host to localhost and Port to 8162, leaving other fields empt ## Available Versions * [TypeScript](typescript/) + * [Python](python/) diff --git a/the-basic-mq/python/.gitignore b/the-basic-mq/python/.gitignore new file mode 100644 index 00000000..383cdd50 --- /dev/null +++ b/the-basic-mq/python/.gitignore @@ -0,0 +1,10 @@ +*.swp +package-lock.json +__pycache__ +.pytest_cache +.env +*.egg-info + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/the-basic-mq/python/README.md b/the-basic-mq/python/README.md new file mode 100644 index 00000000..28ecea4a --- /dev/null +++ b/the-basic-mq/python/README.md @@ -0,0 +1,203 @@ +# The Basic MQ + +![architecture](img/the-basic-mq-arch.png) + +This is an example cdk stack to deploy [static custom domain endpoints with Amazon MQ](https://aws.amazon.com/blogs/compute/creating-static-custom-domain-endpoints-with-amazon-mq/) from Rachel Richardson. + +In this example we have private Amazon MQ brokers behind an internet-facing network load balancer endpoint using a subdomain. + +## Pre Requirements + +This pattern requires you to have a Route53 Hosted Zone already in your account so that you can assign a public URL to your NLB. + +After you setup a hosted zone, you can get the id from the console and replace the value for 'hosted_zone_id' in the cdk stack. Then you need to replace 'zone_name' and 'subdomain_name' with the url you desire in that hosted zone. + +Also, you have to have a certificate for the subdomain before you deploy the stack (to avoid exceeding [ACM yearly certificate limit](https://github.com/aws/aws-cdk/issues/5889)). + +so if your hosted zone id is 1234 and you want your public url to be mq.cdkpatterns.com you would do: + +```python +# Paste Hosted zone ID from Route53 console 'Hosted zone details' +hosted_zone_id = '1234' + +# If zone_name = 'cdkexample.com' and subdomain_name = 'iot', you can connect to the broker by 'iot.cdkexample.com'. +zone_name = 'cdkpatterns.com' +subdomain_name = 'mq' + +# Request and issue a certificate for the subdomain (iot.cdkexample.com in this example) beforehand, and paste ARN. +cert_arn = 'arn:aws:acm:us-east-1:2XXXXXX9:certificate/exxxx8-xxxx-xxxx-xxxx-fxxxxxxxx9' +``` + +After you have replaced these values from a console you can do: + +```bash +cdk deploy +``` + +## Testing broker connectivity + +Once you deploy the stack, you can connect to the broker. +This time we will use [Amazon MQ workshop](https://github.com/aws-samples/amazon-mq-workshop) client application code from re:Invent 2018 +to simplify connectivity test. + +### Step 1. Create an environment in AWS Cloud9 + +Sign in to the AWS Cloud9 console and create an environment. You can leave settings as default. +After AWS Cloud9 creates your environment, you should see a bash shell window for the environment. + +### Step 2. Set up client application + +In the bash shell, clone the repo "amazon-mq-workshop" by running the following command: + +``` +git clone https://github.com/aws-samples/amazon-mq-workshop.git +``` + +Now the code is located on `~/environment/amazon-mq-workshop`. Next, type the following command one by one. + +``` +cd ~/environment/amazon-mq-workshop +./setup.sh +export temp_url="" +echo "url=\"$temp_url\"" >> ~/.bashrc; source ~/.bashrc +``` + +By doing so you can tell the client application where to connect. +Make sure you replace with `` something like `"failover:(ssl://mq.example.com:61617)"` +(the NLB endpoint subdomain you defined in CDK stack). + +### Step 3. Connect + +Run the producer and consumer clients in separate terminal windows. +Run the following command to start the sender: + +``` +java -jar ./bin/amazon-mq-client.jar -url $url -mode sender -type queue -destination workshop.queueA -name Sender-1 +``` + +Open the other terminal and run the following command to start the receiver: + +``` +java -jar ./bin/amazon-mq-client.jar -url $url -mode receiver -type queue -destination workshop.queueA +``` + +If the messages are sent and received successfully across the internet, a log output should be + +(sender) + +``` +ec2-user:~/environment/amazon-mq-workshop (master) $ java -jar ./bin/amazon-mq-client.jar -url $url -mode sender -type queue -destination workshop.queueA -name Sender-1 +[ActiveMQ Task-1] INFO org.apache.activemq.transport.failover.FailoverTransport - Successfully connected to ssl://mq.example.com:61617 +22.08.2020 01:17:53.958 - Sender: sent '[queue://workshop.queueA] [Sender-1] Message number 1' +22.08.2020 01:17:54.975 - Sender: sent '[queue://workshop.queueA] [Sender-1] Message number 2' +22.08.2020 01:17:55.990 - Sender: sent '[queue://workshop.queueA] [Sender-1] Message number 3' +22.08.2020 01:17:57.8 - Sender: sent '[queue://workshop.queueA] [Sender-1] Message number 4' +22.08.2020 01:17:58.27 - Sender: sent '[queue://workshop.queueA] [Sender-1] Message number 5' +``` + +(receiver) + +``` +ec2-user:~/environment/amazon-mq-workshop (master) $ java -jar ./bin/amazon-mq-client.jar -url $url -mode receiver -type queue -destination workshop.queueA +[ActiveMQ Task-1] INFO org.apache.activemq.transport.failover.FailoverTransport - Successfully connected to ssl://mq.example.com:61617 +22.08.2020 01:17:59.717 - Receiver: received '[queue://workshop.queueA] [Sender-1] Message number 1' +22.08.2020 01:17:59.718 - Receiver: received '[queue://workshop.queueA] [Sender-1] Message number 2' +22.08.2020 01:17:59.720 - Receiver: received '[queue://workshop.queueA] [Sender-1] Message number 3' +22.08.2020 01:17:59.721 - Receiver: received '[queue://workshop.queueA] [Sender-1] Message number 4' +22.08.2020 01:17:59.721 - Receiver: received '[queue://workshop.queueA] [Sender-1] Message number 5' +``` + +That's it. You can also check [Lab 4: Testing a Broker Fail-over](https://github.com/aws-samples/amazon-mq-workshop/blob/master/labs/lab-4.md) +to test this solution. + +## Logging into the broker’s ActiveMQ console from a browser + +Create a forwarding tunnel through an SSH connection to the bastion host. +First, you need to add a rule allowing SSH connection from your computer, to the security group which the bastion host belongs to (bastionToMQGroup). +You can retrieve bastionToMQGroup's security group ID and add the rule, and below is an example command in the terminal window. + +``` +SGID=`aws cloudformation describe-stacks --stack-name TheBasicMQStack --region us-east-1 --output json | \ +jq -r '.Stacks[0].Outputs[] | select (.OutputKey == "bastionToMQGroupSGID").OutputValue'` +aws ec2 authorize-security-group-ingress --group-id ${SGID} --protocol tcp --port 22 --cidr YOUR-IP-ADDRESS/32 +``` +Next, push an SSH public key to the bastion host. The key is valid for 60 seconds. + +``` +InstanceID=`aws cloudformation describe-stacks --stack-name TheBasicMQStack --region us-east-1 --output json | \ +jq -r '.Stacks[0].Outputs[] | select (.OutputKey == "bastionInstanceID").OutputValue'` +aws ec2-instance-connect send-ssh-public-key --instance-id ${InstanceID} --instance-os-user ec2-user --ssh-public-key 'file://~/.ssh/id_rsa.pub' --availability-zone us-east-1a +``` + +Finally, create a forwarding tunnel through an SSH connection to the bastion host. + +``` +InstancePublicDNS=`aws cloudformation describe-stacks --stack-name TheBasicMQStack --region us-east-1 --output json | \ +jq -r '.Stacks[0].Outputs[] | select (.OutputKey == "bastionPublicDNS").OutputValue'` +ssh -D 8162 -N -i ~/.ssh/id_rsa ec2-user@${InstancePublicDNS} +``` + +Now you are ready to view broker’s ActiveMQ console from a browser. +Open another window and run `aws mq describe-broker --broker-id myMQ` to get broker's endpoints. +Note only one broker host is active at a time. + +If you use Firefox, go to Firefox Connection Settings. +In the Configure Proxy Access to the Internet section, select Manual proxy configuration, +then set the SOCKS Host to localhost and Port to 8162, leaving other fields empty. +(See "Creating a forwarding tunnel" [here](https://aws.amazon.com/blogs/compute/creating-static-custom-domain-endpoints-with-amazon-mq/).) + +## Useful Commands + +The `cdk.json` file tells the CDK Toolkit how to execute your app. + +This project is set up like a standard Python project. The initialization +process also creates a virtualenv within this project, stored under the .env +directory. To create the virtualenv it assumes that there is a `python3` +(or `python` for Windows) executable in your path with access to the `venv` +package. If for any reason the automatic creation of the virtualenv fails, +you can create the virtualenv manually. + +To manually create a virtualenv on MacOS and Linux: + +``` +$ python3 -m venv .env +``` + +After the init process completes and the virtualenv is created, you can use the following +step to activate your virtualenv. + +``` +$ source .env/bin/activate +``` + +If you are a Windows platform, you would activate the virtualenv like this: + +``` +% .env\Scripts\activate.bat +``` + +Once the virtualenv is activated, you can install the required dependencies. + +``` +$ pip install -r requirements.txt +``` + +At this point you can now synthesize the CloudFormation template for this code. + +``` +$ cdk synth +``` + +To add additional dependencies, for example other CDK libraries, just add +them to your `setup.py` file and rerun the `pip install -r requirements.txt` +command. + +## commands + + * `cdk ls` list all stacks in the app + * `cdk synth` emits the synthesized CloudFormation template + * `cdk deploy` deploy this stack to your default AWS account/region + * `cdk diff` compare deployed stack with current state + * `cdk docs` open CDK documentation + +Enjoy! diff --git a/the-basic-mq/python/app.py b/the-basic-mq/python/app.py new file mode 100644 index 00000000..8e56d0e3 --- /dev/null +++ b/the-basic-mq/python/app.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 + +from aws_cdk import core + +from the_basic_mq.the_basic_mq_stack import TheBasicMQStack + + +app = core.App() +TheBasicMQStack(app, "TheBasicMQStack", env=core.Environment(region="us-east-1")) + +app.synth() diff --git a/the-basic-mq/python/cdk.json b/the-basic-mq/python/cdk.json new file mode 100644 index 00000000..39c301c4 --- /dev/null +++ b/the-basic-mq/python/cdk.json @@ -0,0 +1,7 @@ +{ + "app": "python3 app.py", + "context": { + "@aws-cdk/core:enableStackNameDuplicates": "true", + "aws-cdk:enableDiffNoFail": "true" + } +} diff --git a/the-basic-mq/python/requirements.txt b/the-basic-mq/python/requirements.txt new file mode 100644 index 00000000..d6e1198b --- /dev/null +++ b/the-basic-mq/python/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/the-basic-mq/python/setup.py b/the-basic-mq/python/setup.py new file mode 100644 index 00000000..0938375a --- /dev/null +++ b/the-basic-mq/python/setup.py @@ -0,0 +1,45 @@ +import setuptools + + +with open("README.md") as fp: + long_description = fp.read() + + +setuptools.setup( + name="python", + version="0.0.1", + + description="An empty CDK Python app", + long_description=long_description, + long_description_content_type="text/markdown", + + author="author", + + package_dir={"": "python"}, + packages=setuptools.find_packages(where="python"), + + install_requires=[ + "aws-cdk.core==1.51.0", + ], + + python_requires=">=3.6", + + classifiers=[ + "Development Status :: 4 - Beta", + + "Intended Audience :: Developers", + + "License :: OSI Approved :: Apache Software License", + + "Programming Language :: JavaScript", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + + "Topic :: Software Development :: Code Generators", + "Topic :: Utilities", + + "Typing :: Typed", + ], +) diff --git a/the-basic-mq/python/source.bat b/the-basic-mq/python/source.bat new file mode 100644 index 00000000..8f574429 --- /dev/null +++ b/the-basic-mq/python/source.bat @@ -0,0 +1,13 @@ +@echo off + +rem The sole purpose of this script is to make the command +rem +rem source .env/bin/activate +rem +rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. +rem On Windows, this command just runs this batch file (the argument is ignored). +rem +rem Now we don't need to document a Windows command for activating a virtualenv. + +echo Executing .env\Scripts\activate.bat for you +.env\Scripts\activate.bat diff --git a/the-basic-mq/python/the_basic_mq/__init__.py b/the-basic-mq/python/the_basic_mq/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/the-basic-mq/python/the_basic_mq/the_basic_mq_stack.py b/the-basic-mq/python/the_basic_mq/the_basic_mq_stack.py new file mode 100644 index 00000000..ca7b0d93 --- /dev/null +++ b/the-basic-mq/python/the_basic_mq/the_basic_mq_stack.py @@ -0,0 +1,156 @@ +from aws_cdk import ( + core, + aws_route53 as r53, + aws_route53_targets as r53_targets, + aws_certificatemanager as acm, + aws_ec2 as ec2, + aws_ssm as ssm, + aws_amazonmq as mq, + aws_elasticloadbalancingv2 as elb, + aws_elasticloadbalancingv2_targets as elb_targets, + custom_resources as cr, +) + +# Paste Hosted zone ID from Route53 console 'Hosted zone details' +hosted_zone_id = '1234' + +# If zone_name = 'cdkexample.com' and subdomain_name = 'iot', you can connect to the broker by 'iot.cdkexample.com'. +zone_name = 'cdkexample.com' +subdomain_name = 'iot' + +# Request and issue a certificate for the subdomain (iot.cdkexample.com in this example) beforehand, and paste ARN. +cert_arn = 'arn:aws:acm:us-east-1:228575038959:certificate/e1f36358-f619-4b73-ab08-f8a700cb0f69' +cidr = '10.0.0.0/16' + +# You may use MQTT protocol as well by changing this value to 8883. +broker_port = 61617 +mq_console_port = 8162 + + +class TheBasicMQStack(core.Stack): + + def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: + super().__init__(scope, id, **kwargs) + + zone = r53.HostedZone.from_hosted_zone_attributes(self, 'zone', + hosted_zone_id=hosted_zone_id, + zone_name=zone_name) + + # You may use acm.DnsValidatedCertificate to automate certificate provision. + # However, be careful of ACM yearly certificate limit. + # You will bump into the error after you destroy/deploy the stack over and over again. + # See https://github.com/aws/aws-cdk/issues/5889 for the details. + cert = acm.Certificate.from_certificate_arn(self, 'cert', certificate_arn=cert_arn) + + # MQ needs to be setup in a VPC + vpc = ec2.Vpc(self, 'vpc', + cidr=cidr, + max_azs=2, # Default is all AZs in the region + subnet_configuration=[ + ec2.SubnetConfiguration(name='vpc-public-subnet', + cidr_mask=24, + subnet_type=ec2.SubnetType.PUBLIC), + ec2.SubnetConfiguration(name='vpc-private-subnet', + cidr_mask=24, + subnet_type=ec2.SubnetType.PRIVATE) + ]) + + mq_group = ec2.SecurityGroup(self, 'mq_group', vpc=vpc) + bastion_to_mq_group = ec2.SecurityGroup(self, 'bastion_to_mq_group', vpc=vpc) + + mq_group.add_ingress_rule(peer=ec2.Peer.ipv4(cidr), + connection=ec2.Port.tcp(broker_port), + description='allow OpenWire communication within VPC') + mq_group.add_ingress_rule(peer=ec2.Peer.ipv4(cidr), + connection=ec2.Port.tcp(mq_console_port), + description='allow communication on ActiveMQ console port within VPC') + mq_group.add_ingress_rule(peer=mq_group, + connection=ec2.Port.all_tcp(), + description='allow communication from nlb and other brokers') + + # allow SSH to bastion from anywhere (for debugging) + # bastion_to_mq_group.add_ingress_rule(connection=ec2.Port.all_tcp()) + + core.CfnOutput(self, 'bastionToMQGroupSGID', value=bastion_to_mq_group.security_group_id) + + mq_username = 'admin' + mq_password = 'password1234' + + ssm.StringParameter(self, 'string_parameter', + parameter_name='MQBrokerUserPassword', + string_value='{username},{password}'.format(username=mq_username, password=mq_password)) + + mq_master = mq.CfnBroker.UserProperty(console_access=True, + username=mq_username, + password=mq_password) + + mq_instance = mq.CfnBroker(self, 'mq_instance', + auto_minor_version_upgrade=False, + broker_name='myMQ', + deployment_mode='ACTIVE_STANDBY_MULTI_AZ', + engine_type='ACTIVEMQ', + engine_version='5.15.12', + host_instance_type='mq.t3.micro', + publicly_accessible=False, + users=[mq_master], + subnet_ids=vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE).subnet_ids, + security_groups=[mq_group.security_group_id]) + + nlb_target_group = elb.NetworkTargetGroup(self, 'nlb_target', + vpc=vpc, + port=broker_port, + target_type=elb.TargetType.IP, + protocol=elb.Protocol.TLS, + health_check=elb.HealthCheck(enabled=True, + port=str(mq_console_port), + protocol=elb.Protocol.TCP)) + + # For now there is no way to retrieve private ip addresses of MQ broker instances from aws-amazonmq module. + mq_described = cr.AwsCustomResource(self, 'function', + policy=cr.AwsCustomResourcePolicy.from_sdk_calls( + resources=cr.AwsCustomResourcePolicy.ANY_RESOURCE), + on_create=cr.AwsSdkCall( + physical_resource_id=cr.PhysicalResourceId.of('function'), + service='MQ', + action='describeBroker', + parameters={'BrokerId': mq_instance.broker_name})) + + mq_described.node.add_dependency(mq_instance) + + # Adding private ip addresses of broker instances to target group one by one + for az in range(len(vpc.availability_zones)): + ip = mq_described.get_response_field('BrokerInstances.{az}.IpAddress'.format(az=az)) + nlb_target_group.add_target(elb_targets.IpTarget(ip_address=ip)) + + mq_nlb = elb.NetworkLoadBalancer(self, 'mq_nlb', + vpc=vpc, + internet_facing=True) + + mq_nlb.add_listener('listener', + port=broker_port, + protocol=elb.Protocol.TLS, + certificates=[elb.ListenerCertificate(certificate_arn=cert.certificate_arn)], + default_target_groups=[nlb_target_group]) + + r53.ARecord(self, 'alias_record', + zone=zone, + record_name=subdomain_name, + target=r53.RecordTarget.from_alias(r53_targets.LoadBalancerTarget(mq_nlb))) + + bastion = ec2.BastionHostLinux(self, 'bastion', + vpc=vpc, + instance_name='bastion', + subnet_selection=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), + security_group=bastion_to_mq_group) + + # Allow port forwarding + bastion.instance.add_user_data( + "sudo echo 'GatewayPorts yes' >> /etc/ssh/sshd_config", + "sudo service sshd restart", + ) + + core.CfnOutput(self, 'bastionInstanceID', value=bastion.instance_id) + + core.CfnOutput(self, 'bastionPublicDNS', value=bastion.instance_public_dns_name) + +