Setup Deployment Pipeline

continuous-delivery/pip-install-with-retry.py

@@ -0,0 +1,39 @@
+import time
+import sys
+import subprocess
+
+DOCS = """Given cmdline args, executes: python3 -m pip install [args...]
+Keeps retrying until the new version becomes available in pypi (or we time out)"""
+if len(sys.argv) < 2:
+    sys.exit(DOCS)
+
+RETRY_INTERVAL_SECS = 10
+GIVE_UP_AFTER_SECS = 60 * 15
+
+pip_install_args = [sys.executable, '-m', 'pip', 'install'] + sys.argv[1:]
+
+start_time = time.time()
+while True:
+    print(subprocess.list2cmdline(pip_install_args))
+    result = subprocess.run(pip_install_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+    stdout = result.stdout.decode().strip()
+    if stdout:
+        print(stdout)
+
+    if result.returncode == 0:
+        # success
+        sys.exit(0)
+
+    if "could not find a version" in stdout.lower():
+        elapsed_secs = time.time() - start_time
+        if elapsed_secs < GIVE_UP_AFTER_SECS:
+            # try again
+            print("Retrying in", RETRY_INTERVAL_SECS, "secs...")
+            time.sleep(RETRY_INTERVAL_SECS)
+            continue
+        else:
+            print("Giving up on retries after", int(elapsed_secs), "total secs.")
+
+    # fail
+    sys.exit(result.returncode)

continuous-delivery/publish_to_prod_pypi.yml

@@ -0,0 +1,25 @@
+version: 0.2
+# this image assumes Ubuntu 14.04 base image
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - export PATH=$PATH:$HOME/.local/bin
+      - python3 -m pip install --user --upgrade pip
+      - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six
+  pre_build:
+    commands:
+      - cd aws-iot-device-sdk-python
+      - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
+      - export PKG_VERSION=$(git describe --tags | cut -f2 -dv)
+      - echo "Updating package version to ${PKG_VERSION}"
+      - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py
+  build:
+    commands:
+      - echo Build started on `date`
+      - python3 setup.py sdist bdist_wheel --universal
+      - python3 -m twine upload -r pypi dist/*
+  post_build:
+    commands:
+      - echo Build completed on `date`

continuous-delivery/publish_to_test_pypi.yml

@@ -0,0 +1,25 @@
+version: 0.2
+# this image assumes Ubuntu 14.04 base image
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - export PATH=$PATH:$HOME/.local/bin
+      - python3 -m pip install --user --upgrade pip
+      - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six
+  pre_build:
+    commands:
+      - pypirc=$(aws secretsmanager get-secret-value --secret-id "alpha/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
+      - cd aws-iot-device-sdk-python
+      - export PKG_VERSION=$(git describe --tags | cut -f2 -dv)
+      - echo "Updating package version to ${PKG_VERSION}"
+      - sed --in-place -E "s/__version__ = \".+\"/__version__ = \"${PKG_VERSION}\"/" AWSIoTPythonSDK/__init__.py
+  build:
+    commands:
+      - echo Build started on `date`
+      - python3 setup_test.py sdist bdist_wheel --universal
+      - python3 -m twine upload -r testpypi dist/* --verbose
+  post_build:
+    commands:
+      - echo Build completed on `date`

continuous-delivery/test_prod_pypi.yml

@@ -0,0 +1,28 @@
+version: 0.2
+# this image assumes Ubuntu 14.04 base image
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - python3 -m pip install --upgrade pip
+      - python3 -m pip install --upgrade setuptools
+
+  pre_build:
+    commands:
+      - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem
+      - cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem
+      - key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem
+      - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+  build:
+    commands:
+      - echo Build started on `date`
+      - cd aws-iot-device-sdk-python
+      - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
+      - python3 continuous-delivery/pip-install-with-retry.py --no-cache-dir --user AWSIoTPythonSDK==$CURRENT_TAG_VERSION
+      - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only
+
+  post_build:
+    commands:
+      - echo Build completed on `date`
+

continuous-delivery/test_test_pypi.yml

@@ -0,0 +1,30 @@
+version: 0.2
+# this image assumes Ubuntu 14.04 base image
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - python3 -m pip install --upgrade pip
+      - python3 -m pip install --upgrade setuptools
+
+  pre_build:
+    commands:
+      - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem
+      - cert=$(aws secretsmanager get-secret-value --secret-id "unit-test/certificate" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem
+      - key=$(aws secretsmanager get-secret-value --secret-id "unit-test/privatekey" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem
+      - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+  build:
+    commands:
+      - echo Build started on `date`
+      - cd aws-iot-device-sdk-python
+      - CURRENT_TAG_VERSION=$(git describe --tags | cut -f2 -dv)
+      # this is here because typing isn't in testpypi, so pull it from prod instead
+      - python3 -m pip install typing
+      - python3 continuous-delivery/pip-install-with-retry.py -i https://testpypi.python.org/simple --user AWSIoTPythonSDK-V1==$CURRENT_TAG_VERSION
+      - python3 samples/greengrass/basicDiscovery.py -e ${ENDPOINT} -c /tmp/certificate.pem -k /tmp/privatekey.pem -r /tmp/AmazonRootCA1.pem --print_discover_resp_only
+
+  post_build:
+    commands:
+      - echo Build completed on `date`
+

continuous-delivery/test_version_exists

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -e
+set -x
+# force a failure if there's no tag
+git describe --tags
+# now get the tag
+CURRENT_TAG=$(git describe --tags | cut -f2 -dv)
+# convert v0.2.12-2-g50254a9 to 0.2.12
+CURRENT_TAG_VERSION=$(git describe --tags | cut -f1 -d'-' | cut -f2 -dv)
+# if there's a hash on the tag, then this is not a release tagged commit
+if [ "$CURRENT_TAG" != "$CURRENT_TAG_VERSION" ]; then
+    echo "Current tag version is not a release tag, cut a new release if you want to publish."
+    exit 1
+fi
+
+if python3 -m pip install --no-cache-dir -vvv AWSIoTPythonSDK==$CURRENT_TAG_VERSION; then
+    echo "$CURRENT_TAG_VERSION is already in pypi, cut a new tag if you want to upload another version."
+    exit 1
+fi
+
+echo "$CURRENT_TAG_VERSION currently does not exist in pypi, allowing pipeline to continue."
+exit 0

continuous-delivery/test_version_exists.yml

@@ -0,0 +1,21 @@
+version: 0.2
+#this build spec assumes the ubuntu 14.04 trusty image
+#this build run simply verifies we haven't published something at this tag yet.
+#if we have we fail the build and stop the pipeline, if we haven't we allow the pipeline to run.
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - pip3 install --upgrade setuptools
+  pre_build:
+    commands:
+      - echo Build start on `date`
+  build:
+    commands:
+      - cd aws-iot-device-sdk-python
+      - bash ./continuous-delivery/test_version_exists
+  post_build:
+    commands:
+      - echo Build completed on `date`
+

samples/greengrass/basicDiscovery.py

@@ -47,6 +47,8 @@ def customOnMessage(message):
                     help="Operation modes: %s"%str(AllowedActions))
 parser.add_argument("-M", "--message", action="store", dest="message", default="Hello World!",
                     help="Message to publish")
+#--print_discover_resp_only used for delopyment testing. The test run will return 0 as long as the SDK installed correctly.
+parser.add_argument("-p", "--print_discover_resp_only", action="store_true", dest="print_only", default=False)
 
 args = parser.parse_args()
 host = args.host
@@ -56,6 +58,7 @@ def customOnMessage(message):
 clientId = args.thingName
 thingName = args.thingName
 topic = args.topic
+print_only = args.print_only
 
 if args.mode not in AllowedActions:
     parser.error("Unknown --mode option %s. Must be one of %s" % (args.mode, str(AllowedActions)))
@@ -94,7 +97,7 @@ def customOnMessage(message):
 discoveryInfoProvider.configureCredentials(rootCAPath, certificatePath, privateKeyPath)
 discoveryInfoProvider.configureTimeout(10)  # 10 sec
 
-retryCount = MAX_DISCOVERY_RETRIES
+retryCount = MAX_DISCOVERY_RETRIES if not print_only else 1
 discovered = False
 groupCA = None
 coreInfo = None
@@ -136,6 +139,9 @@ def customOnMessage(message):
         backOffCore.backOff()
 
 if not discovered:
+    # With print_discover_resp_only flag, we only woud like to check if the API get called correctly. 
+    if print_only:
+        sys.exit(0)
     print("Discovery failed after %d retries. Exiting...\n" % (MAX_DISCOVERY_RETRIES))
     sys.exit(-1)
 

setup_test.py

@@ -0,0 +1,38 @@
+# For test deployment with package AWSIoTPythonSDK. The package name has already taken. Therefore we used an
+# alternative name for test pypi.
+# prod_pypi : AWSIoTPythonSDK
+# test_pypi : AWSIoTPythonSDK-V1
+import sys
+sys.path.insert(0, 'AWSIoTPythonSDK')
+import AWSIoTPythonSDK
+currentVersion = AWSIoTPythonSDK.__version__
+
+from distutils.core import setup
+setup(
+    name = 'AWSIoTPythonSDK-V1',
+    packages=['AWSIoTPythonSDK', 'AWSIoTPythonSDK.core',
+              'AWSIoTPythonSDK.core.util', 'AWSIoTPythonSDK.core.shadow', 'AWSIoTPythonSDK.core.protocol',
+              'AWSIoTPythonSDK.core.jobs',
+              'AWSIoTPythonSDK.core.protocol.paho', 'AWSIoTPythonSDK.core.protocol.internal',
+              'AWSIoTPythonSDK.core.protocol.connection', 'AWSIoTPythonSDK.core.greengrass',
+              'AWSIoTPythonSDK.core.greengrass.discovery', 'AWSIoTPythonSDK.exception'],
+    version = currentVersion,
+    description = 'SDK for connecting to AWS IoT using Python.',
+    author = 'Amazon Web Service',
+    author_email = '',
+    url = 'https://github.com/aws/aws-iot-device-sdk-python.git',
+    download_url = 'https://s3.amazonaws.com/aws-iot-device-sdk-python/aws-iot-device-sdk-python-latest.zip',
+    keywords = ['aws', 'iot', 'mqtt'],
+    classifiers = [
+        "Development Status :: 5 - Production/Stable",
+        "Intended Audience :: Developers",
+        "Natural Language :: English",
+        "License :: OSI Approved :: Apache Software License",
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5"
+    ]
+)